From f4dffd0be7acddae9530954a56baf04e20a3c895 Mon Sep 17 00:00:00 2001 From: Byron Wall Date: Sat, 21 Nov 2026 23:16:34 -0500 Subject: [PATCH 1/2] feat(dataflow): context-relay and broad-prop-bundle findings fileSignals turns the legacy report's context-relay pass into two finding rules. report.md counts them and the code viewer marks them under Prop Relay. Co-Authored-By: Claude Opus 5.5 --- client/src/dataflow/contract.generated.ts | 2 +- engine/src/srcly/contracts/file-signals.ts | 7 +++++- engine/src/srcly/ops/file-signals.ts | 25 ++++++++++++++++++- .../context-relay/src/notes/NoteHeader.tsx | 3 +++ .../context-relay/src/notes/NoteList.tsx | 5 ++++ .../context-relay/src/notes/NotesPage.tsx | 14 +++++++++++ .../context-relay/src/notes/notes-context.ts | 6 +++++ .../__fixtures__/context-relay/tsconfig.json | 12 +++++++++ engine/test/srcly/file-signals.test.ts | 12 +++++++++ .../contracts/fileSignals.schema.json | 4 ++- server/app/services/dataflow/models.py | 2 +- server/app/services/dataflow/signals.py | 10 ++++++-- server/app/services/reporting.py | 9 ++++--- server/tests/test_dataflow_signals.py | 2 ++ 14 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 engine/test/srcly/__fixtures__/context-relay/src/notes/NoteHeader.tsx create mode 100644 engine/test/srcly/__fixtures__/context-relay/src/notes/NoteList.tsx create mode 100644 engine/test/srcly/__fixtures__/context-relay/src/notes/NotesPage.tsx create mode 100644 engine/test/srcly/__fixtures__/context-relay/src/notes/notes-context.ts create mode 100644 engine/test/srcly/__fixtures__/context-relay/tsconfig.json diff --git a/client/src/dataflow/contract.generated.ts b/client/src/dataflow/contract.generated.ts index 9352447d..f74f8570 100644 --- a/client/src/dataflow/contract.generated.ts +++ b/client/src/dataflow/contract.generated.ts @@ -613,7 +613,7 @@ export type FileSignalsResult = { }; }[]; findings: { - rule: "impossible-defense" | "repeated-fork"; + rule: "impossible-defense" | "repeated-fork" | "context-relay" | "broad-prop-bundle"; file: string; line: number; column: number; diff --git a/engine/src/srcly/contracts/file-signals.ts b/engine/src/srcly/contracts/file-signals.ts index fa931aa0..6399a047 100644 --- a/engine/src/srcly/contracts/file-signals.ts +++ b/engine/src/srcly/contracts/file-signals.ts @@ -8,7 +8,12 @@ import { generationFields, outcomeSchema } from "./primitives"; * comes from one legacy report (`fullReport()`), so it is slow: minutes on a * big app. Only a batch caller asks for it. */ -export const dataflowRuleSchema = z.enum(["impossible-defense", "repeated-fork"]); +/** + * `context-relay`: a parent that reads context passes context-like props (`theme`, + * `user`, ...) to a same-feature child. `broad-prop-bundle`: it passes three or more + * plain props to one. Both come from the legacy report's context-relay pass. + */ +export const dataflowRuleSchema = z.enum(["impossible-defense", "repeated-fork", "context-relay", "broad-prop-bundle"]); export const dataflowFindingSchema = z.strictObject({ rule: dataflowRuleSchema, diff --git a/engine/src/srcly/ops/file-signals.ts b/engine/src/srcly/ops/file-signals.ts index 37c6a631..6fbda913 100644 --- a/engine/src/srcly/ops/file-signals.ts +++ b/engine/src/srcly/ops/file-signals.ts @@ -25,7 +25,7 @@ export function fileSignalsOp(session: EngineSession, params: { expectedGenerati tier1.inputs.files.map((file) => [rel(file), { impossibleDefenses: 0, relayHops: 0, fanOut: 0, representationChanges: 0, pathDepth: 0 }]), ); const { findings: defenses, unplaced } = impossibleDefenses(tier1, report); - const findings: DataflowFinding[] = [...defenses, ...repeatedForks(report)]; + const findings: DataflowFinding[] = [...defenses, ...repeatedForks(report), ...contextRelays(report)]; for (const finding of defenses) { const metrics = files.get(finding.file); if (metrics) metrics.impossibleDefenses += 1; @@ -132,6 +132,29 @@ function repeatedForks(report: AnalysisReport): DataflowFinding[] { }); } +/** + * One finding per JSX site where a parent that reads context hands a bundle of + * props to a child in the same feature: `context-relay` when the props carry + * shared, context-like names, `broad-prop-bundle` when there are only many. + */ +function contextRelays(report: AnalysisReport): DataflowFinding[] { + return (report.contextRelay ?? []).map((relay) => { + const hooks = relay.contextHooks.map((hook) => `\`${hook}\``).join(", "); + const shared = relay.sharedProps.length > 0; + const named = (shared ? relay.sharedProps : relay.props).map((prop) => `\`${prop}\``).join(", "); + return { + rule: shared ? "context-relay" : "broad-prop-bundle", + file: relay.parentFile, + line: relay.line, + column: relay.column, + endLine: relay.line, + message: shared + ? `Relays ${named} to \`${relay.childComponent}\` (${relay.childFile}) while this file reads ${hooks}. Let the child read them from context.` + : `Passes ${relay.props.length} props (${named}) to \`${relay.childComponent}\` (${relay.childFile}) while this file reads ${hooks}. Pass one model or read shared values from context.`, + }; + }); +} + function oneLine(text: string) { const flat = text.replace(/\s+/g, " ").trim(); return flat.length > 120 ? `${flat.slice(0, 117)}…` : flat; diff --git a/engine/test/srcly/__fixtures__/context-relay/src/notes/NoteHeader.tsx b/engine/test/srcly/__fixtures__/context-relay/src/notes/NoteHeader.tsx new file mode 100644 index 00000000..b4550bb7 --- /dev/null +++ b/engine/test/srcly/__fixtures__/context-relay/src/notes/NoteHeader.tsx @@ -0,0 +1,3 @@ +export function NoteHeader(props: { name: string; count: number; owner: string }) { + return

{props.name} {props.count} {props.owner}

; +} diff --git a/engine/test/srcly/__fixtures__/context-relay/src/notes/NoteList.tsx b/engine/test/srcly/__fixtures__/context-relay/src/notes/NoteList.tsx new file mode 100644 index 00000000..bcd4b4c4 --- /dev/null +++ b/engine/test/srcly/__fixtures__/context-relay/src/notes/NoteList.tsx @@ -0,0 +1,5 @@ +import type { NoteSettings, Workspace } from "./notes-context"; + +export function NoteList(props: { workspace: Workspace; settings: NoteSettings }) { + return ; +} diff --git a/engine/test/srcly/__fixtures__/context-relay/src/notes/NotesPage.tsx b/engine/test/srcly/__fixtures__/context-relay/src/notes/NotesPage.tsx new file mode 100644 index 00000000..b6456874 --- /dev/null +++ b/engine/test/srcly/__fixtures__/context-relay/src/notes/NotesPage.tsx @@ -0,0 +1,14 @@ +import { useWorkspace } from "./notes-context"; +import { NoteHeader } from "./NoteHeader"; +import { NoteList } from "./NoteList"; + +export function NotesPage() { + const workspace = useWorkspace(); + const settings = { wrap: true }; + return ( +
+ + +
+ ); +} diff --git a/engine/test/srcly/__fixtures__/context-relay/src/notes/notes-context.ts b/engine/test/srcly/__fixtures__/context-relay/src/notes/notes-context.ts new file mode 100644 index 00000000..0f8e8e2a --- /dev/null +++ b/engine/test/srcly/__fixtures__/context-relay/src/notes/notes-context.ts @@ -0,0 +1,6 @@ +export type Workspace = { id: string; name: string }; +export type NoteSettings = { wrap: boolean }; + +export function useWorkspace(): Workspace { + return { id: "w1", name: "Notes" }; +} diff --git a/engine/test/srcly/__fixtures__/context-relay/tsconfig.json b/engine/test/srcly/__fixtures__/context-relay/tsconfig.json new file mode 100644 index 00000000..b1cc3cb0 --- /dev/null +++ b/engine/test/srcly/__fixtures__/context-relay/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "bundler", + "jsx": "preserve", + "strict": true, + "noEmit": true, + "skipLibCheck": true + }, + "include": ["src"] +} diff --git a/engine/test/srcly/file-signals.test.ts b/engine/test/srcly/file-signals.test.ts index dff5bbed..61801fa8 100644 --- a/engine/test/srcly/file-signals.test.ts +++ b/engine/test/srcly/file-signals.test.ts @@ -42,6 +42,18 @@ describe("fileSignals carries tsx-data-flow's reporting into srcly (M7)", () => expect(forks[0].message).toContain("`props.type` at 4 sibling branch sites (lines 37, 38, 45, 48)"); }); + it("reports a parent that relays context-like props, or a broad prop bundle, to a same-feature child", () => { + const root = fileURLToPath(new URL("./__fixtures__/context-relay", import.meta.url)); + const source = path.join(root, "src"); + const relay = fileSignalsOp(createEngineSession({ root, source, tsconfig: null, typescriptFrom: chooseTypescript(root, source).from }), {}); + expect(relay.findings.map((f) => `${f.rule} ${f.file}:${f.line}`)).toEqual([ + "broad-prop-bundle src/notes/NotesPage.tsx:10", + "context-relay src/notes/NotesPage.tsx:11", + ]); + expect(relay.findings[1].message).toContain("Relays `workspace`, `settings` to `NoteList`"); + expect(result.findings.some((f) => f.rule === "context-relay" || f.rule === "broad-prop-bundle")).toBe(false); + }); + it("measures every program file, the worst render path per file", () => { expect(result.files.map((f) => f.file)).toEqual(["src/Chart.tsx", "src/DashboardShell.tsx", "src/types.ts", "src/view-models.ts"]); const shell = result.files.find((f) => f.file === "src/DashboardShell.tsx")!.metrics; diff --git a/server/app/services/dataflow/contracts/fileSignals.schema.json b/server/app/services/dataflow/contracts/fileSignals.schema.json index 9864b8a1..5e8fd46a 100644 --- a/server/app/services/dataflow/contracts/fileSignals.schema.json +++ b/server/app/services/dataflow/contracts/fileSignals.schema.json @@ -102,7 +102,9 @@ "type": "string", "enum": [ "impossible-defense", - "repeated-fork" + "repeated-fork", + "context-relay", + "broad-prop-bundle" ] }, "file": { diff --git a/server/app/services/dataflow/models.py b/server/app/services/dataflow/models.py index b9523902..648072c9 100644 --- a/server/app/services/dataflow/models.py +++ b/server/app/services/dataflow/models.py @@ -730,7 +730,7 @@ class AnchorGraphResponse(BaseModel): class DataflowFinding(BaseModel): - rule: Literal["impossible-defense", "repeated-fork"] + rule: Literal["impossible-defense", "repeated-fork", "context-relay", "broad-prop-bundle"] file: str line: int column: int diff --git a/server/app/services/dataflow/signals.py b/server/app/services/dataflow/signals.py index da70286e..338cfbd4 100644 --- a/server/app/services/dataflow/signals.py +++ b/server/app/services/dataflow/signals.py @@ -36,7 +36,13 @@ "dataflow_path_depth": ("pathDepth", "max"), } REPEATED_FORKS = "dataflow_repeated_forks" -RULE_METRIC = {"impossible-defense": "dataflow_impossible_defenses", "repeated-fork": REPEATED_FORKS} +# Rule → the metric whose code-viewer marks show it. Relay findings mark under Prop Relay. +RULE_METRIC = { + "impossible-defense": "dataflow_impossible_defenses", + "repeated-fork": REPEATED_FORKS, + "context-relay": "dataflow_relay_hops", + "broad-prop-bundle": "dataflow_relay_hops", +} @dataclass @@ -249,7 +255,7 @@ def visit(node: dict[str, Any]) -> Optional[dict[str, int]]: def markers_for(path: str, signals: Optional[Signals]) -> dict[str, list[dict]]: - """Code viewer lines behind the impossible-defense and repeated-fork counts.""" + """Code viewer lines behind each finding: the dead guards, repeated forks and prop relays.""" if signals is None or signals.status != "ok": return {} out: dict[str, list[dict]] = {} diff --git a/server/app/services/reporting.py b/server/app/services/reporting.py index fd7eea7f..4dcbdc88 100644 --- a/server/app/services/reporting.py +++ b/server/app/services/reporting.py @@ -667,10 +667,13 @@ def render_dataflow_section(signals: Any, root_path: Path, limit: int = 15) -> s if signals.status != "ok" or signals.result is None: return "\n".join([*lines, f"Not analyzed: {signals.reason}", ""]) result = signals.result - counts = {rule: sum(1 for f in result.findings if f.rule == rule) for rule in ("impossible-defense", "repeated-fork")} + rules = ("impossible-defense", "repeated-fork", "context-relay", "broad-prop-bundle") + counts = {rule: sum(1 for f in result.findings if f.rule == rule) for rule in rules} + relays = counts["context-relay"] + counts["broad-prop-bundle"] lines.append( - f"The TypeScript checker found {counts['impossible-defense']} type-impossible `??`/`?.` guards and " - f"{counts['repeated-fork']} repeated {'fork' if counts['repeated-fork'] == 1 else 'forks'}. Read `dataflow.md` for ranked work packets, candidate edits, " + f"The TypeScript checker found {counts['impossible-defense']} type-impossible `??`/`?.` guards, " + f"{counts['repeated-fork']} repeated {'fork' if counts['repeated-fork'] == 1 else 'forks'} and " + f"{relays} prop {'relay' if relays == 1 else 'relays'} to same-feature children. Read `dataflow.md` for ranked work packets, candidate edits, " "the risk queue and the stop recommendation." ) if result.unplacedDefenses: diff --git a/server/tests/test_dataflow_signals.py b/server/tests/test_dataflow_signals.py index 8db97ef3..be674109 100644 --- a/server/tests/test_dataflow_signals.py +++ b/server/tests/test_dataflow_signals.py @@ -35,6 +35,7 @@ def _result() -> FileSignalsResult: {"rule": "impossible-defense", "file": "src/a.tsx", "line": 8, "column": 5, "endLine": 8, "message": "`y ?? 2` is dead"}, {"rule": "repeated-fork", "file": "src/a.tsx", "line": 10, "column": 3, "endLine": 14, "message": "forks on props.kind"}, {"rule": "impossible-defense", "file": "src/b.ts", "line": 1, "column": 1, "endLine": 1, "message": "`z?.w` is dead"}, + {"rule": "context-relay", "file": "src/a.tsx", "line": 20, "column": 7, "endLine": 20, "message": "Relays `workspace`"}, ], "unplacedDefenses": 0, "markdown": "# Data flow\n", @@ -84,6 +85,7 @@ def test_marks_match_the_counts(): assert len(marks["dataflow_impossible_defenses"]) == values["dataflow_impossible_defenses"] assert len(marks["dataflow_repeated_forks"]) == values["dataflow_repeated_forks"] assert marks["dataflow_repeated_forks"][0] == {"line": 10, "end_line": 14, "note": "dataflow repeated-fork: forks on props.kind"} + assert [m["line"] for m in marks["dataflow_relay_hops"]] == [20] def test_report_dataflow_writes_dataflow_md(tmp_path: Path, monkeypatch): From afc4358b92a9bb10f128f3151ff65e688ce5d650 Mon Sep 17 00:00:00 2001 From: Byron Wall Date: Sat, 21 Nov 2026 23:22:08 -0500 Subject: [PATCH 2/2] docs: retire tsx-dataflow-integration M7 is complete, so the plan retires. Its design rules, decisions and measurements move to docs/dataflow-engine.md. The four data-flow finding rules are in the skill and site. Links into the plan point at 4727bd56. The in-app tier-2 trigger is a backlog task. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +- docs/CODE_MAP.md | 4 +- docs/dataflow-coverage.md | 1 + docs/dataflow-engine.md | 91 ++ docs/initiative-history.json | 20 + .../dataflow-robustness/decision-log.md | 2 +- .../implementation-plan.md | 4 +- .../dataflow-robustness/initiative-map.json | 4 +- docs/intent/dataflow-robustness/research.md | 2 +- .../tsx-dataflow-integration/detailed-plan.md | 682 -------------- .../implementation-plan.md | 186 ---- .../initiative-map.json | 840 ------------------ .../tsx-dataflow-integration/intent-brief.md | 70 -- .../tsx-dataflow-integration/measurements.md | 229 ----- .../tsx-dataflow-integration/research.md | 67 -- .../tsx-dataflow-integration/shape-brief.md | 64 -- .../README.md | 2 +- engine/scripts/bundle.mjs | 4 +- ...-b42af349-7ec7-4573-aa83-c6fb23dfc94f.json | 15 + ...rt-the-data-flow-batch-run-from-the-app.md | 17 + site/src/docs/pages/data-flow.md | 2 +- site/src/docs/pages/design.md | 2 +- site/src/docs/pages/findings.md | 2 +- skills/srcly-code-quality/SKILL.md | 4 +- 24 files changed, 163 insertions(+), 2155 deletions(-) create mode 100644 docs/dataflow-engine.md delete mode 100644 docs/intent/tsx-dataflow-integration/detailed-plan.md delete mode 100644 docs/intent/tsx-dataflow-integration/implementation-plan.md delete mode 100644 docs/intent/tsx-dataflow-integration/initiative-map.json delete mode 100644 docs/intent/tsx-dataflow-integration/intent-brief.md delete mode 100644 docs/intent/tsx-dataflow-integration/measurements.md delete mode 100644 docs/intent/tsx-dataflow-integration/research.md delete mode 100644 docs/intent/tsx-dataflow-integration/shape-brief.md create mode 100644 pgm/data/projects/srcly-backlog/history/20261122T042055895Z-0000-b42af349-7ec7-4573-aa83-c6fb23dfc94f.json create mode 100644 pgm/data/projects/srcly-backlog/nodes/task-start-the-data-flow-batch-run-from-the-app.md diff --git a/AGENTS.md b/AGENTS.md index 157f9ea9..25d66b1a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,7 +44,7 @@ Open work lives in a Product Grid project in `pgm/data` (project `srcly-backlog` ## Engine -`engine/` is the data-flow engine: tsx-data-flow's analysis behind a stdio protocol, a standalone pnpm package with its own lockfile. Provenance is in `engine/UPSTREAM.md`; the plan is `docs/intent/tsx-dataflow-integration/detailed-plan.md`. +`engine/` is the data-flow engine: tsx-data-flow's analysis behind a stdio protocol, a standalone pnpm package with its own lockfile. Provenance is in `engine/UPSTREAM.md`; the design rules and decisions are in `docs/dataflow-engine.md`. - Install and check: `cd engine && pnpm install && pnpm test && pnpm typecheck && pnpm lint`. - Run it from the server in a checkout with `SRCLY_ENGINE_DEV=1` (`dev.sh` and `scripts/dev-server.sh api` set it). The server runs `engine/bin/srcly-engine.ts` through the engine's own `tsx`, ahead of any bundle. @@ -65,7 +65,7 @@ Open work lives in a Product Grid project in `pgm/data` (project `srcly-backlog` - After merging, `survey-main` diffs each push to `main` against its first parent and compares that with the merged PR's own survey, so two PRs that move traces only together surface. It fails and keeps one issue, `Data-flow gate: unacknowledged moves on main`, up to date, and closes it on the next clean push. - Open a data-flow PR, then run `gh pr merge --squash --auto` and stop: auto-merge lands it when the checks pass. Don't run your own multi-repo surveys to prove a rule unless it targets code the gate's corpus doesn't contain. - A new relation, element or proof kind gets one entry in `engine/src/srcly/kinds.ts` (`RELATION_TRAITS`, `ELEMENT_TRAITS`, `PROOF_TRAITS`); the engine does not build until it has one. Rules read its traits (`structural`, `crossCall`, `terminal`, a proof's `strength`), never their own lists of kind names. -- Never edit `engine/src/analysis` to recognise srcly-specific code. The only planned analysis edits are M5's four invocation-identity changes, each with a negative fixture. +- Never edit `engine/src/analysis` to recognise srcly-specific code. An analysis edit that a rule needs comes with a negative fixture. ## Client Package Management diff --git a/docs/CODE_MAP.md b/docs/CODE_MAP.md index 494c7a67..4e04c298 100644 --- a/docs/CODE_MAP.md +++ b/docs/CODE_MAP.md @@ -420,7 +420,7 @@ Graph legibility rules (keep these when you change the layout or canvas): ### Data flow (engine) -The tsx-data-flow analysis lives in `engine/`, a standalone Node package with its own lockfile. Plan: `docs/intent/tsx-dataflow-integration/detailed-plan.md`. +The tsx-data-flow analysis lives in `engine/`, a standalone Node package with its own lockfile. Design rules and decisions: `docs/dataflow-engine.md`. | Feature | Where | | --- | --- | @@ -446,7 +446,7 @@ The tsx-data-flow analysis lives in `engine/`, a standalone Node package with it | `/api/dataflow/*` endpoints | `server/app/routers/dataflow.py` | | Engine `functionEffects` (M6, T14): a walk over the callable's body names each effect as a fact with a witness (families external, input, write, throw, timer, promise; activation direct, after-await, timer, latent), follows first-party callees the checker resolves (memoized, bounded, cycle-safe, aliases followed) and merges their facts behind a `via` chain, names what it cannot follow as frontiers (callback-parameter, dynamic-dispatch, unresolved-callee, recursion, budget), and says how each promise-yielding call is handled; identity from the checker, never a name; TypeScript lib intrinsics taken as pure unless named | `engine/src/analysis/effects/function-effects.ts`, `engine/src/srcly/ops/function-effects.ts`, contract `engine/src/srcly/contracts/function-effects.ts`; fixture `engine/examples/effects-basics/src/cases.ts` (21 cases) and `engine/test/srcly/function-effects.test.ts`; Python `models.py:FunctionEffectsResult` | | Explain an edge (M4, T16): `routePath {route, source?, edge}` maps the field attachment's, frontier's or handoff bridge's `evidencePathElementIds` through the slice into ledger rows (input, transfer, transform, branch, terminal, boundary at a frontier), never inventing a step; the client selects a handoff or field edge by click and the inspector lists its steps, each opening its line | `engine/src/srcly/ops/route-path.ts`, contract `engine/src/srcly/contracts/route-path.ts`, `engine/test/srcly/route-path.test.ts`; client `DependencyCanvas.tsx` (`onSelectEdge`), `FlowView.tsx` (`selectedEdge`, `edgePath`), `FlowInspector.tsx` (`EdgeStepsView`) | -| Data-flow signals (M7): tier-2 `fileSignals` op builds the legacy render-path report once and returns per-file metrics (impossible defenses, relay hops, fan-out, shape changes, path depth), `impossible-defense` and `repeated-fork` finding spans (a defense is placed in the file on its sink's path whose text matches) and `dataflow.md` (the old CLI's work-packets, defensive-ledger, repeated-forks, prop-relay, context-relay and fan-out views, 8 items each); `srcly report --dataflow` and `srcly compare --dataflow` run it; the last result per root is saved, and the app lays the six `dataflow_*` metrics over the tree and marks the code viewer from it while the fingerprint (tree) or file mtime (marks) still match | `engine/src/srcly/ops/file-signals.ts`, `engine/src/srcly/contracts/file-signals.ts`; `server/app/services/dataflow/signals.py` (`collect`, `apply_to_tree`, `markers_for`, `latest_for_file`); `reporting.py` (`scan_tree(dataflow=)`, `render_dataflow_section`); `compare/snapshots.py` (`with_dataflow`); overlay in `scan_jobs.py:_overlay`, marks in `routers/files.py:_dataflow_markers`; catalog group `Data flow` in `metric_catalog.py` | +| Data-flow signals: tier-2 `fileSignals` op builds the legacy render-path report once and returns per-file metrics (impossible defenses, relay hops, fan-out, shape changes, path depth), `impossible-defense`, `repeated-fork`, `context-relay` and `broad-prop-bundle` finding spans (relay rules from the report's context-relay pass; a defense is placed in the file on its sink's path whose text matches) and `dataflow.md` (the old CLI's work-packets, defensive-ledger, repeated-forks, prop-relay, context-relay and fan-out views, 8 items each); `srcly report --dataflow` and `srcly compare --dataflow` run it; the last result per root is saved, and the app lays the six `dataflow_*` metrics over the tree and marks the code viewer from it while the fingerprint (tree) or file mtime (marks) still match | `engine/src/srcly/ops/file-signals.ts`, `engine/src/srcly/contracts/file-signals.ts`; `server/app/services/dataflow/signals.py` (`collect`, `apply_to_tree`, `markers_for`, `latest_for_file`); `reporting.py` (`scan_tree(dataflow=)`, `render_dataflow_section`); `compare/snapshots.py` (`with_dataflow`); overlay in `scan_jobs.py:_overlay`, marks in `routers/files.py:_dataflow_markers`; catalog group `Data flow` in `metric_catalog.py` | | Effects panel (M6, T15): Known facts by family and the places the analysis could not follow together, activation and via chain per fact, how each promise is handled, assumptions one click away, a witness row opens its line; in the code viewer's rail under the findings for the selected scope (else the function enclosing the opened line) and in the data-flow inspector for a selected box's function in anchor mode | `client/src/dataflow/EffectsPanel.tsx` (`createEffectsQuery`), pure wording in `client/src/dataflow/effectsModel.ts` (tested on the fake engine's shapes); hosts `CodeModal.tsx` → `MetricsSidebar.tsx` (`effects` slot) and `viz/deps/flow/FlowInspector.tsx` | | Call identity (T13): the slice walk's invocation stack pairs a callee's return with the call the walk came in by (`pairInvocation`); a first-party callee is walked through its binding, never the bare `argument` link; the ops trust that pairing: a walk leaves a callee only through the call it entered by, and one that started inside it leaves through every call; an `argument` stays at its own call, and only the cross-calls the slice does not pair (callbacks, accessors) keep R7's one-call-site boundary (`facts.ts:needsOneCallSite`, `pairedInSlice`) | `engine/src/analysis/evidence-slice.ts`, `engine/src/srcly/facts.ts`, `engine/src/srcly/ops/{anchor-graph,value-flow}.ts`; fixtures `examples/value-flow-basics/src/{two-call-identity-helper,returns-other-argument}.tsx` | | Field paths: the slice walk carries which field of a value it is after (`fields`, k-limited like the invocation stack). Backward a field read pushes its field and an object pack is entered only through that field (or a spread); forward a packed field pushes it and only a read of that field pops it. A reference, binding or return keeps the path; any other step widens it to the whole value, as does an element reached again with another path. An object literal's `fields` attribute names each property's field by its start, where its `pack-field` relation points; a store path setter's literal keys (`setStore("left", "title", v)`) pack its value field by field into the state, and past a computed key or a predicate the value is the whole field there | `engine/src/analysis/evidence-slice-fields.ts` (`stepFieldPath`), `evidence-slice.ts`, `program-evidence-collector-expression-support.ts` (`objectPackAttributes`); fixture `examples/value-flow-basics/src/field-paths.tsx` | diff --git a/docs/dataflow-coverage.md b/docs/dataflow-coverage.md index 27348b86..c7f94014 100644 --- a/docs/dataflow-coverage.md +++ b/docs/dataflow-coverage.md @@ -118,6 +118,7 @@ These came from the Python data-flow usability audit (`docs/research/python-data ## Engine structure +- Design rules, decisions, measurements and the batch evidence behind reports and the treemap (`fileSignals`): `docs/dataflow-engine.md`. - `flowSignature` reduces each function of a file to line-free keys (origins, ends, origin→end pairs, parameter reads, effects, stops, status). `srcly compare --dataflow` diffs them as "Data-flow changes". Python has no parameter reads or effects (not measured), and a caller of a changed function is not re-traced. - Every evidence kind is classified once in `engine/src/srcly/kinds.ts` (#200). Rules read traits, not lists of kind names. diff --git a/docs/dataflow-engine.md b/docs/dataflow-engine.md new file mode 100644 index 00000000..40251623 --- /dev/null +++ b/docs/dataflow-engine.md @@ -0,0 +1,91 @@ +# Data-flow engine: design + +srcly's data-flow features come from one engine: the tsx-data-flow analysis, moved into `engine/` and run as one lazy Node child per scan root. This page keeps the design rules and the decisions behind them. For where each file lives, read the "Data flow (engine)" section of `docs/CODE_MAP.md`. For what each rule covers, read `docs/dataflow-coverage.md`. For commands, read the "Engine" section of `AGENTS.md`. + +## Shape + +```text +engine/ Node 18+, TypeScript, Zod (the analysis, moved intact) + bin/srcly-engine.ts stdio JSON lines: {id, op, params} -> {id, ok, result | error}, progress frames + src/srcly/session.ts tier 1 session; tier 2 report on demand + src/srcly/ops/*.ts one file per op + src/srcly/contracts/*.ts Zod schemas, the source of truth for every payload +server/app/services/dataflow/ Python owns the child: spawn, reply matching, timeouts, kill +server/app/routers/dataflow.py /api/dataflow/* +client/src/dataflow/ generated contract types, API calls, outcome labels +``` + +Three surfaces read the engine: the route graph in the dependency view, the value ledger from the code viewer, and the effects panel. srcly's batch pipelines (metrics, findings, marks, reports, compare) read it through `fileSignals` and `flowSignature`. + +## Invariants + +1. **The base scan never waits.** The engine starts on first explicit use: Data flow mode, Inspect value flow, the effects panel, or a `--dataflow` CLI flag. The treemap, explorer and import graph are the same with or without Node. +2. **Evidence is bound to a generation.** Every result carries `generation` and `programHash`. A query with an older `expectedGeneration` gets `stale`, and the client does not draw stale rows over current source. +3. **Not analyzed is never 0.** Each surface shows the query outcome by name. A metric the engine did not measure is `None` with an `na_reason`. + +## Session tiers + +The legacy entry point built the full legacy report before it answered anything: 130 s on srcly's `client/`. So the session has two tiers that share one Program. + +- **Tier 1** builds the primary Program and a lazy evidence provider, and finds routes. It answers `routes`, `routeGraph`, `routePath`, `selectExpression`, `valueFlow`, `functionEffects`, `flowSignature` and the anchor ops. `routes` never calls `buildRouteDataInventory`, which alone costs 17.5 s on visual-notes. +- **Tier 2** is the legacy report (`fullReport()`), built only by `fileSignals`. It is slow, so only a batch caller asks for it: `srcly report --dataflow` and `srcly compare --dataflow`. +- `programHash` hashes the sorted `(file, size, mtime)` of the Program's source files, every governing tsconfig, the TypeScript version and the engine version. `status` recomputes it with `stat` calls only. A rebuild bumps `generation`. +- Refresh drops both tiers. There is no incremental invalidation. + +## Outcomes and positions + +Each reply has one outcome: `ok`, `partial` (stops at named frontiers), `not-analyzed` (with a reason: no Node, no config, crash, timeout, memory limit), `unsupported`, `stale`, `cancelled` or `budget` (with counts). Each node, edge and ledger row has an evidence status: `proven`, `partial` or `unsupported`, with the engine's typed reason. Nothing turns these into a score or a percentage. + +All contracts use 1-based lines, 1-based UTF-16 columns and exclusive end columns. Python and the client never convert them. Tree-sitter byte offsets (srcly's own markers) are never mixed with engine spans. + +## One contract, three languages + +The Zod schemas in `engine/src/srcly/contracts/` are the source of truth. `pnpm emit-contracts` writes JSON Schema for Python and `client/src/dataflow/contract.generated.ts`. The Pydantic models are hand-written, and a test checks their fields against the schema. `contracts-current.test.ts` fails when the emitted files drift. Python refuses a child whose protocol version differs; there is no coexistence layer. + +## Containment + +- The child reads only files that the Program owns under the root. Python checks that every `file` parameter resolves under the scan root before it reaches the child. +- The child runs in its own process group and is killed as a group. Its argv is fixed: `node`, the entry path and fixed flags. There is no shell. +- The engine never runs target code. It only parses and type-checks it. + +## Batch evidence (`fileSignals`) + +`fileSignals` builds the tier 2 report once and returns: + +- per TS/TSX file: impossible defenses, relay hops, fan-out, shape changes and path depth. Python adds repeated forks from the findings. These are the six `dataflow_*` catalog metrics (`server/app/services/dataflow/signals.py`), overlay metrics like findings and activity. +- findings, one per site: + - `impossible-defense`: a `??` or `?.` whose guarded type is never null or undefined; + - `repeated-fork`: a component that tests one discriminant at several sibling branch sites; + - `context-relay`: a parent that reads a context hook and passes context-like props (`settings`, `selection`, `on*`, …) to a child in the same feature; + - `broad-prop-bundle`: the same, with three or more plain props. +- `dataflow.md`: the legacy CLI's work packets, defensive ledger, repeated forks, prop relay, context relay and fan-out views. + +The last result per root is saved on disk. The app colors the treemap and marks code-viewer lines from it while the file is unchanged. The app never starts tier 2 itself. + +## Decisions + +| Decision | Choice | What would change it | +| --- | --- | --- | +| TypeScript instance | The bundled TypeScript; the project's copy only when its major.minor matches | A real project that does not parse with the bundled version | +| Tier 2 | Opt-in through `--dataflow` only | Tier 2 under 10 s on a 50K-line app | +| Process | Resident child, closed after 15 idle minutes, at most two alive | Tier 1 under 2 s (per-query spawn), or RSS above 3 GB (a cap of one) | +| Back and Return | A view-state registry (`client/src/navigation/viewState.ts`): snapshots in `sessionStorage` under a token in `history.state`, so a copied URL opens plain | Nothing expected | +| Progress | The client polls `status` every 250 ms | Polling that shows up in profiles (then an SSE endpoint) | + +These were dropped from the plan: + +- `dataflow_defensive_ops`: the legacy count is per sink, and one guard feeds many sinks, so a sum per file counts a guard many times. `dataflow.md`'s defensive ledger lists the guards. +- A `findings/dataflow.py` tool: data-flow findings go to `report.md` and code-viewer marks from the saved `fileSignals` result. They are not in the file-findings panel. + +## Measurements + +Development Mac, Node 22, TypeScript 5.9.3, one cold run each (2026-10-23 and 2026-10-24). + +| Target | Tier 1 | Full legacy report | RSS | +| --- | ---: | ---: | ---: | +| srcly `client/` (26K lines, 126 files) | 3.0 s | 130 s | 2.4 GB | +| visual-notes `app/` (65K lines, 487 files, 43 routes) | 13.9 s | 44 s | 2.0 GB | + +- Cold `hello` is about 0.6 s. A warm `valueFlow` on srcly's client is 1 ms at p50 and 3 ms at p95. +- After the backbone-first work, the first drawn route box on visual-notes `/tasks` takes about 11 s cold and 3.4 s on reopen from the result cache. +- Memory is the binding limit for a resident child. `SRCLY_DATAFLOW_MEMORY_MB` and `pnpm memory-probe` (see `AGENTS.md`) keep it measured. diff --git a/docs/initiative-history.json b/docs/initiative-history.json index a7dfd1ed..7029d94f 100644 --- a/docs/initiative-history.json +++ b/docs/initiative-history.json @@ -213,5 +213,25 @@ ], "revision": "0a1c1133", "follow_ups": [] + }, + { + "title": "Bring tsx-data-flow into srcly", + "slug": "tsx-dataflow-integration", + "outcome": "completed", + "retired_at": "2026-11-21", + "summary": "The tsx-data-flow analysis lives in engine/ as one lazy Node child per scan root. It feeds the route graph, the value ledger with Return, the effects panel, data-flow metrics, findings and code-viewer marks, srcly report --dataflow (dataflow.md work packets) and compare --dataflow, and it ships bundled in the wheel. The strict-tsconfig refusal became nullProof=false. Reconciled against origin/main 4727bd56 plus the retirement PR's context-relay and broad-prop-bundle findings.", + "reason": "All eight milestones shipped. M7 closed on 2026-11-21 (#398, #399, #400, #406 and the retirement PR). Byron approved retirement.", + "documentation": [ + "docs/dataflow-engine.md", + "docs/dataflow-coverage.md", + "docs/CODE_MAP.md", + "AGENTS.md", + "skills/srcly-code-quality/SKILL.md" + ], + "revision": "4727bd56", + "follow_ups": [ + "pgm task-start-the-data-flow-batch-run-from-the-app: start the tier-2 run from the app. Today only the CLI starts it.", + "Dropped: dataflow_defensive_ops (a per-sink count double-counts shared guards) and a findings/dataflow.py tool (findings reach report.md and marks from the saved result)." + ] } ] diff --git a/docs/intent/dataflow-robustness/decision-log.md b/docs/intent/dataflow-robustness/decision-log.md index 2d3b5a2e..f62ea5c7 100644 --- a/docs/intent/dataflow-robustness/decision-log.md +++ b/docs/intent/dataflow-robustness/decision-log.md @@ -8,7 +8,7 @@ last_updated: "2026-10-24" # Decision log -Each decision stays open only to the evidence named in its last column. Decisions R1–R12 came from three rounds of questions to the maintainer on 2026-10-24. The rest are planning defaults. The earlier initiative's D1–D8 (`../tsx-dataflow-integration/detailed-plan.md` §8) still stand unless one is superseded here. +Each decision stays open only to the evidence named in its last column. Decisions R1–R12 came from three rounds of questions to the maintainer on 2026-10-24. The rest are planning defaults. The earlier initiative's D1–D8 (`https://github.com/byronwall/srcly/blob/4727bd56/docs/intent/tsx-dataflow-integration/detailed-plan.md` §8) still stand unless one is superseded here. | Id | Decision | Accepted | Evidence that would reopen it | | --- | --- | --- | --- | diff --git a/docs/intent/dataflow-robustness/implementation-plan.md b/docs/intent/dataflow-robustness/implementation-plan.md index 255c2fd6..4a05ed97 100644 --- a/docs/intent/dataflow-robustness/implementation-plan.md +++ b/docs/intent/dataflow-robustness/implementation-plan.md @@ -27,7 +27,7 @@ Every ticket is one PR stacked on the one before it, starting from `claude/dataf 1. Fixture contract tests. 2. The fake engine in every outcome shape. 3. The real engine on solid-route in Playwright. - 4. Local measurement on srcly `client/` and visual-notes `app/`, recorded in `../tsx-dataflow-integration/measurements.md`. + 4. Local measurement on srcly `client/` and visual-notes `app/`, recorded in `https://github.com/byronwall/srcly/blob/4727bd56/docs/intent/tsx-dataflow-integration/measurements.md`. - **Rollback:** one revert per ticket. Performance tickets sit behind `SRCLY_DATAFLOW_*` switches until two measurements agree. ## Ticket graph @@ -213,7 +213,7 @@ T25 `UPSTREAM.md` as the fork ledger (with T4), and T26 the registry under many ## Status on 2026-10-24: where the stack stands -Done, each as one PR in the stack (`claude/dataflow-t1-box-status` → … → `claude/dataflow-t20-multiple-anchors`): T1, T2, T3, T5, T6, T7, T8, T9, T10, T11, T12 (measured, decided off), T13, T14, T15, T16, T17, T18, T19, T20, T21, T22. Every ticket's backlog entry says what landed and where it was measured (`docs/intent/tsx-dataflow-integration/measurements.md`). +Done, each as one PR in the stack (`claude/dataflow-t1-box-status` → … → `claude/dataflow-t20-multiple-anchors`): T1, T2, T3, T5, T6, T7, T8, T9, T10, T11, T12 (measured, decided off), T13, T14, T15, T16, T17, T18, T19, T20, T21, T22. Every ticket's backlog entry says what landed and where it was measured (`https://github.com/byronwall/srcly/blob/4727bd56/docs/intent/tsx-dataflow-integration/measurements.md`). Not done, and why they can wait: diff --git a/docs/intent/dataflow-robustness/initiative-map.json b/docs/intent/dataflow-robustness/initiative-map.json index 37740b1b..e1ec8f03 100644 --- a/docs/intent/dataflow-robustness/initiative-map.json +++ b/docs/intent/dataflow-robustness/initiative-map.json @@ -58,14 +58,14 @@ { "id": "S2", "type": "doc", - "location": "docs/intent/tsx-dataflow-integration/detailed-plan.md", + "location": "https://github.com/byronwall/srcly/blob/4727bd56/docs/intent/tsx-dataflow-integration/detailed-plan.md", "anchor": "sections 6, 8, 11 (M4-M8, D1-D8, stop rules)", "authority": "repo" }, { "id": "S3", "type": "doc", - "location": "docs/intent/tsx-dataflow-integration/measurements.md", + "location": "https://github.com/byronwall/srcly/blob/4727bd56/docs/intent/tsx-dataflow-integration/measurements.md", "anchor": "M1-M3 tables", "authority": "repo" }, diff --git a/docs/intent/dataflow-robustness/research.md b/docs/intent/dataflow-robustness/research.md index 44101e7e..ab7c5686 100644 --- a/docs/intent/dataflow-robustness/research.md +++ b/docs/intent/dataflow-robustness/research.md @@ -8,7 +8,7 @@ last_updated: "2026-10-24" # Research record -Baseline: the stack top `claude/dataflow-anchor-menus` (`b164b3c`). The source claims were verified by reading the code. The numbers come from `docs/intent/tsx-dataflow-integration/measurements.md`. Nothing new was run for this page. +Baseline: the stack top `claude/dataflow-anchor-menus` (`b164b3c`). The source claims were verified by reading the code. The numbers come from `https://github.com/byronwall/srcly/blob/4727bd56/docs/intent/tsx-dataflow-integration/measurements.md`. Nothing new was run for this page. ## Verified source facts diff --git a/docs/intent/tsx-dataflow-integration/detailed-plan.md b/docs/intent/tsx-dataflow-integration/detailed-plan.md deleted file mode 100644 index 6651f1ad..00000000 --- a/docs/intent/tsx-dataflow-integration/detailed-plan.md +++ /dev/null @@ -1,682 +0,0 @@ ---- -title: "Bring tsx-data-flow into srcly — detailed plan" -slug: "tsx-dataflow-integration" -phase: plan -status: current -last_updated: "2026-10-23" ---- - -# Bring tsx-data-flow into srcly — detailed plan - -**Status:** proposed. Implementation starts only on explicit approval. - -This page is the implementation-level companion to [implementation-plan.md](implementation-plan.md). It keeps the same milestones (M1 to M8) and the same priorities (route graph, ledger, effects, supporting evidence). It adds what an implementer needs: the exact files, contracts, sequencing, tests, and the corrections that reading both codebases forced on the outline plan. - -Everything below was checked against srcly at `b3117e3` (this branch) and tsx-data-flow at `b4f1ba4` plus its uncommitted route-proof edits, on 2026-10-23. Two measurements were taken; everything else is read from source. - -## 1. Review of the outline plan - -The outline plan has the right shape. Its problems are in the places where it assumed something about one codebase that is not true. Each item below changes a milestone, and each is picked up in that milestone's section. - -### 1.1 Corrections that change the work - -| # | Outline assumption | What the code says | Consequence | -| --- | --- | --- | --- | -| R1 | "Add `engine/` to the pnpm workspace." | srcly has no root `package.json` or workspace. `client/` is a standalone package with its own `pnpm-lock.yaml`. tsx-data-flow's `pnpm-workspace.yaml` is also not a workspace (it only holds `allowBuilds`). | `engine/` is a second standalone package beside `client/`, with its own lockfile. CI (`release.yml`) gets an engine install, test and bundle step. See M1 and M8. | -| R2 | "Land tsx-data-flow PRs 1–4 and its local route-proof edits first." | There are seven open PRs (1 to 7), a linear dead-code cleanup stack of about −10.5K lines. The uncommitted edits are 49 files, +2738/−692, and overlap the stack heavily. | Decide the port base before copying (decision D1). Default: commit the route-proof edits on `main`, port that, and drop the cleanup stack. Its frontend deletions become moot once `src/frontend` is left behind, and its engine deletions (about 3.2K lines) can be redone inside `engine/` with the audit script the stack also adds. | -| R3 | "A non-strict tsconfig is a named error." | `resolveProjectConfigs` only warns on a loose config and prefers strict ones. Nothing refuses. | The refusal is new code in the engine's session layer (M1). It must also handle solution files: srcly's own `client/tsconfig.json` is a solution file that points at `tsconfig.app.json`. The existing expansion handles that. | -| R4 | The route graph is proved on `examples/solid-route` and on `/time-blocks`. | Routes are discovered by path convention only (`/routes/` in the path, `route-discovery.ts`). srcly's own client has no routes, so Data flow mode is N/A on srcly itself. `/time-blocks` is `visual-notes/app/src/routes/time-blocks.tsx`: 18 routes, about 65K lines, strict, `~/*` paths. | Name both targets in M2. `visual-notes/app` is the realistic proof, including the per-config program routing that `paths` triggers. The outline's "render srcly's `client/`" is only valid for the ledger and effects tracks. | -| R5 | "Cold ≤ 10 s" with the engine's existing cache. | `createAnalysisCache.ensureBuilt()` builds the Program, then the full legacy report (`analyzer.report()`), then a review report, before answering anything. A full report on srcly's `client/` (26K lines) took 133 s wall. `analyzeRouteData` runs inside `buildReport`, and the totality session is registered in a `WeakMap` keyed by that report. | M1 adds a two-tier session. Tier 1 builds Program, identity index, lazy evidence provider and route discovery, and answers route, value and effects queries. Tier 2 builds the legacy report only for M7's batch signals. The measured split is in section 9. This is composition code, not analysis changes, so "the analysis moves intact" still holds. | -| R6 | One shared enum: proven, partial, not analyzed, unsupported, stale, cancelled, budget. | The engine has three vocabularies: `EvidenceStatus` (proven, partial, unsupported) on elements and relations; `complete / partial / unavailable` on totality and lineage; and typed frontier and gap reasons. | Keep two layers (section 3.4): an *evidence status* per node, edge and row, and a *query outcome* per response. Folding them into one enum would lose the frontier reasons the UI needs. | -| R7 | "Call identity is kept. Two calls to one helper never merge." (must be true) | `argument-binding` and `function-return` relations are shared per callee, and the slice walk has no invocation stack. Two call sites can join today. | Until M5 lands, the ledger must refuse to walk through a callee with more than one call site and emit a frontier row (`unproven-handoff`) instead. That rule is specified in M3 so M3 never shows a false proof. | -| R8 | "Inspect value flow on a selection in the code viewer." | The code viewer never reads text selections. Source is injected as Shiki HTML (`innerHTML`), common indentation is stripped by default (`reduceCommonIndent`), and a selected scope renders only a slice of lines. DOM columns are not source columns. | M3 specifies the mapping from a DOM `Range` to a source span (line, UTF-16 column) through the `span.line` index, the display slice offset and the per-line indent that was removed. The engine resolves the span to AST candidates, so an imprecise selection is recoverable through a picker instead of a wrong answer. | -| R9 | "Return restores filters, metric, expansion, selection and camera." | Camera lives only in d3's zoom state on the SVG, `CanvasApi` has no get/set transform, every new layout refits, graph selection is a local signal, the metric is a context signal, graph filters are a module signal, and Explorer state is local to `Explorer.tsx`. Nothing captures them. | M3 adds a capture/restore registry (`navigation/viewState.ts`) that each owner joins, plus `CanvasApi.getTransform / setTransform` and a fit-suppression flag. Without this, Return cannot be exact. | -| R10 | Data-flow metrics "follow the AGENTS.md metric rules". | Those rules are for analyzer-computed metrics copied in `attach_file_metrics`. The engine runs beside the scan, so its metrics must be *overlay* metrics like activity and findings (`OVERLAY_METRICS`, `result_json_overlaid`, `apply_to_tree`). Two catalog tests pin today's witnessed set and profile leads. | M7 adds a `dataflow` overlay family and updates those two tests deliberately. | -| R11 | Zod contracts in the engine, Pydantic in Python, TS types in the client. | The client has no `zod`, and nothing links the three. | Generate from the Zod source of truth: JSON Schema for Python validation and a `.generated.ts` types file for the client, checked by a currency test like the catalog snapshot (section 7.1). | -| R12 | Bundle the engine with `esbuild`. | The engine resolves the target project's `typescript` at runtime and threads it as `ts`, but about 30 analysis files also `import * as TypeScript from "typescript"` and some call it as a value. Two TypeScript instances with different versions disagree on `SyntaxKind`. | Decision D4: the Program is built with the engine's bundled TypeScript unless the project's version has the same major and minor. The fallback copy ships as a real directory, not inlined, so `lib.*.d.ts` resolution works. | -| R13 | Markers "from finding spans" via `collect_markers`. | `collect_markers` re-runs a tree-sitter analyzer per file; `tests/test_markers.py` asserts marker count equals metric value. | The merge point is `routers/files.py:get_file_markers`, where findings markers are already merged in. Data-flow markers come from the cached batch result, not from re-analysis, and the count test gets a fake-engine case. | -| R14 | Protocol over stdout JSON lines. | The engine warns through `console.warn` (stderr, fine) but any stray `console.log` in analysis code would corrupt the channel. Worker progress is `postMessage`. | The bin entry rebinds `console.log` to stderr and keeps the worker-thread service for cancellation (`SharedArrayBuffer`), so a long query can be cancelled without killing the child. | - -### 1.2 Stale facts in the repository this PR should also fix - -- `docs/CODE_MAP.md` names `GET /api/analysis/dependencies`; the route is `/api/analysis/dependency-graph` (`routers/analysis.py:66`). -- The AGENTS.md metric checklist points at `server/app/services/tree_sitter_analysis.py`, which does not exist; the analyzer is `services/typescript/typescript_analysis.py`. -- The outline plan's anchors say `components/HotSpots/`; the files are `client/src/components/HotSpotItem.tsx` and `HotSpotList.tsx`. The line numbers (130, 134, 135, 145; 88, 109) hold on this branch. -- tsx-data-flow's `test/route-totality-field-proof.test.ts` and `test/route-totality-schedule-field-proof.test.ts` hard-code `/Users/byronwall/Projects/soccer-schedule/app` with no skip guard. They cannot move as-is (M1). - -### 1.3 What the outline gets right and this plan keeps - -- The engine supplies evidence and srcly draws it. No canvas, layout or SPA code moves. -- One lazy stdio child per scan root, owned by Python. -- Data flow mode uses its own contract and never relabels import edges. -- The ledger carries operation order; the graph shows reach. -- Every milestone is a revertible slice, and a released wheel without the bundle shows N/A. -- No purity score, no confidence percentage, N/A is never 0. - -## 2. Architecture - -```text -engine/ Node ≥ 18, TypeScript, Zod (tsx-data-flow analysis, moved intact) - bin/srcly-engine.ts stdio JSON-lines protocol, one worker thread, cancellation - src/srcly/session.ts tier 1: Program + identity index + lazy evidence provider + routes - src/srcly/ops/*.ts status · routes · routeGraph · selectExpression · valueFlow · routePath - functionEffects · fileSignals · workPackets - src/srcly/contracts/*.ts Zod schemas → emit JSON Schema + client .d.ts - │ stdin/stdout, {id, op, params} ↔ {id, ok, result | error} + progress frames -server/app/services/dataflow/ Python owns the child: spawn, locate node, reply matching, timeouts, kill - engine.py · models.py · availability.py · overlay.py · jobs.py -server/app/routers/dataflow.py /api/dataflow/* (status, routes, route-graph, select, value-flow, - route-path, function-effects, file-signals) -server/app/services/findings/dataflow.py finding spans as a findings Tool (markers, file findings) -server/app/services/reporting.py dataflow.md work packets artifact - │ fetch -client/src/dataflow/ contract.generated.ts · api.ts · availability store · outcome labels -client/src/viz/deps/flow/ Data flow mode: contract → ViewModel adapter, legend, URL (flow, field, dest) -client/src/investigation/ pinned preview, ledger, inspector, Return (viewState snapshots) -client/src/components/CodeModal/EffectsPanel.tsx -client/src/navigation/viewState.ts capture/restore registry every surface joins -``` - -Three invariants hold across every layer: - -1. **The base scan never waits.** The engine starts on first explicit use (opening Data flow mode, Inspect value flow, the Effects panel, or `srcly report --dataflow`). The treemap, explorer and import graph are unchanged with or without Node. -2. **Evidence is bound to a generation.** Every result carries `{generation, programHash, tsconfig, typescript}`. Python compares `expectedGeneration` and returns `stale` instead of a mixed answer. The client refuses to overlay stale rows on current source. -3. **Not analyzed is never 0.** Every surface renders the query outcome by name. Metrics are `None` with an `na_reason`, never zero. - -## 3. Engine package (`engine/`) - -### 3.1 What moves, what stays, what is new - -The move is a copy at a pinned SHA (decision D1), recorded in `engine/UPSTREAM.md` with the repository, SHA and the list below. History is not imported; `git subtree` would bring the frontend and 405 scratch directories with it. - -| From tsx-data-flow | To `engine/` | Notes | -| --- | --- | --- | -| `src/analysis/**` (151 files, 37K lines) | `src/analysis/**` | Intact. `evidence-slice-cli.ts` is the only file that imports `cli/args`; it stays because tests use it. | -| `src/project/**` | `src/project/**` | `typescript.ts:packageDir` is recomputed for the bundle (M8). | -| `src/api/**` (contracts, projections, validators) | `src/api/**` | Intact. `report-views.ts` duplicates `REPORT_VIEWS`; leave it. | -| `src/types.ts`, `src/core.ts`, `src/cli/args.ts` | same paths | `core.ts` keeps `analyzeProject` for tier 2 and tests. | -| `src/reports/**` | `src/reports/**` | `format-helpers.ts` and `overview-selectors.ts` are imported by analysis and projections. The Markdown views are tier 2 input for M7 (`work-packets`, `defensive-ledger`, `prop-relay`, `fan-out`, `fan-in`). | -| `src/server/cache.ts`, `source-excerpts.ts`, `analysis-service.ts`, `analysis-worker.ts` | `src/service/**` | Reused for the worker thread and source excerpts. `cache.ts` is split (section 3.3). | -| `examples/solid-route`, `examples/compact-field-proof`, `examples/bad-ish-solid` (+ `reports/` goldens) | `examples/**` | The other examples (`node-cli`, `http-service`, `serverless-handler`, `solid-full-stack`) go only if a test needs them after the port base is settled. | -| `test/**` minus `test/frontend`, `test/server-*.test.ts`, `test/helpers/{http,server-test-context}.ts` | `test/**` | The two soccer-schedule tests are rewritten against `examples/compact-field-proof` or skipped with a reason (R-1.2). | -| `scripts/accept-route-field-proof.ts`, `scripts/route-field-proof-obligations.json`, `scripts/dead-code-audit.ts` | `scripts/**` | The acceptance runner stays the authority for named field proof. | -| `skills/render-path-dataflow-work/SKILL.md` | folded into `skills/srcly-code-quality/SKILL.md` in M7 | Only the Markdown-reading guidance; the rest describes the retired CLI. | -| `docs/analyzer.md`, `CONCEPTS.md`, `docs/route-field-proof-acceptance.md` | `engine/docs/` | Linked from srcly's `docs/CODE_MAP.md`. | - -Left behind: `src/frontend/**`, `src/server.ts`, `src/server-cli.ts`, `src/server/{routes,responses,api-*}.ts`, `src/cli/{help,server-flags}.ts`, `bin/tsx-dataflow-serve.ts`, `vite.frontend.config.ts`, `eslint-plugin-solid`, `@solidjs/testing-library`, `jsdom`, `solid-js` as a dependency, and `tsx` as a runtime dependency (it becomes a devDependency; the bundle never needs it). - -New in `engine/`: - -```text -engine/ - package.json name "@srcly/engine", private, type module; deps: typescript, zod; dev: esbuild, vitest, tsx, typescript-eslint - pnpm-lock.yaml own lockfile, like client/ - tsconfig.json from tsconfig.server.json, include src/** bin/** scripts/** - vitest.config.ts plain node environment (no solid plugin) - bin/srcly-engine.ts protocol loop (section 3.2) - bin/engine-worker.ts imports src/service/analysis-worker - src/srcly/session.ts tier 1 / tier 2 session (section 3.3) - src/srcly/protocol.ts frame types, error codes, progress - src/srcly/ops/ status.ts routes.ts route-graph.ts select-expression.ts value-flow.ts route-path.ts - function-effects.ts file-signals.ts work-packets.ts - src/srcly/contracts/ dataflow-*.ts (Zod, one file per op; shared primitives re-exported from src/api) - scripts/emit-contracts.ts writes server/app/services/dataflow/contracts/*.schema.json and - client/src/dataflow/contract.generated.ts - scripts/bundle.mjs esbuild → server/app/engine/ (M8) - test/srcly/ contract tests per op against examples/solid-route and ../client -``` - -### 3.2 Protocol - -One request per line on stdin, one reply per line on stdout. Frames are UTF-8 JSON, no embedded newlines. The bin entry rebinds `console.log` and `console.info` to stderr before importing any analysis module (R14). - -```text -→ {"id": 1, "op": "hello", "params": {"protocol": 1, "root": "...", "source": "src", "tsconfig": "tsconfig.app.json"}} -← {"id": 1, "ok": true, "result": {"protocol": 1, "engine": "", "node": "v22.21.1", "typescript": "5.9.3", - "typescriptFrom": "project" | "bundled"}} -→ {"id": 2, "op": "routes", "params": {}} -← {"id": 2, "progress": {"phase": "program", "message": "Building TypeScript program for 487 files", "completed": 0, "total": 487}} -← {"id": 2, "progress": {"phase": "identity", ...}} -← {"id": 2, "ok": true, "result": {"generation": 1, "programHash": "…", "tsconfig": "…", "routes": [...]}} -→ {"op": "cancel", "params": {"id": 7}} -← {"id": 7, "ok": false, "error": {"code": "cancelled", "message": "Analysis request was cancelled"}} -``` - -Rules: - -- `id` is a positive integer chosen by Python. Replies carry the request `id`. Progress frames carry it too and may arrive any number of times before the reply. -- `cancel` has no `id` of its own and no reply. It sets the worker's `SharedArrayBuffer` flag exactly as `createAnalysisService.cancel` does today. -- Error codes (one string enum in `protocol.ts`, mirrored in Python): `invalid-request`, `no-tsconfig`, `non-strict-tsconfig`, `no-ts-files`, `typescript-not-found`, `stale-generation`, `not-found`, `unsupported-selection`, `ambiguous-selection`, `cancelled`, `budget-exhausted`, `internal`. -- The child exits on stdin EOF. Python kills the process group after 2 s if it has not. -- One request runs at a time inside the child (the worker is serial), but Python may pipeline requests; the child answers in completion order. -- Payload size is bounded per op (route graph ≤ 8 MiB, ledger ≤ 2 MiB). Over the bound the op returns `budget-exhausted` with the counts it did compute, never a truncated JSON line. - -The existing `analysis-service.ts` request and cancel code is reused as the in-process layer between the protocol loop and the worker. Its `AnalysisOperation` union gains the new op kinds; `workspace`, `file`, `report`, `markdown` and `route-data` remain for tier 2 and tests. - -### 3.3 Session tiers - -`createAnalysisCache.ensureBuilt()` cannot be the engine's entry point, because it builds the full legacy report before answering (R5). `src/srcly/session.ts` replaces it with two tiers that share one Program: - -```ts -export function createEngineSession(args: AnalyzerArgs, progress: AnalyzerProgressReporter) { - // tier 1, built on first query - const analyzer = createAnalyzer(args, progress); // buildProgram + buildIdentityIndex (existing) - const provider = createLazyProgramEvidenceProvider(analyzer.ts, analyzer.program, args.root); - const routes = discoverRoutes(analyzer); // discoverRoute() per source file under /routes/ - const candidates = discoverSolidRouteCandidates(...); - const totality = createRouteDataTotalitySession(ts, program, root, routes, provider, candidates, /* findings */ []); - const generation = { id: 1, programHash: hashProgram(program, args) }; - // tier 2, built lazily by fileSignals / workPackets only - let report: AnalysisReport | null = null; - const fullReport = () => (report ??= analyzer.report()); - return { analyzer, provider, routes, totality, generation, fullReport, refresh }; -} -``` - -- Tier 1 answers `routes`, `routeGraph`, `routePath`, `selectExpression`, `valueFlow` and `functionEffects`. Findings attachments on route nodes are empty in tier 1; they are tier 2 data and arrive in M7. -- Tier 2 answers `fileSignals` and `workPackets`. It is the legacy sink tracer and ranking, and it is slow (section 9). It runs only for the batch job, with its own timeout. -- `routes` lists `routes.map(r => ({key, pathPattern, file, parameters}))` straight from discovery. It must not call `buildRouteDataInventory`: on visual-notes that call alone costs 17.5 s because it builds an exhaustive path graph per route. -- `programHash` is a stable hash over the sorted `(fileName, size, mtimeMs)` of the Program's non-declaration source files, the text of every governing tsconfig, the TypeScript version and the engine version. It is cheap (one `stat` per file) and is recomputed by `status` so Python can detect staleness without rebuilding. A rebuild bumps `generation.id`. -- `refresh` drops both tiers. There is no incremental invalidation (below the cut line). -- The strict refusal (R3) lives here: after `resolveProjectConfigs`, if the primary config has `strictNullChecks` false and no governing config is strict, the session throws `non-strict-tsconfig` with the configs tried. Strict configs beside a loose one keep working as they do today. - -### 3.4 Evidence vocabulary (two layers) - -**Query outcome**, one per reply, in the `result.outcome` field and mirrored by Python and the client: - -| Outcome | Meaning | Shown as | -| --- | --- | --- | -| `ok` | Every requested fact was computed. | normal | -| `partial` | Some facts stop at frontiers or gaps. The result says which. | "Partial" badge plus the frontier rows | -| `not-analyzed` | The engine has no Program for this root (no Node, no tsconfig, non-strict, no TS files, crashed, timed out). `reason` names which. | "Data flow N/A: " | -| `unsupported` | The selection or route is outside what the analysis handles. | the reason text | -| `stale` | The caller's generation is older than the session's. | "Source changed. Refresh." | -| `cancelled` | Superseded or cancelled by the caller. | nothing; the next result replaces it | -| `budget` | A traversal budget or payload bound stopped the walk. Counts are included. | "Stopped at budget: N of M" | - -**Evidence status**, one per node, edge and ledger row: `proven | partial | unsupported`, exactly the engine's `EvidenceStatus`. Frontier nodes and rows also carry the engine's typed reason (`partial-proof`, `budget-exhausted`, `identity-lost`, `ambiguous-target`, `unsupported-relation`, `unsupported-transform`, `dynamic-index`, `renamed-prop`, `multiple-origins`, `evidence-truncated`, `unmapped-occurrence`, `unmapped-terminal`, plus the gap reasons `unsupported-syntax`, `dynamic-dispatch`, `external-code`, `unresolved-symbol`, `runtime-only`, `disconnected`, `unsupported-boundary`, `unproven-handoff`). - -Nothing collapses these into a score or a percentage. - -### 3.5 Source positions - -All contracts use 1-based lines, 1-based UTF-16 columns and exclusive end columns, which is what `sourceLocationSchema` already does. Python and the client never convert; the client's selection mapping (M3) produces these numbers directly from the DOM. Tree-sitter byte offsets (srcly's own markers) are never mixed with engine spans. - -## 4. Python server (`server/app/services/dataflow/`) - -### 4.1 Files - -| File | Responsibility | -| --- | --- | -| `engine.py` | `EngineProcess`: spawn, handshake, reader thread, pending futures keyed by id, per-request timeouts, cancel, kill. `EngineRegistry`: one process per resolved scan root, lazy, idle shutdown, cap. | -| `availability.py` | `locate_node()`, `locate_bundle()`, `probe(root) -> Availability` with the N/A reasons. Pure functions, no process. | -| `models.py` | Pydantic models mirrored from the generated JSON Schema (section 7.1). `Outcome` enum, `EvidenceStatus` enum. | -| `overlay.py` (M7) | `apply_to_tree(tree, signals)` like `findings/overlay.py`: per-file values, folder sums, `na_reasons`, root info. | -| `jobs.py` (M7) | `DataflowJob` per `(root, fingerprint)` like `FindingsJob`; runs `fileSignals` in batches with its own timeout. | -| `../findings/dataflow.py` (M7) | `DataflowTool(Tool)` whose `check()` turns finding spans into `Finding` records so markers and file findings work through the existing overlay. | -| `../../routers/dataflow.py` | The endpoints (4.3). | - -### 4.2 Process ownership - -```python -class EngineProcess: - def __init__(self, root: Path, source: str, tsconfig: str | None, *, node: str, entry: Path): ... - def request(self, op: str, params: dict, *, timeout: float, expected_generation: int | None = None, - on_progress: Callable[[dict], None] | None = None) -> dict: ... - def cancel(self, request_id: int) -> None: ... - def close(self, grace: float = 2.0) -> None: ... -``` - -- Spawn: `Popen([node, entry, "--root", root, ...], stdin=PIPE, stdout=PIPE, stderr=PIPE, start_new_session=True, env={..., "NO_COLOR": "1"})`, the same flags `findings/tools.py:run_process` uses. stderr is drained to a ring buffer of the last 64 lines and included in `engine-crashed` reasons. -- A reader thread parses lines and resolves futures. A progress frame updates `EngineProcess.latest_progress`, which `/api/dataflow/status` serves. -- Timeouts: `hello` 10 s; tier-1 build is part of the first query, so first-query timeouts are `program_timeout = max(60, 0.25 s × source files)` and later queries 30 s; `fileSignals` batches 300 s. All are overridable with `SRCLY_DATAFLOW_TIMEOUT`. -- On timeout Python sends `cancel`; if no `cancelled` reply arrives within 2 s it kills the process group and marks the root `engine-crashed` until the next explicit retry. -- `EngineRegistry` keeps at most two live processes (measured RSS is about 2 GB per Program, section 9). Starting a third closes the least recently used. A process idle for 15 minutes is closed; the next query restarts it, and the client shows the progress UI again. -- Shutdown: `main.py:lifespan` gains `dataflow.engine.registry.close_all()` beside `cancel_all_scans()`, and the SIGINT chain cancels in-flight engine requests first. Today findings subprocesses outlive Ctrl+C until their own timeout; the engine must not. -- Dev mode: `locate_bundle()` prefers `resources.files("app") / "engine" / "srcly-engine.mjs"` (M8), then the repository's `engine/dist/srcly-engine.mjs`, then, when `SRCLY_ENGINE_DEV=1`, runs `engine/bin/srcly-engine.ts` through `engine/node_modules/.bin/tsx`. The dev path is what M1 to M7 use. - -### 4.3 Endpoints - -All handlers are plain `def` (threadpool), return `Response(content=bytes)` for large bodies, and take `path` (the scan root) like `/api/analysis/dependency-graph`. File paths in bodies must resolve inside `path`; otherwise 400. (`/api/files/content` has no such check today; the dataflow endpoints should not copy that.) - -| Endpoint | Milestone | Request | Response | -| --- | --- | --- | --- | -| `GET /api/dataflow/status?path=` | M1 | | `{available, reason?, node?, engine?, generation?, programHash?, progress?}`. Never starts the child. 200 always. | -| `POST /api/dataflow/start` | M1 | `{path}` | Starts the child and tier 1 in the background; returns status. The client polls `status` every 250 ms while `progress` is present. | -| `GET /api/dataflow/routes?path=` | M1 | | `{outcome, generation, routes: [{key, pathPattern, file, absPath, parameters}]}` | -| `GET /api/dataflow/route-graph?path=&route=&source=` | M2 | | the route graph contract (M2) | -| `POST /api/dataflow/select` | M3 | `{path, file, span, text}` | `{outcome, candidates: [{kind, label, span}]}` | -| `POST /api/dataflow/value-flow` | M3 | `{path, file, span, text, direction, expectedGeneration}` | the ledger contract (M3) | -| `GET /api/dataflow/route-path?path=&route=&field=&terminal=` | M4 | | the ledger contract | -| `POST /api/dataflow/function-effects` | M6 | `{path, file, line, column, expectedGeneration}` | the effects contract (M6) | -| `GET /api/scans/{id}/dataflow` | M7 | | batch job status, like `/findings` | -| `DELETE /api/dataflow?path=` | M1 | | closes the child (used by tests and the folder menu's rescan) | - -Status codes: 200 with `outcome` for every analysis state, including `not-analyzed` and `stale`. 400 for bad input or a path outside the root. 404 only for an unknown `route`, `field` or `terminal` id. 503 when the engine crashed and the request was the one that crashed it. - -### 4.4 Tests - -- `tests/fake_engine.py`: a Python script that speaks the protocol (`hello`, `status`, `routes`, scripted delays, a crash flag, a hang flag, a late-reply flag). `EngineProcess` is constructed with `node=sys.executable, entry=fake_engine.py`, so the lifecycle tests need no Node. -- `tests/test_dataflow_engine.py`: handshake, reply matching with out-of-order replies, late reply dropped after timeout, cancel then kill, crash → `engine-crashed` with stderr tail, EOF → child exits, registry cap and idle close, lifespan close. -- `tests/test_dataflow_router.py`: `TestClient(app)` against the fake engine; path containment; `status` never spawns; `stale` on generation mismatch. -- `tests/test_dataflow_smoke.py`: real engine on `engine/examples/solid-route`, skipped when `shutil.which("node")` is None or `engine/node_modules` is missing. Runs in CI (Node 24 is installed there). - -## 5. Client - -### 5.1 Shared module `client/src/dataflow/` - -| File | Responsibility | -| --- | --- | -| `contract.generated.ts` | Types emitted from the engine's Zod contracts. Never hand-edited; a vitest asserts it matches `engine/scripts/emit-contracts.ts` output. | -| `api.ts` | `fetchDataflowStatus`, `startDataflow`, `fetchRoutes`, `fetchRouteGraph`, `selectExpression`, `fetchValueFlow`, `fetchRoutePath`, `fetchFunctionEffects`, each taking `signal`, in the style of `services/fileContent.ts`. | -| `availability.ts` | A per-root store: `availability()` (`unknown \| starting \| ready \| na`), `reason`, `progress`, `generation`; `ensureStarted(root)` posts `start` once and polls `status` at 250 ms until ready or N/A. Lives in `state/` style, outlives views like `scanStore`. | -| `outcome.ts` | `labelFor(outcome, reason)` and `toneFor(status)`: the one place the vocabulary of section 3.4 becomes text and color. | - -### 5.2 Where each surface lives - -| Surface | Files | Entry point | -| --- | --- | --- | -| Data flow mode (M2, M4) | `viz/deps/flow/{adapter.ts, model.ts, FlowLegend.tsx, FlowTools.tsx, RoutePicker.tsx}`; mode switch in `DependencyView.tsx` through ``; URL params in `navigation/location.ts` | Graph view toolbar: "Imports / Data flow" segmented switch beside Detail −/+ | -| Investigation workspace (M3, M4, M5) | `investigation/{InvestigationView.tsx, PinnedPreview.tsx, EvidenceLedger.tsx, EvidenceInspector.tsx, ledgerModel.ts, selectionSpan.ts}`; `navigation/viewState.ts`; `navigation/investigation.ts` | "Inspect value flow" from the code viewer's selection; a destination click in Data flow mode | -| Effects panel (M6) | `components/CodeModal/EffectsPanel.tsx`, `hooks/useFunctionEffects.ts` | A section in `MetricsSidebar` under Findings, keyed on the selected scope | -| Metrics, markers, findings (M7) | nothing new; the catalog and overlay drive `MetricPicker`, `explorerColumns`, treemap color, `useMetricMarkers`, `FileFindings` | | - -### 5.3 View-state capture and restore (`navigation/viewState.ts`) - -Return must restore filters, metric, expansion, selection and camera, and today none of those is captured (R9). The registry is small: - -```ts -export type ViewStateSnapshot = { location: Location; search: string; parts: Record }; -export function registerViewState(key: string, owner: { capture(): unknown; restore(state: unknown): void }): () => void; -export function captureViewState(): ViewStateSnapshot; -export function restoreViewState(snap: ViewStateSnapshot): Promise; -``` - -Owners that register, and what they capture: - -| Key | Owner | Captures | -| --- | --- | --- | -| `metrics` | `MetricsStoreProvider` | `metric`, `excludedPaths` | -| `graph.filters` | `navigation/store.ts` | `graphFilters` | -| `graph.view` | `DependencyView` | `selected`, `expanded` (already in URL), camera transform from a new `CanvasApi.getTransform()`; restore calls `setTransform` and sets `suppressFitOnce` so the next layout does not refit | -| `explorer` | `Explorer.tsx` | tab, sort, columns, expanded ids, scroll top, cohort | -| `treemap` | `Treemap.tsx` | extension filter, maxLoc | -| `code` | `AppContent` + `CodeModal` | open file, scope path, line range, scroll top, active metric overlay | -| `flow` | Data flow mode | route, selected field, destination (also in URL) | - -Snapshots are stored in `sessionStorage` under `srcly-inv-` with the token in the URL (`?inv=`), so Back and Forward through the browser history restore them and a copied URL without the token still opens the investigation with defaults. `HistoryRing` is left alone; it is a recent-places list, not a history stack. - -## 6. Milestones in detail - -Each milestone is one or more stacked PRs with a Conventional Commit title. "Proof" is what must be true before the next PR in the track starts. "Rollback" is one revert. - -### M1 — Engine in srcly answers one query - -**Outcome.** `GET /api/dataflow/routes?path=` returns `/records` through a lazy stdio child. A scan without Node behaves exactly as today and `status` names the reason. - -**Prerequisites in tsx-data-flow (decision D1).** - -1. Commit the 49-file route-proof WIP on `main` behind its own test run (`pnpm verify`). The two soccer-schedule tests get `describe.skipIf(!fs.existsSync(soccerAppRoot))`. -2. Tag the SHA `srcly-port-base`. -3. Leave PRs 1 to 7 open. Record in `engine/UPSTREAM.md` that they were not ported and why. Run `scripts/dead-code-audit.ts` inside `engine/` as the last M1 PR and delete what the frontend removal orphaned (expected: `route-shadow-evidence*`, the non-Solid scope adapters, `program-evidence-loading.ts`, `component-occurrence-identity*`, about 3K lines). Tests must stay green through the deletion. - -**PR 1 — `build(engine): move the tsx-data-flow analysis into engine/`.** - -- Copy per the table in 3.1. `pnpm install` in `engine/`, commit the lockfile. -- `engine/package.json` scripts: `test` (vitest run), `typecheck` (tsc -p tsconfig.json), `lint`, `emit-contracts`, `bundle` (M8). -- Fix the cwd-dependent test (`route-totality-generic-field-proof.test.ts` resolves `examples/compact-field-proof` from cwd) to resolve from `import.meta.url`. -- Golden snapshot: keep the `tsx-dataflow` strings in Markdown output for now; renaming is M7's job when those views are re-exposed. -- `.github/workflows/release.yml`: add `pnpm install --frozen-lockfile` and `pnpm test` in `engine/` beside the client steps; add `engine/pnpm-lock.yaml` to `cache-dependency-path`. -- Proof: `cd engine && pnpm test && pnpm typecheck` pass unchanged (minus the removed frontend and server tests). - -**PR 2 — `feat(engine): stdio protocol and tier-1 session`.** - -- `src/srcly/protocol.ts`, `session.ts`, `ops/status.ts`, `ops/routes.ts`, `bin/srcly-engine.ts`, `bin/engine-worker.ts`. -- `analysis-service.ts` gains the new op kinds and is driven by the protocol loop instead of HTTP. The worker's `perform()` gains `status` and `routes` branches that use the session, not `cache.ensureBuilt()`. -- The strict refusal (3.3) and the `console.log` rebinding (3.2). -- `scripts/emit-contracts.ts` and the first two contracts (`hello`, `routes`). -- Tests `test/srcly/protocol.test.ts` (spawn the bin with `tsx`, drive it over pipes: hello, routes on solid-route, cancel during program build, EOF exit, non-strict refusal on a temp project, no-tsconfig refusal) and `test/srcly/session.test.ts` (tier 1 never calls `report()`; `programHash` changes when a file's mtime changes). - -**PR 3 — `feat(server): python owns a lazy data-flow engine child`.** - -- `services/dataflow/{engine,availability,models}.py`, `routers/dataflow.py` with `status`, `start`, `routes`, `DELETE`. -- `main.py`: register the router before the SPA mount; `lifespan` closes the registry. -- N/A reasons (`availability.py`): `node-not-found` (no `node` on PATH and no `SRCLY_NODE`), `engine-missing` (no bundle and no dev entry), `no-tsconfig`, `non-strict-tsconfig`, `no-ts-files`, `typescript-not-found`, `engine-crashed`, `timed-out`. The first three are decided without spawning; the rest come from the child. -- Tests per 4.4. The real smoke test runs in CI. -- Docs: `docs/CODE_MAP.md` gets a "Data flow (engine)" section with rows for every file above and fixes the stale `dependencies` row; `AGENTS.md` gets an "Engine" section: how to run it (`SRCLY_ENGINE_DEV=1`), how to add an op (contract → emit → Python model → router → client api), and the rule that analysis modules are not edited for srcly-specific recognition. - -**Measurements recorded in `docs/intent/tsx-dataflow-integration/measurements.md`** (started in M1, appended by every milestone): cold `hello` + tier-1 time, RSS after tier 1, for solid-route, srcly `client/` and visual-notes `app/`. - -**Rollback.** Remove `engine/`, `services/dataflow/`, `routers/dataflow.py`, the CI steps. - -### M2 — Route graph in the dependency view - -**Outcome.** With `?view=graph&flow=`, the graph canvas shows the route's component backbone down to terminals. Selecting an origin lights its proven destinations. Any node opens the code viewer at its line. The layout decision (D2) is recorded with screenshots. - -**Engine op `routeGraph {route, source?}`** (`ops/route-graph.ts`, contract `contracts/dataflow-route-graph.ts`). It calls `routeTotalityForRoute(session, routeKey, selectedSource)` then `projectRouteTotality`, and projects the validated `RouteTotality` into srcly's graph style: - -```ts -type FlowNodeKind = "route" | "occurrence" | "control" | "origin" | "field" | "terminal" | "frontier"; -type FlowNode = { id; kind; label; parent: string | null; file: string; absPath: string; line: number; span; - status: EvidenceStatus; collapsedByDefault: boolean; repetition?: "single"|"conditional"|"collection"|"unknown"; - external?: boolean; reason?: FrontierReason }; -type FlowEdgeKind = "render" | "handoff" | "field"; -type FlowEdge = { id; source; target; kind; status: EvidenceStatus; proof: EvidenceProof[]; pathIds?: {elements: string[]; relations: string[]} }; -type RouteGraph = { outcome; generation; route: {key, pathPattern, file}; status: "complete"|"partial"|"unavailable"; - nodes: FlowNode[]; edges: FlowEdge[]; origins: {elementId, role, label, status}[]; - selectedSource: string | null; counts; gaps: {reason, label, location}[] }; -``` - -Mapping from `RouteTotality`: - -| Source | Node or edge | -| --- | --- | -| `route` | the `route` node, root of the hierarchy | -| `occurrenceSurface.occurrences[]` | `occurrence` nodes; `parent = renderParentId ?? parentOccurrenceId ?? route`; `collapsedByDefault = definition.external \|\| hiddenComponentPolicy hides it` | -| `occurrenceSurface.frameworkBoundaries[]` (`portal`, `control-flow`, `collection`, `suspense-async`, `unsupported-ownership`) | `control` nodes with the condition outcome in the label | -| `occurrenceSurface.terminals[]` (`jsx-text`, `dom-attribute`, `style`, `render-expression`) | `terminal` nodes, parent = `ownerOccurrenceId` | -| `occurrenceSurface.renderEdges[]` (`render`, `framework-boundary`, `slot-forward`, `transparent-splice`) | `render` edges, status from the surface (`proven` unless the occurrence surface is `partial`) | -| `bridges[]` | `handoff` edges | -| `occurrenceSurface.origins[]` / `evidenceSlice.origins[]` | `origin` nodes under the route (the pickable sources) | -| `fieldLineage.attachments[]` (only with a `source`) | `field` edges from the origin's `field` node to the consumer's terminal, `pathIds` = `evidencePathElementIds` / `RelationIds` | -| `fieldLineage.frontiers[]` | `frontier` nodes at `stoppedAtElementId`'s location with `reason` | -| `omissions[]`, `gaps[]`, `truncation` | `counts` and `gaps`; the client shows "N occurrences omitted (budget)" on the route node | - -The graph is reach, not order. Nothing in this op invents a step; order lives in M4's ledger. - -Contract tests on `examples/solid-route`: one `RouteFrame` occurrence owns the children, `MetricCard` has one definition and two occurrences, `ViewerCard` never receives records (no `field` edge from the records origin reaches it), and `For` in `RecordTable` is a `control` node of kind collection. - -**Server.** `GET /api/dataflow/route-graph`, model `RouteGraph` in `models.py`. First call per route is slow on real apps (totality is lazy per route, with the per-route `selectedSource` cache); the client shows progress from `status`. - -**Client.** - -- `navigation/location.ts`: `flow?: string` (route key), `field?: string` (origin element id), `dest?: string` (terminal id). `view=graph` stays the view; `flow` switches the graph's mode. `samePlace` ignores `field` and `dest` like it ignores `open`. -- `viz/deps/flow/adapter.ts`: `routeGraphToViewModel(graph, {selectedField, collapsed}) : ViewModel`. Nodes become `ViewNode`s with `region = "center"`, `group = {id: parentOccurrenceId, label}`, so `assignGroups` draws each occurrence as an opened frame with its terminals inside, which is what the existing expand-in-place path already renders. Collapsed-by-default primitives roll into their parent frame's `memberCount`. `field` edges are only emitted for the selected field, as `ViewEdge`s with `weight = 1`, `faint = status !== "proven"`. -- `DependencyView.tsx`: when `location().flow` is set, the resource fetches `route-graph` instead of `dependency-graph`; filters X/T/Y are hidden; the legend swaps to `FlowLegend` (render = solid, handoff = dashed, field = accent, frontier = hatched box, collapsed primitive = dotted outline). Click = select origin or node; double-click / `o` = `props.onOpen(absPath, line)` which reaches `App.openAt` through a new overload that takes an absolute path and a line (today it takes a graph id). Shift-click on a collapsed frame expands it, the same gesture as today. -- `FlowTools.tsx` in ``: the mode switch (Imports / Data flow), the route picker (`RoutePicker.tsx`, built on `DialogShell` like `GoToPalette`, lists `routes` and opens on `/` when in flow mode), and an origin picker for the selected route. -- `availability.ensureStarted(root)` runs when the mode is switched on; until `ready`, the canvas shows `ScanProgress`-style progress from `status.progress` (phase, completed/total). This is the "progress within 250 ms" requirement; the poll interval is 250 ms. -- Vitest: `flow/adapter.test.ts` on a checked-in `routeGraph` fixture captured from solid-route (`engine/test/srcly/__fixtures__/solid-route.route-graph.json`, regenerated by a script like `scripts/capture-dep-graph.sh`). - -**Spike — layout fit (decision D2).** Render solid-route and `/time-blocks` through the adapter with today's `layoutWithHubs` at 1440 and 1024 wide. Record four screenshots in `measurements.md`. Decide: keep the packing (frames per occurrence) or add one top-down rule. The fallback is a `direction: "down"` option on `layoutComponent` that orders levels by hierarchy depth instead of in/center/out, used only when `vm.flow` is set, and coordinated with `dependency-graph-legibility` because both touch `layout.ts`. - -**Proof.** solid-route and `/time-blocks` render; selecting the records origin lights `RecordTable` row terminals and not `ViewerCard`; every node opens the right line; the spike decision is written down. - -**Rollback.** Revert the mode switch, the adapter and the `routeGraph` op. - -### M3 — Value ledger from the code viewer - -**Outcome.** Select `props.node.name` on `HotSpotItem.tsx:135`, choose "Inspect value flow", read an ordered ledger that reaches the JSX text slot, open any row's code, press Return, and be back on the same treemap with the same metric, filters, sidebar state and scroll. - -**Engine: `ops/select-expression.ts`.** Input `{file, span, text}`. It finds the smallest AST node whose `[getStart, getEnd)` covers the span, in the file's `SourceFile` from the session's Program (or the routed program when `paths` apply). It walks up while the parent is one of the supported expression kinds and has the same text; it walks down when the span is wider than one expression and the text matches a child. Supported kinds in M3: `Identifier`, `PropertyAccessExpression`, `ElementAccessExpression` with a literal index, `CallExpression` whose callee is a Solid accessor (zero-argument call of a signal, memo or resource), `JsxExpression`, `BinaryExpression` with `??` or `||`, `ConditionalExpression`, `NonNullExpression`, `ParenthesizedExpression`. Anything else is `unsupported-selection` with the kind name. It returns `candidates` (the covering node and its supported ancestors, innermost first) so the client can show a picker when more than one fits the span exactly; `text` must equal the candidate's source text or the candidate is dropped (this is the "shared source spans keep distinct call context" rule from the research). - -**Engine: `ops/value-flow.ts`.** Input `{file, span, text, direction: "forward", expectedGeneration}`. - -1. Resolve the selection as above. Reject `stale-generation` first. -2. Find the proven program element for the AST node: the same lookup `RouteTotalityFieldProofIndex.element(node, kind)` does, generalised: ask `provider.facts.fileCandidates(file)` for a fact whose location key equals the node's `file:startLine:startCol:endLine:endCol` and whose kind is in `{field-read, value, alias, call-result, parameter, jsx-occurrence}`. If none: `unsupported-selection` with reason `no-evidence-element` (the collector did not see this expression; name it, do not guess). -3. Run `queryEvidenceSlice(provider, {seed: element.id, direction: "forward", boundaryPolicy: DEFAULT_BOUNDARY_POLICY, terminalPolicy: {roles: ["render", "component-occurrence", "side-effect", "return"], ...}})`. -4. Order the slice into rows: a depth-first walk from the seed over `relations` grouped by `from`, following relation kinds in this order of preference `field-input, consumer-value, alias, references, for-element, for-callback-parameter, show-when, show-render-parameter, component-prop, component-prop-binding, render-terminal, renders, performs, pack-field, map-*, filter-result, find-result, includes-membership, join-source, accessor-call`. Cross-call kinds (`invokes`, `argument`, `argument-binding`, `function-call`, `function-return`, `return-value`, `callback-parameter`) are followed only when the callee has exactly one `invokes` relation in the provider (one call site); otherwise the walk stops with a `boundary` row, reason `unproven-handoff` (R7). M5 replaces this rule with invocation pairing. -5. Row kinds: `input` (the seed), `transfer` (alias, field-read, references, consumer-value), `fallback` (an element with `operationKind: "default"` or a `??`/`||` binary), `branch` (`show-when`, `type-narrow`, conditional), `transform` (map, filter, pack, pick, parse, validate, serialize, selection), `terminal` (an element with a `terminalRoles` entry), `boundary` (a gap or a refused cross-call), each with `{id, kind, label, file, absPath, span, status, elementId, relationId, proof, typeText?, note?}`. `typeText` comes from `checker.typeToString` at the element's node, so the inspector can show "`string | undefined` → `string`" across a fallback. -6. Value properties stay separate from connectivity: a `fallback` row says what `??` keeps (`0`, `""`) and `||` replaces; a `branch` row from `NonNullExpression` is labelled "assertion, not a check". -7. Budgets and counts come from `slice.coverage` and go into `outcome: "budget"` when exhausted. - -Output contract `contracts/dataflow-value-flow.ts`: - -```ts -type Ledger = { outcome; generation; programHash; selection: {file, span, text, kind}; - rows: LedgerRow[]; frontiers: {rowId, reason, label, location}[]; - terminals: string[]; coverage: {elements, relations, budget: {limit, used, exhausted}}; - bindings: {tsconfig, typescript, sourceFingerprints: {file, size, mtimeMs}[]} }; -``` - -Fixtures in `engine/examples/value-flow-basics/` (strict tsconfig, Solid JSX): `direct-property.tsx` (a prop read reaches a JSX text slot), `same-name-negative.tsx` (two bindings with a `name` field; only the selected one reaches its slot), `null-vs-finite.tsx` (`x ?? 0` keeps zero; a null check proves nothing about finiteness; `v!` is not a check), `two-call-helper.tsx` (the same helper called twice; M3 expects a `boundary` row, M5 expects two distinct paths). - -**Server.** `POST /api/dataflow/select` and `POST /api/dataflow/value-flow` with path containment and `expectedGeneration`. `models.py` gains `Ledger`, `LedgerRow`. - -**Client: selection in the code viewer (`investigation/selectionSpan.ts`).** A `selectionchange` listener scoped to `.code-modal-content` maps the DOM `Range` to a source span: - -- line = `displayStartLine() + index of the span.line ancestor`; -- column = 1 + the removed indent for that line + the UTF-16 offset of the range boundary inside the line's text content. `reduceCommonIndent` must return the per-line removed width (today it returns only the text), a small pure change with a test; -- the selected `text` is taken from `rawCode()` at that span, not from the DOM, and the two must match or the selection is rejected with "Selection crosses markup". -- When the selection is empty (a click), the span is the caret; the engine's candidates give the innermost expression under it. - -The code viewer header gets an "Inspect value flow" button, enabled when a selection exists and `availability()` is not `na`. Pressing it calls `select`; one candidate opens the investigation, several open a small `Popover` listing them with their text and kind. - -**Client: investigation workspace.** `InvestigationView.tsx` replaces the main area (treemap/graph) while `?inv=` is in the URL; the header keeps the location bar. Layout at 1440: pinned preview (source lines around the selection, `CodePane` in read-only slice mode) on the left, ledger in the middle, inspector on the right; at 1024 the inspector collapses under the ledger. Keyboard: ↑/↓ rows, Enter opens the row's code in `CodeModal`, Space toggles the inspector, Esc returns focus to the ledger, `Backspace` = Return. The status line names the outcome (3.4) and the generation. - -- `ledgerModel.ts`: pure; groups consecutive `transfer` rows that are plain aliases into one row with an expand toggle; never groups fallbacks, branches, joins, terminals or boundaries. -- `EvidenceInspector.tsx`: span, `typeText`, proof list (kind, detail, locations as links), assumptions, stop reason, and the bindings block (tsconfig, TypeScript version, generation). -- `navigation/investigation.ts`: `openInvestigation(request)` captures `captureViewState()`, stores it under a token, pushes `?inv=`; `returnFromInvestigation()` pops and calls `restoreViewState`. Back and Forward go through popstate and the same restore. -- Staleness: the result's `sourceFingerprints` are compared against `/api/files/content` headers (`Last-Modified` is not enough; add `X-Srcly-Size` and `X-Srcly-Mtime` to that endpoint) before marking rows on the preview; on mismatch the ledger shows "Source changed. Refresh." and offers `refresh`. - -**Acceptance (anchors on this branch).** - -| Selection | Expected | -| --- | --- | -| `HotSpotItem.tsx:135` `props.node.name` | input → `props.node` field read → JSX text slot (`terminal`, proven) | -| `HotSpotItem.tsx:134` `title={props.node.name}` | its own ledger ends at a `dom-attribute` terminal; it does not appear in the line-135 ledger | -| `HotSpotItem.tsx:130` `props.rank ?? "–"` | a `fallback` row: `??` keeps `0`; status proven | -| `HotSpotItem.tsx:145` the metric expression | `partial`: `formatMetricWithUnit` is a cross-call with many call sites → `boundary` (`unproven-handoff`); `value()!` is labelled an assertion | -| `HotSpotList.tsx:88` `item.node` | M3 result: `boundary` at `row(...)` because `row` has two call sites (lines 88 and 109). This is the negative that M5 turns positive. | -| A changed imported helper with the selected file untouched | `stale` after `status` reports a new `programHash` | - -Journey: start from a non-default metric, filters, expanded explorer and a zoomed treemap; inspect; open a row; Return. Everything restores. Repeat with the graph view and a camera transform. - -**Proof.** The 135 ledger is exact. If it is not, the engine is fixed before any further UI work (the shape brief's rule). - -**Rollback.** Revert `investigation/`, `viewState.ts`, the two ops. - -### M4 — One path explanation from the graph - -**Engine op `routePath {route, field, terminal}`** (`ops/route-path.ts`). Find the attachment in `fieldLineage.attachments` whose `origin.elementId` is `field` and whose `terminalIds[0]` is `terminal`. Its `evidencePathElementIds` and `evidencePathRelationIds` are already ordered; map them through the slice's elements and relations into the same `LedgerRow`s as M3, reusing the row classifier. `transformationIds` become `transform` rows with their `kind`. A frontier for that origin becomes a `boundary` row at the end. If no attachment exists for the pair, the reply is `not-found`, never a guessed path. - -**Client.** In Data flow mode, clicking a lit destination (or pressing Enter on it) calls `openInvestigation({kind: "route-path", route, field, terminal})`. The investigation header shows "Route /time-blocks · field … → terminal …" and Return restores `flow`, `field`, `dest`, selection and camera through the same snapshot. - -**Proof.** On `/time-blocks`, origin → destination → ledger shows the proven operations in order with their owners, and frontiers where the lineage stops. The ledger and the graph agree on status for every edge. - -### M5 — Cross-file invocation identity - -**Problem.** `argument-binding` ties every argument of every call site to the same parameter element, and `function-return` ties the parameter's result back to every `call-result`. A forward walk from line 88 therefore also reaches line 109's `HotSpotItem`. `For` recognition requires `each` to be a property access and the callback to have one parameter; `shown()` is a call and the callback has two. `componentPropValueMode` returns null for a plain identifier (`node={node}`). - -**Engine changes (the only analysis-module edits in the whole program, each with a negative fixture):** - -1. `program-evidence-collector-call-support.ts:processCall`: record the call id on `argument-binding` and `function-return` relations (`attributes.callId`). Existing proofs are unchanged. -2. `evidence-slice.ts` traversal: carry an invocation stack. Entering a callee through an `argument-binding` with `callId = C` pushes `C`; a `function-return` or `return-value` relation is followed only to the `call-result` of the top of the stack, which pops it. This is 1-CFA on the slice walk, bounded by `maxDepth`. Relations without `callId` (older facts) keep the M3 rule: follow only when the callee has one call site. -3. `connectSolidFor`: accept `each` as any expression that resolves to a proven element (a call like `shown()` has a `call-result` element) and a callback with one or two parameters; the second (`index`) gets a `for-callback-parameter` relation too so index reads are not frontiers. -4. `componentPropValueMode`: a plain `Identifier` initializer whose symbol resolves to a parameter or a `const` becomes a new mode `"identifier-alias"`, handled by the component binding verifier like `scalar-alias` without a `sourceFieldName`. - -Each change keeps the acceptance runner green on `examples/compact-field-proof` (G01 to G20) and adds fixtures: `two-call-identity-helper.tsx` (two calls of `id(x)`; paths stay separate), `returns-other-argument.tsx` (`pick(a, b)` returns `b`; `a` must not reach the slot), `for-over-call-two-params.tsx`, `plain-identifier-prop.tsx`. - -**Client.** `ledgerModel.ts` renders a cross-call as a helper frame: a collapsed row "through `row` (HotSpotList.tsx:61)" that expands to the callee's rows with the invocation marked. Branch expansion shows both `Show` arms as sibling groups when the slice has them. - -**Acceptance.** `HotSpotList.tsx:88` `item.node` → `row(item.node, …)` → parameter `node` → `` → `props.node` → `props.node.name` (135) → JSX text, every row proven. Line 109's call to `row` does not appear. The upstream `rankHotSpots` chain stays a frontier. - -**Decision D3 (branch graph beside the ledger).** Only if an M5 path reads worse as rows than as a small graph. Evidence: the two-arm `Show` fixture rendered both ways. - -### M6 — Effects panel - -**Engine.** New modules under `src/analysis/effects/` following the research plan: `contract.ts` (records: scope, callable, call site, evidence, summary, relation, witness, projection), `collect.ts` (callables, exact call occurrences with their resolved targets or an incomplete-target reason, direct facts, bounded origins), `models.ts` (the short list of modelled intrinsics: `String.prototype.split`, array `slice` / `sort`, canonical `console.*`, canonical `fetch`, `setTimeout` with a literal callback, the `Promise` constructor with a literal executor, no-argument `Promise.resolve`; identity-checked through the checker, never by name), `propagate.ts` (SCC order, finite monotone union over `family × region × activation`, witnesses by bounded BFS). Op `functionEffects {file, line, column}` resolves the enclosing callable in the session's Program and reports `{callable, facts, frontiers, coverage: "complete-for-supported-scope" | "partial" | "not-analyzed", activation, resultHandling, witnesses, assumptions}`. It reuses `processProgramCallBoundary`'s classification for network, filesystem, stdout and exit, and treats its labels as evidence only when the callee identity is compiler-resolved. - -Fixture `engine/examples/effects-basics/` with the 21 cases from the research table (closed leaf, imported leaf reference, direct and alias calls, fresh slice then sort, input alias then sort, transitive parameter write, fresh-then-borrowed child write as a partial frontier, latent returned arrow, IIFE, timer, Promise executor, async log after await, discarded / awaited / returned promise, local arrow named `fetch`, parameter named `fetch`, log then unknown callback, mutual recursion with and without a seed). The fixture includes an astral character before a call and CRLF and BOM variants, and asserts the selected substring, not only offsets. - -**Server.** `POST /api/dataflow/function-effects` with `expectedGeneration`; model `FunctionEffects`. - -**Client.** `EffectsPanel.tsx` as a `MetricsSidebar` section under Findings, keyed on the selected scope's start line. It shows Known (facts grouped by family: external behaviour, ambient inputs, writes by target class, allocation, throws) and Partial (frontiers with reasons) together, the activation channel and result handling per call, and assumptions always visible. A witness row opens its span in the pane with a neutral mark (not the metric overlay's hot tone). Historical `gitRef` content shows "Effects unavailable at a ref". - -**Acceptance (lines on this branch).** `client/src/utils/dataProcessing.ts:81 extractFilePath`: a closed primitive-transformation candidate under standard-intrinsic assumptions, split site named. `client/src/utils/hotSpotRanking.ts:57 rankHotSpots`: a call to `metricDef` and a transitive read of the module-level `byId`; library operations may stay partial. `client/src/services/scanJobs.ts:88 startScan`: a network witness through `fetch`, one direct await, the returned promise adopted from `expectJson` (`:70`). `expectJson`: two alternative JSON await sites and an explicit throw in an async body; neither branch is claimed to execute. - -**Rollback.** Revert `effects/`, the op and the panel. - -### M7 — Supporting evidence in srcly's pipelines - -**Engine op `fileSignals {files?: string[]}`** (tier 2). Builds `fullReport()` once, then per file: metric sums over `report.sinks` in that file and finding spans. Op `workPackets {}` renders `renderMarkdownView(report, {view: "work-packets"})` plus `defensive-ledger`, `prop-relay`, `fan-out` and `fan-in`, with the `tsx-dataflow` strings replaced by `srcly` and the regen footer replaced by the `srcly report` command. - -**Metrics (overlay family `DATAFLOW_METRICS` in `analysis_types.py`, added to `OVERLAY_METRICS`):** - -| Metric id | From `SinkMetrics` | Catalog | -| --- | --- | --- | -| `dataflow_impossible_defenses` | Σ `impossibleDefenseCount` | group Data flow, `higher-worse`, sum, `witness=True` (the worst sink), `color_domain (0, 1, 5)`, weights `frontend 1.0, typescript 1.0` | -| `dataflow_defensive_ops` | Σ `actionableDefensiveOperationCount` | `higher-worse`, sum, `color_domain (0, 5, 20)`, no weights | -| `dataflow_path_depth` | max `maximumPathDepth` | `higher-worse`, max (`_MAX_ROLLUP`), `witness=True` | -| `dataflow_representation_changes` | Σ `representationChurn` | `higher-worse`, sum | -| `dataflow_relay_hops` | Σ `helperHops` over sinks whose path crosses a component boundary | `higher-worse`, sum | -| `dataflow_fan_out` | `reachableSinks` of the file's widest value | `neutral`, Detail group (size-like) | - -`MetricSpec.languages` special-cases `DATAFLOW_METRICS` to `["ts", "tsx"]` like it does `FINDINGS_METRICS`. `tests/test_metric_catalog.py::test_witnessed_metrics_are_complexity_and_components` is widened to include the two witnessed data-flow metrics, and `test_profile_lead_is_the_heaviest_weight` is checked: `frontend` keeps `component_loc` as its lead, so the weights above stay below it. - -**Overlay.** `dataflow/jobs.py:DataflowJob(root, fingerprint)` starts beside `start_findings()` in `ScanJob.run` and `from_cache` only when `availability.probe(root)` says the engine can run and the user has opened a data-flow surface at least once in this session or passed `--dataflow` (decision D5: never by default, because tier 2 costs minutes). `scan_jobs.result_json_overlaid` adds the job identity to its cache signature. `dataflow/overlay.apply_to_tree` sets the six metrics per file, folder sums or maxes, and `na_reasons` ("Data flow not analyzed: ") on every TS/TSX file when the job is N/A. Non-TS files get the ordinary "Not measured for Python files" reason through the catalog's `languages`. - -**Findings and markers.** `findings/dataflow.py:DataflowTool` runs only when the job result exists (it never starts the engine). Its `check()` turns each finding span into a `Finding(path, line, end_line, column, rule, message, severity="warning", tool="dataflow")` with rules `impossible-defense`, `repeated-fork`, `context-relay`, `broad-prop-bundle`. Markers for the six metrics come from the same spans and are merged in `routers/files.py:get_file_markers` from the cached job (never by re-running the engine per file). `tests/test_markers.py` gets a fake-engine case asserting the marker count equals the metric value. - -**Report.** `reporting.scan_tree` gains `dataflow=` (off by default; `srcly report --dataflow`), `write_report` writes `dataflow.md` (work packets) and lists it in `enrich_manifest_with_artifacts` with `agent_load: top_n`. `build_metrics_schema` already picks the catalog up. `skills/srcly-code-quality/SKILL.md` gets a "Data flow" section with the reading order from `render-path-dataflow-work`. - -**Compare.** `compare/diff.COMPARE_METRICS` derives from the catalog, so the six metrics appear with their direction once `compare/snapshots._with_checks` applies the overlay for a side. Compare uses portable tools only, so this is gated on the same `--dataflow` opt-in and shows N/A otherwise. - -**Proof.** The treemap colors by impossible defenses on `examples/bad-ish-solid`; the code viewer marks the lines behind each count; `srcly report --dataflow` writes `dataflow.md`; snapshots regenerated with `SRCLY_UPDATE_SNAPSHOTS=1`. - -### M8 — Engine ships in the wheel - -- `engine/scripts/bundle.mjs`: `esbuild bin/srcly-engine.ts bin/engine-worker.ts --bundle --platform=node --format=esm --packages=external --outdir=../server/app/engine --sourcemap=external --minify-syntax`, then copy a trimmed `typescript` (`package.json`, `lib/typescript.js`, `lib/lib.*.d.ts`) to `server/app/engine/node_modules/typescript/` and write `server/app/engine/VERSION.json` `{engine, typescript, builtAt}`. `zod` is bundled (`--packages=external` is dropped for everything except `typescript`). -- `project/typescript.ts:loadTypescript`: the fallback base becomes `path.join(bundleDir, "node_modules")` resolved from `import.meta.url`, and D4's version rule picks project vs bundled. -- `server/app/.gitignore`: add `engine` beside `static`. The hatch `include = ["app/**"]` already ships it. -- `build-srcly.sh`: `cd engine && pnpm install (if needed) && pnpm bundle` before `uv build`. -- `release.yml`: the engine steps from M1 plus `scripts/test_package.py` additions: the bundle exists, `VERSION.json` parses, and a clean-venv smoke runs `srcly`'s Python `EngineProcess` against a copy of `engine/examples/solid-route` and gets `/records`. -- Record wheel size before and after in `measurements.md`. Expected: the trimmed TypeScript is about 15 MB uncompressed, 3 to 4 MB in the wheel. -- README: "Data flow needs Node 18 or newer on PATH. Without it the feature shows N/A and nothing else changes." - -**Rollback.** Drop the bundle step; wheels show N/A again. - -## 7. Cross-cutting concerns - -### 7.1 One contract, three languages - -The Zod schemas in `engine/src/srcly/contracts/` are the source of truth. `engine/scripts/emit-contracts.ts` runs `z.toJSONSchema` (Zod 4) per op and writes: - -- `server/app/services/dataflow/contracts/.schema.json`, loaded by `models.py` tests to assert that every Pydantic model's field set equals the schema's `properties` (the Pydantic models are hand-written for readability; the test keeps them honest). -- `client/src/dataflow/contract.generated.ts`, produced from `z.infer` through a small type printer (or `json-schema-to-typescript` as a dev dependency of `engine/`). - -`engine/test/srcly/contracts-current.test.ts` regenerates both into a temp dir and diffs them against the committed files, the same discipline as `tests/test_metric_catalog.py::test_client_catalog_snapshot_is_current`. The regenerate command is `cd engine && pnpm emit-contracts`, recorded in AGENTS.md. - -Contracts are versioned by `protocol` in `hello`. Python refuses a child whose protocol differs and reports `engine-protocol-mismatch`; there is no coexistence layer. - -### 7.2 Identity and staleness, end to end - -| Layer | Carries | Checks | -| --- | --- | --- | -| Engine | `generation.id`, `programHash`, per-result `sourceFingerprints` | `expectedGeneration` on every tier-1 query; `status` recomputes `programHash` with `stat` only | -| Python | passes `expectedGeneration` through; caches nothing semantic | returns `stale` when the child says so; closes the child when the scan root's fingerprint (`cache.fingerprint`) changes and a surface is open | -| Client | `generation` on every displayed result; `sourceFingerprints` vs `/api/files/content` size and mtime headers | refuses to mark stale rows on current source; offers Refresh, which posts `DELETE /api/dataflow` then `start` | - -A rescan from the folder menu closes the child for that root. Compare's worktree checkouts never start an engine unless `--dataflow` is set. - -### 7.3 Performance plan - -Targets from the shape brief, with what this plan does about each: - -| Target | Mechanism | -| --- | --- | -| Progress within 250 ms | `status` is answered from Python memory; the client polls at 250 ms and renders the engine's last progress frame. The child emits a progress frame at every `AnalyzerProgressReporter` step. | -| Base scan time unchanged | The engine starts only on explicit use; tier 2 is opt-in (D5). `ScanJob.run` never awaits it. | -| Cold first result ≤ 10 s | Tier 1 is 3 s on srcly's client and 14 s on visual-notes (section 9). The first `routeGraph` on a real route adds the totality build for that route, unmeasured until M2. The 10 s target holds for srcly-sized apps and is a stated miss for 65K-line apps; the plan reports it honestly in the UI (progress with phase names) rather than hiding it. | -| Warm first row ≤ 750 ms | Tier-1 queries on a built session reuse the lazy provider's fact index; measured in M3. | -| Preview and Return ≤ 150 ms p95 | Return is a client-only restore from `sessionStorage`; no request. The pinned preview reuses the already fetched file content. | -| Memory | About 2 GB RSS per Program (section 9). Registry cap of two processes and a 15-minute idle close. The status endpoint reports RSS from `process.memoryUsage()` so the measurement stays visible. | - -Tier 2 (`fileSignals`) is the known slow path: 130 s on srcly's client and 44 s on visual-notes for the full report. It runs only in the batch job with a 300 s timeout, so a user never waits on it interactively, and the metrics are N/A with "still analyzing" until it completes (the same pattern as findings). - -### 7.4 Security and containment - -- The child reads only files the Program owns under `root`, which `cache.ts:resolveSourcePath` already enforces for excerpts (realpath, no `..`, size cap); `selectExpression` and `valueFlow` use the same resolver. -- Python validates every `file` parameter resolves under the scan root before it reaches the child. -- The child runs with `start_new_session=True` and is killed as a group. No shell, no user-supplied argv: `node`, the entry path and fixed flags only. -- The engine never executes target code; it only parses and type-checks it. - -### 7.5 Documentation that lands with the code - -- `docs/CODE_MAP.md`: a "Data flow (engine)" section (rows per file in sections 3 to 5) and the API table rows; fix the stale `dependencies` row in M1. -- `AGENTS.md`: an "Engine" section (run, test, add an op, regenerate contracts, never add srcly-specific recognition rules to analysis modules, bundle only in `build-srcly.sh`); fix the stale `tree_sitter_analysis.py` path. -- `engine/UPSTREAM.md`, `engine/README.md` (how the protocol works, how to run the acceptance runner), moved `docs/analyzer.md` and `CONCEPTS.md`. -- `docs/intent/tsx-dataflow-integration/measurements.md`: every number this program records. -- `README.md`: the Node requirement and the N/A behaviour (M8). - -## 8. Decision log - -The defaults below were accepted by the user on 2026-10-23. Each stays open only to the evidence named in its last column. - -| Id | Decision | Default (accepted) | Evidence that would change it | -| --- | --- | --- | --- | -| D1 | Port base in tsx-data-flow | Commit the route-proof WIP on `main`, tag `srcly-port-base`, port that, drop the seven cleanup PRs, re-run the dead-code audit inside `engine/` | If `pnpm verify` fails on the WIP for reasons that take more than a day, port `main` at `b4f1ba4` instead and re-apply the WIP inside `engine/` as the first engine PR | -| D2 | Route layout | Existing packing with occurrence frames | The M2 screenshots: if terminals of one occurrence land in different columns, add the `direction: "down"` rule | -| D3 | Branch graph beside the ledger | Rows with expansion | An M5 two-arm path that is misread in a review | -| D4 | TypeScript instance | Bundled TypeScript for the Program; the project's copy only when its major.minor matches the bundled one | A real project that fails to parse with the bundled version (then prefer the project's and record the mismatch in `hello`) | -| D5 | Tier 2 batch job | Opt-in: a data-flow surface opened this session, or `--dataflow` | Tier 2 under 10 s on a 50K-line app after the engine's own performance work (out of scope here) | -| D6 | Resident vs per-query process | Resident, idle close at 15 minutes, cap 2 | Tier 1 under 2 s on realistic apps would make per-query spawn viable; measured RSS above 3 GB would force a cap of 1 | -| D7 | Investigation URL | `?inv=` with the verbose snapshot in `sessionStorage` | Nothing expected; a shareable URL restores semantic focus with defaults by design | -| D8 | Progress transport | Polling `status` at 250 ms | If polling shows up in profiles, add an SSE endpoint like `/api/scans/{id}/events` | - -## 9. Measurements taken for this plan - -Machine: the development Mac, Node v22.21.1, tsx-data-flow at `b4f1ba4` plus WIP, TypeScript 5.9.3 resolved from the target project. One cold run each; no warm numbers yet. - -| Target | Source files | Program build | Identity index | Full legacy report | Route inventory | RSS after | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| srcly `client/` (26K lines) | 126 | 1.2 s | 1.8 s | 130.0 s | 2 ms (no routes) | 2.4 GB | -| visual-notes `app/` (65K lines, 43 routes) | 487 | 4.3 s | 9.5 s | 44.2 s | 17.5 s | 2.0 GB | - -What these numbers decide: - -- Tier 1 (program + identity index) is 3.0 s and 13.9 s. That is the cold cost a user sees before the first route list or ledger. It meets the 10 s target for srcly-sized apps and misses it for visual-notes; the UI shows phased progress either way. -- The full report is the wrong entry point for interactive queries (R5). srcly's client, which is finding-dense, is three times slower than an app two and a half times its size. -- `buildRouteDataInventory` must not be on the `routes` path (3.3). -- Memory is the binding constraint for a resident child (D6). - -Still to measure, by milestone: first `routeGraph` per route on visual-notes (M2); warm `valueFlow` p95 over 20 queries and cold `hello` to first row (M3); `functionEffects` on the three anchors (M6); tier 2 on `examples/bad-ish-solid` and the wheel size (M7, M8). - -## 10. Sequencing and parallel work - -```text -M1 ──┬── Track A: M2 (route graph) ──── M4 (route path; needs M3's ledger UI) - ├── Track B: M3 (ledger) ────────── M5 (invocation identity) - ├── Track C: M6 (effects) - └── Track D: M7 (signals, metrics, markers, report) M8 (wheel) after any track has a user-visible result -``` - -- M1 is three PRs in one worktree, merged in order. Nothing else starts before PR 3 is on `main`, because every track needs `EngineProcess` and the contract emitter. -- Tracks A to D are separate worktrees. They touch disjoint files except: `location.ts` (A and B both add params; B adds `inv`, A adds `flow`, `field`, `dest`), `analysis-service.ts` op union (every track adds a kind), `models.py` and `routers/dataflow.py` (every track adds a model and a route), `docs/CODE_MAP.md` and `AGENTS.md`. Each track adds its own lines; rebases are mechanical. -- M4 needs M3's `InvestigationView` and `ledgerModel`; it is track A's second PR but waits for M3's first PR to merge. -- M5 edits analysis modules. No other track edits them, so M5 cannot conflict; its fixtures gate it. -- M8 can start as soon as M1 is in, as a parallel track, because the bundle is independent of the ops. Releasing it before M2 would ship a wheel whose only visible feature is `status`, which is harmless; but the plan ships M8 after the first user-visible track to keep the wheel-size cost tied to a benefit. -- PR screenshots: every PR that changes UI attaches before/after screenshots with `gh pr edit --attach` from an ignored `tmp/` folder, per the user's global instructions. Suggested engine fixture renders go in `measurements.md`, not the repo tree. - -Relative sizes, for planning only: M1 large (the move plus two new layers), M2 large (adapter, mode, spike), M3 extra large (two engine ops, selection mapping, workspace, view-state registry), M4 small, M5 medium (analysis edits with fixtures), M6 extra large (a new analysis pass), M7 large (six integration points), M8 medium. - -## 11. Risks and stop rules - -| Risk | Signal | Response | -| --- | --- | --- | -| Generic value evidence cannot prove the line-135 slot exactly | The M3 acceptance table has a wrong row | Stop UI work in track B; fix the collector or the walk; a toy-only fixture passing does not count | -| False attachment across shared helpers before M5 | A ledger shows a `proven` row through a callee with more than one call site | The M3 rule in step 4 is a bug, fix before merging | -| Cold time on large apps makes Data flow mode feel broken | Tier 1 over 30 s on a target the user cares about | Scope the Program: honour `--source` as a sub-folder of `src`, or offer a per-route `tsconfig` pick; never silently fall back to a loose config | -| Memory: two 2 GB children on a laptop | RSS reported by `status` above 3 GB total | Cap one process; close on `DELETE` when the user leaves the surface | -| TypeScript version skew (D4) | `hello.typescriptFrom` is `bundled` while the project pins a newer TypeScript and parse diagnostics appear | Prefer the project's copy for that major and record the decision | -| Two-instance SyntaxKind mismatch | Any `TypeScript.isX` on a node from the other instance | The M1 audit of the ~30 static imports routes every value use through the loaded `ts`; the dead-code audit removes `program-evidence-loading.ts` | -| Protocol corruption | A non-JSON stdout line | The bin rebinds `console.log`; Python logs the line to stderr capture and reports `engine-crashed` with it | -| Wheel size | More than 10 MB added | Trim `lib.*.d.ts` to the ES2022 and DOM set the engine needs, or make the fallback an optional download; record either way | -| Layout churn with `dependency-graph-legibility` | Both change `layout.ts` | The D2 rule is a new option, not a change to existing packing; coordinate the PR order with that initiative | -| The outline's "analysis moves intact" vs the M5 edits | Any edit to `src/analysis` outside M5's four named changes | Reject in review; recognition rules for srcly's own code never go into analysis modules | - -Stop and correct immediately on: a false host match in effects, a lost async effect, a flattened latent closure, an incorrect private-mutation claim, an erased unknown, a non-converging recursive summary, a `proven` edge whose proof locations do not contain the span, or any surface that shows `0` for a not-analyzed value. - -## 12. Acceptance summary - -| Milestone | Must be true | -| --- | --- | -| M1 | `routes` on solid-route lists `/records`; no Node → identical scan and `status.reason = node-not-found`; fake-engine lifecycle tests pass; engine tests pass unchanged | -| M2 | solid-route and `/time-blocks` render in Data flow mode; the records origin lights `RecordTable` terminals and not `ViewerCard`; nodes open the right lines; D2 recorded with screenshots | -| M3 | The six-row acceptance table in M3 holds; Return restores metric, filters, explorer, treemap, graph camera and selection; keyboard-only journey works at 1440 and 1024 | -| M4 | On `/time-blocks`, origin → destination → ledger agrees with the graph edge status; a missing attachment is `not-found` | -| M5 | Line 88 → line 135 proven through `row`; line 109 absent; the four new fixtures and the compact-field-proof obligations pass | -| M6 | The three anchors show the research's required evidence and acceptable unknowns; the 21-case fixture passes; Known and Partial render together | -| M7 | Six catalog metrics appear in Color (and Rank by where rankable); treemap colors by impossible defenses on bad-ish-solid; markers equal counts; `dataflow.md` is written; snapshots regenerated | -| M8 | Clean-venv `uvx` smoke on a solid-route copy answers `/records`; wheel size recorded; README states the Node requirement | - -See also the [intent](intent-brief.md), [shape](shape-brief.md), [outline plan](implementation-plan.md), [research](research.md) and [initiative-map.json](initiative-map.json). diff --git a/docs/intent/tsx-dataflow-integration/implementation-plan.md b/docs/intent/tsx-dataflow-integration/implementation-plan.md deleted file mode 100644 index 79cc4bee..00000000 --- a/docs/intent/tsx-dataflow-integration/implementation-plan.md +++ /dev/null @@ -1,186 +0,0 @@ ---- -title: "Bring tsx-data-flow into srcly — implementation plan" -slug: "tsx-dataflow-integration" -phase: plan -status: current -last_updated: "2026-10-23" ---- - -# Bring tsx-data-flow into srcly — implementation plan - -**Status:** proposed. Implementation starts only on explicit approval. - -The file-level design, contracts, measurements and the corrections from reviewing both codebases are in [detailed-plan.md](detailed-plan.md). Read that page before starting any milestone. - -## Plan at a glance - -M1 moves the engine into `engine/` and proves srcly's server can query it. After that, four tracks run in parallel. They share only the engine process and the evidence states. - -| Track | Milestones | -| --- | --- | -| A, route graph | M2, then M4 (which also needs M3) | -| B, ledger | M3, then M5 | -| C, effects | M6 | -| D, supporting evidence | M7 | - -M8 bundles the engine into the wheel. Until then, a released wheel shows data flow as N/A, so every milestone can ship. - -#102 proposed an HTTP child on a loopback port. Because everything is in one repository, this plan uses a stdio JSON-lines child instead. It needs no port discovery and no open socket, and Python owns its whole lifetime. Inside the child, the existing worker and cache are reused. - -## Implementation strategy - -- **Primary seam:** `server/app/services/dataflow/engine.py` is the only code that starts, queries, cancels, or stops Node. A fake engine script speaks the same protocol for tests. -- **Fast local loop:** `cd engine && pnpm vitest run test/srcly`. These are contract tests against `examples/solid-route` and srcly's `client/`. The Python side is `cd server && uv run pytest tests/test_dataflow_engine.py`. -- **Proof ladder:** - 1. A fake engine for lifecycle failures: crash, hang, late reply, missing node. - 2. The real engine from `engine/`. - 3. The bundled engine in a clean-venv wheel (M8). -- **Rollback:** data flow appears only when the engine answers. Each milestone reverts as one PR. -- **Evidence states:** proven, partial, not analyzed, unsupported, stale, cancelled, budget. They are one enum, shared by the engine contract and the Python models. -- **Targets on srcly `client/`, measured in M3 and M4:** preview and return p95 ≤ 150 ms, progress ≤ 250 ms, warm first row ≤ 750 ms, cold ≤ 10 s. The base scan time must not regress. - -## Milestone 1: Engine in srcly answers one query - -- **Change — Move the engine into `engine/`.** - - Land tsx-data-flow PRs 1–4 and its local route-proof edits first, then copy from that pinned SHA. - - Bring over `src/analysis`, `src/project`, `src/api`, the analysis service, worker, and cache, plus examples and tests. - - Leave behind `src/frontend`, the HTTP routes, the Markdown views (M7 restores the useful ones), and the serve CLI. - - Add `engine/` to the pnpm workspace. Its tests and typecheck must pass unchanged. -- **Change — `engine/bin/srcly-engine.ts`.** - - Protocol: `{id, op, params}` → `{id, ok, result|error}`, plus `cancel`. - - Ops: `status`, and `routes` (via `createAnalyzer` → `buildRouteDataInventory`). Results carry `{generation, programHash, tsconfig}`. - - A non-strict tsconfig is a named error. -- **Change — `services/dataflow/engine.py` and `routers/dataflow.py`.** - - One lazy child per scan root. Replies are matched by id, late replies are dropped, and superseded requests are cancelled. - - On shutdown or Ctrl+C, kill the process group. - - Named N/A reasons: `node-not-found`, `no-tsconfig`, `non-strict-tsconfig`, `no-ts-files`, `engine-crashed`, `timed-out`. - - Endpoints: `GET /api/dataflow/status` and `GET /api/dataflow/routes`. - - Fake-engine lifecycle tests, plus one real smoke test that skips when there is no Node. -- **Change — Docs:** `docs/CODE_MAP.md` rows, and an `engine/` section in `AGENTS.md`. - -### Desired end state - -- `/api/dataflow/routes` lists the routes of `solid-route`. -- A scan without Node behaves exactly as today, with the reason shown. - -## Milestone 2: Route graph in the dependency view - -- **Change — Engine op `routeGraph {route}`.** - - It projects `routeTotalityForRoute` → `projectRouteTotality` → a data-flow contract. - - Nodes: `route`, `occurrence`, `control`, `field`, `terminal`, `frontier`, each with a parent and a span. - - Edges: `render`, `handoff`, `field`, each with a proof status. - - UI primitives are marked `collapsedByDefault`. - - Contract tests on `solid-route`. -- **Change — Server:** `GET /api/dataflow/route-graph`, with a Pydantic model next to `DepGraph`. -- **Change — Client `viz/deps` Data flow mode.** - - A toolbar mode switch, `flow=` in `navigation/location.ts`, and routes in the go-to palette. - - A contract adapter into the existing `viewModel`. Field reach uses isolate. Nodes open through `openAt`. - - Its own legend, and no import edges. - - Vitest for the adapter. -- **Spike — Layout fit.** - - Decision required: keep the existing packing, or add a route layout rule. - - Evidence to gather: screenshots of `solid-route` and `/time-blocks` at 1440 and 1024 wide. - - Fallback: one top-down rule in `layout.ts`, scoped to this mode and coordinated with `dependency-graph-legibility`. - -### Desired end state - -- Selecting a field lights its proven destinations. Nodes open the code viewer at their line. The layout decision is recorded. - -## Milestone 3: Value ledger from the code viewer - -- **Change — Engine: `expression-selection.ts`, `value-investigation.ts`, and op `valueFlow`.** - - Input is a span (1-based lines, UTF-16 columns, exclusive end) plus its text. It resolves to one AST occurrence, or returns AST choices when ambiguous. - - Supported transfers: static identifier and property reads, intrinsic JSX slots, and `??` / `||`. - - Output is ordered rows with element and relation ids, frontiers, and budgets. - - Fixtures: direct flow, a same-name negative, and a null-versus-finite negative. -- **Change — Server:** `POST /api/dataflow/value-flow`. It checks containment, compares against the expected generation, and returns `stale` when they differ. -- **Change — Client `investigation/`.** - - "Inspect value flow" on a selection in the code viewer opens a pinned preview, then the workspace: header, locator, ledger, and inspector. - - `navigation/investigation.ts` stores the return snapshot. Back, Forward, and Return restore it. - - Keyboard navigation for the rows. -- **Acceptance:** - - `HotSpotItem.tsx:135` reaches its JSX slot. - - Line 134 stays distinct. - - Line 130 keeps zero. - - Line 145 is partial. - - `value()!` is not treated as a check. - - A changed imported helper marks the result stale. - -### Desired end state - -- From srcly scanning itself, the ledger is correct and Return restores the treemap. It works at 1440 and 1024 wide and by keyboard alone. -- If the line 135 proof isn't exact, the engine is fixed before any further UI work. - -## Milestone 4: One path explanation from the graph - -- **Change — Engine op `routePath {route, field, terminal}`.** It returns route lineage as ledger rows. A missing ordered fact becomes a frontier row, never an invented step. -- **Change — Client.** A graph destination opens the ledger, and Return restores the route, field, and camera. - -### Desired end state - -- On `/time-blocks`, field → destination → ledger shows the proven operations, their owners, and the frontiers. - -## Milestone 5: Cross-file invocation identity - -- **Change — Engine.** - - Pair arguments, parameters, and returns per invocation in `program-value-summary*`, `program-callback-return.ts`, and `evidence-slice.ts`. - - Support `For` over `shown()` with two callback parameters, and plain `node={node}` handoffs. -- **Change — Client.** Helper frames and branch expansion. Fallbacks, joins, and terminals stay explicit. - -### Desired end state - -- `HotSpotList.tsx:88` is proven through `row` to `HotSpotItem.tsx:135`, and line 109 stays isolated. -- Two-call identity-helper and returns-other-argument fixtures pass. - -## Milestone 6: Effects panel - -- **Change — Engine effects pass, op `functionEffects`.** - - Built on the shared Program. - - Call occurrences, transitive effects with argument substitution, and finite recursive summaries. - - Activation channels, result handling, and coverage. -- **Change — `CodeModal/EffectsPanel.tsx`** in the metrics rail. Known and Partial show together. Witnesses open their spans. Marks are neutral. - -### Desired end state - -- `extractFilePath` is shown as a closed candidate, with its assumptions. -- `rankHotSpots` shows its catalog read. -- `startScan` shows its network witness and how its await and returned promise are handled. - -## Milestone 7: Supporting evidence in srcly's pipelines - -- **Change — Engine batch op `fileSignals`:** per-file metrics and finding spans. -- **Change — Server.** - - A `dataflow` tool in `services/findings/`. - - Catalog entries for burden, defenses, impossible defenses, relay, and fan-out/in, following the AGENTS.md metric rules. - - Markers from finding spans. - - A `dataflow.md` work-packets artifact. - - The agent skill merged into `skills/srcly-code-quality`. - - Compare picks these up through the catalog. - -### Desired end state - -- The treemap colors by impossible defenses, the code viewer marks the lines behind each count, and `srcly report` includes the work packets. - -## Milestone 8: Engine ships in the wheel - -- **Change — `build-srcly.sh`:** an esbuild bundle in `server/app/engine/`, with a fallback copy of `typescript`. The project's own `typescript` is preferred when present. -- **Change — Release check:** record the wheel size before and after, and run a `uvx` smoke test in a clean venv on `solid-route`. - -### Desired end state - -- With Node 18+ installed, users get every data-flow feature with no npm step. Without Node, they get a named N/A. - -## Open decisions and spikes - -- **Resident vs per-query process.** - - Evidence: M3 cold and warm timings. - - Fallback: spawn one child per query over the same protocol. Only `engine.py` changes. -- **Layout fit:** the M2 spike above. -- **Branch graph beside the ledger:** add one only if an M5 path reads worse as rows. - -## Below the cut line - -- The async execution view (`watchScan`, #103 M2), and turning a path or effects result into a work packet. -- Links from import-graph files to the routes they feed. -- Python effects, runtime traces, incremental invalidation, and saved evidence artifacts. -- Narrow layouts beyond keeping the source, Return, and status visible. diff --git a/docs/intent/tsx-dataflow-integration/initiative-map.json b/docs/intent/tsx-dataflow-integration/initiative-map.json deleted file mode 100644 index 025263f7..00000000 --- a/docs/intent/tsx-dataflow-integration/initiative-map.json +++ /dev/null @@ -1,840 +0,0 @@ -{ - "schema_version": "0.5.0", - "initiative": { - "title": "Bring tsx-data-flow into srcly", - "slug": "tsx-dataflow-integration", - "scale": "program", - "current_phase": "execution", - "last_updated": "2026-11-21", - "summary": "One engine (tsx-data-flow analysis moved into engine/, one lazy Node child) feeds three srcly surfaces: route -> terminal graph in the dependency viewer (headline), value-path ledger from graph destinations or code-viewer expressions (PR #102), and a function-effects panel (PR #103); findings/metrics/markers/reports follow. detailed-plan.md adds the file-level design, contracts, two-tier engine session, view-state registry and the corrections a code review of both repositories forced.", - "status": "active", - "next_action": "M7 shipped in part (2026-11-21): fileSignals op, report --dataflow (dataflow.md, report.md section), six dataflow_* metrics with compare --dataflow, impossible-defense and repeated-fork findings with code-viewer marks, skill section. Left: context-relay and broad-prop-bundle finding rules, dataflow_defensive_ops, and an in-app way to start the tier-2 run (today only the CLI starts it). Retire the plan when those land or are dropped." - }, - "artifacts": { - "intent_brief": { - "path": "intent-brief.md", - "status": "current", - "updated_at": "2026-10-23" - }, - "brainstorm": { - "path": "brainstorm.md", - "status": "not-requested", - "updated_at": null - }, - "shape_brief": { - "path": "shape-brief.md", - "status": "current", - "updated_at": "2026-10-23" - }, - "implementation_plan": { - "path": "implementation-plan.md", - "status": "current", - "updated_at": "2026-10-23" - }, - "research": { - "path": "research.md", - "status": "current", - "updated_at": "2026-10-23", - "supersedes": [ - "PR #102", - "PR #103" - ] - }, - "detailed_plan": { - "path": "detailed-plan.md", - "status": "current", - "updated_at": "2026-10-23" - }, - "measurements": { - "path": "measurements.md", - "status": "current", - "updated_at": "2026-10-23" - } - }, - "sources": [ - { - "id": "S1", - "type": "transcript", - "location": "conversation", - "anchor": "initial request 2026-10-23", - "authority": "user" - }, - { - "id": "S2", - "type": "repo", - "location": "~/Projects/tsx-data-flow/INTENT.md", - "anchor": "product priority order; code map primary surface", - "authority": "user" - }, - { - "id": "S3", - "type": "repo", - "location": "~/Projects/tsx-data-flow/README.md", - "anchor": "Views table; Explore it in the browser", - "authority": "repo" - }, - { - "id": "S4", - "type": "repo", - "location": "~/Projects/tsx-data-flow/docs/application-structure.md", - "anchor": "Boundaries; API surface", - "authority": "repo" - }, - { - "id": "S5", - "type": "repo", - "location": "~/Projects/tsx-data-flow/docs/performance-baseline.md", - "anchor": "10K/50K/200K table", - "authority": "repo" - }, - { - "id": "S6", - "type": "repo", - "location": "~/Projects/tsx-data-flow/docs/intent/field-trajectories-and-cleanup/", - "anchor": "intent + shape briefs", - "authority": "user" - }, - { - "id": "S7", - "type": "repo", - "location": "docs/CODE_MAP.md", - "anchor": "Findings row; Metrics end-to-end; Code viewer; Graph views", - "authority": "repo" - }, - { - "id": "S8", - "type": "repo", - "location": "AGENTS.md", - "anchor": "Add / extend a metric", - "authority": "repo" - }, - { - "id": "S9", - "type": "repo", - "location": "https://github.com/byronwall/srcly/blob/0a1c1133/docs/intent/imported-quality-evidence/shape-brief.md", - "anchor": "import artifacts before building engines", - "authority": "user" - }, - { - "id": "qa-round-1", - "type": "qa", - "location": "conversation", - "anchor": "replies to 3 open questions, 2026-10-23", - "authority": "user" - }, - { - "id": "qa-round-2", - "type": "qa", - "location": "conversation", - "anchor": "route -> terminal graph is the real want, 2026-10-23", - "authority": "user" - }, - { - "id": "S10", - "type": "repo", - "location": "client/src/viz/deps/contract.ts", - "anchor": "DependencyGraphContract", - "authority": "repo" - }, - { - "id": "PR102", - "type": "doc", - "location": "https://github.com/byronwall/srcly/pull/102", - "anchor": "docs/research/data-flow-integration-2026-10-23 + docs/intent/data-flow-integration/implementation-plan.md (condensed into research.md)", - "authority": "inferred" - }, - { - "id": "PR103", - "type": "doc", - "location": "https://github.com/byronwall/srcly/pull/103", - "anchor": "docs/research/effects-async-test-seams-2026-10-23 + docs/intent/effects-async-test-seams/implementation-plan.md (condensed into research.md)", - "authority": "inferred" - }, - { - "id": "qa-round-3", - "type": "qa", - "location": "conversation", - "anchor": "use PRs 102/103 as context, 2026-10-23", - "authority": "user" - }, - { - "id": "qa-round-4", - "type": "qa", - "location": "conversation", - "anchor": "run route-graph and ledger proofs in parallel; write plan", - "authority": "user" - } - ], - "intent": { - "claims": [ - { - "id": "C1", - "statement": "srcly is the home and base layer; tsx-data-flow features merge into it.", - "kind": "goal", - "sources": [ - "S1" - ], - "confidence": "high" - }, - { - "id": "C2", - "statement": "srcly's speed must be preserved; data flow never delays the base render.", - "kind": "constraint", - "sources": [ - "S1", - "S5" - ], - "confidence": "high" - }, - { - "id": "C3", - "statement": "Value order (revised): route -> terminal data-flow graph in the dependency viewer first; findings/metrics/markers/reports support it.", - "kind": "priority", - "sources": [ - "S1" - ], - "confidence": "high" - }, - { - "id": "C4", - "statement": "Analysis correctness (compiler-backed identity, strict tsconfig) outranks everything else in tsx-data-flow.", - "kind": "constraint", - "sources": [ - "S2", - "S3" - ], - "confidence": "high" - }, - { - "id": "C5", - "statement": "Markdown output exists for agent handoff: dense, grounded, concise next action.", - "kind": "goal", - "sources": [ - "S2" - ], - "confidence": "high" - }, - { - "id": "C6", - "statement": "tsx-data-flow's own overview/report pages are transitional and need not survive.", - "kind": "interpretation", - "sources": [ - "S2", - "S1" - ], - "confidence": "medium" - }, - { - "id": "C7", - "statement": "Several Pro threads will detail branches and update this map.", - "kind": "decision", - "sources": [ - "S1" - ], - "confidence": "high" - }, - { - "id": "C8", - "statement": "Single repository: engine moves into srcly (workspace/monorepo layout only if needed).", - "kind": "decision", - "sources": [ - "qa-round-1" - ], - "confidence": "high" - }, - { - "id": "C9", - "statement": "tsx-dataflow-serve UI goes away; focus on best versions of features in srcly.", - "kind": "decision", - "sources": [ - "qa-round-1" - ], - "confidence": "high" - }, - { - "id": "C10", - "statement": "Distribution delegated to agent: choose most robust. Chosen: esbuild bundle in the wheel, system node>=18, project typescript with bundled fallback, N/A without node.", - "kind": "decision", - "sources": [ - "qa-round-1" - ], - "confidence": "medium" - }, - { - "id": "C11", - "statement": "The real want is the route -> terminal data viz (Route Totality) shown in srcly dependency viewer.", - "kind": "goal", - "sources": [ - "qa-round-2" - ], - "confidence": "high" - }, - { - "id": "C12", - "statement": "All tsx-data-flow type checking and analysis logic is to be maintained.", - "kind": "constraint", - "sources": [ - "qa-round-2" - ], - "confidence": "high" - }, - { - "id": "C13", - "statement": "srcly has better visuals and selection and should own the graph experience.", - "kind": "decision", - "sources": [ - "qa-round-2" - ], - "confidence": "high" - }, - { - "id": "C14", - "statement": "Value investigation: exact expression -> ordered evidence ledger + inspector, with Return/Back restoring overview.", - "kind": "goal", - "sources": [ - "PR102", - "qa-round-3" - ], - "confidence": "high" - }, - { - "id": "C15", - "statement": "Function effects panel in code viewer: effects, ambient reads, writes, async handling, unknowns with witnesses; no purity score.", - "kind": "goal", - "sources": [ - "PR103", - "qa-round-3" - ], - "confidence": "high" - }, - { - "id": "C16", - "statement": "Connectivity, coverage and value properties are separate; Not analyzed != 0; evidence bound to Program text/config/generation.", - "kind": "constraint", - "sources": [ - "PR102", - "PR103" - ], - "confidence": "high" - }, - { - "id": "C17", - "statement": "Invocation identity must be preserved; shared helper calls never join.", - "kind": "constraint", - "sources": [ - "PR102" - ], - "confidence": "high" - }, - { - "id": "C18", - "statement": "Never relabel import or folder edges as data flow or execution.", - "kind": "constraint", - "sources": [ - "PR102", - "PR103" - ], - "confidence": "high" - }, - { - "id": "C19", - "statement": "Existing route canvas has empty evidence-node/edge arrays; it cannot show operation order, so ordered paths use a ledger.", - "kind": "fact", - "sources": [ - "PR102" - ], - "confidence": "high" - } - ], - "themes": [ - "route -> terminal graph as headline", - "evidence from engine, picture from srcly", - "base layer vs enrichment layer", - "one vocabulary (metrics/findings/markers)" - ], - "conflicts": [ - { - "id": "X1", - "statement": "tsx-data-flow says the code map is its primary surface; srcly's primary surface is the map/treemap with a modal code viewer.", - "resolution": "srcly's treemap stays primary; the code viewer absorbs code-map overlays (path on source, markers, where-used)." - }, - { - "id": "X2", - "statement": "PR102 says do not merge the repositories and use a prepared external checkout (--value-analyzer).", - "resolution": "User decision (single repo) wins; the prepared-checkout idea survives as dev mode running engine/ from the srcly checkout." - }, - { - "id": "X3", - "statement": "PR102 REC-017 avoids a semantic graph on the repository map and keeps the component canvas separate; user wants the route graph in the dependency viewer.", - "resolution": "Data-flow mode uses its own contract in the same canvas, never overlaid on import edges; ordered path detail lives in the ledger." - }, - { - "id": "X4", - "statement": "PR102 wants one resident lazy Node child; PR103 starts with a one-shot JSON command and defers a resident service.", - "resolution": "Shape adopts one lazy child shared by all query families (Program built once); keep a one-shot command for headless report/tests; confirm with measured cold cost." - } - ], - "human_questions": [] - }, - "brainstorm": { - "status": "not-requested", - "requested_count": 7, - "options": [], - "selected_option": null - }, - "shape": { - "status": "current", - "appetite": "Program delivered as independently revertible slices, one srcly home per slice.", - "problem": "Two tools with overlapping shells; srcly is fast but tree-sitter-only, tsx-data-flow is deep but slow and has its own UI.", - "recommended_shape": "Engine supplies evidence, srcly supplies the picture: engine (in engine/, bundled in wheel) emits a data-flow graph in srcly contract style (route -> occurrence -> step hierarchy + typed proven edges); srcly's dependency view renders it in a Data flow mode with its canvas, focus, expand-in-place, hover, keyboard and code opening. Findings/metrics/markers/reports flow through existing srcly pipelines.", - "feature_mapping": [ - { - "tsx_feature": "Route Totality / Data trajectories graph (route -> component occurrences -> field lineage -> terminals, frontiers)", - "srcly_home": "client/src/viz/deps Data flow mode + new data-flow graph contract", - "status": "planned", - "priority": 1 - }, - { - "tsx_feature": "Route discovery, occurrence surface, field lineage/proof, value summaries, type checking (src/analysis)", - "srcly_home": "engine/ package, moved intact", - "status": "planned", - "priority": 1 - }, - { - "tsx_feature": "Field reach / selected-path projections, isolation with boundary stubs", - "srcly_home": "deps focus + isolate for reach; selected path opens the value ledger", - "status": "planned", - "priority": 1 - }, - { - "tsx_feature": "Source evidence dialog / popovers", - "srcly_home": "CodeModal opened at node line", - "status": "planned", - "priority": 1 - }, - { - "tsx_feature": "Render-path findings, defensive-ledger, impossible defenses, repeated forks, context relay", - "srcly_home": "findings/ dataflow tool + markers", - "status": "planned", - "priority": 4 - }, - { - "tsx_feature": "Burden, path depth, representation changes, defensive ops, prop relay, fan-out/in", - "srcly_home": "metric_catalog.py entries (TS/TSX)", - "status": "planned", - "priority": 4 - }, - { - "tsx_feature": "work-packets Markdown, render-path-dataflow-work skill", - "srcly_home": "reporting.py artifacts + skills/srcly-code-quality", - "status": "planned", - "priority": 4 - }, - { - "tsx_feature": "--compare baseline", - "srcly_home": "compare/diff.COMPARE_METRICS via catalog", - "status": "planned", - "priority": 4 - }, - { - "tsx_feature": "Code map symbol uses / component-refs", - "srcly_home": "CodeModal where-used; deps symbol focus", - "status": "evaluate" - }, - { - "tsx_feature": "boundary-report, junctions, path-families, inline-preview", - "srcly_home": "finding rules or report-only", - "status": "evaluate" - }, - { - "tsx_feature": "Route Totality canvas/camera/layout/disclosure UI (~9.6K lines), component topology, world map", - "srcly_home": "replaced by srcly deps canvas; not migrated", - "status": "drop" - }, - { - "tsx_feature": "tsx-dataflow-serve overview, file rows, report pages", - "srcly_home": "none (explorer/Hot Spots cover it); retired", - "status": "drop" - }, - { - "tsx_feature": "Expression selection + value investigation (new engine API, per-invocation pairing)", - "srcly_home": "engine/ + server value-flow router + client investigation/ (PinnedPreview, EvidenceLedger, EvidenceInspector)", - "status": "planned", - "priority": 2, - "source": "PR102" - }, - { - "tsx_feature": "Function effects pass (new)", - "srcly_home": "engine/ effects + CodeModal EffectsPanel", - "status": "planned", - "priority": 3, - "source": "PR103" - }, - { - "tsx_feature": "Bounded async execution slice", - "srcly_home": "client/src/viz/effects (later)", - "status": "later", - "source": "PR103" - } - ], - "success_moves": [ - "Emit route evidence in srcly's graph-contract style (hierarchy + typed proven edges) in its own Data flow mode, never on import edges.", - "Explain paths once: graph destinations and code-viewer expressions both open the same ordered ledger.", - "One lazy Node child owns one Program for all query families; base scan never waits on it.", - "Bind evidence to Program text, tsconfig and generation; name not-analyzed, partial, stale and budget states.", - "Prove each family on real anchors (solid-route; HotSpotItem.tsx:135 and HotSpotList.tsx:88/109; extractFilePath/rankHotSpots/startScan) before growing UI.", - "Coordinate deps layout changes with dependency-graph-legibility." - ], - "first_proof": { - "question": "Can srcly present one route's field reach in its dependency canvas and one exact value path as an ordered ledger, both from compiler-backed evidence?", - "artifact": "examples/solid-route rendered in the current deps canvas; HotSpotItem.tsx:135 props.node.name ledger from the code viewer.", - "decision_rule": "Graph: extend packing if legible, else add one route layout rule. Ledger: if generic evidence cannot prove the JSX slot exactly (and keep line 134 distinct, rank default preserving zero), fix the engine before UI." - }, - "rabbit_holes": [ - "Porting compiler-API analysis to Python.", - "Embedding tsx-data-flow's SPA or route-totality graph wholesale.", - "200K-LOC repos: engine takes ~120s; needs scoping (changed files, per-package) or a skip rule.", - "Monorepos with many tsconfigs: bound projects like MAX_TSC_PROJECTS.", - "Wheel size from bundled TypeScript fallback; measure in first proof.", - "Porting tsx-data-flow's graph layout/camera instead of using srcly's.", - "Fields with dozens of destinations rendered as a wall instead of grouped broad-use.", - "Merging separate helper invocations (HotSpotList row helper, line 88 vs 109).", - "Treating effects as a numeric risk metric." - ], - "open_decisions": [], - "performance_targets": { - "preview_return_p95_ms": 150, - "progress_ms": 250, - "warm_first_row_ms": 750, - "cold_first_result_s": 10, - "source": "PR102 proposals" - } - }, - "implementation": { - "status": "planned", - "repository_snapshot": { - "root": "srcly", - "revision": "b3117e3", - "worktree_status": "clean", - "subsystems": [ - "server/app/services/findings", - "server/app/services/metric_catalog.py", - "server/app/services/markers.py", - "server/app/services/reporting.py", - "server/app/services/compare", - "client/src/components/CodeModal", - "client/src/viz/deps" - ], - "test_commands": [ - "cd server && uv run pytest", - "cd client && pnpm test" - ], - "deployment_surfaces": [ - "PyPI srcly wheel (bundles engine JS)" - ] - }, - "dependencies": [ - { - "id": "node-runtime", - "name": "Node >=18 + TypeScript", - "seam": "server/app/services/dataflow/engine.py (stdio JSON lines)", - "first_proof_role": "real (dev checkout); fake engine script for lifecycle tests", - "proof_ladder": [ - "fake engine script", - "real engine from engine/", - "bundled engine in wheel (m8)" - ] - } - ], - "fast_local_loop": { - "command": "cd engine && pnpm vitest run test/srcly", - "python": "cd server && uv run pytest tests/test_dataflow_engine.py", - "network": false - }, - "milestones": [ - { - "id": "m1", - "title": "Engine in srcly answers one query", - "outcome": "GET /api/dataflow/routes lists examples/solid-route routes via a lazy stdio Node child owned by server/app/services/dataflow/engine.py.", - "dependencies_introduced": [ - "node>=18 runtime", - "engine/ pnpm workspace package" - ], - "proof": "pnpm --filter engine test; uv run pytest tests/test_dataflow_engine.py (fake engine lifecycle + real smoke)", - "valid_intermediate_state": "No UI; base scan unchanged; status names N/A reasons.", - "rollback": "Remove engine/ and routers/dataflow.py.", - "status": "completed", - "evidence": [ - "#104, #105, #106 merged; engine/bin/srcly-engine.ts, server/app/services/dataflow/engine.py." - ], - "track": "foundation" - }, - { - "id": "m2", - "title": "Route graph in the dependency view", - "outcome": "Data flow mode renders a route's component backbone; field selection lights proven destinations; nodes open code.", - "dependencies_introduced": [], - "proof": "engine contract tests on solid-route; client adapter vitest; screenshots of solid-route and /time-blocks; layout spike decision.", - "valid_intermediate_state": "Mode hidden when engine N/A.", - "rollback": "Revert mode switch + routeGraph op.", - "status": "completed", - "evidence": [ - "#107, #110, #111, #112, #130 merged; engine/src/srcly/ops/route-graph.ts, client/src/viz/deps/flow/." - ], - "track": "A" - }, - { - "id": "m3", - "title": "Value ledger from the code viewer", - "outcome": "Inspect value flow on HotSpotItem.tsx:135 shows an ordered ledger; Return restores overview.", - "dependencies_introduced": [], - "proof": "engine fixtures + HotSpotItem anchor contract tests; client investigation tests; browser journey 1440/1024 with Back/Forward/Return.", - "valid_intermediate_state": "Entry hidden when engine N/A.", - "rollback": "Revert investigation/ + valueFlow op.", - "status": "completed", - "evidence": [ - "#108, #109, #114, #115, #116, #117 merged." - ], - "track": "B" - }, - { - "id": "m4", - "title": "One path explanation from the graph", - "outcome": "Graph destination opens the same ledger from route lineage; Return restores graph.", - "dependencies_introduced": [], - "proof": "routePath contract test on /time-blocks; browser journey.", - "valid_intermediate_state": "Graph still works without ledger link.", - "rollback": "Revert routePath op and link.", - "status": "completed", - "evidence": [ - "#136 merged; GET /api/dataflow/route-path." - ], - "track": "A+B" - }, - { - "id": "m5", - "title": "Cross-file invocation identity", - "outcome": "HotSpotList.tsx:88 proven to HotSpotItem.tsx:135 through row; line 109 isolated.", - "dependencies_introduced": [], - "proof": "precision fixtures (two-call identity helper, returns-other-arg) + anchor tests.", - "valid_intermediate_state": "M3 ledger unaffected for local paths.", - "rollback": "Revert pairing; paths fall back to frontier rows.", - "status": "completed", - "evidence": [ - "#132 merged." - ], - "track": "B" - }, - { - "id": "m6", - "title": "Effects panel", - "outcome": "CodeModal Effects panel for extractFilePath, rankHotSpots, startScan with witnesses and coverage.", - "dependencies_introduced": [], - "proof": "compact semantic fixture + three anchor contract tests; panel scenario test.", - "valid_intermediate_state": "Panel shows unavailable without engine.", - "rollback": "Revert EffectsPanel + functionEffects op.", - "status": "completed", - "evidence": [ - "#134, #135 merged; POST /api/dataflow/function-effects." - ], - "track": "C" - }, - { - "id": "m7", - "title": "Supporting evidence in srcly's pipelines", - "outcome": "Data-flow findings, catalog metrics, markers, report work packets, compare.", - "dependencies_introduced": [], - "proof": "test_metric_catalog snapshots; markers count tests; report artifact test.", - "valid_intermediate_state": "Metrics N/A when engine absent.", - "rollback": "Remove dataflow tool + catalog entries.", - "status": "active", - "evidence": [], - "track": "D" - }, - { - "id": "m8", - "title": "Engine ships in the wheel", - "outcome": "esbuild bundle + fallback typescript in wheel; uvx smoke in clean venv.", - "dependencies_introduced": [ - "esbuild bundle in server/app/engine" - ], - "proof": "build-srcly.sh then clean-venv smoke on examples/solid-route; wheel size recorded.", - "valid_intermediate_state": "Pre-M8 wheels show N/A.", - "rollback": "Drop bundle step.", - "status": "completed", - "evidence": [ - "#149 merged; engine/scripts/bundle.mjs." - ], - "track": "release" - } - ], - "cut_line": [ - "async execution view (#103 M2)", - "path/effects -> work packet", - "import-graph -> route links", - "Python effects, runtime tracing, incremental invalidation", - "saved evidence artifacts", - "mobile layouts" - ], - "open_decisions": [ - { - "id": "D1", - "question": "Port base in tsx-data-flow", - "evidence": "pnpm verify on the committed WIP", - "fallback": "port main b4f1ba4 and re-apply the WIP inside engine/", - "status": "default-accepted", - "accepted_at": "2026-10-23" - }, - { - "id": "D2", - "question": "Route layout rule vs existing packing", - "evidence": "M2 screenshots at 1440 and 1024", - "fallback": "direction: down option on layoutComponent, flow mode only", - "status": "default-accepted", - "accepted_at": "2026-10-23" - }, - { - "id": "D3", - "question": "Branch graph beside the ledger", - "evidence": "M5 two-arm Show path", - "fallback": "rows with expansion", - "status": "default-accepted", - "accepted_at": "2026-10-23" - }, - { - "id": "D4", - "question": "TypeScript instance for the Program", - "evidence": "hello.typescriptFrom and parse diagnostics on real projects", - "fallback": "prefer the project's copy for that major", - "status": "default-accepted", - "accepted_at": "2026-10-23" - }, - { - "id": "D5", - "question": "Tier-2 batch job default", - "evidence": "tier-2 time on 50K-line apps", - "fallback": "opt-in via a data-flow surface or --dataflow", - "status": "default-accepted", - "accepted_at": "2026-10-23" - }, - { - "id": "D6", - "question": "Resident vs per-query process", - "evidence": "tier-1 time and RSS (2 GB measured)", - "fallback": "per-query spawn, same protocol; cap 1 process", - "status": "default-accepted", - "accepted_at": "2026-10-23" - }, - { - "id": "D7", - "question": "Investigation URL", - "evidence": "none expected", - "fallback": "?inv= with sessionStorage snapshot", - "status": "default-accepted", - "accepted_at": "2026-10-23" - }, - { - "id": "D8", - "question": "Progress transport", - "evidence": "profiles of 250 ms status polling", - "fallback": "SSE like /api/scans/{id}/events", - "status": "default-accepted", - "accepted_at": "2026-10-23" - } - ], - "prerequisites": [ - "Commit tsx-data-flow's route-proof WIP on main, tag srcly-port-base, port that SHA; the seven cleanup PRs are not ported (D1).", - "Skip-guard the two soccer-schedule tests before the copy." - ], - "parallel_tracks": { - "after": "m1", - "tracks": { - "A": [ - "m2", - "m4" - ], - "B": [ - "m3", - "m5" - ], - "C": [ - "m6" - ], - "D": [ - "m7" - ] - } - }, - "measurements": { - "recorded_at": "2026-10-23", - "path": "measurements.md", - "tier1_ms": { - "srcly-client": 3005, - "visual-notes-app": 13865 - }, - "full_report_ms": { - "srcly-client": 129962, - "visual-notes-app": 44166 - }, - "route_inventory_ms": { - "srcly-client": 2, - "visual-notes-app": 17494 - }, - "rss_mb": { - "srcly-client": 2419, - "visual-notes-app": 2023 - } - } - }, - "history": [ - { - "date": "2026-10-23", - "event": "Initiative created; intent and shape drafted from the user's request plus both repos' docs.", - "phase": "shape" - }, - { - "date": "2026-10-23", - "event": "Resolved D1 (single repo, engine/ package), D2 (bundle in wheel, delegated), D3 (retire serve UI). Shape updated; no downstream artifacts invalidated.", - "phase": "shape" - }, - { - "date": "2026-10-23", - "event": "Intent revised: route -> terminal graph in dependency viewer is the headline; shape, mapping priorities and first proof regenerated.", - "phase": "shape" - }, - { - "date": "2026-10-23", - "event": "Absorbed PR #102 (value investigation) and #103 (function effects): added ledger and effects surfaces, shared engine runtime, evidence rules, real anchors; recorded conflicts X2-X4.", - "phase": "shape" - }, - { - "date": "2026-10-23", - "event": "D4 resolved: route-graph and ledger proofs run in parallel. Implementation plan written (m1 foundation, tracks A-D, m8 packaging).", - "phase": "plan" - }, - { - "date": "2026-10-23", - "event": "Condensed: #102/#103 research folded into research.md; intent, shape and plan tightened. Implementation awaits explicit user approval.", - "phase": "plan" - }, - { - "date": "2026-10-23", - "event": "Detailed plan and measurements added after a code review of srcly and tsx-data-flow; fourteen corrections (R1-R14) and eight decisions (D1-D8) recorded.", - "phase": "plan" - }, - { - "date": "2026-10-23", - "event": "User accepted the D1-D8 defaults; PR #101 remains docs only, M1 awaits separate approval.", - "phase": "plan" - }, - { - "date": "2026-11-12", - "change": "Set true milestone statuses from merged PRs: M1-M6 and M8 completed; M7 still planned." - }, - { - "date": "2026-11-21", - "change": "M7 shipped in part: fileSignals, report/compare --dataflow, data-flow metrics, findings and marks, skill. Context-relay and broad-prop-bundle rules and an in-app trigger remain." - } - ], - "closure": null -} diff --git a/docs/intent/tsx-dataflow-integration/intent-brief.md b/docs/intent/tsx-dataflow-integration/intent-brief.md deleted file mode 100644 index 0975b71c..00000000 --- a/docs/intent/tsx-dataflow-integration/intent-brief.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: "Bring tsx-data-flow into srcly" -slug: "tsx-dataflow-integration" -phase: intent -status: current -last_updated: "2026-10-23" ---- - -# Bring tsx-data-flow into srcly - -## My read - -What you want most from tsx-data-flow is its route → terminal data-flow graph, shown in srcly's dependency viewer. Its compiler-backed analysis is the asset to keep: TypeScript type checking, symbol identity, field lineage, and proof frontiers. srcly has the better visuals and selection. So the engine supplies the evidence, and srcly draws it. - -The same engine answers two more questions from the code viewer: -- What happened to this value? An ordered path ledger answers this. -- What does this function touch? An effects panel answers this. - -Findings, metrics, marks, and reports bring the evidence into srcly's existing surfaces. Throughout, srcly stays the fast base layer. - -## What matters most - -- The route → terminal graph in the dependency viewer is the headline. -- The analysis moves whole. Evidence is never approximated or invented. -- srcly owns visuals, selection, navigation, and return. -- Evidence states are honest: proven, partial, not analyzed, and stale are distinct, and not analyzed is never 0. -- The engine never delays the treemap or the import graph. - -## The experience you appear to want - -You open srcly on a SolidStart app, and the treemap and import graph appear as they do today. In Data flow mode you pick a route and see its components down to the terminal render points. You select a field, and its proven destinations light up. You pick one destination, and a ledger explains the path step by step. Any step opens the code. From any line in the code viewer, you can ask the same of a value or a function. Return puts you back exactly where you were. - -## Boundaries - -### Must be true - -- Content comes from compiler-backed evidence with exact source spans. Evidence is tied to the Program, the tsconfig, and the generation that produced it. -- Call identity is kept. Two calls to one helper never merge. -- Without Node, a tsconfig, or TS files, data flow is N/A with a reason, and nothing else changes. - -### Must be avoided - -- No Python port and no second TypeScript analyzer. -- No relabeling import edges as data flow or execution. -- No purity, testability, or tangle score. -- Don't carry over tsx-data-flow's canvas, layout, or SPA. - -## What seems settled - -- One repository: the engine moves to `engine/`, and tsx-dataflow-serve goes away. -- The engine ships bundled in the wheel and runs on the system `node`. -- The route-graph and ledger proofs run in parallel, after a shared engine milestone. -- Priority order: - 1. the route graph; - 2. the ledger, which also explains graph paths; - 3. effects; - 4. findings, metrics, and reports. -- Implementation starts only with your explicit approval. - -## Current reality that matters - -- tsx-data-flow's route canvas has no operation-order evidence, which is why the ledger carries path detail. -- Two engine capabilities are new work: generic expression selection, and per-invocation tracking. -- tsx-data-flow has four open cleanup PRs and uncommitted route-proof edits. These land before the move. - -## Next step after confirmation - -Wait for your approval, then start M1. - -Evidence for all of this is in [research.md](research.md). The shape is in [shape-brief.md](shape-brief.md), and the steps are in [implementation-plan.md](implementation-plan.md). diff --git a/docs/intent/tsx-dataflow-integration/measurements.md b/docs/intent/tsx-dataflow-integration/measurements.md deleted file mode 100644 index ad72eee2..00000000 --- a/docs/intent/tsx-dataflow-integration/measurements.md +++ /dev/null @@ -1,229 +0,0 @@ ---- -title: "Bring tsx-data-flow into srcly — measurements" -slug: "tsx-dataflow-integration" -phase: plan -status: current -last_updated: "2026-10-23" ---- - -# Measurements - -Every number this initiative records. Append a dated row per milestone; never overwrite an earlier one. - -## 2026-10-23 — planning baseline - -Machine: development Mac. Node v22.21.1. tsx-data-flow `b4f1ba4` plus the uncommitted route-proof edits, run from its checkout with `node --import tsx`. TypeScript 5.9.3 resolved from each target project. One cold run per target, wall clock from `performance.now()`. - -| Target | Source files | `buildProgram` | `buildIdentityIndex` | `analyzer.report()` (full legacy report) | `buildRouteDataInventory` | RSS after | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| srcly `client/` (`tsconfig.app.json`, 26K lines) | 126 | 1,209 ms | 1,796 ms | 129,962 ms | 2 ms (0 routes) | 2,419 MB | -| visual-notes `app/` (`tsconfig.json`, 65K lines) | 487 | 4,323 ms | 9,542 ms | 44,166 ms | 17,494 ms (43 routes) | 2,023 MB | - -A second run of the CLI end to end (`pnpm analyze --root srcly/client --source src --tsconfig tsconfig.app.json --view overview --format json`) took 2 min 13 s wall and wrote a 421 KB overview. - -Routes discovered on visual-notes include `/time-blocks`, `/tasks`, `/canvas`, `/docs/[id]`, `/api/auth/[...solidauth]` and 38 others. - -Script: a throwaway `time-phases.ts` that called `parseArgs`, `createAnalyzer`, `analyzer.report()` and `buildRouteDataInventory` in sequence and printed `process.memoryUsage().rss`. It is not checked in; M1 adds an equivalent `engine/scripts/time-phases.ts`. - -## 2026-10-23 — M1: cold `hello` and tier 1 through the protocol - -Machine: development Mac. Node v22.21.1. Engine at `srcly-port-base` (`0a814cd`) plus the PR 2 session. `engine/scripts/time-tier1.ts` spawns `bin/srcly-engine.ts` through `tsx`, sends `hello`, then `routes` (tier 1: Program, identity index, route discovery; no legacy report), then `status`. One cold run per target; RSS is the engine process (protocol thread plus worker) from `status`. - -| Target | `hello` (spawn to reply) | `routes` (tier 1) | Routes | TypeScript | RSS after | -| --- | ---: | ---: | ---: | --- | ---: | -| `engine/examples/solid-route` | 568 ms | 942 ms | 1 | 5.9.3 bundled | 524 MB | -| srcly `client/` (`tsconfig.app.json`) | 557 ms | 2,265 ms | 0 | 5.9.3 bundled (no `client/node_modules` in this checkout) | 592 MB | -| visual-notes `app/` (`tsconfig.json`) | 691 ms | 10,392 ms | 43 | 5.9.3 project | 1,733 MB | - -Tier 1 on visual-notes is 10.4 s against the 13.9 s the planning baseline summed for program plus identity index, and it misses the 10 s cold target by a little. RSS after tier 1 is below the baseline's because the legacy report never runs. - -## 2026-10-24 — M2: first `routeGraph` per route, and decision D2 - -Same machine and Node. Engine `routeGraph` called in process right after tier 1 (cold totality for that route; `route-graph.ts` projection included). - -| Target · route | tier 1 | first `routeGraph` | Payload | Nodes | Edges | Engine-omitted items | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| solid-route · `/records` | 461 ms | 172 ms | 24.7 KB | 33 | 13 | 0 | -| visual-notes · `/time-blocks` | 14,206 ms | 34,691 ms | 359 KB | 418 | 325 | 258 | -| visual-notes · `/tasks` | 10,186 ms | 31,432 ms | 270 KB | 327 | 250 | 184 | - -Through the UI, a cold `/time-blocks` (saved scan, engine start, tier 1, first `routeGraph`) took 68 s from page load to the first drawn box; the progress panel showed each phase meanwhile. - -Before the session aligned route identity, every visual-notes route came back `unavailable` ("No unique proven Solid route candidate matched"): route discovery records `export default Page;` at the identifier, the Solid scope adapter at the `const Page = () => …` declaration, and totality matches them by span. Upstream's full-report path fails the same way. `engine/src/srcly/route-identity.ts` aligns the span through the checker in the session; `src/analysis` is unchanged. - -**D2 (route layout): keep today's packing with occurrence frames.** The adapter puts every flow node in the center region, makes its render parent its frame (`group.opened`), and adds hidden containment edges so columns follow hierarchy depth. In all four screenshots each frame's boxes stay together in one column block: ViewerCard's five terminals, RecordRow's four, and WeeklyTimeBlocksCalendar's thirteen children each read as one framed column. The packer folds adjacent depth levels into one column when the viewport is short, so a frame can share its parent's column, but no occurrence's terminals were split from each other. No `direction: "down"` rule and no `layout.ts` change. Screenshots (attached to the M2 PR, kept in the ignored `tmp/`): `d2-solid-route-1440.png`, `d2-solid-route-1024.png`, `d2-time-blocks-1440.png`, `d2-time-blocks-1024.png`. - -## 2026-10-24 — M3: value-flow ledger on srcly `client/` - -`engine/scripts/time-value-flow.ts` spawns the engine on srcly's `client/` (`tsconfig.app.json`), sends `hello`, then `valueFlow` on `HotSpotItem.tsx:135` (`props.node.name`), then 20 warm `valueFlow` queries cycling over the 135, 130, 145 and `HotSpotList.tsx:88` anchors. - -| Measure | Value | -| --- | ---: | -| Cold spawn → `hello` → first ledger (tier 1 plus the lazy provider's fact index) | 11,877 ms | -| Warm `valueFlow` p50 / p95 / max over 20 | 1 / 3 / 24 ms | -| RSS after | 1,109 MB | - -The cold number misses the 10 s target on srcly's own client: about 2.3 s is tier 1 and the rest is the evidence provider building its fact index on the first query that needs facts. Warm queries are far under the 750 ms first-row target. - -Acceptance anchors (`engine/test/srcly/value-flow.test.ts`, located by text, not line number): - -| Selection | Ledger | -| --- | --- | -| `HotSpotItem.tsx:135` `props.node.name` | `ok`: input `props.node.name` (proven) → transfer `props.node` (proven) → terminal "JSX render slot" (proven). Exact. | -| `HotSpotItem.tsx:134` `title={props.node.name}` | `ok`: its own ledger ending at a "DOM attribute or property" terminal; no row of it appears in the 135 ledger. | -| `HotSpotItem.tsx:130` `props.rank ?? "–"` | `ok`: a proven fallback row, "?? keeps 0, \"\" and false…". | -| `HotSpotItem.tsx:145` the metric expression | `ok`, not the plan's `partial`: `formatMetricWithUnit` has exactly one call site in the program on this branch (the plan assumed many), so it is a `transform` row. `value()!` is a branch row labelled "assertion, not a check". The boundary rule itself is exercised by line 88 and by `examples/value-flow-basics/src/two-call-helper.tsx`. | -| `HotSpotList.tsx:88` `item.node` | `partial`: one boundary row at `row(item.node, …)`, `unproven-handoff`, "row has 2 call sites (…:88, …:109)". | - -Call sites are counted from the TypeScript program (calls resolving to the callee's symbol; a non-call reference makes the callers unknown), not from the lazy provider, which only knew `invokes` relations from files it had loaded and first reported `formatMetricWithUnit` correctly by accident. - -## 2026-10-24 — T1: per-box status in `routeGraph` - -Same machine and Node. Engine `routeGraph` called in process right after tier 1, counting boxes by status before and after `route-graph.ts` gave each box its own status (`markPartialBoxes`). - -| Target · route | Boxes | Proven before | Partial before | Proven after | Partial after | Located gaps | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| solid-route `/records` | 33 | 33 | 0 | 32 | 1 (the route box: `external-code` in `data/*.ts`) | 2 of 2 | -| visual-notes `/time-blocks` | 418 | 1 | 417 | 207 | 211 | 303 of 303 | - -Before, every box on `/time-blocks` took the surface's `partial` status (only the origin kept its own). After, the 211 partial boxes are the ones that own a located gap or omission: 97 `unresolved-symbol`, 96 `external-code`, 16 `budget-exhausted`, 1 `unsupported-ownership`, plus the route box for the incomplete surface. Every gap had a location, so the R14 fallback (the occurrence's own omissions) was not needed. On `/records` the route's two gaps sit in `data/records.ts` and `data/viewer.ts`, outside every drawn box, so they belong to the route box and all 32 component boxes read proven. - -## 2026-10-24 — T8: where the time goes (`engine/scripts/time-phases.ts`) - -Same machine and Node, in process (`pnpm time-phases --root ~/Projects/visual-notes/app --route /time-blocks --route /tasks --anchor …`). The engine's own test suite ran beside this measurement, so the absolute numbers are higher than M1/M2 (tier 1 was 10.4 s there); the shares are what this run is for. - -| Target | Tier 1 | Provider fact collection | Facts | RSS | -| --- | ---: | ---: | ---: | ---: | -| solid-route | 437 ms | 68 ms | 380 | 399 MB | -| visual-notes `app/` | 19,768 ms | 16,567 ms | 110,990 | 1,673 MB | - -Per route on visual-notes (`/tasks` first, so it pays every warm-up; `/time-blocks` second): - -| Route | candidate | surface | context | slice | bridges | lineage | other | totality | project | nodes | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| `/tasks` | 2 | 54 | 1,790 | 8,133 | 3 | 0 | 1,807 | 11,789 | 12 | 327 | -| `/time-blocks` | 0 | 23 | 321 | 180 | 2 | 0 | 6 | 533 | 9 | 418 | - -`other` is the finding attachment after the record is built. `anchorGraph` on five visual-notes anchors is 3–28 ms each once the facts exist (the TimeBlockEditorDialog `props.block` anchor draws 83 boxes in 28 ms). - -**What this decides (the T8 → T9/T10 question).** The cold path to a first box is tier 1 (≈10 s on a quiet machine), then the provider's fact collection (16.6 s, every file in the program, before any route), then the first route's slice (8.1 s) and context continuity (1.8 s). Fact collection is over half of the time after tier 1, and the route backbone itself (surface 54 ms) cannot start before it: route candidates come from the fact index. So T10 goes first, on fact collection and then the slice, and T9 follows. The budget R4 (10 s cold) needs tier 1 plus facts plus a surface under 10 s; today tier 1 alone is near it, so the disk cache (T11) is the lever for reopen and T10 for the cold path. - -## 2026-10-24 — T10: profile-led edits to fact collection and the slice - -Same machine, in process (`pnpm time-phases --root ~/Projects/visual-notes/app --route /tasks --route /time-blocks`), a quiet machine this time. The CPU profile behind the edits (`node --cpu-prof` over the same script) put 43% of the run in the TypeScript checker, 20% in garbage collection, and inside the engine's own code: `locationFor` 4.7 s (of which `path.relative`/`path.resolve` 4.4 s), `safeType` 6.3 s (3.8 s of it hydrating facts for the HTTP-bridge pass, which never reads type text; 2.6 s in `isHandlerLike` asking every parameter), `getFullyQualifiedName` 2.1 s, the identity index 5.2 s of tier 1 (its own `relative` per expression). - -| Phase (visual-notes) | Before (T8) | After T10 | -| --- | ---: | ---: | -| Tier 1 | 10,115 ms | 8,038 ms | -| Provider fact collection (110,990 facts) | 15,109 ms | 6,703 ms | -| `/tasks` first: slice | 9,322 ms | 3,912 ms | -| `/tasks` first: context continuity | 1,741 ms | 2,251 ms | -| `/tasks` first: whole totality | 12,901 ms | 7,967 ms | -| `/time-blocks` second: whole totality | 533 ms | 512 ms | -| Cold path to the first `/tasks` box (tier 1 + facts + totality) | 38.1 s | 22.7 s | - -Every result is byte-identical before and after (route graphs for `/time-blocks` and `/tasks`, two anchor graphs and a ledger on visual-notes; `/records`, an anchor and a ledger on solid-route; two anchors and two ledgers on srcly `client/`), and the engine suite with G01–G20 passes. The edits are listed in `engine/UPSTREAM.md`. - -**What is left on the cold path.** Tier 1 (8 s, the TypeScript program and the identity index) and fact collection (6.7 s, the checker's `getSymbolAtLocation` for every identifier) are now the two halves; the first slice (3.9 s) re-walks every file once to collect relations (`ProgramEvidenceRelationFrontierLoader.relationSource`). The next T10 round is that relation source, per file on demand; T11's cache serves a reopen without any of it. - -## 2026-10-24 — T9: the backbone first - -In process (`pnpm time-phases`, after tier 1): `/time-blocks` surface 134 ms (687 boxes before folding), `/tasks` 358 ms, from facts collected for the route file alone plus the occurrence surface, before the program-wide provider (6.7 s) and the slice. - -Through the UI on visual-notes `/time-blocks`, cold (saved scan, engine start, tier 1, then the backbone; a Playwright page timing from navigation): - -| | Before (M2) | After T9 | -| --- | ---: | ---: | -| First drawn box | 68 s | 10.9–11.4 s (the backbone, "tracing evidence…") | -| Complete graph (origins, handoffs, gaps) | 68 s | 25.7 s | -| Warm reopen, first box | — | 3.9 s | - -Found on the way: the flow view's start effect read the recorded status and so posted `/start` again on every status write, hundreds of times a second while the engine started; and `routes` was asked again on every poll. Both fixed here (one `/start`, one `routes`, one `route-surface`, one `route-graph` per cold load). - -R4 asks for ≤ 10 s cold. The first box is now at the edge of it on this machine; tier 1 (8 s) is what is left in front of the backbone, and T11's cache answers a reopen without it. - -## 2026-10-24 — T11: the result cache on reopen - -visual-notes `/time-blocks` through the UI, a Playwright page timing from navigation. The first run builds and writes the cache (684 KB for this root after one route); the second closes the engine (`DELETE /api/dataflow`) and starts a new process, which rehashes the last build's 487 inputs and serves the cached complete graph as its generation 1 while tier 1 builds behind it. - -| | First drawn box | What it is | -| --- | ---: | --- | -| Cold (T9) | 12.4 s | the backbone, evidence still tracing | -| Reopen, hash unchanged | 3.4 s | the complete graph, from disk | - -Of the 3.4 s, about 1 s is the page and the saved scan and about 2 s the engine spawn and `hello`; the cached `routeGraph` itself answers in milliseconds. R4's reopen target (3 s to the first box) is met for the engine's share and missed by the page's. A changed source hashes differently and serves nothing until the build answers (`result-cache.test.ts`). - -## 2026-10-24 — T13: call identity - -The slice walk carries an invocation stack (M5 edits 1–2, as 1-CFA on the walk). Acceptance anchors (`engine/test/srcly/{anchor-graph,value-flow}.test.ts`): - -| Selection | Before | After | -| --- | --- | --- | -| basics `two-call-identity-helper.tsx` `props.a` | boundary at `id` (2 call sites) | `ok`: through `id`'s parameter and return to its own call's slot at line 11; line 12 absent | -| basics `two-call-identity-helper.tsx` `value` (inside `id`) | boundary | boundary at both calls (`unproven-handoff`): leaving a helper two calls own stays ambiguous (R7) | -| basics `returns-other-argument.tsx` `props.first` | `ok` through the bare `argument` link (a false proof) | `partial`: a dead end at `pick`'s `a`, which nothing returns; no terminal | -| basics `two-call-helper.tsx` `props.a` | boundary at `shout` | `partial`: enters `shout`, dead end at `value.toUpperCase()` (external) | -| srcly `HotSpotList.tsx:88` `item.node` | boundary at `row` (2 call sites: 88, 109) | `ok`: `node` at 61 → `` at 62, "handed to a component"; line 109 never appears | - -Line 88 → 135 would still need the plain-identifier-prop edit (below the cut line, R3), so the walk ends where `node` is handed to `HotSpotItem`, a terminal, not a false proof. - -## 2026-10-24 — T12: a Program scoped to one route's import closure (decided off) - -`engine/scripts/time-scoped-program.ts` (one route per process, `--max-old-space-size=8192`: holding the full and a scoped Program together overflows the default heap) on visual-notes: - -| Route | files full → scoped | tier 1 full → scoped | first `routeGraph` full → scoped | same result | -| --- | ---: | ---: | ---: | --- | -| `/time-blocks` | 487 → 34 | 8,035 → 8,823 ms | 15,119 → 3,707 ms | no: the `TimeBlockEditorDialog.tsx:58` `props.block` anchor differs (same box count, different boxes) | -| `/tasks` | 487 → 36 | 7,753 → 10,191 ms | 15,242 → 3,680 ms | yes | - -**Decision.** Tier 1 gains nothing from scoping: TypeScript still resolves and type-checks the same library and `node_modules` surface for 34 roots as for 487. The gain is in fact collection and the slice (15 s to 3.7 s), which T9's backbone and T11's cache already cover for the first box and the reopen, and one anchor differs from the full Program, which the stop rule turns scoping off for. The `rootFiles` option stays as the spike's instrument and nothing in the engine uses it. - -## 2026-10-24 — T16: explain an edge - -`routePath` turns one lit edge into ordered ledger rows. A handoff's rows are the bridge's own `evidencePathElementIds`; a field edge's rows are its accepted transfers (C01-C12) chained from the origin to the consumer value, because a field attachment's `evidencePathElementIds` holds only the origin. Elements the projected slice left out are read from the provider's fact index; nothing is invented. - -| Case | Edge | Rows | Time after the graph is up | -| --- | --- | --- | --- | -| compact-field-proof `/projects`, origin `readFile(...)` | field `projects[*].ownerName` → ProjectDetails | 14 (origin, 11 carries and transforms through `data()?.projects.find(...)`, `current()`, the prop, the field, terminal `props.project.ownerName` at projects.tsx:13) | under 100 ms | -| compact-field-proof `/projects`, same origin | handoff → `props.value` | 26 | under 100 ms | -| solid-route `/records`, origin `createResource(loadViewer)` | three handoffs | 6 each, every row after the first carrying its relation | under 100 ms | - -The field edge ends at the consumer value inside the component body, one hop past the occurrence box the graph draws the edge to; the inspector names the endpoints from the drawn edge, so the header still reads origin → box. - -**visual-notes `/time-blocks` and `/tasks`**: no origin reaches a proven render site and field lineage is N/A ("no unique compiler-backed filesystem origin"), so there is no lit edge to explain there. The proof the ticket named on `/time-blocks` therefore reduces to the honesty tickets' finding; the edge explanation is proven on the two examples and in the browser smoke (`a lit edge, clicked, lists its steps in order in the inspector`). - -## 2026-10-24 — T17: Return through the view-state registry - -`navigation/viewState.ts`: owners register what the URL does not carry; the navigation store captures them when an entry is left (a push, or Back and Forward through popstate) and restores them when an entry is landed on, from sessionStorage under a token in `history.state`. Owners: `graph.view` (selection, selected edge, reach mode, camera), `metrics` (metric, excluded paths), `graph.filters`. The data-flow graph gets a Return button and Alt+←. - -Return p95 over 20 returns on solid-route (`client/e2e/return-latency.spec.ts`, `SRCLY_MEASURE=1`; route graph zoomed with a box selected → anchor through the code viewer → Return, Forward, repeated; measured from the click to the camera, selection and status all back): - -| p50 | p95 | max | -| --- | --- | --- | -| 55 ms | 61 ms | 65 ms | - -Target was p95 ≤ 150 ms. The route graph is re-fetched from the engine's result cache on every return; the camera is held through the backbone-first draw and the complete graph that replaces it (T9), and through the resize observer's first fit, until the user's first gesture. - -## 2026-10-24 — T19: file and folder anchors through exports - -`fileExports` lists a file's or folder's exported declarations from the checker (`getExportsOfModule`, aliases resolved), with the line to anchor at; the client draws one box per export and traces an export only when it is opened (R18). - -| Target (srcly `client/`) | Files | Exports | Traceable | `fileExports` time | -| --- | --- | --- | --- | --- | -| `src/utils/metricsStore.tsx` | 1 | 28 | 18 | 1 ms | -| `src/utils` | 20 | 122 | 85 | 5 ms | -| `src/navigation` | 11 | 67 | 50 | 1 ms | -| `src` | 157 | 731 | 504 | 10 ms | - -Tier 1 for `client/` was 2.2 s. `src/utils` is over the 50-export budget, so it opens as 20 file boxes with counts; opening `color.ts` shows its one export. On solid-route `src/components` (5 exports in 5 files), opening `MetricCard` adds its forward trace (3 boxes) in a frame of its own next to the other exports. - -## To record next - -| Milestone | Numbers | -| --- | --- | -| M1 | cold `hello` plus tier 1 on solid-route, srcly `client/`, visual-notes `app/`; RSS after tier 1 | -| M2 | first `routeGraph` per route on visual-notes (`/time-blocks`, `/tasks`); payload sizes; four layout screenshots at 1440 and 1024 | -| M3 | warm `valueFlow` p95 over 20 queries; cold `hello` to first ledger row; Return p95 over 20 returns | -| M6 | `functionEffects` on the three anchors | -| M7 | tier 2 on `examples/bad-ish-solid` and on srcly `client/` from the batch job | -| M8 | wheel size before and after; clean-venv smoke time | diff --git a/docs/intent/tsx-dataflow-integration/research.md b/docs/intent/tsx-dataflow-integration/research.md deleted file mode 100644 index 38756743..00000000 --- a/docs/intent/tsx-dataflow-integration/research.md +++ /dev/null @@ -1,67 +0,0 @@ -# Research record - -This page condenses two Pro research runs from 2026-10-23, originally PRs #102 (value investigation) and #103 (function effects). The full reports stay in their Pro conversations. This page keeps only what changes the plan. - -- Value investigation: [Pro conversation](https://chatgpt.com/c/6adc0a99-0844-83e9-b309-74996708456a), branch `codex/data-flow-integration-research`. -- Function effects: [Pro conversation](https://chatgpt.com/c/6adc0dbe-1934-83e9-ae3e-99777939ff03), branch `codex/effects-async-test-seams-research`. -- Baselines: srcly `aea1015`, tsx-data-flow `b4f1ba4` plus its local route-proof edits. No code, tests, builds, or browser journeys ran. The source claims below were verified by reading the code; latency and usability are unmeasured. - -## Verified source facts - -| Fact | Where | Consequence | -| --- | --- | --- | -| The route canvas returns empty evidence-node and evidence-edge arrays. It projects component surfaces, not operation order. | tsx `route-totality-display-layout.ts` | Path order lives in the ledger, not on the graph. | -| Exact selected-file proof requires a proven `node:fs/promises.readFile` source at an exact location. | tsx `route-totality-field-proof-index.ts` | Arbitrary expressions need a new selection API. | -| Some carrier rules check named functions and source paths. | tsx `program-evidence-carrier.ts` | Don't present them as generic support. | -| Generic call edges share parameter and return facts, and slice traversal has no invocation stack. | tsx `program-evidence-collector-jsx-support.ts`, `evidence-slice.ts` | Calls to the same helper can join. This is an inference, so M5 needs a negative fixture. | -| `For` handling needs a property-access `each` and one plain callback parameter. Plain `node={node}` falls outside value-mode recognition. | tsx `connectSolidFor`, `componentPropValueMode` | The `HotSpotList` → `HotSpotItem` path needs new support (M5). | -| The cache builds the full report before answering, and reads source separately from the Program. | tsx `server/cache.ts` | Queries need setup without the report, and excerpts taken from the Program's own text. | -| The CLI prints the requested port, not the bound port. | tsx `server-cli.ts` | This is moot with a stdio child. | -| Call occurrences are recorded. The fetch boundary is classified by name. | tsx `processCall`, `processProgramCallBoundary` | Effects can build on call records, but name labels never certify purity. | -| The worker `request` is promise-returning without `async`. | tsx `analysis-service.ts` | The engine already has a correlation and cancellation pattern to reuse. | -| srcly's edges are import and usage edges: unused imports kept, same-file edges skipped. | srcly `deps/typescript_deps.py`, `deps/common.py` | Never relabel them as data flow or execution. | -| Recent places don't store camera, filters, scroll, or focus. | srcly `navigation/location.ts`, `history.ts` | Ledger Return needs its own snapshot. | -| Lines 134 and 135 are separate name reads, with the rank default at 130 and the metric at 145. Lines 88 and 109 call the same `row` helper. | srcly `HotSpotItem.tsx`, `HotSpotList.tsx` | These are the M3 and M5 anchors. | -| `rankHotSpots` reads the module-level catalog through `metricDef`. `startScan` awaits once and adopts `expectJson`'s promise. | srcly `hotSpotRanking.ts`, `scanJobs.ts` | These are the M6 anchors. | - -## Evidence semantics - -**Value flow** -- Connectivity, coverage, and value properties are separate answers. -- `??` keeps 0 and `""`; `||` replaces them. -- Nullability is not finiteness, schema validity, or a domain rule. -- An assertion (`x!`) is not a runtime check. -- A target field's identity is separate from collection-membership dependencies. -- An accepted path links ordered adjacent element and relation ids to the exact terminal. Containment or references alone prove nothing. - -**Effects** -- Facts and coverage are stored separately, as complete-for-supported-scope, partial, or not analyzed. There is never a `pure` boolean, a confidence percentage, or a risk score. -- The categories are external behavior, ambient inputs, writes (input, shared, fresh-local, unknown target), allocation and identity, throws, and analysis limits. -- There are two activation channels: - - invocation-caused, which includes an async function's own continuations; - - deferred-triggered, for registered or scheduled callbacks. - A returned closure stays latent. -- Result handling is recorded separately: awaited, adopted, chained, aggregated, discarded, or unknown. `void p` does not handle rejection. -- `Promise.all`, `allSettled`, `race`, and `forEach(async)` keep their real meanings. -- Results are may-effects. There are no claims about determinism, termination, or race freedom. - -## Recommendation dispositions - -| Range | Disposition | Milestone | -| --- | --- | --- | -| REC-001–008, 011 | Build | M3 (workspace, pinned preview, binding to the Program, return state, lazy child, selection API, ledger, evidence states) | -| REC-009, 010 | Build | M5 (cross-file path, helper and branch expansion) | -| REC-012–014 | Explore | The process decision (P1), the branch graph (P3), and packaging (now M8) | -| REC-015 | Defer | New languages, runtime instrumentation | -| REC-016–018 | Avoid | Embedding an iframe or shared frontend, a semantic graph over import edges, a Python port | -| EAT-001–010, 013–015 | Build | M6 | -| EAT-011, 012 | Build, gated | Below the cut line (`watchScan` execution view) | -| EAT-016–021 | Explore | Solid activation models, the worker service, Python lifecycle, container precision, Jelly/CodeQL comparison, tangle signals | -| EAT-022–026 | Defer | Runtime traces, caching, a shared package, whole-heap proof, coupling to route UI | -| EAT-027–030 | Avoid | Purity or testability scores, name-only host rules, relabeled import edges, mass refactors | - -Two changes come from this initiative's decisions, not from the research. The research said to keep the repositories separate and use a prepared checkout. The user chose one repository, so `engine/` lives in srcly. The research also proposed an HTTP child on a loopback port. With one repository, a stdio child is simpler. - -## External references - -[CodeQL path queries](https://codeql.github.com/docs/writing-codeql-queries/creating-path-queries/) · [CodeQL JS call graph](https://codeql.github.com/docs/codeql-language-guides/codeql-library-for-javascript/#call-graph) · [JetBrains data-flow inspection](https://www.jetbrains.com/help/idea/analyzing-data-flow.html) · [TypeScript soundness note](https://www.typescriptlang.org/docs/handbook/type-compatibility.html#a-note-on-soundness) · [ECMAScript Promise constructor](https://tc39.es/ecma262/multipage/control-abstraction-objects.html#sec-promise-constructor) · [typescript-eslint `ignoreVoid`](https://typescript-eslint.io/rules/no-floating-promises/#ignorevoid) · [uv tools](https://docs.astral.sh/uv/guides/tools/) diff --git a/docs/intent/tsx-dataflow-integration/shape-brief.md b/docs/intent/tsx-dataflow-integration/shape-brief.md deleted file mode 100644 index 925855ff..00000000 --- a/docs/intent/tsx-dataflow-integration/shape-brief.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -title: "Bring tsx-data-flow into srcly — shape brief" -slug: "tsx-dataflow-integration" -phase: shape -status: current -last_updated: "2026-10-23" ---- - -# Bring tsx-data-flow into srcly — what we are adding - -**Outcome:** One engine adds three things to srcly: route data flow in the dependency viewer, value paths, and function effects. - -**Primary flow:** Graph → Data flow → route → field → destination → path ledger → code → Return. - -## Feature scope - -```text -TS/TSX semantic evidence in srcly -├── PLANNED ADDITIONS -│ ├── Engine: engine/ package, one lazy stdio Node child, bundled in the wheel -│ │ └── Analysis moved intact; one Program shared by all queries; stale results refused -│ ├── 1. Route graph: Data flow mode in the dependency view -│ │ ├── Own contract (route → occurrence → step), never import edges -│ │ ├── UI primitives collapsed; field reach lights proven destinations -│ │ └── Frontier nodes keep their cause; any node → code viewer -│ ├── 2. Path ledger, opened from a graph destination or any code-viewer expression -│ │ ├── Ordered rows: input, transfer, fallback, branch, transform, terminal, boundary -│ │ ├── Evidence inspector: span, types, proof, assumptions, stop reason -│ │ └── Back / Forward / Return restore the overview exactly -│ ├── 3. Effects panel in the code viewer -│ │ └── Effects, ambient reads, writes, async handling; Known + Partial; witnesses -│ └── 4. Supporting: findings, markers, catalog metrics, report work packets, compare -├── EVALUATE BEFORE COMMITTING -│ ├── A route layout rule vs the existing packing -│ └── A small branch graph beside the ledger -└── LATER POSSIBILITIES - └── An async execution view; path/effects → agent work packet -``` - -**Dropped:** all of tsx-dataflow-serve. - -## Behavior - -| Situation | Expected result | -| --- | --- | -| Engine cold or busy | Base views work; progress shows within 250 ms. | -| Not analyzed, unsupported, stale, or over budget | The state is named, never shown as "0". | -| One helper, two calls (`HotSpotList` 88 vs 109) | The calls never join. | -| Network call plus an unresolved call | "Network · Partial". | -| Return from the ledger | Filters, metric, expansion, selection, and camera are restored. | - -## Decisions and boundaries - -**Appetite:** A big program of revertible slices. Each query family proves itself on real code before its UI grows. - -**Key decision:** The engine supplies evidence and srcly draws it. The graph shows reach. The ledger shows order, because the old canvas has no ordering evidence. Graph paths and code-viewer expressions open the same ledger. srcly's Python server owns a stdio child, so there is no port and the Program is built once. - -**Boundary:** No relabeled import edges, and no second analyzer. Layout changes coordinate with `dependency-graph-legibility`. - -**First proof:** Two proofs run in parallel after M1. -- **Route graph:** **Try:** render `examples/solid-route` in the current dependency canvas. **Observe:** field reach is legible, and nodes open the right lines. **Decide:** keep the existing packing, or add one route layout rule. -- **Ledger:** **Try:** inspect `HotSpotItem.tsx:135` `props.node.name`. **Observe:** it reaches the JSX slot, line 134 stays distinct, and the rank default keeps zero. **Decide:** if the proof isn't exact, fix the engine before building UI. - -See the [intent](intent-brief.md), [plan](implementation-plan.md), [research](research.md), and [initiative-map.json](initiative-map.json). diff --git a/docs/research/dataflow-pattern-coverage-2026-10/README.md b/docs/research/dataflow-pattern-coverage-2026-10/README.md index 0bdf0c0d..79e7e2fe 100644 --- a/docs/research/dataflow-pattern-coverage-2026-10/README.md +++ b/docs/research/dataflow-pattern-coverage-2026-10/README.md @@ -154,7 +154,7 @@ The rules remove stops, but whole-export traces have several stops each, and 40% ### Open product decision -`user-flow-workbench` (Solid, `strict: false`) is refused by design (R3 in `docs/intent/tsx-dataflow-integration/detailed-plan.md`): a loose tsconfig makes null verdicts unsound. Data flow itself does not need `strictNullChecks`. One option is to analyze loose projects with `strictNullChecks` forced on and mark their null verdicts as unproven. Byron decides whether to do that. +`user-flow-workbench` (Solid, `strict: false`) is refused by design (R3 in `https://github.com/byronwall/srcly/blob/4727bd56/docs/intent/tsx-dataflow-integration/detailed-plan.md`): a loose tsconfig makes null verdicts unsound. Data flow itself does not need `strictNullChecks`. One option is to analyze loose projects with `strictNullChecks` forced on and mark their null verdicts as unproven. Byron decides whether to do that. ### Gaps that remain diff --git a/engine/scripts/bundle.mjs b/engine/scripts/bundle.mjs index 4157e32d..f8b52da4 100644 --- a/engine/scripts/bundle.mjs +++ b/engine/scripts/bundle.mjs @@ -1,7 +1,7 @@ -// Bundle the engine into the wheel (docs/intent/tsx-dataflow-integration/detailed-plan.md, M8): +// Bundle the engine into the wheel (docs/dataflow-engine.md): // `srcly-engine.mjs` and the worker it starts, the Python helper, and a trimmed // copy of the engine's TypeScript, which stays a real directory so the -// Program finds its `lib.*.d.ts` files (decision D4). +// Program finds its `lib.*.d.ts` files (TypeScript instance, docs/dataflow-engine.md). import { build } from "esbuild"; import fs from "node:fs"; import path from "node:path"; diff --git a/pgm/data/projects/srcly-backlog/history/20261122T042055895Z-0000-b42af349-7ec7-4573-aa83-c6fb23dfc94f.json b/pgm/data/projects/srcly-backlog/history/20261122T042055895Z-0000-b42af349-7ec7-4573-aa83-c6fb23dfc94f.json new file mode 100644 index 00000000..eadf3575 --- /dev/null +++ b/pgm/data/projects/srcly-backlog/history/20261122T042055895Z-0000-b42af349-7ec7-4573-aa83-c6fb23dfc94f.json @@ -0,0 +1,15 @@ +{ + "id": "b42af349-7ec7-4573-aa83-c6fb23dfc94f", + "kind": "node.created", + "actor": { + "kind": "agent", + "id": "token:6176ad", + "label": "Agent token ...6176ad" + }, + "createdAt": "2026-11-22T04:20:55.895Z", + "summary": "Created \"Start the data-flow batch run from the app\" under \"Data flow: engine structure\".", + "nodeIds": [ + "task-start-the-data-flow-batch-run-from-the-app", + "area-data-flow-engine-structure" + ] +} diff --git a/pgm/data/projects/srcly-backlog/nodes/task-start-the-data-flow-batch-run-from-the-app.md b/pgm/data/projects/srcly-backlog/nodes/task-start-the-data-flow-batch-run-from-the-app.md new file mode 100644 index 00000000..236424b1 --- /dev/null +++ b/pgm/data/projects/srcly-backlog/nodes/task-start-the-data-flow-batch-run-from-the-app.md @@ -0,0 +1,17 @@ +--- +id: task-start-the-data-flow-batch-run-from-the-app +label: Start the data-flow batch run from the app +type: task +status: planned +parent: area-data-flow-engine-structure +--- +# Start the data-flow batch run from the app + +**Evidence.** Today only `srcly report --dataflow` and `srcly compare --dataflow` run tier 2 (`fileSignals`). The app colors the treemap and marks the code viewer from the last saved CLI result (`server/app/services/dataflow/signals.py:latest_for_file`, `scan_jobs.py:_overlay`). Without a CLI run, the six `dataflow_*` metrics are N/A in the app. The retired tsx-dataflow-integration plan (decision D5, at https://github.com/byronwall/srcly/blob/4727bd56/docs/intent/tsx-dataflow-integration/detailed-plan.md) planned a `DataflowJob` beside `start_findings()`. It would start once a data-flow surface was opened in the session, with `GET /api/scans/{id}/dataflow` for status. + +**Files.** `server/app/services/scan_jobs.py`, `server/app/services/dataflow/signals.py`, a new job and route, and a client trigger (the metric picker's Data flow group or the N/A reason). + +**Cost.** Tier 2 takes minutes on a big app (130 s on srcly's client), so the run must be explicit and must not block the scan. + +**Done when.** From the app, a user can start the run on a TS root and see it progress. When it finishes, the treemap colors by Impossible Defenses with no CLI step. + diff --git a/site/src/docs/pages/data-flow.md b/site/src/docs/pages/data-flow.md index 5f33fb57..04001a22 100644 --- a/site/src/docs/pages/data-flow.md +++ b/site/src/docs/pages/data-flow.md @@ -25,7 +25,7 @@ uvx srcly report . --out .srcly --dataflow The engine traces every render path in the TypeScript and TSX program, from a prop, signal, or local to the JSX that renders it. This takes minutes on a large app, so it runs only when you ask. It adds: - `dataflow.md`: ranked work packets with candidate edits, the risk queue and a stop recommendation, then the defensive ledger, repeated forks, prop relay, context relay, and fan-out. -- Two finding rules in the Data flow section of `report.md`. `impossible-defense` is a `??` or `?.` whose guarded value the types say is never null or undefined. `repeated-fork` is a component that tests the same discriminant at several sibling branch sites, including Solid `` and ``. +- Four finding rules in the Data flow section of `report.md`. `impossible-defense` is a `??` or `?.` whose guarded value the types say is never null or undefined. `repeated-fork` is a component that tests the same discriminant at several sibling branch sites, including Solid `` and ``. `context-relay` and `broad-prop-bundle` are a parent that reads a context hook and passes context-like props, or three or more plain props, to a child in the same feature. - Six metrics per TS/TSX file in the Data flow group: impossible defenses, repeated forks, prop relay, shape changes, path depth, and value fan-out. After the run, the app colors the treemap and marks the code viewer with them while the files stay unchanged. `uvx srcly compare --dataflow` runs the engine on both sides and diffs these metrics. diff --git a/site/src/docs/pages/design.md b/site/src/docs/pages/design.md index 9747bbf0..9659e2f6 100644 --- a/site/src/docs/pages/design.md +++ b/site/src/docs/pages/design.md @@ -34,7 +34,7 @@ This prevents missing analysis from looking like improvement. Rankings can use d ## Use the right relationship for the question -The TypeScript and TSX dependency graph shows imports. Data flow traces modeled values, function effects, or supported routes. An import edge does not prove value movement; a data-flow edge is not a general runtime trace. The [integration brief](https://github.com/byronwall/srcly/blob/main/docs/intent/tsx-dataflow-integration/intent-brief.md) and [logic visibility brief](https://github.com/byronwall/srcly/blob/0bfb05ba/docs/intent/dataflow-logic-visibility/intent-brief.md) explain the boundary. +The TypeScript and TSX dependency graph shows imports. Data flow traces modeled values, function effects, or supported routes. An import edge does not prove value movement; a data-flow edge is not a general runtime trace. The [engine design](https://github.com/byronwall/srcly/blob/main/docs/dataflow-engine.md) and [logic visibility brief](https://github.com/byronwall/srcly/blob/0bfb05ba/docs/intent/dataflow-logic-visibility/intent-brief.md) explain the boundary. Symbols mode shows which functions, classes, fields, or types reference a symbol. Edges carry a resolution label: exact, typed, or name-based. This gives more detail than imports without claiming value flow. See the [symbol graph brief](https://github.com/byronwall/srcly/blob/0bfb05ba/docs/intent/symbol-graph/intent-brief.md). diff --git a/site/src/docs/pages/findings.md b/site/src/docs/pages/findings.md index 0d6d87cd..879ad761 100644 --- a/site/src/docs/pages/findings.md +++ b/site/src/docs/pages/findings.md @@ -14,7 +14,7 @@ Srcly selects checks based on the files and tools it finds. Checks run independe | Srcly TypeScript rules | TS files are outside project checker coverage | A selected built-in pattern matched, or that rule set did not cover the file. | | Reachability and Solid rules | Supported TypeScript graph and patterns are available | Entry-point detection or language coverage may not match the project. Configure entries when needed. | | ast-grep rules | `.srcly/rules/` files exist and ast-grep is available | The optional binary or rule run is unavailable. | -| Data-flow rules (`impossible-defense`, `repeated-fork`) | You pass `--dataflow` to `report` or `compare`, and Node 18+ is available | The engine is missing, the root is not TypeScript, or the run failed. The reason is in `report.md`. | +| Data-flow rules (`impossible-defense`, `repeated-fork`, `context-relay`, `broad-prop-bundle`) | You pass `--dataflow` to `report` or `compare`, and Node 18+ is available | The engine is missing, the root is not TypeScript, or the run failed. The reason is in `report.md`. | The exact checks vary by project. Srcly runs `tsc` per detected TypeScript project, expands solution configurations, runs ESLint when configured, and runs Ruff for Python. Its built-in TS checks include selected patterns such as explicit `any`, suppression directives without a reason, and non-null assertion chains when project tools do not cover those files. It also reports selected reachability, Solid, and await-in-loop signals. Optional ast-grep YAML rules can add project-specific checks. See [Configuration](../configuration/) for setup. diff --git a/skills/srcly-code-quality/SKILL.md b/skills/srcly-code-quality/SKILL.md index 9c138911..5d2fe977 100644 --- a/skills/srcly-code-quality/SKILL.md +++ b/skills/srcly-code-quality/SKILL.md @@ -104,10 +104,12 @@ Progress, version banners and cache warnings go to stderr, so stdout stays parse - A `## Data flow` section in `report.md` that lists each finding by line. - Metrics per TS/TSX file: `dataflow_impossible_defenses`, `dataflow_repeated_forks`, `dataflow_relay_hops`, `dataflow_representation_changes`, `dataflow_path_depth` and `dataflow_fan_out`. The app colors the treemap and marks the code viewer from them while the files stay unchanged. -The two finding rules: +The finding rules: - `impossible-defense`: a `??` or `?.` whose guarded value the types say is never null or undefined. Remove it only when the runtime contract agrees with the types (no external JSON, no unchecked cast, no SSR first render). - `repeated-fork`: a component tests the same discriminant at several sibling branch sites (ternary, `&&`, Solid `` or ``). Split it into one sub-component per branch, and move each branch-only value into its sub-component. +- `context-relay`: a parent that reads a context hook passes context-like props (`settings`, `selection`, `on*` handlers and similar) to a child in the same feature. Let the child read them from context. The code viewer marks it under Prop Relay. +- `broad-prop-bundle`: the same, with three or more plain props. Pass one model, or read shared values from context. Read `dataflow.md` in this order, and fix one slice at a time: