Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions client/src/utils/normalizeTree.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,12 @@ export const DEFAULT_METRICS = {
churn_lines: null,
finding_count: null,
error_count: null,
dataflow_impossible_defenses: null,
dataflow_repeated_forks: null,
dataflow_relay_hops: null,
dataflow_fan_out: null,
dataflow_representation_changes: null,
dataflow_path_depth: null,
na_reason: "",
na_reasons: null,
analysis_status: "ok",
Expand All @@ -70,6 +76,7 @@ export const DEFAULT_NODE = {
end_line: 0,
activity: null,
findings: null,
dataflow: null,
} as const;

type RawNode = {
Expand Down
22 changes: 21 additions & 1 deletion server/app/models.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from typing import List
from typing import Any, List
from pydantic import BaseModel, Field, PrivateAttr

class Witness(BaseModel):
Expand Down Expand Up @@ -92,6 +92,15 @@ class Metrics(BaseModel):
finding_count: int | None = None
error_count: int | None = None

# Data-flow signals (``services/dataflow/signals``), laid over the tree
# after ``srcly report --dataflow``: TS/TSX files and folders; scopes N/A.
dataflow_impossible_defenses: int | None = None
dataflow_repeated_forks: int | None = None
dataflow_relay_hops: int | None = None
dataflow_fan_out: int | None = None
dataflow_representation_changes: int | None = None
dataflow_path_depth: int | None = None

na_reason: str = ""
# Per-metric reasons that override ``na_reason`` (activity has its own:
# "Not a Git repository", "Shallow clone", ...).
Expand Down Expand Up @@ -151,6 +160,13 @@ class FindingsInfo(BaseModel):
finding_count: int = 0


class DataflowInfo(BaseModel):
"""Whether the data-flow signals ran for a tree (root node only, after ``--dataflow``)."""

status: str # "ok" | "na"
reason: str = ""


class AnalysisInfo(BaseModel):
"""Where a CLI command's tree came from (root node only; set by ``reporting.scan_tree``)."""

Expand All @@ -177,8 +193,12 @@ class Node(BaseModel):
activity: ActivityInfo | None = None
# Root only: whether the checks finished, and which tools ran.
findings: FindingsInfo | None = None
# Root only: whether the data-flow signals ran (``services/dataflow/signals``).
dataflow: DataflowInfo | None = None
# Root only, CLI only (never on the wire): whether ``reporting.scan_tree`` reused the saved scan.
_analysis: AnalysisInfo | None = PrivateAttr(default=None)
# Root only, CLI only: the data-flow signals ``scan_tree(dataflow=True)`` laid over the tree.
_dataflow: Any = PrivateAttr(default=None)

model_config = {
"populate_by_name": True
Expand Down
8 changes: 8 additions & 0 deletions server/app/routers/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ def get_file_markers(
else:
markers = collect_markers(str(file_path))
markers.update(_findings_markers(file_path))
markers.update(_dataflow_markers(file_path))
except HTTPException:
raise
except Exception as e:
Expand All @@ -115,6 +116,13 @@ def _findings_markers(file_path: Path) -> dict:
return findings.markers_for(str(file_path), _findings_job_result(file_path))


def _dataflow_markers(file_path: Path) -> dict:
"""Lines behind the data-flow counts, from the last ``--dataflow`` result, while the file is unchanged."""
from app.services.dataflow import signals

return signals.markers_for(str(file_path.resolve()), signals.latest_for_file(file_path))


@router.get("/findings")
def get_file_findings(path: str = Query(..., description="Absolute path to the file")):
"""
Expand Down
16 changes: 16 additions & 0 deletions server/app/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,14 @@ def _run_headless(argv: list[str]) -> None:
default=[],
help="Also rank this file class. May be repeated.",
)
report_parser.add_argument(
"--dataflow",
action="store_true",
help=(
"Also run the TypeScript data-flow engine (needs Node 18+): data-flow metrics and findings, "
"and dataflow.md with ranked work packets. Takes minutes on a big app."
),
)
report_parser.add_argument("--tree-depth", type=_positive_int, default=3, help="Maximum depth in tree.summary.json.")
report_parser.add_argument("--tree-top", type=_positive_int, default=6, help="Maximum children per tree level.")
report_parser.add_argument("--include-tree", action="store_true", help="Also write/include the full raw tree.json artifact.")
Expand Down Expand Up @@ -434,6 +442,11 @@ def _run_headless(argv: list[str]) -> None:
help="Include both full trees and dependency graphs in the JSON.",
)
compare_parser.add_argument("--refresh", action="store_true", help="Rescan both sides instead of using the cache.")
compare_parser.add_argument(
"--dataflow",
action="store_true",
help="Also diff the data-flow metrics (runs the TypeScript data-flow engine on both sides; minutes on a big app).",
)
compare_parser.add_argument("--limit", type=_positive_int, default=40, help="Maximum files in the markdown summary.")

fix_parser = subparsers.add_parser(
Expand Down Expand Up @@ -521,6 +534,7 @@ def _dispatch(args) -> None:
quadrant=args.quadrant,
quadrant_cohort=args.quadrant_cohort,
include_classes=("code", *args.include_class),
dataflow=args.dataflow,
)
# Static reference (metric schema, agent guide) and the findings_top prefix are artifacts,
# not per-run evidence; `report --out` still writes them.
Expand Down Expand Up @@ -548,6 +562,7 @@ def _dispatch(args) -> None:
quadrant=args.quadrant,
quadrant_cohort=args.quadrant_cohort,
include_classes=("code", *args.include_class),
dataflow=args.dataflow,
)
written = reporting.write_artifacts(payload, out_dir, output_format=output_format)
if args.verbose:
Expand Down Expand Up @@ -622,6 +637,7 @@ def _run_compare(args) -> None:
head,
include_sides=args.include_sides,
force=args.refresh,
dataflow=args.dataflow,
on_phase=lambda phase: print(f"srcly compare: {phase}", file=sys.stderr, flush=True),
)
except GitError as exc:
Expand Down
14 changes: 13 additions & 1 deletion server/app/services/analysis_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,8 +205,20 @@ def iter_executable_scopes(funcs):
# Lint and type-check findings: from external tools (``services/findings``),
# laid over the tree when it is served, per file.
FINDINGS_METRICS = frozenset({"finding_count", "error_count"})
# Data-flow signals from the engine's tier-2 report (``services/dataflow/signals``):
# TypeScript and TSX only, laid over the tree when a ``--dataflow`` run left a result.
DATAFLOW_METRICS = frozenset(
{
"dataflow_impossible_defenses",
"dataflow_repeated_forks",
"dataflow_relay_hops",
"dataflow_fan_out",
"dataflow_representation_changes",
"dataflow_path_depth",
}
)
# Filled after the scan, so never "not measured for <language>".
OVERLAY_METRICS = ACTIVITY_METRICS | FINDINGS_METRICS
OVERLAY_METRICS = ACTIVITY_METRICS | FINDINGS_METRICS | DATAFLOW_METRICS

# Means over a file's functions; with no functions there is nothing to average.
MEAN_OVER_FUNCTIONS = frozenset({"complexity_avg", "average_function_length"})
Expand Down
10 changes: 8 additions & 2 deletions server/app/services/compare/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
resolve_auto_base,
resolve_ref,
)
from app.services.compare.snapshots import Side, load_side
from app.services.compare.snapshots import Side, load_side, with_dataflow
from app.services.scan_progress import NULL_OBSERVER, ScanObserver
from app.services.scan_workers import CancelToken

Expand Down Expand Up @@ -98,12 +98,14 @@ def build_compare(
on_phase: Callable[[str], None] = lambda _phase: None,
timer: Optional[StepTimer] = None,
activity_window: Optional[str] = None,
dataflow: bool = False,
) -> dict[str, Any]:
"""
Compare ``base`` → ``head`` for the git repo containing ``path``.

``include_sides`` adds both full trees and graphs (what the UI draws);
the CLI omits them by default to keep the JSON small.
the CLI omits them by default to keep the JSON small. ``dataflow`` also
runs the data-flow engine on both sides, so its metrics are diffed.
"""
timer = timer or StepTimer()

Expand Down Expand Up @@ -137,6 +139,10 @@ def phase(name: str) -> None:
for side in (base_side, head_side):
result = activity.collect(side.ref.root(repo), activity_window, rev=side.ref.sha)
activity.apply_to_tree(side.tree, result, lambda path: path)
if dataflow:
phase("dataflow")
for side in (base_side, head_side):
with_dataflow(repo, side)

phase("diffing")
payload: dict[str, Any] = {
Expand Down
22 changes: 22 additions & 0 deletions server/app/services/compare/snapshots.py
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,28 @@ def _with_checks(tree_json: dict[str, Any], root: Path, *, fingerprint: Optional
return findings.apply_to_tree(tree_json, result)


def with_dataflow(repo: Path, side: Side) -> None:
"""
Lay data-flow metrics over a loaded side (``compare --dataflow``). A
commit side gets its own throwaway checkout, since the side cache keeps no
files; its engine is closed after. Like compare's checks, a checkout has
no ``node_modules``, so both sides use srcly's bundled TypeScript.
"""
from app.services.dataflow import signals

def apply(root: Path, *, fingerprint: Optional[str], close: bool) -> None:
result = signals.collect(root, fingerprint=fingerprint, close_after=close)
root_str = str(root.resolve())
signals.apply_to_tree(side.tree, result, lambda p: os.path.join(root_str, p) if p else root_str)

if side.ref.is_worktree or side.ref.sha is None:
root = side.ref.root(repo)
apply(root, fingerprint=cache.fingerprint(root), close=False)
return
with snapshot_checkout(repo, side.ref.sha) as checkout:
apply(checkout, fingerprint=None, close=True)


def load_side(
repo: Path,
ref: ResolvedRef,
Expand Down
Loading
Loading