From 87cdb76abe21011dbeab32c75f354cfd3f59079f Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Wed, 26 Aug 2026 16:31:12 -0400 Subject: [PATCH] Add source-backed opportunity ranking foundation --- CONTEXT.md | 4 + docs/architecture.md | 2 + .../ecosystem-opportunity-ranking.ts | 420 ++++++++++++++++++ .../ecosystem-opportunity-ranking.test.ts | 140 ++++++ 4 files changed, 566 insertions(+) create mode 100644 src/server/knowledge/ecosystem-opportunity-ranking.ts create mode 100644 tests/integration/ecosystem-opportunity-ranking.test.ts diff --git a/CONTEXT.md b/CONTEXT.md index 0dffbeb0a..7c27cd6c6 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -20,6 +20,10 @@ _Avoid_: candidate, lead, alert The global, versioned catalog of reusable security concepts and source-backed relationships used for retrieval, classification, and strategy. It never owns project observations, evidence, assertions, or findings. _Avoid_: Investigation Graph, project graph, fact database +**Ecosystem Signal**: +An immutable, source-backed measurement about a reusable repository, package, release, configuration, or ecosystem subject used for opportunity ranking. It never claims that a project Target is vulnerable. +_Avoid_: Research Observation, Finding, risk score, target fact + **Knowledge Concept**: A reusable security subject with one stable lowercase `namespace:value` ID, one controlled kind, typed external identifiers, and source references. Weaknesses, attack patterns, techniques, controls, protocols, tools, commands, and standards are Knowledge Concepts. _Avoid_: project fact, finding, copied taxonomy row diff --git a/docs/architecture.md b/docs/architecture.md index 411374ef9..6362a03ff 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -138,6 +138,8 @@ The pinned `@mastra/lance` package carries a local patch for three adapter defec The explicit Security Knowledge Graph remains in SQLite for versioned, reusable concepts; controlled predicates; source-backed relationships; typed external identifiers; tool I/O; tool groups; group membership; and weighted tool relationships. Reusable IDs use lowercase `namespace:value` keys. Seed publication synchronizes the owned catalog revision so renamed or retired seed edges do not survive indefinitely. `knowledge_tool_groups`, `knowledge_tool_group_members`, and `knowledge_tool_relationships` are seeded from curated common-shell transitions plus existing `docs/tools/*` `category` and `related_tools` frontmatter. Eval scoring treats a relationship match as positive sequence-coherence evidence; a missing edge remains unmodeled rather than becoming an exclusive allowlist failure. Skill Markdown chunking and semantic document retrieval use Mastra RAG, while deterministic keyword and concept-graph traversal remain local and explicit. +Pinned global source snapshots may yield immutable Ecosystem Signals for opportunity ranking. Ranking applies license, revision, reproducibility, disclosure, isolation, egress, reset/teardown, and authorization-readiness gates before arithmetic; missing evidence holds a candidate instead of becoming zero. Eligible candidates are ordered only inside comparable cohorts using the published seven-dimension vector. The total never appears without its contributions, confidence, source-signal references, and unweighted change/disclosed-history overlays, and it never asserts that a project Target is vulnerable. Selecting a candidate is the boundary that creates project-owned Targets and subsequent Research Observations; global signals themselves never cross into project memory as deployed-target facts. + The project Investigation Graph is an assertion layer over existing records, not another owner of Targets, Artifacts, Findings, Research Observations, Tasks, Attack Paths, Tool Runs, messages, memory, or reusable security knowledge. A Research Observation preserves measured or directly seen behavior, structured inputs and outputs, measurements, external identifiers, versioned scores, actor, time, and precise citations before interpretation. The user-facing Research Map projects canonical records, cited threads and messages, external sources, reusable-concept references, current Investigation Assertions, and Research Priorities through one coherent relational snapshot. The write model resolves canonical records through project-local Investigation Entities and stores append-only Assertions, coordinate-only role-bearing Citations, and rule-versioned Derivations with ordered inputs. Evidence state (`observed`, `derived`, `proposed`, `contradicted`, or `rejected`) stays separate from assertion lifecycle (`current`, `withdrawn`, or `superseded`). Revision is an optimistic, transactional replacement that retains the predecessor and its citations. SQLite and PostgreSQL relational queries define correctness. See [ADR 0001](./adr/0001-investigation-graph-as-assertion-layer.md). A Research Priority is an unresolved, citation-backed current assertion ranked for follow-up. Its deterministic score weights objective relevance (25%), evidence gap (20%), expected information gain (20%), target importance (15%), inverse predicate cost (8%), inverse predicate risk (7%), and authorization readiness (5%). Authorization readiness comes from the durable target ledger. Deliberately turning a Research Priority into a Task uses the existing Task workflow and a unique assertion-task receipt; it never schedules work, creates an approval, runs a tool, or promotes a Finding. diff --git a/src/server/knowledge/ecosystem-opportunity-ranking.ts b/src/server/knowledge/ecosystem-opportunity-ranking.ts new file mode 100644 index 000000000..de54791b4 --- /dev/null +++ b/src/server/knowledge/ecosystem-opportunity-ranking.ts @@ -0,0 +1,420 @@ +import { createHash } from "node:crypto"; + +export const OPPORTUNITY_DIMENSION_WEIGHTS = { + "exposure-adoption": 0.2, + "researchable-surface": 0.2, + reproducibility: 0.15, + "parallel-density": 0.15, + "disclosure-maturity": 0.1, + "portfolio-diversity": 0.1, + "operational-safety": 0.1, +} as const; + +export const OPPORTUNITY_GATE_IDS = [ + "license-and-terms", + "exact-revision", + "reproducible-deployment", + "disclosure-route", + "isolation", + "egress", + "reset-and-teardown", + "authorization-readiness", +] as const; + +export type OpportunityDimension = keyof typeof OPPORTUNITY_DIMENSION_WEIGHTS; +export type OpportunityGateId = (typeof OPPORTUNITY_GATE_IDS)[number]; +export type OpportunityGateStatus = "pass" | "fail" | "unknown"; + +export type EcosystemSignal = { + id: string; + subjectId: string; + sourceId: string; + sourceRecordId: string; + artifactId: string; + retrievedAt: string; + effectiveAt?: string; + fieldPath: string; + name: string; + rawValue: string | number | boolean | null; + normalizedValue: number | null; + unit: string; + transformation: string; + cohort: string; + confidence: number; + freshness: "fresh" | "stale" | "withdrawn"; + caveats: string[]; +}; + +export type OpportunityGate = { + id: OpportunityGateId; + status: OpportunityGateStatus; + evidenceSignalIds: string[]; + reason: string; +}; + +export type OpportunityDimensionInput = { + dimension: OpportunityDimension; + normalizedValue: number | null; + confidence: number; + evidenceSignalIds: string[]; + missingReason?: string; +}; + +export type OpportunityOverlay = { + id: "change-pressure" | "disclosed-history"; + summary: string; + evidenceSignalIds: string[]; +}; + +export type OpportunityCandidateInput = { + id: string; + displayName: string; + cohort: string; + subjectIds: string[]; + gates: OpportunityGate[]; + dimensions: OpportunityDimensionInput[]; + overlays: OpportunityOverlay[]; +}; + +export type OpportunityRankingInput = { + snapshotId: string; + asOf: string; + sourceSnapshotIds: string[]; + normalizationRevision: string; + knownAnswerCutoff: string; + signals: EcosystemSignal[]; + candidates: OpportunityCandidateInput[]; +}; + +export type RankedOpportunityCandidate = OpportunityCandidateInput & { + status: "ranked" | "held" | "rejected"; + gateFailures: OpportunityGateId[]; + unknownGates: OpportunityGateId[]; + missingDimensions: OpportunityDimension[]; + weightedContributions: Partial>; + total: number | null; + evidenceConfidence: number | null; + cohortRank: number | null; +}; + +export type OpportunityRankingSnapshot = { + snapshotId: string; + asOf: string; + sourceSnapshotIds: string[]; + normalizationRevision: string; + knownAnswerCutoff: string; + digest: `sha256:${string}`; + candidates: RankedOpportunityCandidate[]; +}; + +/** + * Builds an explainable queue snapshot. The result is ordering evidence, never + * a vulnerability, safety, or authorization verdict for a deployed Target. + */ +export function rankEcosystemOpportunities( + input: OpportunityRankingInput, +): OpportunityRankingSnapshot { + assertNonEmpty(input.snapshotId, "snapshotId"); + assertTimestamp(input.asOf, "asOf"); + assertTimestamp(input.knownAnswerCutoff, "knownAnswerCutoff"); + assertNonEmpty(input.normalizationRevision, "normalizationRevision"); + assertUniqueNonEmpty(input.sourceSnapshotIds, "sourceSnapshotIds"); + + const signals = validateSignals(input.signals); + const signalIds = new Set(signals.map((signal) => signal.id)); + const candidateIds = new Set(); + const candidates = input.candidates.map((candidate) => { + assertNonEmpty(candidate.id, "candidate.id"); + if (candidateIds.has(candidate.id)) { + throw new Error(`Duplicate opportunity candidate "${candidate.id}".`); + } + candidateIds.add(candidate.id); + return evaluateCandidate(candidate, signalIds); + }); + + assignCohortRanks(candidates); + const snapshot = { + snapshotId: input.snapshotId, + asOf: input.asOf, + sourceSnapshotIds: [...input.sourceSnapshotIds], + normalizationRevision: input.normalizationRevision, + knownAnswerCutoff: input.knownAnswerCutoff, + candidates, + }; + const digest = + `sha256:${createHash("sha256").update(stableJson(snapshot)).digest("hex")}` as const; + return { ...snapshot, digest }; +} + +function evaluateCandidate( + candidate: OpportunityCandidateInput, + signalIds: ReadonlySet, +): RankedOpportunityCandidate { + assertNonEmpty(candidate.displayName, `${candidate.id}.displayName`); + assertNonEmpty(candidate.cohort, `${candidate.id}.cohort`); + assertUniqueNonEmpty(candidate.subjectIds, `${candidate.id}.subjectIds`); + + const gates = orderedGates(candidate, signalIds); + const dimensions = orderedDimensions(candidate, signalIds); + for (const overlay of candidate.overlays) { + assertNonEmpty(overlay.summary, `${candidate.id}.${overlay.id}.summary`); + assertEvidenceReferences( + overlay.evidenceSignalIds, + signalIds, + `${candidate.id}.${overlay.id}`, + ); + } + + const gateFailures = gates + .filter((gate) => gate.status === "fail") + .map((gate) => gate.id); + const unknownGates = gates + .filter((gate) => gate.status === "unknown") + .map((gate) => gate.id); + const missingDimensions = dimensions + .filter((dimension) => dimension.normalizedValue === null) + .map((dimension) => dimension.dimension); + const status = + gateFailures.length > 0 + ? "rejected" + : unknownGates.length > 0 || missingDimensions.length > 0 + ? "held" + : "ranked"; + + const weightedContributions: Partial> = + {}; + let total: number | null = null; + let evidenceConfidence: number | null = null; + if (status === "ranked") { + total = 0; + evidenceConfidence = 0; + for (const dimension of dimensions) { + const value = dimension.normalizedValue as number; + const weight = OPPORTUNITY_DIMENSION_WEIGHTS[dimension.dimension]; + weightedContributions[dimension.dimension] = roundScore(value * weight); + total += value * weight; + evidenceConfidence += dimension.confidence * weight; + } + total = roundScore(total); + evidenceConfidence = roundScore(evidenceConfidence); + } + + return { + ...candidate, + gates, + dimensions, + status, + gateFailures, + unknownGates, + missingDimensions, + weightedContributions, + total, + evidenceConfidence, + cohortRank: null, + }; +} + +function orderedGates( + candidate: OpportunityCandidateInput, + signalIds: ReadonlySet, +): OpportunityGate[] { + const byId = new Map(); + for (const gate of candidate.gates) { + if (byId.has(gate.id)) + throw new Error(`${candidate.id} has duplicate gate "${gate.id}".`); + assertNonEmpty(gate.reason, `${candidate.id}.${gate.id}.reason`); + assertEvidenceReferences( + gate.evidenceSignalIds, + signalIds, + `${candidate.id}.${gate.id}`, + ); + byId.set(gate.id, gate); + } + for (const gateId of OPPORTUNITY_GATE_IDS) { + if (!byId.has(gateId)) + throw new Error(`${candidate.id} is missing gate "${gateId}".`); + } + return OPPORTUNITY_GATE_IDS.map( + (gateId) => byId.get(gateId) as OpportunityGate, + ); +} + +function orderedDimensions( + candidate: OpportunityCandidateInput, + signalIds: ReadonlySet, +): OpportunityDimensionInput[] { + const byId = new Map(); + for (const dimension of candidate.dimensions) { + if (byId.has(dimension.dimension)) { + throw new Error( + `${candidate.id} has duplicate dimension "${dimension.dimension}".`, + ); + } + assertUnitInterval( + dimension.confidence, + `${candidate.id}.${dimension.dimension}.confidence`, + ); + if (dimension.normalizedValue === null) { + assertNonEmpty( + dimension.missingReason, + `${candidate.id}.${dimension.dimension}.missingReason`, + ); + if (dimension.evidenceSignalIds.length > 0) { + throw new Error( + `${candidate.id}.${dimension.dimension} cannot cite a value while marked missing.`, + ); + } + } else { + assertUnitInterval( + dimension.normalizedValue, + `${candidate.id}.${dimension.dimension}.normalizedValue`, + ); + assertEvidenceReferences( + dimension.evidenceSignalIds, + signalIds, + `${candidate.id}.${dimension.dimension}`, + ); + if (dimension.evidenceSignalIds.length === 0) { + throw new Error( + `${candidate.id}.${dimension.dimension} needs source-backed evidence.`, + ); + } + } + byId.set(dimension.dimension, dimension); + } + for (const dimension of Object.keys( + OPPORTUNITY_DIMENSION_WEIGHTS, + ) as OpportunityDimension[]) { + if (!byId.has(dimension)) + throw new Error(`${candidate.id} is missing dimension "${dimension}".`); + } + return ( + Object.keys(OPPORTUNITY_DIMENSION_WEIGHTS) as OpportunityDimension[] + ).map((dimension) => byId.get(dimension) as OpportunityDimensionInput); +} + +function validateSignals(signals: EcosystemSignal[]): EcosystemSignal[] { + const ids = new Set(); + for (const signal of signals) { + assertNonEmpty(signal.id, "signal.id"); + if (ids.has(signal.id)) + throw new Error(`Duplicate ecosystem signal "${signal.id}".`); + ids.add(signal.id); + for (const [name, value] of Object.entries({ + subjectId: signal.subjectId, + sourceId: signal.sourceId, + sourceRecordId: signal.sourceRecordId, + artifactId: signal.artifactId, + fieldPath: signal.fieldPath, + name: signal.name, + unit: signal.unit, + transformation: signal.transformation, + cohort: signal.cohort, + })) { + assertNonEmpty(value, `${signal.id}.${name}`); + } + assertTimestamp(signal.retrievedAt, `${signal.id}.retrievedAt`); + if (signal.effectiveAt) + assertTimestamp(signal.effectiveAt, `${signal.id}.effectiveAt`); + assertUnitInterval(signal.confidence, `${signal.id}.confidence`); + if (signal.normalizedValue !== null) { + assertUnitInterval( + signal.normalizedValue, + `${signal.id}.normalizedValue`, + ); + } + } + return signals; +} + +function assignCohortRanks(candidates: RankedOpportunityCandidate[]): void { + const cohorts = new Map(); + for (const candidate of candidates) { + if (candidate.status !== "ranked") continue; + const cohort = cohorts.get(candidate.cohort) ?? []; + cohort.push(candidate); + cohorts.set(candidate.cohort, cohort); + } + for (const cohort of cohorts.values()) { + cohort + .sort( + (left, right) => + (right.total as number) - (left.total as number) || + (right.evidenceConfidence as number) - + (left.evidenceConfidence as number) || + valueFor(right, "portfolio-diversity") - + valueFor(left, "portfolio-diversity") || + valueFor(right, "reproducibility") - + valueFor(left, "reproducibility") || + valueFor(right, "operational-safety") - + valueFor(left, "operational-safety") || + left.id.localeCompare(right.id), + ) + .forEach((candidate, index) => { + candidate.cohortRank = index + 1; + }); + } +} + +function valueFor( + candidate: RankedOpportunityCandidate, + dimension: OpportunityDimension, +): number { + return ( + candidate.dimensions.find((item) => item.dimension === dimension) + ?.normalizedValue ?? 0 + ); +} + +function assertEvidenceReferences( + references: string[], + signalIds: ReadonlySet, + context: string, +): void { + if (new Set(references).size !== references.length) { + throw new Error(`${context} has duplicate evidence signal references.`); + } + for (const id of references) { + if (!signalIds.has(id)) + throw new Error( + `${context} references unknown ecosystem signal "${id}".`, + ); + } +} + +function assertUniqueNonEmpty(values: string[], name: string): void { + if (values.length === 0) throw new Error(`${name} must not be empty.`); + for (const value of values) assertNonEmpty(value, name); + if (new Set(values).size !== values.length) + throw new Error(`${name} must not contain duplicates.`); +} + +function assertNonEmpty(value: string | undefined, name: string): void { + if (!value?.trim()) throw new Error(`${name} must be non-empty.`); +} + +function assertTimestamp(value: string, name: string): void { + if (!Number.isFinite(Date.parse(value))) + throw new Error(`${name} must be an ISO-compatible timestamp.`); +} + +function assertUnitInterval(value: number, name: string): void { + if (!Number.isFinite(value) || value < 0 || value > 1) { + throw new Error(`${name} must be between 0 and 1.`); + } +} + +function roundScore(value: number): number { + return Math.round(value * 1_000_000_000_000) / 1_000_000_000_000; +} + +function stableJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.entries(value) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, child]) => `${JSON.stringify(key)}:${stableJson(child)}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "null"; +} diff --git a/tests/integration/ecosystem-opportunity-ranking.test.ts b/tests/integration/ecosystem-opportunity-ranking.test.ts new file mode 100644 index 000000000..e4cca7694 --- /dev/null +++ b/tests/integration/ecosystem-opportunity-ranking.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, it } from "vitest"; + +import { + OPPORTUNITY_DIMENSION_WEIGHTS, + OPPORTUNITY_GATE_IDS, + type OpportunityCandidateInput, + rankEcosystemOpportunities, +} from "../../src/server/knowledge/ecosystem-opportunity-ranking"; + +const signal = { + id: "signal:repo:litellm:release", + subjectId: "repo:github.com/berriai/litellm", + sourceId: "github-repository", + sourceRecordId: "BerriAI/litellm@01234567", + artifactId: "artifact-github-repository-response", + retrievedAt: "2026-08-26T12:00:00.000Z", + fieldPath: "release.tag_name", + name: "supported-release", + rawValue: "v1.80.0", + normalizedValue: 0.8, + unit: "evidence-rubric", + transformation: "documented=0.67; verified=1", + cohort: "self-hosted-ai-gateway", + confidence: 0.9, + freshness: "fresh" as const, + caveats: ["release metadata does not prove deployed reachability"], +}; + +function candidate( + id: string, + options: { + gateStatus?: "pass" | "fail" | "unknown"; + missingDimension?: string; + } = {}, +): OpportunityCandidateInput { + return { + id, + displayName: id, + cohort: "self-hosted-ai-gateway", + subjectIds: [signal.subjectId], + gates: OPPORTUNITY_GATE_IDS.map((gateId) => ({ + id: gateId, + status: gateId === "egress" ? (options.gateStatus ?? "pass") : "pass", + evidenceSignalIds: [signal.id], + reason: `${gateId} was checked against the pinned source snapshot`, + })), + dimensions: Object.keys(OPPORTUNITY_DIMENSION_WEIGHTS).map((dimension) => ({ + dimension: dimension as keyof typeof OPPORTUNITY_DIMENSION_WEIGHTS, + normalizedValue: + dimension === options.missingDimension + ? null + : id === "candidate-a" + ? 0.8 + : 0.7, + confidence: 0.9, + evidenceSignalIds: + dimension === options.missingDimension ? [] : [signal.id], + ...(dimension === options.missingDimension + ? { missingReason: "the source has not published a comparable value" } + : {}), + })), + overlays: [ + { + id: "change-pressure", + summary: + "Recent releases are visible but do not affect the weighted total.", + evidenceSignalIds: [signal.id], + }, + ], + }; +} + +describe("ecosystem opportunity ranking", () => { + it("ranks eligible candidates within a cohort while preserving the score vector", () => { + const snapshot = rankEcosystemOpportunities({ + snapshotId: "opportunity-snapshot-2026-08-26", + asOf: "2026-08-26T12:00:00.000Z", + sourceSnapshotIds: ["github-repository@01234567"], + normalizationRevision: "opportunity-ranking-v1", + knownAnswerCutoff: "2026-08-26T12:00:00.000Z", + signals: [signal], + candidates: [candidate("candidate-b"), candidate("candidate-a")], + }); + + expect(snapshot.digest).toMatch(/^sha256:[0-9a-f]{64}$/); + expect(snapshot.candidates).toEqual([ + expect.objectContaining({ + id: "candidate-b", + status: "ranked", + total: 0.7, + cohortRank: 2, + }), + expect.objectContaining({ + id: "candidate-a", + status: "ranked", + total: 0.8, + cohortRank: 1, + }), + ]); + expect(snapshot.candidates[1]?.weightedContributions).toEqual({ + "exposure-adoption": 0.16, + "researchable-surface": 0.16, + reproducibility: 0.12, + "parallel-density": 0.12, + "disclosure-maturity": 0.08, + "portfolio-diversity": 0.08, + "operational-safety": 0.08, + }); + }); + + it("keeps hard-gate failures and missing evidence out of the numeric queue", () => { + const snapshot = rankEcosystemOpportunities({ + snapshotId: "opportunity-snapshot-2026-08-26", + asOf: "2026-08-26T12:00:00.000Z", + sourceSnapshotIds: ["github-repository@01234567"], + normalizationRevision: "opportunity-ranking-v1", + knownAnswerCutoff: "2026-08-26T12:00:00.000Z", + signals: [signal], + candidates: [ + candidate("unsafe-candidate", { gateStatus: "fail" }), + candidate("missing-candidate", { + missingDimension: "parallel-density", + }), + ], + }); + + expect(snapshot.candidates[0]).toMatchObject({ + status: "rejected", + gateFailures: ["egress"], + total: null, + cohortRank: null, + }); + expect(snapshot.candidates[1]).toMatchObject({ + status: "held", + missingDimensions: ["parallel-density"], + total: null, + cohortRank: null, + }); + }); +});