From a10a0e25aa2a459763ba3921fcfccb390a444a11 Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Wed, 26 Aug 2026 16:34:49 -0400 Subject: [PATCH 1/2] Gate findings on demonstrated security impact --- CONTEXT.md | 1 + docs/architecture.md | 2 + src/server/graph/impact-validation.ts | 311 ++++++++++++++++++++ src/server/graph/index.ts | 84 +++--- tests/integration/impact-validation.test.ts | 115 ++++++++ 5 files changed, 475 insertions(+), 38 deletions(-) create mode 100644 src/server/graph/impact-validation.ts create mode 100644 tests/integration/impact-validation.test.ts diff --git a/CONTEXT.md b/CONTEXT.md index 0dffbeb0a..dd607d9c0 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -156,6 +156,7 @@ _Avoid_: hidden gold, judge assertion - A **Research Observation** may indicate several **Knowledge Concepts** through proposed, cited Investigation Assertions without becoming a **Finding**. - A **Research Observation** may preserve several external identifiers and versioned score assessments; each remains attributable to the Observation's citations and time. - A **Research Observation** may cite a message from another project thread when that discussion materially supports or contextualizes it. +- A **Research Observation** becomes eligible for promotion to a **Finding** only after cited validation demonstrates a reproducible protected security effect under recorded authorization; rejected leads and coverage records remain distinct outcomes. - An **Investigation Entity** references a canonical project record when one exists instead of copying that record into the **Investigation Graph**. - An **Investigation Assertion** may be supported, contradicted, derived, revised, rejected, or left unresolved without changing the canonical record it discusses. - An **Investigation Citation** identifies why an **Investigation Assertion** exists; an **Artifact** remains the durable evidence object. diff --git a/docs/architecture.md b/docs/architecture.md index 411374ef9..0edbd1cda 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -140,6 +140,8 @@ The explicit Security Knowledge Graph remains in SQLite for versioned, reusable The project Investigation Graph is an assertion layer over existing records, not another owner of Targets, Artifacts, Findings, Research Observations, Tasks, Attack Paths, Tool Runs, messages, memory, or reusable security knowledge. A Research Observation preserves measured or directly seen behavior, structured inputs and outputs, measurements, external identifiers, versioned scores, actor, time, and precise citations before interpretation. The user-facing Research Map projects canonical records, cited threads and messages, external sources, reusable-concept references, current Investigation Assertions, and Research Priorities through one coherent relational snapshot. The write model resolves canonical records through project-local Investigation Entities and stores append-only Assertions, coordinate-only role-bearing Citations, and rule-versioned Derivations with ordered inputs. Evidence state (`observed`, `derived`, `proposed`, `contradicted`, or `rejected`) stays separate from assertion lifecycle (`current`, `withdrawn`, or `superseded`). Revision is an optimistic, transactional replacement that retains the predecessor and its citations. SQLite and PostgreSQL relational queries define correctness. See [ADR 0001](./adr/0001-investigation-graph-as-assertion-layer.md). +Impact validation is a deterministic promotion boundary over those records. An anomaly remains a Research Observation until cited Artifacts and Investigation Assertions demonstrate a protected read/write, cross-account effect, privilege change, secret exposure, integrity loss, deletion, availability loss, or another concrete security effect under recorded authorization and a reproducible Target Recipe/configuration. The gate preserves rejected leads, coverage records, and inconclusive observations as separate outcomes; only `finding-ready` decisions may feed the Evidence Interface's Finding creation path. + A Research Priority is an unresolved, citation-backed current assertion ranked for follow-up. Its deterministic score weights objective relevance (25%), evidence gap (20%), expected information gain (20%), target importance (15%), inverse predicate cost (8%), inverse predicate risk (7%), and authorization readiness (5%). Authorization readiness comes from the durable target ledger. Deliberately turning a Research Priority into a Task uses the existing Task workflow and a unique assertion-task receipt; it never schedules work, creates an approval, runs a tool, or promotes a Finding. The former generic security-graph repository is retired. Historical database tables may remain so existing local data is not destructively dropped, but no product path writes them and they are not authoritative. The only graph ownership boundaries are the global Security Knowledge Graph and each project's Investigation Graph. diff --git a/src/server/graph/impact-validation.ts b/src/server/graph/impact-validation.ts new file mode 100644 index 000000000..4aa9d46d3 --- /dev/null +++ b/src/server/graph/impact-validation.ts @@ -0,0 +1,311 @@ +export const SECURITY_EFFECT_TYPES = [ + "protected-read", + "protected-write", + "cross-account-effect", + "privilege-change", + "secret-exposure", + "integrity-loss", + "deletion", + "availability-loss", + "other-demonstrated-effect", +] as const; + +export type SecurityEffectType = (typeof SECURITY_EFFECT_TYPES)[number]; + +export type ValidatedSecurityEffect = { + type: SecurityEffectType; + summary: string; + protectedOperation: string; + beforeState: string; + afterState: string; + evidenceArtifactIds: string[]; + assertionIds: string[]; + sourcePrincipalId?: string; + affectedPrincipalId?: string; +}; + +export type ImpactValidationInput = { + conclusion: + | "validated-security-effect" + | "rejected-lead" + | "coverage-only" + | "inconclusive"; + observationIds: string[]; + targetIds: string[]; + evidenceArtifactIds: string[]; + assertionIds: string[]; + authorizationIds: string[]; + effects: ValidatedSecurityEffect[]; + reproduction: { + attempts: number; + successes: number; + independentRuns: number; + targetRecipeDigest?: string; + configurationId?: string; + }; + controls: Array<{ + kind: "baseline" | "negative-control" | "cross-account" | "fixed-revision"; + summary: string; + evidenceArtifactIds: string[]; + }>; + rejectionReason?: string; + coverageScope?: string; + remainingGaps?: string[]; +}; + +export type ImpactValidationDecision = { + disposition: + | "finding-ready" + | "rejected-lead" + | "coverage-record" + | "research-observation"; + promotionAllowed: boolean; + securityEffectDemonstrated: boolean; + reasons: string[]; + missingRequirements: string[]; + findingMetadata?: { + validationProtocol: "impact-validation-v1"; + observationIds: string[]; + assertionIds: string[]; + authorizationIds: string[]; + securityEffectTypes: SecurityEffectType[]; + reproduction: ImpactValidationInput["reproduction"]; + controlKinds: ImpactValidationInput["controls"][number]["kind"][]; + }; +}; + +/** + * Determines whether cited observations have crossed the Finding boundary. + * This function never creates a Finding or grants authorization by itself. + */ +export function decideImpactValidation( + input: ImpactValidationInput, +): ImpactValidationDecision { + assertUniqueNonEmpty(input.observationIds, "observationIds"); + assertUniqueNonEmpty(input.targetIds, "targetIds"); + assertUnique(input.evidenceArtifactIds, "evidenceArtifactIds"); + assertUnique(input.assertionIds, "assertionIds"); + assertUnique(input.authorizationIds, "authorizationIds"); + assertReproduction(input.reproduction); + validateEffects(input.effects); + validateControls(input.controls); + + if (input.conclusion === "rejected-lead") { + assertNonEmpty(input.rejectionReason, "rejectionReason"); + if (input.effects.length > 0) { + throw new Error( + "A rejected lead cannot retain validated security effects.", + ); + } + return { + disposition: "rejected-lead", + promotionAllowed: false, + securityEffectDemonstrated: false, + reasons: [input.rejectionReason as string], + missingRequirements: [], + }; + } + + if (input.conclusion === "coverage-only") { + assertNonEmpty(input.coverageScope, "coverageScope"); + if (input.effects.length > 0) { + throw new Error( + "A coverage record cannot retain validated security effects.", + ); + } + return { + disposition: "coverage-record", + promotionAllowed: false, + securityEffectDemonstrated: false, + reasons: [input.coverageScope as string], + missingRequirements: [], + }; + } + + if (input.conclusion === "inconclusive") { + if (input.effects.length > 0) { + throw new Error( + "An inconclusive validation cannot retain validated security effects.", + ); + } + return { + disposition: "research-observation", + promotionAllowed: false, + securityEffectDemonstrated: false, + reasons: input.remainingGaps?.length + ? [...input.remainingGaps] + : ["Impact remains unestablished."], + missingRequirements: input.remainingGaps?.length + ? [...input.remainingGaps] + : ["a reproducible protected security effect"], + }; + } + + const missingRequirements: string[] = []; + if (input.effects.length === 0) + missingRequirements.push("at least one demonstrated security effect"); + if (input.evidenceArtifactIds.length === 0) { + missingRequirements.push("durable evidence artifacts"); + } + if (input.assertionIds.length === 0) { + missingRequirements.push("citation-backed Investigation Assertions"); + } + if (input.authorizationIds.length === 0) { + missingRequirements.push("target authorization references"); + } + if (input.reproduction.successes < 1) { + missingRequirements.push("at least one successful reproduction"); + } + if (input.reproduction.independentRuns < 1) { + missingRequirements.push("an independently reset run"); + } + if (!input.reproduction.targetRecipeDigest) { + missingRequirements.push("the reproducing Target Recipe digest"); + } + if (!input.reproduction.configurationId) { + missingRequirements.push("the reproducing configuration identity"); + } + if (input.controls.length === 0) { + missingRequirements.push( + "a baseline, negative, cross-account, or fixed-revision control", + ); + } + for (const effect of input.effects) { + for (const artifactId of effect.evidenceArtifactIds) { + if (!input.evidenceArtifactIds.includes(artifactId)) { + missingRequirements.push( + `effect evidence ${artifactId} in the validation artifact set`, + ); + } + } + for (const assertionId of effect.assertionIds) { + if (!input.assertionIds.includes(assertionId)) { + missingRequirements.push( + `effect assertion ${assertionId} in the validation assertion set`, + ); + } + } + } + + const uniqueMissing = [...new Set(missingRequirements)]; + if (uniqueMissing.length > 0) { + return { + disposition: "research-observation", + promotionAllowed: false, + securityEffectDemonstrated: false, + reasons: [ + "The claimed effect has not crossed the evidence and reproduction gate.", + ], + missingRequirements: uniqueMissing, + }; + } + + return { + disposition: "finding-ready", + promotionAllowed: true, + securityEffectDemonstrated: true, + reasons: input.effects.map((effect) => effect.summary), + missingRequirements: [], + findingMetadata: { + validationProtocol: "impact-validation-v1", + observationIds: [...input.observationIds], + assertionIds: [...input.assertionIds], + authorizationIds: [...input.authorizationIds], + securityEffectTypes: [ + ...new Set(input.effects.map((effect) => effect.type)), + ], + reproduction: { ...input.reproduction }, + controlKinds: [...new Set(input.controls.map((control) => control.kind))], + }, + }; +} + +function validateEffects(effects: ValidatedSecurityEffect[]): void { + for (const [index, effect] of effects.entries()) { + assertNonEmpty(effect.summary, `effects.${index}.summary`); + assertNonEmpty( + effect.protectedOperation, + `effects.${index}.protectedOperation`, + ); + assertNonEmpty(effect.beforeState, `effects.${index}.beforeState`); + assertNonEmpty(effect.afterState, `effects.${index}.afterState`); + assertUniqueNonEmpty( + effect.evidenceArtifactIds, + `effects.${index}.evidenceArtifactIds`, + ); + assertUniqueNonEmpty(effect.assertionIds, `effects.${index}.assertionIds`); + if (effect.beforeState === effect.afterState) { + throw new Error( + `effects.${index} must demonstrate an observable state change.`, + ); + } + if (effect.type === "cross-account-effect") { + assertNonEmpty( + effect.sourcePrincipalId, + `effects.${index}.sourcePrincipalId`, + ); + assertNonEmpty( + effect.affectedPrincipalId, + `effects.${index}.affectedPrincipalId`, + ); + if (effect.sourcePrincipalId === effect.affectedPrincipalId) { + throw new Error( + `effects.${index} cross-account principals must be distinct.`, + ); + } + } + } +} + +function validateControls(controls: ImpactValidationInput["controls"]): void { + for (const [index, control] of controls.entries()) { + assertNonEmpty(control.summary, `controls.${index}.summary`); + assertUniqueNonEmpty( + control.evidenceArtifactIds, + `controls.${index}.evidenceArtifactIds`, + ); + } +} + +function assertReproduction( + reproduction: ImpactValidationInput["reproduction"], +): void { + for (const [name, value] of Object.entries({ + attempts: reproduction.attempts, + successes: reproduction.successes, + independentRuns: reproduction.independentRuns, + })) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`reproduction.${name} must be a non-negative integer.`); + } + } + if (reproduction.successes > reproduction.attempts) { + throw new Error("reproduction.successes cannot exceed attempts."); + } + if (reproduction.independentRuns > reproduction.attempts) { + throw new Error("reproduction.independentRuns cannot exceed attempts."); + } + if ( + reproduction.targetRecipeDigest && + !/^sha256:[0-9a-f]{64}$/.test(reproduction.targetRecipeDigest) + ) { + throw new Error( + "reproduction.targetRecipeDigest must be a lowercase SHA-256 digest.", + ); + } +} + +function assertUniqueNonEmpty(values: string[], name: string): void { + if (values.length === 0) throw new Error(`${name} must not be empty.`); + assertUnique(values, name); + for (const value of values) assertNonEmpty(value, name); +} + +function assertUnique(values: string[], name: string): void { + if (new Set(values).size !== values.length) + throw new Error(`${name} must not contain duplicates.`); +} + +function assertNonEmpty(value: string | undefined, name: string): void { + if (!value?.trim()) throw new Error(`${name} must be non-empty.`); +} diff --git a/src/server/graph/index.ts b/src/server/graph/index.ts index 788dde1ff..0f16509e5 100644 --- a/src/server/graph/index.ts +++ b/src/server/graph/index.ts @@ -1,46 +1,54 @@ export { - ASSERTION_EPISTEMIC_STATUSES, - ASSERTION_LIFECYCLE_STATUSES, - ASSERTION_POLARITIES, - ASSERTION_REFERENCE_TYPES, - CITATION_ROLES, - CITATION_SOURCE_TYPES, - createInvestigationEntity, - createTaskFromResearchPriority, - INVESTIGATION_PREDICATES, - type InvestigationAssertion, - type InvestigationCitation, - type InvestigationCitationInput, - type InvestigationEntity, - type RememberAssertionInput, - type ResearchPriority, - type ReviseAssertionInput, - rankResearchPriorities, - recallInvestigationAssertions, - rememberInvestigationAssertion, - reviseInvestigationAssertion, + decideImpactValidation, + type ImpactValidationDecision, + type ImpactValidationInput, + SECURITY_EFFECT_TYPES, + type SecurityEffectType, + type ValidatedSecurityEffect, +} from "./impact-validation"; +export { + ASSERTION_EPISTEMIC_STATUSES, + ASSERTION_LIFECYCLE_STATUSES, + ASSERTION_POLARITIES, + ASSERTION_REFERENCE_TYPES, + CITATION_ROLES, + CITATION_SOURCE_TYPES, + createInvestigationEntity, + createTaskFromResearchPriority, + INVESTIGATION_PREDICATES, + type InvestigationAssertion, + type InvestigationCitation, + type InvestigationCitationInput, + type InvestigationEntity, + type RememberAssertionInput, + type ResearchPriority, + type ReviseAssertionInput, + rankResearchPriorities, + recallInvestigationAssertions, + rememberInvestigationAssertion, + reviseInvestigationAssertion, } from "./investigation-assertions"; export { - buildProjectResearchGraph, - type ProjectResearchGraph, - RESEARCH_GRAPH_NODE_LIMIT, - type ResearchGraphEdge, - type ResearchGraphNode, - type ResearchGraphNodeKind, + buildProjectResearchGraph, + type ProjectResearchGraph, + RESEARCH_GRAPH_NODE_LIMIT, + type ResearchGraphEdge, + type ResearchGraphNode, + type ResearchGraphNodeKind, } from "./research-graph-view"; export { - type ResearchMapSnapshot, - readResearchMapSnapshot, + type ResearchMapSnapshot, + readResearchMapSnapshot, } from "./research-map-snapshot"; export { - RESEARCH_OBSERVATION_KINDS, - type RecordedResearchObservation, - type RecordResearchObservationInput, - type ResearchExternalIdentifier, - type ResearchMeasurement, - type ResearchObservation, - type ResearchObservationKind, - type ResearchScore, - recallResearchObservations, - recordResearchObservation, + RESEARCH_OBSERVATION_KINDS, + type RecordedResearchObservation, + type RecordResearchObservationInput, + type ResearchExternalIdentifier, + type ResearchMeasurement, + type ResearchObservation, + type ResearchObservationKind, + type ResearchScore, + recallResearchObservations, + recordResearchObservation, } from "./research-observations"; diff --git a/tests/integration/impact-validation.test.ts b/tests/integration/impact-validation.test.ts new file mode 100644 index 000000000..bac23ad4c --- /dev/null +++ b/tests/integration/impact-validation.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, it } from "vitest"; + +import { + decideImpactValidation, + type ImpactValidationInput, +} from "../../src/server/graph"; + +function validation(): ImpactValidationInput { + return { + conclusion: "validated-security-effect", + observationIds: ["observation-cross-account-write"], + targetIds: ["target-owned-api"], + evidenceArtifactIds: [ + "artifact-request", + "artifact-response", + "artifact-control", + ], + assertionIds: ["assertion-cross-account-write"], + authorizationIds: ["authorization-owned-api"], + effects: [ + { + type: "cross-account-effect", + summary: "A viewer from account A changed protected account B state.", + protectedOperation: "update account B notification destination", + beforeState: "destination=owner-b@example.test", + afterState: "destination=viewer-a@example.test", + evidenceArtifactIds: ["artifact-request", "artifact-response"], + assertionIds: ["assertion-cross-account-write"], + sourcePrincipalId: "account-a-viewer", + affectedPrincipalId: "account-b-owner", + }, + ], + reproduction: { + attempts: 2, + successes: 2, + independentRuns: 1, + targetRecipeDigest: `sha256:${"a".repeat(64)}`, + configurationId: "two-account-fixture-v1", + }, + controls: [ + { + kind: "cross-account", + summary: + "The same operation is denied when the account binding is preserved.", + evidenceArtifactIds: ["artifact-control"], + }, + ], + }; +} + +describe("impact validation", () => { + it("allows Finding promotion only after a cited reproducible protected effect", () => { + const decision = decideImpactValidation(validation()); + + expect(decision).toMatchObject({ + disposition: "finding-ready", + promotionAllowed: true, + securityEffectDemonstrated: true, + missingRequirements: [], + findingMetadata: { + validationProtocol: "impact-validation-v1", + observationIds: ["observation-cross-account-write"], + securityEffectTypes: ["cross-account-effect"], + controlKinds: ["cross-account"], + }, + }); + }); + + it("keeps surprising behavior as an observation when impact evidence is incomplete", () => { + const input = validation(); + input.evidenceArtifactIds = []; + input.assertionIds = []; + input.authorizationIds = []; + input.reproduction = { attempts: 1, successes: 0, independentRuns: 0 }; + + expect(decideImpactValidation(input)).toMatchObject({ + disposition: "research-observation", + promotionAllowed: false, + securityEffectDemonstrated: false, + missingRequirements: expect.arrayContaining([ + "durable evidence artifacts", + "citation-backed Investigation Assertions", + "target authorization references", + "at least one successful reproduction", + "an independently reset run", + ]), + }); + }); + + it("preserves rejected leads and coverage without manufacturing Findings", () => { + const rejected: ImpactValidationInput = { + ...validation(), + conclusion: "rejected-lead", + effects: [], + rejectionReason: + "The apparent write was only a cached response; durable state did not change.", + }; + const coverage: ImpactValidationInput = { + ...validation(), + conclusion: "coverage-only", + effects: [], + coverageScope: + "Viewer-to-admin protected writes were checked across all documented routes.", + }; + + expect(decideImpactValidation(rejected)).toMatchObject({ + disposition: "rejected-lead", + promotionAllowed: false, + }); + expect(decideImpactValidation(coverage)).toMatchObject({ + disposition: "coverage-record", + promotionAllowed: false, + }); + }); +}); From 14fc0fe4c2071bf52a91f4bde4c66411ca937aea Mon Sep 17 00:00:00 2001 From: Dan Levy Date: Wed, 26 Aug 2026 16:36:11 -0400 Subject: [PATCH 2/2] Keep graph exports focused --- src/server/graph/index.ts | 88 +++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/src/server/graph/index.ts b/src/server/graph/index.ts index 0f16509e5..4fce766b6 100644 --- a/src/server/graph/index.ts +++ b/src/server/graph/index.ts @@ -1,54 +1,54 @@ export { - decideImpactValidation, - type ImpactValidationDecision, - type ImpactValidationInput, - SECURITY_EFFECT_TYPES, - type SecurityEffectType, - type ValidatedSecurityEffect, + decideImpactValidation, + type ImpactValidationDecision, + type ImpactValidationInput, + SECURITY_EFFECT_TYPES, + type SecurityEffectType, + type ValidatedSecurityEffect, } from "./impact-validation"; export { - ASSERTION_EPISTEMIC_STATUSES, - ASSERTION_LIFECYCLE_STATUSES, - ASSERTION_POLARITIES, - ASSERTION_REFERENCE_TYPES, - CITATION_ROLES, - CITATION_SOURCE_TYPES, - createInvestigationEntity, - createTaskFromResearchPriority, - INVESTIGATION_PREDICATES, - type InvestigationAssertion, - type InvestigationCitation, - type InvestigationCitationInput, - type InvestigationEntity, - type RememberAssertionInput, - type ResearchPriority, - type ReviseAssertionInput, - rankResearchPriorities, - recallInvestigationAssertions, - rememberInvestigationAssertion, - reviseInvestigationAssertion, + ASSERTION_EPISTEMIC_STATUSES, + ASSERTION_LIFECYCLE_STATUSES, + ASSERTION_POLARITIES, + ASSERTION_REFERENCE_TYPES, + CITATION_ROLES, + CITATION_SOURCE_TYPES, + createInvestigationEntity, + createTaskFromResearchPriority, + INVESTIGATION_PREDICATES, + type InvestigationAssertion, + type InvestigationCitation, + type InvestigationCitationInput, + type InvestigationEntity, + type RememberAssertionInput, + type ResearchPriority, + type ReviseAssertionInput, + rankResearchPriorities, + recallInvestigationAssertions, + rememberInvestigationAssertion, + reviseInvestigationAssertion, } from "./investigation-assertions"; export { - buildProjectResearchGraph, - type ProjectResearchGraph, - RESEARCH_GRAPH_NODE_LIMIT, - type ResearchGraphEdge, - type ResearchGraphNode, - type ResearchGraphNodeKind, + buildProjectResearchGraph, + type ProjectResearchGraph, + RESEARCH_GRAPH_NODE_LIMIT, + type ResearchGraphEdge, + type ResearchGraphNode, + type ResearchGraphNodeKind, } from "./research-graph-view"; export { - type ResearchMapSnapshot, - readResearchMapSnapshot, + type ResearchMapSnapshot, + readResearchMapSnapshot, } from "./research-map-snapshot"; export { - RESEARCH_OBSERVATION_KINDS, - type RecordedResearchObservation, - type RecordResearchObservationInput, - type ResearchExternalIdentifier, - type ResearchMeasurement, - type ResearchObservation, - type ResearchObservationKind, - type ResearchScore, - recallResearchObservations, - recordResearchObservation, + RESEARCH_OBSERVATION_KINDS, + type RecordedResearchObservation, + type RecordResearchObservationInput, + type ResearchExternalIdentifier, + type ResearchMeasurement, + type ResearchObservation, + type ResearchObservationKind, + type ResearchScore, + recallResearchObservations, + recordResearchObservation, } from "./research-observations";