Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 25 additions & 25 deletions plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,7 @@ import scanDraftSchema from "../../schemas/tools/scan-draft.schema.json";
import type { ArtifactContext, DeepReducerContext } from "./artifact-context.js";
import {
parsePersistedScanDraft,
parseScanDraft,
saveScanDraftCheckpoint,
type ScanDraftInput
} from "./artifact-scan-draft.js";
import {
loadArtifactZodSchema,
Expand All @@ -21,7 +19,12 @@ import {
writeJsonAtomic,
type DeepScanArtifacts
} from "./deep-scan/artifacts.js";
import { reconcileDeepReduction } from "./deep-scan/artifact-validation.js";
import {
parseDeepReduction,
reconcileDeepReduction,
type DeepReductionInput,
type DeepReductionSources,
} from "./deep-scan/artifact-validation.js";

const schemaDocuments = [
commonSchema,
Expand All @@ -39,15 +42,7 @@ export const deepReductionInputSchema = loadArtifactZodSchema(
schemaDocuments,
reducerSchema.$id,
"reductionInput"
) as ZodType<ScanDraftInput>;

interface DeepReducerInputs {
discoveries: {
workerId: string;
result: ScanDraftInput;
}[];
previous: ScanDraftInput | null;
}
) as ZodType<DeepReductionInput>;

interface BoundReducer {
artifacts: DeepScanArtifacts;
Expand All @@ -56,18 +51,19 @@ interface BoundReducer {
scanId?: string;
}

/** Return complete Standard results without exposing their artifact locations. */
/** Read the findings and scan context assigned to this reducer. */
export async function getCodexSecurityDeepReducerInputs(
context: ArtifactContext
): Promise<DeepReducerInputs> {
): Promise<DeepReductionSources> {
return withLogicalReducerErrors(context, async () => {
const bound = bindDeepReducer(context);
const discoveries = await Promise.all(bound.state.claimedWorkers.map(async (worker) => {
await requireRegularFile(worker.resultPath, bound.artifacts.workersRoot);
const result = parseStoredScanDraft(
await readJsonObject(worker.resultPath),
"Accepted Standard worker " + worker.id,
bound.scanId
bound.scanId,
parsePersistedScanDraft
);
if (result.complete === false) throw new Error("An assigned Standard worker wrote only a checkpoint, not a complete result.");
result.findings = result.findings.map((finding, index) => ({
Expand All @@ -77,7 +73,8 @@ export async function getCodexSecurityDeepReducerInputs(
sourceFindingIds: [`${worker.id}:${index}`],
},
}));
return { workerId: worker.id, result };
const { coverage: _coverage, ...reduction } = result;
return { workerId: worker.id, result: reduction };
}));
const previous = await readPreviousReduction(bound);
const scanId = bound.scanId ?? previous?.scanId ?? discoveries[0]?.result.scanId;
Expand All @@ -97,7 +94,7 @@ export async function getCodexSecurityDeepReducerInputs(
});
}

/** Validate and durably replace this reducer's complete semantic Standard result. */
/** Check and save the reducer's finished result. */
export async function recordCodexSecurityDeepReduction(
context: ArtifactContext,
input: unknown
Expand All @@ -107,7 +104,8 @@ export async function recordCodexSecurityDeepReduction(
}> {
return withLogicalReducerErrors(context, async () => {
const bound = bindDeepReducer(context);
let reduction = parseScanDraft(input as ScanDraftInput);
const submitted = deepReductionInputSchema.parse(input);
let reduction = parseDeepReduction(submitted);
if (reduction.complete === false) throw new Error("Deep reduction is only a checkpoint, not a complete result.");
const inputs = await getCodexSecurityDeepReducerInputs(context);
const expectedScanId = bound.scanId
Expand Down Expand Up @@ -164,25 +162,27 @@ function bindDeepReducer(context: ArtifactContext): BoundReducer {

async function readPreviousReduction(
bound: BoundReducer
): Promise<ScanDraftInput | null> {
): Promise<DeepReductionInput | null> {
const { previousReducerResultPath } = bound.state;
if (!previousReducerResultPath) return null;
await requireRegularFile(previousReducerResultPath, bound.artifacts.dedupRoot);
return parseStoredScanDraft(
await readJsonObject(previousReducerResultPath),
"The previous accepted Deep reduction",
bound.scanId
bound.scanId,
(value) => parseDeepReduction(value, true)
);
}

function parseStoredScanDraft(
function parseStoredScanDraft<Result extends DeepReductionInput>(
value: Record<string, unknown>,
label: string,
expectedScanId?: string
): ScanDraftInput {
let parsed: ScanDraftInput;
expectedScanId: string | undefined,
parse: (input: Record<string, unknown>) => Result
): Result {
let parsed: Result;
try {
parsed = parsePersistedScanDraft(value);
parsed = parse(value);
} catch (error) {
const detail = error instanceof Error ? error.message : String(error);
throw new Error(label + " has an invalid Standard scan result: " + detail, { cause: error });
Expand Down
53 changes: 44 additions & 9 deletions plugins/codex-security/mcp-app/src/artifact-scan-draft.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ interface PreparedScanDraft {

type PublishScanDraft = (
draft: PreparedScanDraft,
expectedDigest: string,
expectedDigest: string | undefined,
checkpoint: ScanDraftInput,
) => Promise<void>;

Expand Down Expand Up @@ -88,7 +88,11 @@ export async function recordCodexSecurityScanDraft(

for (;;) {
signal?.throwIfAborted();
const preserved = await preserveScanDraft(context, parsed, false);
// Deep results are ready to save. Do not merge older drafts or
// checkpoints into them.
const preserved = context.mode === "deep" && parsed.complete !== false
? { input: parsed, previousDigest: undefined }
: await preserveScanDraft(context, parsed, false);
const reconciled = preserved.input;
const contract = requireObject(
context.targetContract,
Expand All @@ -104,7 +108,7 @@ export async function recordCodexSecurityScanDraft(
);
const target = buildTarget(context, contract, trustedTarget);
const scope = buildScope(context, trustedScope, reconciled.scope);
const findings = buildFindings(reconciled.findings);
const findings = buildFindings(reconciled.findings, context.mode);
const coverage = buildCoverage(
context,
contract,
Expand Down Expand Up @@ -190,9 +194,10 @@ export async function recordCodexSecurityScanDraftViaWorkbench(
draftPath,
"--checkpoint-path",
checkpointPath,
"--expected-draft-digest",
expectedDigest,
];
if (expectedDigest !== undefined) {
arguments_.push("--expected-draft-digest", expectedDigest);
}
if (context.handoffClaimToken) {
arguments_.push("--claim-token", context.handoffClaimToken);
}
Expand Down Expand Up @@ -268,7 +273,7 @@ export async function recordCodexSecurityWorkerScanDraft(
/** Keep the semantic input before any replaceable worker or canonical artifact. */
export async function saveScanDraftCheckpoint(
context: ArtifactContext,
input: ScanDraftInput,
input: Omit<ScanDraftInput, "coverage">,
updateHead = true,
): Promise<void> {
const { handoffClaimToken: _claim, ...snapshot } = input;
Expand Down Expand Up @@ -514,7 +519,7 @@ async function readCurrentCheckpoints(
return checkpoints.map(({ input }) => input);
}

function scanDraftCheckpointName(input: ScanDraftInput): string {
function scanDraftCheckpointName(input: Omit<ScanDraftInput, "coverage">): string {
const { handoffClaimToken: _claim, ...snapshot } = input;
return createHash("sha256").update(JSON.stringify(snapshot)).digest("hex") + ".json";
}
Expand Down Expand Up @@ -1388,7 +1393,7 @@ function buildScope(
};
}

function buildFindings(findings: JsonObject[]): JsonObject[] {
function buildFindings(findings: JsonObject[], mode?: string): JsonObject[] {
const generatedIdentities = findings.map((finding, index) => {
if (finding.identity !== undefined) return undefined;
const candidateId = (finding.extensions as JsonObject | undefined)
Expand Down Expand Up @@ -1421,7 +1426,7 @@ function buildFindings(findings: JsonObject[]): JsonObject[] {
);
}

return findings.map((finding, index) => {
const identified: JsonObject[] = findings.map((finding, index) => {
const generatedIdentity = generatedIdentities[index];
if (generatedIdentity === undefined) return { ...finding };
const identity: JsonObject = { anchor: generatedIdentity.anchor };
Expand All @@ -1441,6 +1446,36 @@ function buildFindings(findings: JsonObject[]): JsonObject[] {
identity,
};
});
if (mode !== "deep") return identified;

// Keep both findings when workers reuse an ID.
// Add a numeric suffix to make each ID unique.
const reserved = new Set(identified.map(scanFindingIdentity));
const used = new Set<string>();
return identified.map((finding) => {
const key = scanFindingIdentity(finding);
if (!used.has(key)) {
used.add(key);
return finding;
}
const identity = finding.identity as JsonObject;
const baseInstance = identity.instance ?? "saved";
let suffix = 2;
const distinct: JsonObject & { identity: JsonObject } = {
...finding, identity: { ...identity },
};
do {
distinct.identity.instance = `${baseInstance}-${suffix}`;
suffix += 1;
} while (reserved.has(scanFindingIdentity(distinct)) || used.has(scanFindingIdentity(distinct)));
const provenance = finding.provenance as JsonObject;
distinct.provenance = {
...provenance,
preservedIdentity: provenance.preservedIdentity ?? structuredClone(identity),
};
used.add(scanFindingIdentity(distinct));
return distinct;
});
}

function buildCoverage(
Expand Down
Loading
Loading