Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import {
createStoredPassiveAuthSurface,
PassiveAuthSurfaceArtifactError,
PassiveAuthSurfaceAttributionError,
PassiveAuthSurfaceAuthorizationError,
type StoredPassiveArtifactRef,
} from "../../../../../../server/recon";
Expand Down Expand Up @@ -38,6 +39,9 @@ export async function POST(request: Request, context: Context) {
if (error instanceof PassiveAuthSurfaceArtifactError) {
return notFound(error.message);
}
if (error instanceof PassiveAuthSurfaceAttributionError) {
return notFound(error.message);
}
if (
error instanceof Error &&
/required|must be|allows?|unique/i.test(error.message)
Expand Down
272 changes: 213 additions & 59 deletions src/server/evidence/artifact-service.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { createHash } from "node:crypto";

import { createId } from "../../lib/ids";
import { type Queryable, withDatabase } from "../db/client";
import { type Queryable, withDatabase, withTransaction } from "../db/client";
import { notifyProjectChanged } from "../projects/project-events";
import { readResearchRunContext } from "../research/run-context";
import { createObjectStorageClient, type ObjectStorageClient } from "../storage";
Expand Down Expand Up @@ -45,8 +45,16 @@ export type CreateArtifactInput = {
inlineFallback?: boolean;
maxInlineBytes?: number;
metadata?: Record<string, unknown>;
attributionPolicy?: "project-thread-task-target";
};

export class ArtifactAttributionGuardError extends Error {
constructor() {
super("Artifact attribution changed before persistence.");
this.name = "ArtifactAttributionGuardError";
}
}

export type CreateUploadedObjectArtifactInput = {
projectId: string;
threadId?: string;
Expand Down Expand Up @@ -128,13 +136,10 @@ export function createArtifactService(config: ArtifactServiceConfig = {}) {
const now = config.now ?? (() => new Date());
const randomId = config.randomId ?? (() => createId("artifact"));

const tryWriteStorage = async (input: {
const prepareStorage = async (input: {
projectId: string;
threadId?: string;
content: string | Uint8Array;
contentType: string;
name: string;
metadata?: Record<string, string>;
}) => {
if (!storage) {
return {
Expand All @@ -145,20 +150,12 @@ export function createArtifactService(config: ArtifactServiceConfig = {}) {

await storage.ensureBucket();
const key = `projects/${input.projectId}/threads/${input.threadId ?? "default"}/evidence/${now().getTime()}-${shortObjectKeyId(randomId())}-${sanitizeObjectKeySegment(input.name)}`;
await storage.putObject({
key,
body: input.content,
contentType: input.contentType,
metadata: {
projectId: input.projectId,
threadId: input.threadId ?? "",
...(input.metadata ?? {}),
},
});
return {
storageBucket: storage.bucket,
storageKey: key,
storageMode: storage.mode ?? "object-storage",
storageMode: (storage.mode ?? "object-storage") as
| NonNullable<ObjectStorageClient["mode"]>
| "object-storage",
};
};

Expand All @@ -184,11 +181,9 @@ export function createArtifactService(config: ArtifactServiceConfig = {}) {
const allowInlineFallback = input.inlineFallback ?? !isBinary;
const maxInlineBytes = input.maxInlineBytes ?? DEFAULT_MAX_INLINE_BYTES;

const { storageBucket, storageKey, storageMode } = await tryWriteStorage({
const { storageBucket, storageKey, storageMode } = await prepareStorage({
projectId: input.projectId,
threadId: input.threadId,
content: contentForStorage,
contentType,
name,
});

Expand All @@ -202,47 +197,71 @@ export function createArtifactService(config: ArtifactServiceConfig = {}) {
inlineContent = contentString.slice(0, maxInlineBytes);
}

const artifactId = await withDatabase(async (db: Queryable) => {
const { threadId, targetId, taskId, findingId, agentGenerated, toolRunId, metadata } =
input;
const rows = await db.query<{ id: string }>(
`INSERT INTO artifacts (project_id, thread_id, task_id, finding_id, kind, name, content_type, storage_bucket, storage_key, size_bytes, sha256, inline_content, agent_generated, tool_run_id, metadata)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15::jsonb)
RETURNING id`,
[
input.projectId,
threadId ?? null,
taskId ?? null,
findingId ?? null,
kind,
name,
contentType,
storageBucket ?? null,
storageKey ?? null,
sizeBytes,
sha256,
storageKey ? null : inlineContent,
agentGenerated ?? true,
toolRunId ?? null,
JSON.stringify({
...(metadata ?? {}),
...(researchRunContext ?? {}),
source: input.source,
targetId,
targetIds: targetScope.kind === "targets" ? targetScope.targetIds : [],
targetScope: targetScope.kind === "targets" ? "targets" : "project",
storageMode: storageKey ? (storageMode ?? "object-storage") : "inline",
...(storageKey
? {}
: { inlineBytes: new TextEncoder().encode(inlineContent ?? "").byteLength }),
...(storageKey || inlineContent === contentString ? {} : { inlineTruncated: true }),
sizeBytes,
createdAt: timestamp,
let wroteStorageObject = false;
let artifactId: string | null = null;
const row = {
input,
kind,
name,
contentType,
storageBucket,
storageKey,
storageMode,
sizeBytes,
sha256,
inlineContent,
contentString,
timestamp,
targetIds: targetScope.kind === "targets" ? targetScope.targetIds : [],
targetScope: targetScope.kind === "targets" ? ("targets" as const) : ("project" as const),
researchRunContext,
};
const writeStorageObject = async () => {
if (!storageKey || !storage) return;
await storage.putObject({
key: storageKey,
body: contentForStorage,
contentType,
metadata: {
projectId: input.projectId,
threadId: input.threadId ?? "",
},
});
wroteStorageObject = true;
};

if (input.attributionPolicy !== "project-thread-task-target") {
await writeStorageObject();
const rows = await withDatabase((db) => insertArtifactRow(db, row));
artifactId = rows.rows[0]?.id ?? null;
} else {
try {
artifactId = await withDatabase(async (db, database) =>
withTransaction(db, async (tx) => {
if (database.backend === "postgres") {
await lockArtifactAttribution(tx, input);
}
const rows = await insertArtifactRow(tx, row);
const persistedId = rows.rows[0]?.id ?? null;
if (!persistedId) throw new ArtifactAttributionGuardError();
await writeStorageObject();
return persistedId;
}),
],
);
return rows.rows[0]?.id ?? null;
});
);
} catch (error) {
if (wroteStorageObject && storageKey && storage) {
try {
await storage.deleteObject({
key: storageKey,
...(storageBucket ? { bucket: storageBucket } : {}),
});
} catch {
// Preserve the persistence error; an orphaned object has no database or index visibility.
}
}
throw error;
}
}

if (!artifactId) {
throw new Error("Failed to create artifact row");
Expand Down Expand Up @@ -515,6 +534,141 @@ export function createArtifactService(config: ArtifactServiceConfig = {}) {
};
}

type ArtifactRowInsert = {
input: CreateArtifactInput;
kind: NonNullable<CreateArtifactInput["kind"]>;
name: string;
contentType: string;
storageBucket?: string;
storageKey?: string;
storageMode?: "s3" | "filesystem" | "object-storage";
sizeBytes: number;
sha256: string;
inlineContent: string | null;
contentString: string;
timestamp: string;
targetIds: string[];
targetScope: "targets" | "project";
researchRunContext: ReturnType<typeof readResearchRunContext>;
};

async function insertArtifactRow(db: Queryable, row: ArtifactRowInsert) {
const {
input,
kind,
name,
contentType,
storageBucket,
storageKey,
storageMode,
sizeBytes,
sha256,
inlineContent,
contentString,
timestamp,
targetIds,
targetScope,
researchRunContext,
} = row;
const values = [
input.projectId,
input.threadId ?? null,
input.taskId ?? null,
input.findingId ?? null,
kind,
name,
contentType,
storageBucket ?? null,
storageKey ?? null,
sizeBytes,
sha256,
storageKey ? null : inlineContent,
input.agentGenerated ?? true,
input.toolRunId ?? null,
JSON.stringify({
...(input.metadata ?? {}),
...(researchRunContext ?? {}),
source: input.source,
targetId: input.targetId,
targetIds,
targetScope,
storageMode: storageKey ? (storageMode ?? "object-storage") : "inline",
...(storageKey
? {}
: { inlineBytes: new TextEncoder().encode(inlineContent ?? "").byteLength }),
...(storageKey || inlineContent === contentString ? {} : { inlineTruncated: true }),
sizeBytes,
createdAt: timestamp,
}),
];

if (input.attributionPolicy !== "project-thread-task-target") {
return db.query<{ id: string }>(
`INSERT INTO artifacts (project_id, thread_id, task_id, finding_id, kind, name, content_type, storage_bucket, storage_key, size_bytes, sha256, inline_content, agent_generated, tool_run_id, metadata)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15::jsonb)
RETURNING id`,
values,
);
}
if (!input.targetId) throw new ArtifactAttributionGuardError();

return db.query<{ id: string }>(
`INSERT INTO artifacts (project_id, thread_id, task_id, finding_id, kind, name, content_type, storage_bucket, storage_key, size_bytes, sha256, inline_content, agent_generated, tool_run_id, metadata)
SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15::jsonb
WHERE EXISTS (
SELECT 1 FROM targets
WHERE project_id = $1 AND id = $16
)
AND ($2::text IS NULL OR EXISTS (
SELECT 1 FROM chat_threads
WHERE project_id = $1 AND id = $2
))
AND ($3::text IS NULL OR EXISTS (
SELECT 1 FROM tasks
WHERE project_id = $1
AND id = $3
AND metadata->>'targetId' = $16
AND ((thread_id IS NULL AND $2::text IS NULL) OR thread_id = $2)
))
RETURNING id`,
[...values, input.targetId],
);
}

async function lockArtifactAttribution(db: Queryable, input: CreateArtifactInput) {
if (!input.targetId) throw new ArtifactAttributionGuardError();
const target = await db.query<{ id: string }>(
`SELECT id FROM targets
WHERE project_id = $1 AND id = $2
FOR UPDATE`,
[input.projectId, input.targetId],
);
if (!target.rows[0]) throw new ArtifactAttributionGuardError();

if (input.threadId) {
const thread = await db.query<{ id: string }>(
`SELECT id FROM chat_threads
WHERE project_id = $1 AND id = $2
FOR UPDATE`,
[input.projectId, input.threadId],
);
if (!thread.rows[0]) throw new ArtifactAttributionGuardError();
}

if (input.taskId) {
const task = await db.query<{ id: string }>(
`SELECT id FROM tasks
WHERE project_id = $1
AND id = $2
AND metadata->>'targetId' = $3
AND ((thread_id IS NULL AND $4::text IS NULL) OR thread_id = $4)
FOR UPDATE`,
[input.projectId, input.taskId, input.targetId, input.threadId ?? null],
);
if (!task.rows[0]) throw new ArtifactAttributionGuardError();
}
}

function resolveGeneratedEvidenceScope(
input: {
targetId?: string;
Expand Down
1 change: 1 addition & 0 deletions src/server/evidence/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ export {
setArtifactService,
} from "./artifact-runtime";
export {
ArtifactAttributionGuardError,
type ArtifactServiceConfig,
type CreateArtifactInput,
type CreateArtifactResult,
Expand Down
1 change: 1 addition & 0 deletions src/server/recon/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ export {
MAX_PASSIVE_AUTH_SOURCE_ARTIFACTS,
MAX_PASSIVE_AUTH_SOURCE_BYTES,
PassiveAuthSurfaceArtifactError,
PassiveAuthSurfaceAttributionError,
PassiveAuthSurfaceAuthorizationError,
STORED_PASSIVE_ARTIFACT_SOURCES,
type StoredPassiveArtifactRef,
Expand Down
1 change: 1 addition & 0 deletions src/server/recon/passive-auth-surface.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ export async function persistPassiveAuthSurface(
source: "passive-recon",
indexForRag: true,
agentGenerated: true,
attributionPolicy: "project-thread-task-target",
metadata: {
workflow: "passive-auth-surface-v1",
rawArtifactIds: summary.rawArtifactIds,
Expand Down
Loading