fix(open-knowledge): MCP writes reflect on-disk truth — reconcile…

.changeset/prd-6832-beta-disk-authority.md

@@ -0,0 +1,16 @@
+---
+"@inkeep/open-knowledge-server": patch
+"@inkeep/open-knowledge": patch
+---
+
+fix(open-knowledge): MCP writes reflect on-disk truth — reconcile out-of-band edits instead of silently clobbering them (PRD-6832)
+
+An OK MCP read could authoritatively return content older than the bytes on disk, with no warning. The root cause was upstream of the read: a doc loaded in the server holds stale in-memory CRDT state, an out-of-band edit (a script, `git pull`, manual edit) makes disk newer, and the next MCP write serializes the stale CRDT over the newer file — making disk itself stale, so the next read faithfully returns bad bytes. In one incident this gave an agent a stale spec scope and cost a multi-file revert.
+
+This change makes disk authoritative on the write/reconcile path:
+
+- **Reconcile-before-apply (L1).** Before a content write (`write_document` / `edit_document` / `edit_frontmatter`) or a `rename` applies, the server compares the on-disk bytes to the last-synced base and, on divergence, ingests the disk edit first (through the existing sanctioned file-watcher path) so the agent's edit lands on top of current reality. Both edits survive.
+- **Store-time backstop (L3).** For the residual few-millisecond window where disk changes between the reconcile check and the store, the store re-checks disk before overwriting. On divergence it aborts the overwrite (disk wins) and the handler returns a hard `urn:ok:error:disk-divergence` (409) — the agent's edit was NOT applied; re-read and retry (a retry re-applies exactly once). This is the only guard for `undo` / `rollback`, which have no L1.
+- **Heads-up on reconcile.** When a write reconciles an out-of-band edit, the success response carries a `disk-edit-reconciled` warning (`structuredContent.contentDivergence`) so the agent re-reads to see the combined result. Observational — the write still landed and both edits are on disk.
+
+Human-editor (browser) writes are unaffected: the backstop only fires on agent-triggered stores, so in-progress typing is never reverted.

packages/app/tests/integration/disk-authority-reconcile.test.ts

@@ -0,0 +1,125 @@
+import { afterEach, describe, expect, test } from 'bun:test';
+import { randomUUID } from 'node:crypto';
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import {
+  agentPatch,
+  agentWriteMd,
+  createTestServer,
+  pollUntil,
+  readTestDoc,
+  type TestServer,
+} from './test-harness.ts';
+
+let server: TestServer | undefined;
+
+afterEach(async () => {
+  if (server) {
+    await server.cleanup();
+    server = undefined;
+  }
+});
+
+async function frontmatterPatch(port: number, docName: string, patch: Record<string, unknown>) {
+  return fetch(`http://localhost:${port}/api/frontmatter-patch`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ docName, patch }),
+  });
+}
+
+async function renamePath(port: number, fromPath: string, toPath: string) {
+  return fetch(`http://localhost:${port}/api/rename-path`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ kind: 'file', fromPath, toPath }),
+  });
+}
+
+describe('PRD-6832 β L1: agent write reconciles a newer out-of-band disk edit', () => {
+  test('write_document append: the native edit is NOT clobbered + FR3 warning fires', async () => {
+    server = await createTestServer({ debounce: 50, maxDebounce: 200 });
+    const { port, contentDir } = server;
+    const docName = `reconcile-append-${randomUUID()}`;
+    const filePath = join(contentDir, `${docName}.md`);
+
+    await agentWriteMd(port, '# V1 from agent\n\nbody-v1\n', { docName, position: 'replace' });
+    await pollUntil(() => readTestDoc(contentDir, docName).includes('body-v1'));
+
+    writeFileSync(filePath, '# V2 NATIVE OUT-OF-BAND EDIT\n\nbody-v2-native\n', 'utf-8');
+
+    const res = await fetch(`http://localhost:${port}/api/agent-write-md`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        docName,
+        markdown: 'appended-by-agent-still-on-v1\n',
+        position: 'append',
+      }),
+    });
+    expect(res.status).toBe(200);
+    const body = (await res.json()) as { warning?: { kind?: string } };
+    expect(body.warning?.kind).toBe('disk-edit-reconciled');
+
+    const after = readTestDoc(contentDir, docName);
+    expect(after).toContain('body-v2-native'); // out-of-band edit preserved (no clobber)
+    expect(after).toContain('appended-by-agent-still-on-v1'); // agent edit applied on top
+  });
+
+  test('edit_document find/replace: runs against the live (disk-reflecting) content', async () => {
+    server = await createTestServer({ debounce: 50, maxDebounce: 200 });
+    const { port, contentDir } = server;
+    const docName = `reconcile-patch-${randomUUID()}`;
+    const filePath = join(contentDir, `${docName}.md`);
+
+    await agentWriteMd(port, '# Doc\n\nBANANA here\n', { docName, position: 'replace' });
+    await pollUntil(() => readTestDoc(contentDir, docName).includes('BANANA'));
+
+    writeFileSync(filePath, '# Doc\n\nBANANA here\n\nnative-extra-line\n', 'utf-8');
+
+    await agentPatch(port, 'BANANA', 'CHERRY', docName);
+
+    const after = readTestDoc(contentDir, docName);
+    expect(after).toContain('CHERRY'); // patch applied against the reconciled content
+    expect(after).not.toContain('BANANA'); // the find target was replaced
+    expect(after).toContain('native-extra-line'); // out-of-band edit preserved
+  });
+
+  test('edit_frontmatter: the native body edit is preserved while the FM patch applies', async () => {
+    server = await createTestServer({ debounce: 50, maxDebounce: 200 });
+    const { port, contentDir } = server;
+    const docName = `reconcile-fm-${randomUUID()}`;
+    const filePath = join(contentDir, `${docName}.md`);
+
+    await agentWriteMd(port, '# Doc\n\nbody-original\n', { docName, position: 'replace' });
+    await pollUntil(() => readTestDoc(contentDir, docName).includes('body-original'));
+
+    writeFileSync(filePath, '# Doc\n\nbody-original\n\nnative-body-line\n', 'utf-8');
+
+    const res = await frontmatterPatch(port, docName, { title: 'New Title' });
+    expect(res.status).toBe(200);
+
+    const after = readTestDoc(contentDir, docName);
+    expect(after).toContain('New Title'); // FM patch applied
+    expect(after).toContain('native-body-line'); // out-of-band body edit preserved
+  });
+
+  test('rename: the renamed doc carries the newer out-of-band disk content', async () => {
+    server = await createTestServer({ debounce: 50, maxDebounce: 200 });
+    const { port, contentDir } = server;
+    const fromDoc = `reconcile-rename-from-${randomUUID()}`;
+    const toDoc = `reconcile-rename-to-${randomUUID()}`;
+    const fromPath = join(contentDir, `${fromDoc}.md`);
+
+    await agentWriteMd(port, '# V1\n\nbody-v1\n', { docName: fromDoc, position: 'replace' });
+    await pollUntil(() => readTestDoc(contentDir, fromDoc).includes('body-v1'));
+
+    writeFileSync(fromPath, '# V2 NATIVE OUT-OF-BAND EDIT\n\nbody-v2-native\n', 'utf-8');
+
+    const res = await renamePath(port, `${fromDoc}.md`, `${toDoc}.md`);
+    expect(res.status).toBe(200);
+
+    const after = readTestDoc(contentDir, toDoc);
+    expect(after).toContain('body-v2-native'); // the newer disk edit moved with the rename
+  });
+});

packages/app/tests/integration/disk-divergence-backstop.test.ts

@@ -0,0 +1,150 @@
+import { afterEach, describe, expect, test } from 'bun:test';
+import { randomUUID } from 'node:crypto';
+import {
+  agentWriteMd,
+  awaitDocQuiescence,
+  createTestClient,
+  createTestServer,
+  pollUntil,
+  readTestDoc,
+  type TestServer,
+} from './test-harness.ts';
+
+const INJECTED_MARKER = 'native-divergence-injected';
+
+let server: TestServer | undefined;
+
+afterEach(async () => {
+  delete process.env.OK_TEST_STORE_DIVERGENCE;
+  if (server) {
+    await server.cleanup();
+    server = undefined;
+  }
+});
+
+async function writeMd(
+  port: number,
+  markdown: string,
+  opts: { docName: string; position: 'append' | 'prepend' | 'replace' },
+) {
+  return fetch(`http://localhost:${port}/api/agent-write-md`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ markdown, ...opts }),
+  });
+}
+
+async function agentUndoRaw(
+  port: number,
+  opts: { docName: string; connectionId: string; scope?: 'last' | 'session' },
+) {
+  return fetch(`http://localhost:${port}/api/agent-undo`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      docName: opts.docName,
+      connectionId: opts.connectionId,
+      scope: opts.scope ?? 'last',
+    }),
+  });
+}
+
+describe('PRD-6832 β L3: store-time divergence backstop', () => {
+  test('reverts on TOCTOU divergence (409); disk wins; retry re-applies exactly once', async () => {
+    server = await createTestServer({ debounce: 50, maxDebounce: 200 });
+    const { port, contentDir } = server;
+    const docName = `l3-content-${randomUUID()}`;
+
+    const seed = await writeMd(port, '# V1\n\nbody-v1\n', { docName, position: 'replace' });
+    expect(seed.status).toBe(200);
+    await pollUntil(() => readTestDoc(contentDir, docName).includes('body-v1'));
+
+    process.env.OK_TEST_STORE_DIVERGENCE = docName;
+
+    const attempt1 = await writeMd(port, 'AGENT-APPEND-XYZ\n', {
+      docName,
+      position: 'append',
+    });
+    expect(attempt1.status).toBe(409);
+    const body1 = (await attempt1.json()) as { type?: string };
+    expect(body1.type).toBe('urn:ok:error:disk-divergence');
+
+    const afterRevert = readTestDoc(contentDir, docName);
+    expect(afterRevert).toContain(INJECTED_MARKER);
+    expect(afterRevert).not.toContain('AGENT-APPEND-XYZ');
+
+    delete process.env.OK_TEST_STORE_DIVERGENCE;
+    const attempt2 = await writeMd(port, 'AGENT-APPEND-XYZ\n', {
+      docName,
+      position: 'append',
+    });
+    expect(attempt2.status).toBe(200);
+
+    const afterRetry = readTestDoc(contentDir, docName);
+    expect(afterRetry).toContain(INJECTED_MARKER);
+    expect(afterRetry).toContain('AGENT-APPEND-XYZ');
+    expect(afterRetry.split('AGENT-APPEND-XYZ').length - 1).toBe(1);
+  });
+
+  test('undo: L3 reverts on TOCTOU divergence (409); native survives; undo NOT applied', async () => {
+    server = await createTestServer({ debounce: 50, maxDebounce: 200 });
+    const { port, contentDir } = server;
+    const docName = `l3-undo-${randomUUID()}`;
+
+    await agentWriteMd(port, '# Base\n\nbase-body\n', {
+      docName,
+      position: 'replace',
+      agentId: 'u1',
+    });
+    await pollUntil(() => readTestDoc(contentDir, docName).includes('base-body'));
+    await new Promise((r) => setTimeout(r, 700));
+    await agentWriteMd(port, 'UNDO-ME-LINE\n', { docName, position: 'append', agentId: 'u1' });
+    await pollUntil(() => readTestDoc(contentDir, docName).includes('UNDO-ME-LINE'));
+
+    process.env.OK_TEST_STORE_DIVERGENCE = docName;
+
+    const undoRes = await agentUndoRaw(port, { docName, connectionId: 'agent-u1', scope: 'last' });
+    expect(undoRes.status).toBe(409);
+    const body = (await undoRes.json()) as { type?: string };
+    expect(body.type).toBe('urn:ok:error:disk-divergence');
+
+    const afterRevert = readTestDoc(contentDir, docName);
+    expect(afterRevert).toContain(INJECTED_MARKER);
+    expect(afterRevert).not.toContain('base-body');
+
+    delete process.env.OK_TEST_STORE_DIVERGENCE;
+    await agentWriteMd(port, 'RECOVERY-LINE\n', { docName, position: 'append', agentId: 'u1' });
+    await pollUntil(() => readTestDoc(contentDir, docName).includes('RECOVERY-LINE'));
+    const afterRecovery = readTestDoc(contentDir, docName);
+    expect(afterRecovery).toContain(INJECTED_MARKER);
+    expect(afterRecovery).toContain('RECOVERY-LINE');
+  });
+
+  test('gate: an unmarked human/client store is NEVER reverted by L3', async () => {
+    server = await createTestServer({ debounce: 50, maxDebounce: 200 });
+    const { port, contentDir } = server;
+    const docName = `l3-gate-human-${randomUUID()}`;
+
+    const seed = await writeMd(port, '# V1\n\nseed-body\n', { docName, position: 'replace' });
+    expect(seed.status).toBe(200);
+    await pollUntil(() => readTestDoc(contentDir, docName).includes('seed-body'));
+
+    const client = await createTestClient(port, docName);
+    try {
+      await pollUntil(() => client.ytext.toString().includes('seed-body'), 5000);
+
+      process.env.OK_TEST_STORE_DIVERGENCE = docName;
+
+      const HUMAN_MARK = 'HUMAN-EDIT-NOT-REVERTED';
+      client.doc.transact(() => {
+        client.ytext.insert(client.ytext.length, `\n${HUMAN_MARK}\n`);
+      });
+      await awaitDocQuiescence(client.doc, { timeoutMs: 3000 });
+
+      await pollUntil(() => readTestDoc(contentDir, docName).includes(HUMAN_MARK), 8000);
+    } finally {
+      delete process.env.OK_TEST_STORE_DIVERGENCE;
+      await client.cleanup();
+    }
+  });
+});

packages/core/src/handoff/urn-ipc-registry.ts

@@ -52,6 +52,7 @@ export const URN_HTTP_ONLY: ReadonlySet<ProblemType> = new Set<ProblemType>([
   'urn:ok:error:frontmatter-malformed',
   'urn:ok:error:no-active-session',
   'urn:ok:error:too-many-agent-sessions',
+  'urn:ok:error:disk-divergence',
   'urn:ok:error:doc-in-conflict',
   'urn:ok:error:no-conflict-tracked',
   'urn:ok:error:doc-not-found',

packages/core/src/index.ts

@@ -494,6 +494,8 @@ export {
   DiffLineSchema,
   type DiffSuccess,
   DiffSuccessSchema,
+  type DiskEditReconciledWarning,
+  DiskEditReconciledWarningSchema,
   type DocumentListEntry,
   DocumentListEntrySchema,
   type DocumentListSuccess,
@@ -772,6 +774,8 @@ export {
   UploadRequestSchema,
   type WorkspaceSuccess,
   WorkspaceSuccessSchema,
+  type WriteWarning,
+  WriteWarningSchema,
 } from './schemas/api/index.ts';
 export {
   CC1_CHANNEL_BRANCH_SWITCHED,

packages/core/src/schemas/api/_envelope.ts

@@ -63,6 +63,7 @@ export const ProblemTypeSchema = z.enum([
   'urn:ok:error:frontmatter-malformed',
   'urn:ok:error:no-active-session',
   'urn:ok:error:too-many-agent-sessions',
+  'urn:ok:error:disk-divergence',
   'urn:ok:error:doc-not-found',
   'urn:ok:error:doc-already-exists',
   'urn:ok:error:doc-not-open',

packages/core/src/schemas/api/agent-write.ts

@@ -89,10 +89,28 @@ export const OrphanHintSchema = z
   .loose() satisfies StandardSchemaV1;
 export type OrphanHint = z.infer<typeof OrphanHintSchema>;
 
+export const DiskEditReconciledWarningSchema = z
+  .object({
+    kind: z.literal('disk-edit-reconciled'),
+    intendedBytes: z.number().int().nonnegative(),
+    actualBytes: z.number().int().nonnegative(),
+    byteDelta: z.number().int(),
+    hint: z.string().optional(),
+  })
+  .loose() satisfies StandardSchemaV1;
+export type DiskEditReconciledWarning = z.infer<typeof DiskEditReconciledWarningSchema>;
+
+export const WriteWarningSchema = z.discriminatedUnion('kind', [
+  ContentDivergenceWarningSchema,
+  DiskEditReconciledWarningSchema,
+]);
+export type WriteWarning = z.infer<typeof WriteWarningSchema>;
+
 export const AgentWriteSuccessSchema = z
   .object({
     timestamp: z.string().min(1),
     summary: SummaryResponseFieldSchema.optional(),
+    warning: WriteWarningSchema.optional(),
   })
   .loose() satisfies StandardSchemaV1;
 export type AgentWriteSuccess = z.infer<typeof AgentWriteSuccessSchema>;
@@ -104,7 +122,7 @@ export const AgentWriteMdSuccessSchema = z
     systemSubscriberCount: z.number().int().nonnegative(),
     hints: z.array(OrphanHintSchema).optional(),
     summary: SummaryResponseFieldSchema.optional(),
-    warning: ContentDivergenceWarningSchema.optional(),
+    warning: WriteWarningSchema.optional(),
   })
   .loose() satisfies StandardSchemaV1;
 export type AgentWriteMdSuccess = z.infer<typeof AgentWriteMdSuccessSchema>;
@@ -115,7 +133,7 @@ export const AgentPatchSuccessSchema = z
     subscriberCount: z.number().int().nonnegative(),
     systemSubscriberCount: z.number().int().nonnegative(),
     summary: SummaryResponseFieldSchema.optional(),
-    warning: ContentDivergenceWarningSchema.optional(),
+    warning: WriteWarningSchema.optional(),
   })
   .loose() satisfies StandardSchemaV1;
 export type AgentPatchSuccess = z.infer<typeof AgentPatchSuccessSchema>;
@@ -147,6 +165,7 @@ export const FrontmatterPatchSuccessSchema = z
     systemSubscriberCount: z.number().int().nonnegative(),
     appliedKeys: z.array(z.string()),
     summary: SummaryResponseFieldSchema.optional(),
+    warning: WriteWarningSchema.optional(),
   })
   .loose() satisfies StandardSchemaV1;
 export type FrontmatterPatchSuccess = z.infer<typeof FrontmatterPatchSuccessSchema>;