fix(sync): keep a graph-scoped seal's _meta subject-atomic across durable pages (#1788)

packages/agent/src/sync/auth/request-build.ts

@@ -182,7 +182,15 @@ export async function buildSyncRequestEnvelope(params: BuildSyncRequestParams):
   const requestedLimit = Number.isSafeInteger(limit)
     ? Math.max(1, Math.min(limit, SYNC_BYTE_BUDGET_MAX_ROWS))
     : SYNC_PAGE_SIZE;
-  const useByteBudgetPage = !includeSharedMemory && phase === 'data' && requestedLimit > SYNC_PAGE_SIZE;
+  // Advertise byte-budget page mode for durable DATA and META (#1916/#1923).
+  // Additive/rolling-upgrade safe both directions: an OLD responder ignores the
+  // meta pageMode (its meta path is not byte-budget-gated → serves legacy meta),
+  // and a NEW responder treats a request WITHOUT meta pageMode as non-negotiated
+  // (plain meta serializer). The signed `limit` still rides the 500-row legacy
+  // cap below, so digests stay wire-compatible.
+  const useByteBudgetPage = !includeSharedMemory
+    && (phase === 'data' || phase === 'meta')
+    && requestedLimit > SYNC_PAGE_SIZE;
   const assetUals = rawAssetUals === undefined ? undefined : requireExactAssetUals(rawAssetUals);
 
   if (!needsAuth) {

packages/agent/src/sync/requester/page-fetch.ts

@@ -300,6 +300,16 @@ export async function fetchSyncPages(params: FetchSyncPagesParams): Promise<Sync
   // A transient failure merely makes the remainder of this phase conservative;
   // it never changes offsets or responder-session identity.
   const safePageSize = Math.min(syncPageSize, SYNC_REQUEST_SAFE_PAGE_SIZE);
+  // Byte-budget pagination: a SHORT page is NOT EOF — only an empty response is
+  // (see the loop's EOF checks). This is a REQUESTER-SIDE default derived from
+  // `syncPageSize > SYNC_PAGE_SIZE` (the fetch wrapper passes
+  // SYNC_REQUEST_PAGE_SIZE=8192 for every phase), NOT a wire-negotiated
+  // capability. Durable meta relies on it: since #1916 the responder byte-caps
+  // durable-meta pages, so a page can be short for byte reasons; a requester
+  // that treated "short = EOF" for meta could end the phase early. Every
+  // testnet-canary+ requester uses 8192 here, so short≠EOF holds for meta and
+  // data alike; a pre-canary requester using the 500-row cap is the only one
+  // that would regress, and only on an oversized (>4 MiB) meta subject.
   const usesByteBudgetPagination = syncPageSize > SYNC_PAGE_SIZE;
   let activePageSize = syncPageSize;
   let successfulPageSize = syncPageSize;

packages/agent/src/sync/responder/sync-handler.ts

@@ -25,6 +25,7 @@ import {
   createResponderSyncRowListMemo,
   createResponderSubGraphRegistrationMemo,
   createResponderSwmAdmissionMemo,
+  DurableMetaPageFrameError,
   readCatalogPage,
   readDurableDataPage,
   readDurableMetaPage,
@@ -562,6 +563,13 @@ export function registerSyncHandler(params: RegisterSyncHandlerParams): void {
       request.pageMode === SYNC_BYTE_BUDGET_PAGE_MODE &&
       hintedPageRows > limit;
     const durableDataLimit = usesByteBudgetPage ? hintedPageRows : limit;
+    // Durable meta negotiated its byte-budget page mode on the wire (#1916 /
+    // #1923). The subject-atomic byte-fit in readDurableMetaPage already bounds
+    // the page ≤ budget for BOTH modes, so this only selects the belt-and-
+    // suspenders response serializer and records the explicit contract.
+    const usesMetaByteBudget = !isWorkspace &&
+      phase === 'meta' &&
+      request.pageMode === SYNC_BYTE_BUDGET_PAGE_MODE;
     if (!contextGraphId || typeof contextGraphId !== 'string') {
       // Count this early return too — it short-circuits before limiter.run, so
       // without this it would never reach the syncResponseTotal{ok}/{error}
@@ -778,10 +786,35 @@ export function registerSyncHandler(params: RegisterSyncHandlerParams): void {
             refreshRowList: session?.refreshRowList,
             refreshGeneration: session?.refreshGeneration,
             assetUals,
+            maxResponseBytes: SYNC_BYTE_BUDGET_RESPONSE_BYTES,
+            // NON-NEGOTIATED legacy requesters (no wire `pageMode`) must fail
+            // loud on an oversized `_meta` subject rather than receive a byte-fit
+            // SHORT page they would read as EOF — silent metadata loss + a #1788
+            // seal split. Negotiated (testnet-canary+) requesters keep the
+            // verified byte-fit behavior (empty=EOF pagination, so short≠EOF).
+            oversizedSubjectPolicy: usesMetaByteBudget ? 'byte-fit' : 'fail-loud',
           });
           const queryDurationMs = Date.now() - queryStartedAt;
           const serializeStartedAt = Date.now();
-          const serialized = serializeResponderRows(rows);
+          // Byte-cap the durable-meta response (#1916) exactly like durable data:
+          // the subject-atomic extend can return a whole (or oversized) subject,
+          // so serialize within the frame budget rather than emitting unbounded
+          // N-Quads. The extend keeps every valid seal well under the budget, so
+          // this only ever truncates a pathological oversized subject.
+          //
+          // Pagination contract: durable meta uses byte-budget pagination where a
+          // SHORT page is NOT EOF — only an empty page is. The requester's
+          // short≠EOF handling is a requester-side default (page-fetch:
+          // syncPageSize=8192 > SYNC_PAGE_SIZE), and since #1923 it is ALSO
+          // negotiated on the wire via `pageMode` (usesMetaByteBudget). The
+          // subject-atomic byte-fit in readDurableMetaPage already bounds every
+          // page ≤ budget AND to whole subjects, so both the negotiated
+          // (byte-budget serializer) and the non-negotiated (plain serializer)
+          // branches are frame-safe and never split a subject; the gate here just
+          // honours the explicit contract.
+          const serialized = usesMetaByteBudget
+            ? serializeResponderRowsWithinByteBudget(rows, SYNC_BYTE_BUDGET_RESPONSE_BYTES)
+            : serializeResponderRows(rows);
           if (serialized) nquads.push(serialized);
           const serializeDurationMs = Date.now() - serializeStartedAt;
           logFirstPageDetail(() => `Sync responder durable meta for "${contextGraphId}": auth=${authDurationMs}ms query=${queryDurationMs}ms serialize=${serializeDurationMs}ms`);
@@ -909,6 +942,21 @@ export function registerSyncHandler(params: RegisterSyncHandlerParams): void {
         );
         throw new QuietRetryableHandlerError(err.message);
       }
+      if (err instanceof DurableMetaPageFrameError) {
+        // Loud, non-retryable failure (#1788/#1916): an oversized `_meta` subject
+        // cannot be served frame-safe to a non-negotiated legacy requester, and
+        // byte-fitting it would be a silent short=EOF metadata loss. Retrying
+        // cannot help — surface it as a hard error so the round fails visibly
+        // rather than completing with partial metadata. Root fix: #1921.
+        getMetrics().syncResponseTotal.add(1, { outcome: 'error' });
+        span.setAttribute('dkg.sync_response_outcome', 'error');
+        logWarn(
+          createOperationContext('sync'),
+          `Sync responder cannot serve durable meta frame-safe to a non-negotiated (legacy) `
+          + `requester for "${contextGraphId}" from peer ${peerId}: ${err.message}`,
+        );
+        throw err;
+      }
       getMetrics().syncResponseTotal.add(1, { outcome: 'error' });
       throw err;
     });

packages/agent/test/sync-byte-budget-pages.test.ts

@@ -60,6 +60,65 @@ describe('byte-budget sync pagination', () => {
     expect(request.requesterSignatureR).toMatch(/^0x/);
   });
 
+  // #1916: durable META now negotiates byte-budget paging exactly like durable
+  // DATA. These two cases pin the request-builder's meta advertisement directly:
+  // a regression dropping 'meta' from the useByteBudgetPage condition would
+  // silently break the wire negotiation, and the handler-level tests (which
+  // hand-craft the pageMode field) would not catch it.
+  it('advertises the byte-budget page mode for a durable meta request above the legacy cap', async () => {
+    const wallet = ethers.Wallet.createRandom();
+    const signedLimits: number[] = [];
+    const encoded = await buildSyncRequestEnvelope({
+      contextGraphId: CG_ID,
+      offset: 0,
+      limit: SYNC_REQUEST_PAGE_SIZE,
+      includeSharedMemory: false,
+      targetPeerId: REMOTE_PEER_ID,
+      requesterPeerId: LOCAL_PEER_ID,
+      phase: 'meta',
+      needsAuth: true,
+      computeSyncDigest: (_cg, _offset, limit) => {
+        signedLimits.push(limit);
+        return new Uint8Array(32);
+      },
+      getIdentityId: async () => 0n,
+      claimedAgentAddress: wallet.address,
+      claimedAgentPrivateKey: wallet.privateKey,
+    });
+
+    const request = JSON.parse(new TextDecoder().decode(encoded));
+    // The larger hint rides while the signed legacy limit stays 500-row capped,
+    // so digests remain wire-compatible with an old responder.
+    expect(signedLimits).toEqual([SYNC_PAGE_SIZE]);
+    expect(request.limit).toBe(SYNC_PAGE_SIZE);
+    expect(request.pageMode).toBe(SYNC_BYTE_BUDGET_PAGE_MODE);
+    expect(request.pageRowsHint).toBe(SYNC_REQUEST_PAGE_SIZE);
+  });
+
+  it('does not advertise byte-budget paging for a durable meta request at the legacy cap', async () => {
+    const wallet = ethers.Wallet.createRandom();
+    const encoded = await buildSyncRequestEnvelope({
+      contextGraphId: CG_ID,
+      offset: 0,
+      limit: SYNC_PAGE_SIZE,
+      includeSharedMemory: false,
+      targetPeerId: REMOTE_PEER_ID,
+      requesterPeerId: LOCAL_PEER_ID,
+      phase: 'meta',
+      needsAuth: true,
+      computeSyncDigest: () => new Uint8Array(32),
+      getIdentityId: async () => 0n,
+      claimedAgentAddress: wallet.address,
+      claimedAgentPrivateKey: wallet.privateKey,
+    });
+
+    const request = JSON.parse(new TextDecoder().decode(encoded));
+    // At the 500-row cap there is no larger page to negotiate, so the responder
+    // must see an unmodified legacy meta request (no pageMode field).
+    expect(request.pageMode).toBeUndefined();
+    expect(request.pageRowsHint).toBeUndefined();
+  });
+
   it('continues after an old responder returns a short legacy page', async () => {
     const requested: Array<{ offset: number; limit: number }> = [];
     let sends = 0;

packages/agent/test/sync-responder-concurrent-interleaving.test.ts

@@ -10,6 +10,12 @@ import {
   createResponderGraphListMemo,
   createResponderSubGraphRegistrationMemo,
 } from '../src/sync/responder/graph-plan.js';
+import {
+  SYNC_BYTE_BUDGET_MAX_ROWS,
+  SYNC_BYTE_BUDGET_PAGE_MODE,
+  SYNC_BYTE_BUDGET_RESPONSE_BYTES,
+  SYNC_PAGE_SIZE,
+} from '../src/dkg-agent-constants.js';
 import {
   DKG_NS,
   lineGraphsFromNquads,
@@ -447,10 +453,14 @@ describe('sync responder pagination interleaving', () => {
     const cgId = 'oversized-durable-meta';
     const cgPrefix = `did:dkg:context-graph:${cgId}`;
     const metaGraph = `${cgPrefix}/_meta`;
-    // Rows keyed on the CG entity subject survive readDurableMetaRows filtering.
+    // Three DISTINCT admitted subjects (one row each), so the page boundary
+    // falls on a subject boundary and the fallback pages 2 + 1. Since #1788 a
+    // single subject is emitted atomically and would never split across pages,
+    // so distinct subjects are required to exercise paging here. Activity-prefix
+    // subjects survive readDurableMetaRows filtering.
     await store.insert(Array.from({ length: 3 }, (_, i) => ({
       graph: metaGraph,
-      subject: cgPrefix,
+      subject: `did:dkg:activity:${cgId}-${i}`,
       predicate: `http://schema.org/p${i.toString().padStart(3, '0')}`,
       object: `"meta-${i.toString().padStart(3, '0')}"`,
     })));
@@ -479,6 +489,84 @@ describe('sync responder pagination interleaving', () => {
     expect(new Set(linesFromNquads(`${first}\n${second}`)).size).toBe(3);
   });
 
+  // Handler-level (through registerSyncHandler): the durable-meta wire branch
+  // handles an oversized admitted subject DIFFERENTLY by negotiation, and both
+  // outcomes are frame-safe with no silent metadata loss (#1788/#1916):
+  //  - NEGOTIATED (byte-budget pageMode): the subject-atomic byte-fit chunks the
+  //    oversized subject under the frame and pages to completion (empty=EOF, so a
+  //    short page is not EOF). A regression returning the whole >4 MiB subject
+  //    would fail (pageCount 1 + over-budget bytes).
+  //  - LEGACY (no pageMode): a legacy requester reads a short page as EOF, so
+  //    byte-fitting would silently drop the rest of the subject; instead the
+  //    responder FAILS LOUD. A regression byte-fitting it would fail (no throw).
+  const oversizedMetaStore = (cgId: string): { store: OxigraphStore; rows: Quad[]; subject: string } => {
+    const store = new OxigraphStore();
+    const metaGraph = `did:dkg:context-graph:${cgId}/_meta`;
+    const subject = `did:dkg:activity:${cgId}-big`;
+    const bigLiteral = `"${'y'.repeat(60_000)}"`; // ~60 KB per row (under the 65535 literal cap)
+    const rows: Quad[] = Array.from({ length: 80 }, (_, i) => ({
+      graph: metaGraph,
+      subject,
+      predicate: `${DKG_NS}p${String(i).padStart(3, '0')}`,
+      object: bigLiteral,
+    })); // ~4.8 MB total > the 4 MiB budget
+    return { store, rows, subject };
+  };
+
+  it('durable-meta handler byte-caps an oversized subject under the frame and pages to completion — negotiated (byte-budget pageMode) (#1916)', async () => {
+    const cgId = 'oversized-meta-frame-neg';
+    const { store, rows } = oversizedMetaStore(cgId);
+    await store.insert(rows);
+    const cap = registerTestSyncHandler(store, { syncPageSize: SYNC_PAGE_SIZE });
+    const base = {
+      contextGraphId: cgId,
+      includeSharedMemory: false,
+      phase: 'meta' as const,
+      limit: SYNC_PAGE_SIZE,
+      syncSessionId: `${cgId}-session`,
+      pageMode: SYNC_BYTE_BUDGET_PAGE_MODE,
+      pageRowsHint: SYNC_BYTE_BUDGET_MAX_ROWS,
+    };
+
+    const enc = new TextEncoder();
+    let offset = 0;
+    let delivered = 0;
+    let pageCount = 0;
+    for (let guard = 0; guard < 100; guard += 1) {
+      const resp = await cap.invoke({ ...base, offset });
+      const n = resp === '' ? 0 : linesFromNquads(resp).length;
+      if (n === 0) break;
+      // Frame-safety: every response stays within the byte budget.
+      expect(enc.encode(resp).byteLength).toBeLessThanOrEqual(SYNC_BYTE_BUDGET_RESPONSE_BYTES);
+      pageCount += 1;
+      delivered += n;
+      offset += n;
+    }
+    // Chunked (byte cap engaged, not one oversized frame) and every row delivered.
+    expect(pageCount).toBeGreaterThan(1);
+    expect(delivered).toBe(rows.length);
+    await store.close();
+  });
+
+  it('durable-meta handler FAILS LOUD on an oversized subject for a legacy (no pageMode) requester, never a silent short page (#1788)', async () => {
+    const cgId = 'oversized-meta-frame-legacy';
+    const { store, rows } = oversizedMetaStore(cgId);
+    await store.insert(rows);
+    const cap = registerTestSyncHandler(store, { syncPageSize: SYNC_PAGE_SIZE });
+    // No pageMode ⇒ non-negotiated legacy requester. Byte-fitting would return a
+    // short page it reads as EOF (silent loss + #1788 split); the responder must
+    // instead surface a hard, explicit failure — not a successful short response.
+    await expect(cap.invoke({
+      contextGraphId: cgId,
+      includeSharedMemory: false,
+      phase: 'meta',
+      limit: SYNC_PAGE_SIZE,
+      offset: 0,
+      syncSessionId: `${cgId}-session`,
+    })).rejects.toThrow(/cannot be served frame-safe/);
+    await store.close();
+  });
+
   it('falls back to store-bounded paging for an oversized shared-memory meta snapshot', async () => {
     const store = new OxigraphStore();
     const cgId = 'oversized-swm-meta';
@@ -700,11 +788,15 @@ describe('sync responder pagination interleaving', () => {
     const cgPrefix = `did:dkg:context-graph:${cgId}`;
     const metaGraph = `${cgPrefix}/_meta`;
     const rows: Quad[] = [];
+    // 100 DISTINCT admitted subjects (one row each): since #1788 a single
+    // subject is emitted atomically, so a deep window into ONE subject is no
+    // longer meaningful — distinct subjects let the deep page address a subject
+    // boundary. Activity-prefix subjects survive durable-meta admission.
     for (let index = 0; index < 100; index++) {
       const padded = index.toString().padStart(3, '0');
       rows.push({
         graph: metaGraph,
-        subject: cgPrefix,
+        subject: `did:dkg:activity:m${padded}`,
         predicate: `http://schema.org/p${padded}`,
         object: `"meta-${padded}"`,
       });
@@ -717,9 +809,12 @@ describe('sync responder pagination interleaving', () => {
     });
     await store.insert(rows);
 
-    // The durable-meta read is now store-bounded (subject-membership filter
-    // pushed into the store via EXISTS), so a deep page is a paged store query.
-    const probe = watchBoundedPageQuery(store, metaGraph, 90, 5);
+    // The durable-meta read is store-bounded (subject-membership filter pushed
+    // into the store via EXISTS), so a deep page is a paged store query. Durable
+    // meta reads `limit + 1` rows to detect a subject straddling the page
+    // boundary (#1788) and serves at most `limit` when the boundary is clean, so
+    // the store query's LIMIT is 6 here while the served page stays 5.
+    const probe = watchBoundedPageQuery(store, metaGraph, 90, 6);
     const cap = registerTestSyncHandler(store, { syncPageSize: 5 });
     const out = await cap.invoke({
       contextGraphId: cgId,