From 316344a5769edbe3143647fdac229beb191e1818 Mon Sep 17 00:00:00 2001 From: Korey Kassir Date: Wed, 22 Jul 2026 19:42:41 -0400 Subject: [PATCH 1/9] feat: implement gateway access-check tokens and related endpoint for sharee visibility --- .../realtime-gateway-access-and-reconnect.md | 9 ++ packages/core/src/client/use-db-sync.ts | 8 +- .../core/src/server/core-routes-plugin.ts | 3 + .../src/server/gateway-access-check.spec.ts | 105 ++++++++++++++++ .../core/src/server/gateway-access-check.ts | 65 ++++++++++ .../core/src/server/short-lived-token.spec.ts | 79 ++++++++++++ packages/core/src/server/short-lived-token.ts | 114 ++++++++++++++++++ 7 files changed, 380 insertions(+), 3 deletions(-) create mode 100644 .changeset/realtime-gateway-access-and-reconnect.md create mode 100644 packages/core/src/server/gateway-access-check.spec.ts create mode 100644 packages/core/src/server/gateway-access-check.ts diff --git a/.changeset/realtime-gateway-access-and-reconnect.md b/.changeset/realtime-gateway-access-and-reconnect.md new file mode 100644 index 0000000000..df1917d2c8 --- /dev/null +++ b/.changeset/realtime-gateway-access-and-reconnect.md @@ -0,0 +1,9 @@ +--- +"@agent-native/core": minor +--- + +Realtime sync: two hosted-gateway follow-ups. Both are opt-in and leave apps without hosted-realtime config unchanged. + +- Hosted SSE reconnect now sends the client's cursor (`&since=`) on the gateway stream URL, so the gateway's connect-time catch-up replays events written during the reconnect gap immediately instead of deferring them to the next poll. First connect (cursor 0) is unaffected. +- New gateway access-check tokens (`signGatewayAccessToken`/`verifyGatewayAccessToken`, exported from `./server/short-lived-token`): per-project HMAC key, a `typ` discriminator (not interchangeable with subscribe or media tokens), and the full access query (`resourceType`/`resourceId`/`userEmail`/`orgId`) bound into the signature so the app authenticates the params, not just the caller. +- New endpoint `GET /_agent-native/can-see` mounted by core-routes. The hosted Realtime Gateway has no copy of an app's shareable-resource registry, so it calls this to resolve sharee visibility: the endpoint verifies a gateway access-check token against the app's per-project secret, runs the app's registry-based `resolveAccess`, and returns `{ allowed }`. Fails closed (`allowed: false`) on an unknown resource type or lookup error; 404 when the app has no realtime secret; `Cache-Control: private, no-store`. diff --git a/packages/core/src/client/use-db-sync.ts b/packages/core/src/client/use-db-sync.ts index c6268753de..e04cf75920 100644 --- a/packages/core/src/client/use-db-sync.ts +++ b/packages/core/src/client/use-db-sync.ts @@ -330,9 +330,11 @@ class SyncTransport { private get activeSseUrl(): string | false { if (this.mode === "hosted" && this.gateway) { - return this.token - ? `${this.gateway.sseUrl}?token=${encodeURIComponent(this.token)}` - : this.gateway.sseUrl; + if (!this.token) return this.gateway.sseUrl; + const base = `${this.gateway.sseUrl}?token=${encodeURIComponent(this.token)}`; + // Cursor lets the gateway replay the reconnect gap on connect instead of + // deferring it to the next poll; 0 on first connect means nothing to replay. + return this.versionRef > 0 ? `${base}&since=${this.versionRef}` : base; } return this.sseUrl; } diff --git a/packages/core/src/server/core-routes-plugin.ts b/packages/core/src/server/core-routes-plugin.ts index d2cc0a2d98..bb4c863c8f 100644 --- a/packages/core/src/server/core-routes-plugin.ts +++ b/packages/core/src/server/core-routes-plugin.ts @@ -158,6 +158,7 @@ import { markDefaultPluginProvided, trackPluginInit, } from "./framework-request-handler.js"; +import { createGatewayAccessCheckHandler } from "./gateway-access-check.js"; import { getAppBasePath, getOrigin } from "./google-oauth.js"; import { createGoogleRealtimeSessionHandler } from "./google-realtime-session.js"; import { @@ -1420,6 +1421,8 @@ export function createCoreRoutesPlugin( `${P}/realtime-token`, createRealtimeTokenHandler(), ); + // Sharee visibility check for the hosted gateway + getH3App(nitroApp).use(`${P}/can-see`, createGatewayAccessCheckHandler()); // SSE if (!options.disableSSE) { diff --git a/packages/core/src/server/gateway-access-check.spec.ts b/packages/core/src/server/gateway-access-check.spec.ts new file mode 100644 index 0000000000..39bacddb1d --- /dev/null +++ b/packages/core/src/server/gateway-access-check.spec.ts @@ -0,0 +1,105 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { signGatewayAccessToken } from "./short-lived-token.js"; + +const mockResolveAccess = vi.hoisted(() => vi.fn()); + +vi.mock("h3", () => ({ + defineEventHandler: (h: any) => h, + getMethod: (e: any) => e.method ?? "GET", + getQuery: (e: any) => e.query ?? {}, + setResponseStatus: (e: any, s: number) => { + e.status = s; + }, + setResponseHeader: (e: any, k: string, v: string) => { + e.headers = e.headers ?? {}; + e.headers[k] = v; + }, +})); +vi.mock("../sharing/access.js", () => ({ resolveAccess: mockResolveAccess })); +vi.mock("./request-context.js", () => ({ + runWithRequestContext: (_ctx: any, fn: any) => fn(), +})); + +const SECRET = "per-project-hmac-secret"; +const CLAIMS = { + projectId: "proj_a", + resourceType: "document", + resourceId: "doc-1", + userEmail: "sharee@example.com", + orgId: "org-1", +}; + +async function invoke(event: Record) { + const { createGatewayAccessCheckHandler } = + await import("./gateway-access-check.js"); + const handler = createGatewayAccessCheckHandler() as any; + const e = { headers: {} as Record, ...event }; + const body = await handler(e); + return { e, body }; +} + +describe("gateway access-check endpoint", () => { + beforeEach(() => { + vi.resetModules(); + process.env.AGENT_NATIVE_REALTIME_HMAC_SECRET = SECRET; + mockResolveAccess.mockReset(); + }); + afterEach(() => { + delete process.env.AGENT_NATIVE_REALTIME_HMAC_SECRET; + }); + + it("returns allowed:true and forwards the signed query to resolveAccess", async () => { + mockResolveAccess.mockResolvedValue({ role: "viewer" }); + const token = signGatewayAccessToken(CLAIMS, SECRET); + const { body } = await invoke({ query: { token } }); + expect(body).toEqual({ allowed: true }); + expect(mockResolveAccess).toHaveBeenCalledWith( + "document", + "doc-1", + { userEmail: "sharee@example.com", orgId: "org-1" }, + { skipResourceBody: true }, + ); + }); + + it("returns allowed:false when the app denies (null)", async () => { + mockResolveAccess.mockResolvedValue(null); + const token = signGatewayAccessToken(CLAIMS, SECRET); + const { body } = await invoke({ query: { token } }); + expect(body).toEqual({ allowed: false }); + }); + + it("fails closed when resolveAccess throws (unknown resource type)", async () => { + mockResolveAccess.mockRejectedValue( + new Error("Unknown shareable resource"), + ); + const token = signGatewayAccessToken(CLAIMS, SECRET); + const { body } = await invoke({ query: { token } }); + expect(body).toEqual({ allowed: false }); + }); + + it("401s a token signed with the wrong key and never checks access", async () => { + const token = signGatewayAccessToken(CLAIMS, "wrong-secret"); + const { e } = await invoke({ query: { token } }); + expect(e.status).toBe(401); + expect(mockResolveAccess).not.toHaveBeenCalled(); + }); + + it("401s a missing token", async () => { + const { e } = await invoke({ query: {} }); + expect(e.status).toBe(401); + }); + + it("404s when no realtime secret is configured", async () => { + delete process.env.AGENT_NATIVE_REALTIME_HMAC_SECRET; + const token = signGatewayAccessToken(CLAIMS, SECRET); + const { e } = await invoke({ query: { token } }); + expect(e.status).toBe(404); + }); + + it("405s a non-GET request", async () => { + const token = signGatewayAccessToken(CLAIMS, SECRET); + const { e } = await invoke({ method: "POST", query: { token } }); + expect(e.status).toBe(405); + }); +}); diff --git a/packages/core/src/server/gateway-access-check.ts b/packages/core/src/server/gateway-access-check.ts new file mode 100644 index 0000000000..e46c6f7058 --- /dev/null +++ b/packages/core/src/server/gateway-access-check.ts @@ -0,0 +1,65 @@ +/** + * `GET /_agent-native/can-see?token=` — the hosted + * Realtime Gateway has no copy of this app's shareable-resource registry, so it + * cannot resolve sharee visibility itself. It signs a token with the app's + * per-project HMAC secret (binding the full access query) and asks here; the + * app runs `resolveAccess` and answers `{ allowed }`. The token is the auth — + * only a holder of the per-project secret can mint it. + */ + +import { + defineEventHandler, + getMethod, + getQuery, + type H3Event, + setResponseHeader, + setResponseStatus, +} from "h3"; + +import { resolveAccess } from "../sharing/access.js"; +import { getRealtimeSigningSecret } from "./realtime-token.js"; +import { runWithRequestContext } from "./request-context.js"; +import { verifyGatewayAccessToken } from "./short-lived-token.js"; + +export function createGatewayAccessCheckHandler() { + return defineEventHandler(async (event: H3Event) => { + setResponseHeader(event, "Cache-Control", "private, no-store"); + + if (getMethod(event) !== "GET") { + setResponseStatus(event, 405); + return { error: "Method not allowed" }; + } + + const secret = getRealtimeSigningSecret(); + if (!secret) { + setResponseStatus(event, 404); + return { error: "Realtime gateway not configured" }; + } + + const token = getQuery(event).token; + const verified = + typeof token === "string" + ? verifyGatewayAccessToken(token, secret) + : ({ ok: false, reason: "missing" } as const); + if (!verified.ok) { + setResponseStatus(event, 401); + return { error: "Unauthorized" }; + } + + const { resourceType, resourceId, userEmail, orgId } = verified; + return runWithRequestContext({ userEmail, orgId }, async () => { + try { + const access = await resolveAccess( + resourceType, + resourceId, + { userEmail, orgId }, + { skipResourceBody: true }, + ); + return { allowed: access != null }; + } catch { + // Unknown resource type or lookup failure: fail closed. + return { allowed: false }; + } + }); + }); +} diff --git a/packages/core/src/server/short-lived-token.spec.ts b/packages/core/src/server/short-lived-token.spec.ts index 5009324ad6..33b62de9ed 100644 --- a/packages/core/src/server/short-lived-token.spec.ts +++ b/packages/core/src/server/short-lived-token.spec.ts @@ -1,8 +1,10 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import { + signGatewayAccessToken, signRealtimeSubscribeToken, signShortLivedToken, + verifyGatewayAccessToken, verifyRealtimeSubscribeToken, verifyShortLivedToken, } from "./short-lived-token.js"; @@ -184,3 +186,80 @@ describe("realtime subscribe token", () => { ).not.toThrow(); }); }); + +describe("gateway access-check token", () => { + const KEY_A = "project-a-hmac-secret"; + const KEY_B = "project-b-hmac-secret"; + const claims = { + projectId: "proj_a", + resourceType: "document", + resourceId: "doc-1", + userEmail: "sharee@example.com", + orgId: "org-1", + }; + + afterEach(() => vi.useRealTimers()); + + it("round-trips the bound access query", () => { + const token = signGatewayAccessToken(claims, KEY_A); + expect(verifyGatewayAccessToken(token, KEY_A)).toEqual({ + ok: true, + projectId: "proj_a", + resourceType: "document", + resourceId: "doc-1", + userEmail: "sharee@example.com", + orgId: "org-1", + }); + }); + + it("rejects a token signed with a different key", () => { + const token = signGatewayAccessToken(claims, KEY_A); + expect(verifyGatewayAccessToken(token, KEY_B)).toEqual({ + ok: false, + reason: "bad_signature", + }); + }); + + it("rejects a tampered resourceId (query is signed, not just the caller)", () => { + const token = signGatewayAccessToken(claims, KEY_A); + const [payload, sig] = token.split(".", 2); + const decoded = JSON.parse( + Buffer.from( + payload.replace(/-/g, "+").replace(/_/g, "/"), + "base64", + ).toString("utf8"), + ); + decoded.resourceId = "doc-victim"; + const forgedPayload = Buffer.from(JSON.stringify(decoded)) + .toString("base64") + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=+$/g, ""); + expect(verifyGatewayAccessToken(`${forgedPayload}.${sig}`, KEY_A)).toEqual({ + ok: false, + reason: "bad_signature", + }); + }); + + it("is not interchangeable with a realtime subscribe token", () => { + const subscribe = signRealtimeSubscribeToken( + { projectId: "proj_a", owner: "u@example.com" }, + KEY_A, + ); + expect(verifyGatewayAccessToken(subscribe, KEY_A)).toEqual({ + ok: false, + reason: "wrong_type", + }); + }); + + it("rejects an expired token", () => { + vi.useFakeTimers(); + vi.setSystemTime(1_000_000_000_000); + const token = signGatewayAccessToken({ ...claims, ttlSeconds: 30 }, KEY_A); + vi.setSystemTime(1_000_000_000_000 + 31_000); + expect(verifyGatewayAccessToken(token, KEY_A)).toEqual({ + ok: false, + reason: "expired", + }); + }); +}); diff --git a/packages/core/src/server/short-lived-token.ts b/packages/core/src/server/short-lived-token.ts index 2802385ffd..1899fa7d68 100644 --- a/packages/core/src/server/short-lived-token.ts +++ b/packages/core/src/server/short-lived-token.ts @@ -324,3 +324,117 @@ export function verifyRealtimeSubscribeToken( exp: claims.exp, }; } + +// ── Gateway access-check tokens ────────────────────────────────────────────── +// +// The hosted gateway has no access to an app's shareable-resource registry, so +// it cannot resolve sharee visibility itself. It signs one of these with the +// app's per-project key and calls the app's `/_agent-native/can-see`, which runs +// `resolveAccess` and answers. The full access query is bound into the token so +// the app authenticates the params, not merely the caller. + +/** Payload `typ` discriminator for gateway access-check tokens. */ +export const GATEWAY_ACCESS_TOKEN_TYPE = "rt-access-check"; +/** Short TTL: minted per check, used immediately server-to-server. */ +const DEFAULT_GATEWAY_ACCESS_TTL_SECONDS = 60; + +/** Inputs for {@link signGatewayAccessToken}. */ +export interface GatewayAccessClaims { + projectId: string; + resourceType: string; + resourceId: string; + /** App end-user whose visibility of the resource is being checked. */ + userEmail: string; + orgId?: string; + ttlSeconds?: number; +} + +interface DecodedGatewayAccessClaims { + typ: string; + projectId: string; + resourceType: string; + resourceId: string; + userEmail: string; + orgId?: string; + exp: number; +} + +/** Result of {@link verifyGatewayAccessToken}; on success carries the bound query. */ +export type GatewayAccessVerifyResult = + | { + ok: true; + projectId: string; + resourceType: string; + resourceId: string; + userEmail: string; + orgId?: string; + } + | { ok: false; reason: string }; + +/** Mint a gateway access-check token, signed with the app's per-project `key`. */ +export function signGatewayAccessToken( + claims: GatewayAccessClaims, + key: string, +): string { + if (!key) throw new Error("signGatewayAccessToken requires a key"); + const ttl = claims.ttlSeconds ?? DEFAULT_GATEWAY_ACCESS_TTL_SECONDS; + const payload: DecodedGatewayAccessClaims = { + typ: GATEWAY_ACCESS_TOKEN_TYPE, + projectId: claims.projectId, + resourceType: claims.resourceType, + resourceId: claims.resourceId, + userEmail: claims.userEmail, + exp: Math.floor(Date.now() / 1000) + ttl, + }; + if (claims.orgId) payload.orgId = claims.orgId; + const payloadStr = base64UrlEncode(JSON.stringify(payload)); + return `${payloadStr}.${hmacB64(payloadStr, key)}`; +} + +/** Verify a gateway access-check token against the app's per-project `key`. */ +export function verifyGatewayAccessToken( + token: string, + key: string, +): GatewayAccessVerifyResult { + if (!key) return { ok: false, reason: "no_key" }; + if (typeof token !== "string" || !token.includes(".")) { + return { ok: false, reason: "malformed" }; + } + const [payloadStr, sig] = token.split(".", 2); + if (!payloadStr || !sig) return { ok: false, reason: "malformed" }; + + if (!timingSafeEqualB64(sig, hmacB64(payloadStr, key))) { + return { ok: false, reason: "bad_signature" }; + } + + let claims: DecodedGatewayAccessClaims; + try { + claims = JSON.parse(base64UrlDecode(payloadStr).toString("utf8")); + } catch { + return { ok: false, reason: "bad_payload" }; + } + + if (claims.typ !== GATEWAY_ACCESS_TOKEN_TYPE) { + return { ok: false, reason: "wrong_type" }; + } + if (typeof claims.exp !== "number") + return { ok: false, reason: "bad_payload" }; + if (claims.exp * 1000 < Date.now()) return { ok: false, reason: "expired" }; + if ( + !claims.projectId || + !claims.resourceType || + !claims.resourceId || + !claims.userEmail + ) { + return { ok: false, reason: "bad_payload" }; + } + + return { + ok: true, + projectId: claims.projectId, + resourceType: claims.resourceType, + resourceId: claims.resourceId, + userEmail: claims.userEmail, + orgId: claims.orgId, + }; +} From bf6ce6c2c6a2efa66c8035109ef0404e87375dd2 Mon Sep 17 00:00:00 2001 From: Korey Kassir Date: Thu, 23 Jul 2026 10:13:00 -0400 Subject: [PATCH 2/9] feat: enhance hosted SSE reconnect logic and add tests for stream URL rebuilding --- .../realtime-gateway-access-and-reconnect.md | 2 +- .../use-db-sync.gateway-reconnect.spec.tsx | 103 ++++++++++++++++++ packages/core/src/client/use-db-sync.ts | 33 +++--- 3 files changed, 119 insertions(+), 19 deletions(-) create mode 100644 packages/core/src/client/use-db-sync.gateway-reconnect.spec.tsx diff --git a/.changeset/realtime-gateway-access-and-reconnect.md b/.changeset/realtime-gateway-access-and-reconnect.md index df1917d2c8..e826122329 100644 --- a/.changeset/realtime-gateway-access-and-reconnect.md +++ b/.changeset/realtime-gateway-access-and-reconnect.md @@ -4,6 +4,6 @@ Realtime sync: two hosted-gateway follow-ups. Both are opt-in and leave apps without hosted-realtime config unchanged. -- Hosted SSE reconnect now sends the client's cursor (`&since=`) on the gateway stream URL, so the gateway's connect-time catch-up replays events written during the reconnect gap immediately instead of deferring them to the next poll. First connect (cursor 0) is unaffected. +- Hosted SSE reconnect now sends the client's cursor (`&since=`) on the gateway stream URL, so the gateway's connect-time catch-up replays events written during the reconnect gap immediately instead of deferring them to the next poll. First connect (cursor 0) is unaffected. Because a `since=` only takes effect when a new `EventSource` is constructed (the browser's own auto-reconnect reuses the URL frozen at construction), the hosted transport now owns reconnects: on a stream error it closes the stream and schedules its own reconnect so the next connect rebuilds the URL from the current cursor, keeping the token on transient errors and reminting on a closed stream. Local mode keeps native EventSource reconnect. - New gateway access-check tokens (`signGatewayAccessToken`/`verifyGatewayAccessToken`, exported from `./server/short-lived-token`): per-project HMAC key, a `typ` discriminator (not interchangeable with subscribe or media tokens), and the full access query (`resourceType`/`resourceId`/`userEmail`/`orgId`) bound into the signature so the app authenticates the params, not just the caller. - New endpoint `GET /_agent-native/can-see` mounted by core-routes. The hosted Realtime Gateway has no copy of an app's shareable-resource registry, so it calls this to resolve sharee visibility: the endpoint verifies a gateway access-check token against the app's per-project secret, runs the app's registry-based `resolveAccess`, and returns `{ allowed }`. Fails closed (`allowed: false`) on an unknown resource type or lookup error; 404 when the app has no realtime secret; `Cache-Control: private, no-store`. diff --git a/packages/core/src/client/use-db-sync.gateway-reconnect.spec.tsx b/packages/core/src/client/use-db-sync.gateway-reconnect.spec.tsx new file mode 100644 index 0000000000..5e88052049 --- /dev/null +++ b/packages/core/src/client/use-db-sync.gateway-reconnect.spec.tsx @@ -0,0 +1,103 @@ +// @vitest-environment happy-dom + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + _resetSyncTransportRegistryForTests, + subscribeSyncEvents, +} from "./use-db-sync"; + +/** Minimal EventSource stand-in that records every constructed instance so the + * test can inspect the URL each connect was built with. */ +class FakeEventSource { + static readonly CONNECTING = 0; + static readonly OPEN = 1; + static readonly CLOSED = 2; + static instances: FakeEventSource[] = []; + readyState = FakeEventSource.CONNECTING; + onopen: (() => void) | null = null; + onerror: (() => void) | null = null; + onmessage: ((message: { data: string }) => void) | null = null; + addEventListener(): void {} + constructor(readonly url: string) { + FakeEventSource.instances.push(this); + } + close(): void { + this.readyState = FakeEventSource.CLOSED; + } +} + +describe("hosted SSE reconnect ownership", () => { + beforeEach(() => { + vi.useFakeTimers(); + FakeEventSource.instances = []; + _resetSyncTransportRegistryForTests(); + vi.stubGlobal("EventSource", FakeEventSource); + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes("/_agent-native/realtime-token")) { + return { + ok: true, + json: async () => ({ token: "tok-1", ttlSeconds: 600 }), + }; + } + return { ok: true, json: async () => ({ version: 0, events: [] }) }; + }), + ); + ( + window as unknown as { __AGENT_NATIVE_CONFIG__: unknown } + ).__AGENT_NATIVE_CONFIG__ = { + realtime: { transport: "hosted", gatewayBaseUrl: "https://gw.example" }, + }; + }); + + afterEach(() => { + _resetSyncTransportRegistryForTests(); + delete (window as unknown as { __AGENT_NATIVE_CONFIG__?: unknown }) + .__AGENT_NATIVE_CONFIG__; + vi.unstubAllGlobals(); + vi.useRealTimers(); + }); + + it("rebuilds the stream URL with the current cursor on a browser-managed reconnect", async () => { + const unsub = subscribeSyncEvents({ onEvents: () => {} }); + + // Mint resolves, then the first stream opens with the token and no cursor. + await vi.advanceTimersByTimeAsync(200); + const first = FakeEventSource.instances.at(-1)!; + expect(first.url).toContain("token=tok-1"); + expect(first.url).not.toContain("since="); + + first.readyState = FakeEventSource.OPEN; + first.onopen?.(); + + // A delivered batch advances the transport cursor to 100. + first.onmessage?.({ + data: JSON.stringify({ + type: "batch", + version: 100, + events: [ + { version: 100, source: "app-state", type: "change", key: "*" }, + ], + }), + }); + + // Transient (CONNECTING) error: the browser would auto-reconnect this same + // instance with its frozen URL. We must own it and close the stream. + first.readyState = FakeEventSource.CONNECTING; + first.onerror?.(); + expect(first.readyState).toBe(FakeEventSource.CLOSED); + + // The owned reconnect builds a NEW stream carrying the current cursor and + // the still-valid token (no re-mint on a transient error). + await vi.advanceTimersByTimeAsync(1500); + const second = FakeEventSource.instances.at(-1)!; + expect(second).not.toBe(first); + expect(second.url).toContain("since=100"); + expect(second.url).toContain("token=tok-1"); + + unsub(); + }); +}); diff --git a/packages/core/src/client/use-db-sync.ts b/packages/core/src/client/use-db-sync.ts index e04cf75920..ad6762b7e3 100644 --- a/packages/core/src/client/use-db-sync.ts +++ b/packages/core/src/client/use-db-sync.ts @@ -659,26 +659,23 @@ class SyncTransport { }; source.onerror = () => { this.setSseConnected(false); - // When the browser gives up permanently (HTTP error → readyState - // CLOSED), it won't auto-reconnect. Drop the ref so a later - // connectEvents() (on focus/visibility) can establish a fresh stream; - // otherwise the non-null closed `eventSource` blocks reconnection and - // we'd be stuck on polling-only forever. + if (this.mode === "hosted" && this.gateway) { + // Browser auto-reconnect reuses the URL frozen at construction, so it + // would replay from a stale `since`. Own the reconnect so the next + // connect rebuilds activeSseUrl from the current versionRef. CLOSED also + // refreshes the token (expired/rotated/deploy); CONNECTING keeps it. A + // successful reconnect resets the count in onopen; a hard-down gateway + // trips the threshold and health-gates to local. + if (source.readyState === EventSource.CLOSED) this.token = null; + this.closeEvents(); + this.onGatewayTransientFailure(); + if (this.mode === "hosted") this.scheduleGatewayReconnect(); + return; + } + // Local mode: native EventSource reconnect is fine. Drop a CLOSED ref so a + // later connectEvents() (focus/visibility) can establish a fresh stream. if (source.readyState === EventSource.CLOSED) { this.eventSource = null; - if (this.mode === "hosted" && this.gateway) { - // A closed gateway stream is most likely an expired token or a - // request-timeout/deploy cycle. Re-mint and reconnect with jitter; - // this is NOT the poll-401 cooldown path. Each closed stream counts - // toward the unhealthy threshold so a hard-down gateway (or one - // rejecting our tokens) health-gates to local instead of looping - // mint+connect forever; a successful reconnect resets the count in - // onopen above. - this.token = null; - this.onGatewayTransientFailure(); - if (this.mode === "hosted") this.scheduleGatewayReconnect(); - return; - } } this.schedulePoll(); }; From 4ea76876e29c2269bc07b002d9fba04f226ece3d Mon Sep 17 00:00:00 2001 From: Korey Kassir Date: Thu, 23 Jul 2026 11:23:34 -0400 Subject: [PATCH 3/9] feat: enhance gateway access-check tokens and improve SSE reconnect logic --- .../realtime-gateway-access-and-reconnect.md | 4 +-- .../use-db-sync.gateway-reconnect.spec.tsx | 29 +++++++++++++++++++ packages/core/src/client/use-db-sync.ts | 3 ++ .../src/server/gateway-access-check.spec.ts | 17 +++++++++++ .../core/src/server/gateway-access-check.ts | 16 +++++----- .../core/src/server/short-lived-token.spec.ts | 20 +++++++++++++ packages/core/src/server/short-lived-token.ts | 8 +++++ 7 files changed, 88 insertions(+), 9 deletions(-) diff --git a/.changeset/realtime-gateway-access-and-reconnect.md b/.changeset/realtime-gateway-access-and-reconnect.md index e826122329..e30558a62b 100644 --- a/.changeset/realtime-gateway-access-and-reconnect.md +++ b/.changeset/realtime-gateway-access-and-reconnect.md @@ -4,6 +4,6 @@ Realtime sync: two hosted-gateway follow-ups. Both are opt-in and leave apps without hosted-realtime config unchanged. -- Hosted SSE reconnect now sends the client's cursor (`&since=`) on the gateway stream URL, so the gateway's connect-time catch-up replays events written during the reconnect gap immediately instead of deferring them to the next poll. First connect (cursor 0) is unaffected. Because a `since=` only takes effect when a new `EventSource` is constructed (the browser's own auto-reconnect reuses the URL frozen at construction), the hosted transport now owns reconnects: on a stream error it closes the stream and schedules its own reconnect so the next connect rebuilds the URL from the current cursor, keeping the token on transient errors and reminting on a closed stream. Local mode keeps native EventSource reconnect. -- New gateway access-check tokens (`signGatewayAccessToken`/`verifyGatewayAccessToken`, exported from `./server/short-lived-token`): per-project HMAC key, a `typ` discriminator (not interchangeable with subscribe or media tokens), and the full access query (`resourceType`/`resourceId`/`userEmail`/`orgId`) bound into the signature so the app authenticates the params, not just the caller. +- Hosted SSE reconnect now sends the client's cursor (`&since=`) on the gateway stream URL, so the gateway's connect-time catch-up replays events written during the reconnect gap immediately instead of deferring them to the next poll. First connect (cursor 0) is unaffected. Because a `since=` only takes effect when a new `EventSource` is constructed (the browser's own auto-reconnect reuses the URL frozen at construction), the hosted transport now owns reconnects: on a stream error it closes the stream and schedules its own reconnect so the next connect rebuilds the URL from the current cursor, keeping the token on transient errors and reminting on a closed stream. A late error from a replaced (stale) stream is ignored so it cannot tear down the current one. Local mode keeps native EventSource reconnect. +- New gateway access-check tokens (`signGatewayAccessToken`/`verifyGatewayAccessToken`, exported from `./server/short-lived-token`): per-project HMAC key, a `typ` discriminator (not interchangeable with subscribe or media tokens), and the full access query (`resourceType`/`resourceId`/`userEmail`/`orgId`) bound into the signature so the app authenticates the params, not just the caller. `verifyGatewayAccessToken` also accepts an optional expected `projectId` to bind the channel (mirroring `verifyRealtimeSubscribeToken`); the `can-see` endpoint passes its own project id when known. - New endpoint `GET /_agent-native/can-see` mounted by core-routes. The hosted Realtime Gateway has no copy of an app's shareable-resource registry, so it calls this to resolve sharee visibility: the endpoint verifies a gateway access-check token against the app's per-project secret, runs the app's registry-based `resolveAccess`, and returns `{ allowed }`. Fails closed (`allowed: false`) on an unknown resource type or lookup error; 404 when the app has no realtime secret; `Cache-Control: private, no-store`. diff --git a/packages/core/src/client/use-db-sync.gateway-reconnect.spec.tsx b/packages/core/src/client/use-db-sync.gateway-reconnect.spec.tsx index 5e88052049..df3e4ef2c5 100644 --- a/packages/core/src/client/use-db-sync.gateway-reconnect.spec.tsx +++ b/packages/core/src/client/use-db-sync.gateway-reconnect.spec.tsx @@ -100,4 +100,33 @@ describe("hosted SSE reconnect ownership", () => { unsub(); }); + + it("ignores a late error from a replaced (stale) stream", async () => { + const unsub = subscribeSyncEvents({ onEvents: () => {} }); + await vi.advanceTimersByTimeAsync(200); + const first = FakeEventSource.instances.at(-1)!; + first.readyState = FakeEventSource.OPEN; + first.onopen?.(); + + // Force a reconnect so `first` is replaced by `second`. + first.readyState = FakeEventSource.CONNECTING; + first.onerror?.(); + await vi.advanceTimersByTimeAsync(1500); + const second = FakeEventSource.instances.at(-1)!; + expect(second).not.toBe(first); + second.readyState = FakeEventSource.OPEN; + second.onopen?.(); + + const countBefore = FakeEventSource.instances.length; + // The stale `first` fires a late error: the guard must ignore it so the + // healthy `second` is neither closed nor triggers a spurious reconnect. + first.readyState = FakeEventSource.CONNECTING; + first.onerror?.(); + await vi.advanceTimersByTimeAsync(1500); + + expect(second.readyState).toBe(FakeEventSource.OPEN); + expect(FakeEventSource.instances.length).toBe(countBefore); + + unsub(); + }); }); diff --git a/packages/core/src/client/use-db-sync.ts b/packages/core/src/client/use-db-sync.ts index ad6762b7e3..6d6f4a7942 100644 --- a/packages/core/src/client/use-db-sync.ts +++ b/packages/core/src/client/use-db-sync.ts @@ -658,6 +658,9 @@ class SyncTransport { this.schedulePoll(); }; source.onerror = () => { + // A replaced/closed source can still fire late; ignore it so it can't + // flip the connected state or tear down the current stream. + if (this.eventSource !== source) return; this.setSseConnected(false); if (this.mode === "hosted" && this.gateway) { // Browser auto-reconnect reuses the URL frozen at construction, so it diff --git a/packages/core/src/server/gateway-access-check.spec.ts b/packages/core/src/server/gateway-access-check.spec.ts index 39bacddb1d..2a64774c12 100644 --- a/packages/core/src/server/gateway-access-check.spec.ts +++ b/packages/core/src/server/gateway-access-check.spec.ts @@ -47,6 +47,7 @@ describe("gateway access-check endpoint", () => { }); afterEach(() => { delete process.env.AGENT_NATIVE_REALTIME_HMAC_SECRET; + delete process.env.BUILDER_PROJECT_ID; }); it("returns allowed:true and forwards the signed query to resolveAccess", async () => { @@ -69,6 +70,22 @@ describe("gateway access-check endpoint", () => { expect(body).toEqual({ allowed: false }); }); + it("401s and skips access when the token's project id mismatches this app", async () => { + process.env.BUILDER_PROJECT_ID = "proj_other"; + const token = signGatewayAccessToken(CLAIMS, SECRET); // projectId proj_a + const { e } = await invoke({ query: { token } }); + expect(e.status).toBe(401); + expect(mockResolveAccess).not.toHaveBeenCalled(); + }); + + it("allows when the token's project id matches this app's", async () => { + process.env.BUILDER_PROJECT_ID = "proj_a"; + mockResolveAccess.mockResolvedValue({ role: "viewer" }); + const token = signGatewayAccessToken(CLAIMS, SECRET); + const { body } = await invoke({ query: { token } }); + expect(body).toEqual({ allowed: true }); + }); + it("fails closed when resolveAccess throws (unknown resource type)", async () => { mockResolveAccess.mockRejectedValue( new Error("Unknown shareable resource"), diff --git a/packages/core/src/server/gateway-access-check.ts b/packages/core/src/server/gateway-access-check.ts index e46c6f7058..851804885c 100644 --- a/packages/core/src/server/gateway-access-check.ts +++ b/packages/core/src/server/gateway-access-check.ts @@ -1,10 +1,8 @@ /** - * `GET /_agent-native/can-see?token=` — the hosted - * Realtime Gateway has no copy of this app's shareable-resource registry, so it - * cannot resolve sharee visibility itself. It signs a token with the app's - * per-project HMAC secret (binding the full access query) and asks here; the - * app runs `resolveAccess` and answers `{ allowed }`. The token is the auth — - * only a holder of the per-project secret can mint it. + * `GET /_agent-native/can-see` — the hosted Realtime Gateway's sharee-visibility + * check. Verifies a gateway access-check token (rationale in + * short-lived-token.ts), runs the app's own `resolveAccess`, answers + * `{ allowed }`, and fails closed. */ import { @@ -17,6 +15,7 @@ import { } from "h3"; import { resolveAccess } from "../sharing/access.js"; +import { getBuilderBranchProjectId } from "./builder-browser.js"; import { getRealtimeSigningSecret } from "./realtime-token.js"; import { runWithRequestContext } from "./request-context.js"; import { verifyGatewayAccessToken } from "./short-lived-token.js"; @@ -37,9 +36,12 @@ export function createGatewayAccessCheckHandler() { } const token = getQuery(event).token; + // Sync, env-only: binds the token's channel when this app's project id is + // known, and no-ops (undefined) for scoped-secret apps where it isn't. + const expectedProjectId = getBuilderBranchProjectId() || undefined; const verified = typeof token === "string" - ? verifyGatewayAccessToken(token, secret) + ? verifyGatewayAccessToken(token, secret, expectedProjectId) : ({ ok: false, reason: "missing" } as const); if (!verified.ok) { setResponseStatus(event, 401); diff --git a/packages/core/src/server/short-lived-token.spec.ts b/packages/core/src/server/short-lived-token.spec.ts index 33b62de9ed..01a41ef9df 100644 --- a/packages/core/src/server/short-lived-token.spec.ts +++ b/packages/core/src/server/short-lived-token.spec.ts @@ -262,4 +262,24 @@ describe("gateway access-check token", () => { reason: "expired", }); }); + + it("binds the projectId channel when an expected value is provided", () => { + const token = signGatewayAccessToken(claims, KEY_A); // projectId proj_a + expect(verifyGatewayAccessToken(token, KEY_A, "proj_a")).toMatchObject({ + ok: true, + projectId: "proj_a", + }); + expect(verifyGatewayAccessToken(token, KEY_A, "proj_b")).toEqual({ + ok: false, + reason: "wrong_project", + }); + }); + + it("skips the projectId check when no expected value is given", () => { + const token = signGatewayAccessToken(claims, KEY_A); + expect(verifyGatewayAccessToken(token, KEY_A)).toMatchObject({ ok: true }); + expect(verifyGatewayAccessToken(token, KEY_A, undefined)).toMatchObject({ + ok: true, + }); + }); }); diff --git a/packages/core/src/server/short-lived-token.ts b/packages/core/src/server/short-lived-token.ts index 1899fa7d68..04a67c491e 100644 --- a/packages/core/src/server/short-lived-token.ts +++ b/packages/core/src/server/short-lived-token.ts @@ -395,6 +395,7 @@ export function signGatewayAccessToken( export function verifyGatewayAccessToken( token: string, key: string, + expectedProjectId?: string, ): GatewayAccessVerifyResult { if (!key) return { ok: false, reason: "no_key" }; if (typeof token !== "string" || !token.includes(".")) { @@ -428,6 +429,13 @@ export function verifyGatewayAccessToken( ) { return { ok: false, reason: "bad_payload" }; } + // Optional channel binding, mirroring verifyRealtimeSubscribeToken. The + // per-project key already scopes verification to one app; this is belt-and- + // suspenders for a future multi-tenant secret store. Skipped when the caller + // can't cheaply resolve its own project id (scoped-secret apps). + if (expectedProjectId && claims.projectId !== expectedProjectId) { + return { ok: false, reason: "wrong_project" }; + } return { ok: true, From ab356697e83c3e445dac48422915f5684caf3cc4 Mon Sep 17 00:00:00 2001 From: Korey Kassir Date: Thu, 23 Jul 2026 12:23:20 -0400 Subject: [PATCH 4/9] feat: implement DB-assigned version allocation and related tests for AppSyncState --- .../realtime-gateway-access-and-reconnect.md | 1 + .../src/server/db-assigned-versions.spec.ts | 348 ++++++++++++++++++ packages/core/src/server/poll.ts | 278 +++++++++++++- 3 files changed, 618 insertions(+), 9 deletions(-) create mode 100644 packages/core/src/server/db-assigned-versions.spec.ts diff --git a/.changeset/realtime-gateway-access-and-reconnect.md b/.changeset/realtime-gateway-access-and-reconnect.md index e30558a62b..be69b8acb2 100644 --- a/.changeset/realtime-gateway-access-and-reconnect.md +++ b/.changeset/realtime-gateway-access-and-reconnect.md @@ -7,3 +7,4 @@ Realtime sync: two hosted-gateway follow-ups. Both are opt-in and leave apps wit - Hosted SSE reconnect now sends the client's cursor (`&since=`) on the gateway stream URL, so the gateway's connect-time catch-up replays events written during the reconnect gap immediately instead of deferring them to the next poll. First connect (cursor 0) is unaffected. Because a `since=` only takes effect when a new `EventSource` is constructed (the browser's own auto-reconnect reuses the URL frozen at construction), the hosted transport now owns reconnects: on a stream error it closes the stream and schedules its own reconnect so the next connect rebuilds the URL from the current cursor, keeping the token on transient errors and reminting on a closed stream. A late error from a replaced (stale) stream is ignored so it cannot tear down the current one. Local mode keeps native EventSource reconnect. - New gateway access-check tokens (`signGatewayAccessToken`/`verifyGatewayAccessToken`, exported from `./server/short-lived-token`): per-project HMAC key, a `typ` discriminator (not interchangeable with subscribe or media tokens), and the full access query (`resourceType`/`resourceId`/`userEmail`/`orgId`) bound into the signature so the app authenticates the params, not just the caller. `verifyGatewayAccessToken` also accepts an optional expected `projectId` to bind the channel (mirroring `verifyRealtimeSubscribeToken`); the `can-see` endpoint passes its own project id when known. - New endpoint `GET /_agent-native/can-see` mounted by core-routes. The hosted Realtime Gateway has no copy of an app's shareable-resource registry, so it calls this to resolve sharee visibility: the endpoint verifies a gateway access-check token against the app's per-project secret, runs the app's registry-based `resolveAccess`, and returns `{ allowed }`. Fails closed (`allowed: false`) on an unknown resource type or lookup error; 404 when the app has no realtime secret; `Cache-Control: private, no-store`. +- New `AppSyncStateOptions.dbAssignedVersions` (default off): allocate durable-event versions from the app's Postgres DB — a one-row allocator advanced with `GREATEST(v + 1, epoch_ms_now)` inside the same autocommit insert statement — instead of the per-writer in-memory clock counter. Hosted realtime has multiple writers per app DB (the app's serverless instances plus gateway instances), where clock skew can assign a LOWER version to a LATER event and a client whose cursor passed the higher value filters the later event out permanently; DB allocation serializes on the allocator row's lock so version order equals commit order across all writers. Buffer/emit defer until the allocated version returns (no provisional version ever reaches clients); a deterministic-id dedupe loser adopts the winner's version via `ON CONFLICT DO UPDATE ... RETURNING`; on DB failure the writer falls back to clock allocation (logged) so the in-process fast path never stalls. Versions stay epoch-ms scale, so existing cursors, seeds, and lag metrics are unaffected. The default instance enables this automatically when `AGENT_NATIVE_REALTIME_TRANSPORT=hosted` with a gateway URL; self-hosted apps are byte-identical. Postgres only; ignored on SQLite. diff --git a/packages/core/src/server/db-assigned-versions.spec.ts b/packages/core/src/server/db-assigned-versions.spec.ts new file mode 100644 index 0000000000..baf333a832 --- /dev/null +++ b/packages/core/src/server/db-assigned-versions.spec.ts @@ -0,0 +1,348 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { AppSyncState, POLL_CHANGE_EVENT } from "./poll.js"; + +/** + * Emulates the Postgres side of the DB-assigned version allocator: ddl-guard + * probes report everything as existing (no DDL), the seed is accepted, and the + * allocating INSERT advances a shared one-row allocator with + * GREATEST(v + 1, now, floor) — including the ON CONFLICT winner-version + * semantics for duplicate deterministic ids. + */ +function makeAllocatorDb(shared?: { v: number; ids: Map }) { + const state = shared ?? { v: 0, ids: new Map() }; + const log: Array<{ sql: string; args: unknown[] }> = []; + const db = { + state, + log, + failAllocation: false, + async execute(query: string | { sql: string; args?: unknown[] }) { + const sql = typeof query === "string" ? query : query.sql; + const args = typeof query === "string" ? [] : (query.args ?? []); + log.push({ sql, args }); + if ( + sql.includes("information_schema.tables") || + sql.includes("pg_indexes") + ) { + return { rows: [{ "1": 1 }], rowsAffected: 0 }; + } + if (sql.includes("INSERT INTO sync_version")) { + if (state.v === 0) state.v = Date.now(); + return { rows: [], rowsAffected: 1 }; + } + if (sql.includes("WITH alloc")) { + if (db.failAllocation) throw new Error("neon unavailable"); + const floor = Number(args[0]); + const id = String(args[1]); + const existing = state.ids.get(id); + // The allocator row advances even for a conflict loser (burnt gap). + state.v = Math.max(state.v + 1, Date.now(), floor); + if (existing !== undefined) { + return { rows: [{ version: existing }], rowsAffected: 1 }; + } + state.ids.set(id, state.v); + return { rows: [{ version: state.v }], rowsAffected: 1 }; + } + // Legacy INSERT / DELETE prune / anything else. + return { rows: [], rowsAffected: 0 }; + }, + }; + return db; +} + +function baseEvent(extra: Record = {}) { + return { source: "app-state", type: "change", key: "k", ...extra }; +} + +async function flush() { + // The first gated record awaits the whole ensure chain (ddl-guard probes + + // seed) — macrotask turns drain arbitrarily deep microtask chains. + for (let i = 0; i < 3; i++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + } +} + +describe("dbAssignedVersions", () => { + beforeEach(() => { + process.env.AGENT_NATIVE_SYNC_EVENTS_ENABLE_IN_TESTS = "1"; + }); + afterEach(() => { + delete process.env.AGENT_NATIVE_SYNC_EVENTS_ENABLE_IN_TESTS; + vi.useRealTimers(); + }); + + it("gate off: recordChange stays synchronous and never touches the allocator", () => { + const db = makeAllocatorDb(); + const s = new AppSyncState({ + getDb: () => db as never, + isPostgres: () => true, + }); + s.recordChange(baseEvent()); + // Synchronous contract: the event is visible before any await. + expect(s.getChangesSince(0).events).toHaveLength(1); + expect(db.log.some((q) => q.sql.includes("sync_version"))).toBe(false); + }); + + it("SQLite + gate on: falls through to the synchronous clock path", () => { + const db = makeAllocatorDb(); + const s = new AppSyncState({ + getDb: () => db as never, + isPostgres: () => false, + dbAssignedVersions: true, + }); + s.recordChange(baseEvent()); + expect(s.getChangesSince(0).events).toHaveLength(1); + expect(db.log.some((q) => q.sql.includes("sync_version"))).toBe(false); + }); + + it("gated: emits ONLY the DB-allocated version, never a provisional clock value", async () => { + const db = makeAllocatorDb(); + const s = new AppSyncState({ + getDb: () => db as never, + isPostgres: () => true, + dbAssignedVersions: true, + }); + const emitted: number[] = []; + s.getPollEmitter().on(POLL_CHANGE_EVENT, (e: { version: number }) => { + emitted.push(e.version); + }); + + s.recordChange(baseEvent()); + // Deferred emit: nothing is visible synchronously in gated mode. + expect(s.getChangesSince(0).events).toHaveLength(0); + await flush(); + + const events = s.getChangesSince(0).events; + expect(events).toHaveLength(1); + expect(events[0].version).toBe(db.state.v); + expect(emitted).toEqual([db.state.v]); + expect(s.getVersion()).toBe(db.state.v); + // The seed ran during ensure. + expect(db.log.some((q) => q.sql.includes("INSERT INTO sync_version"))).toBe( + true, + ); + }); + + it("two skewed writers sharing one allocator get strictly increasing versions", async () => { + vi.useFakeTimers(); + const T = 1_800_000_000_000; + const shared = { v: 0, ids: new Map() }; + const dbA = makeAllocatorDb(shared); + const dbB = makeAllocatorDb(shared); + const a = new AppSyncState({ + getDb: () => dbA as never, + isPostgres: () => true, + dbAssignedVersions: true, + }); + const b = new AppSyncState({ + getDb: () => dbB as never, + isPostgres: () => true, + dbAssignedVersions: true, + }); + + // Writer A has a fast clock (60s ahead) and writes FIRST. + vi.setSystemTime(T + 60_000); + a.recordChange(baseEvent({ key: "from-a" })); + await vi.advanceTimersByTimeAsync(0); + const versionA = a.getChangesSince(0).events[0]?.version; + + // Writer B's clock is 60s behind but its event is LATER. Under clock + // allocation this would invert; the shared allocator forbids it. + vi.setSystemTime(T); + b.recordChange(baseEvent({ key: "from-b" })); + await vi.advanceTimersByTimeAsync(0); + const versionB = b.getChangesSince(0).events[0]?.version; + + expect(versionA).toBeGreaterThan(0); + expect(versionB).toBeGreaterThan(versionA); + }); + + it("deterministic-id dedupe loser adopts the winner's version", async () => { + const shared = { v: 0, ids: new Map() }; + const a = new AppSyncState({ + getDb: () => makeAllocatorDb(shared) as never, + isPostgres: () => true, + dbAssignedVersions: true, + deterministicEventIds: true, + }); + const b = new AppSyncState({ + getDb: () => makeAllocatorDb(shared) as never, + isPostgres: () => true, + dbAssignedVersions: true, + deterministicEventIds: true, + }); + + // Same logical out-of-band write detected by both instances. + a.recordChange(baseEvent(), { dedupeKey: "app-state|500" }); + await flush(); + b.recordChange(baseEvent(), { dedupeKey: "app-state|500" }); + await flush(); + + const winner = a.getChangesSince(0).events[0]?.version; + const loser = b.getChangesSince(0).events[0]?.version; + expect(winner).toBeGreaterThan(0); + expect(loser).toBe(winner); + expect(shared.ids.size).toBe(1); // one durable row + }); + + it("falls back to clock versions when allocation fails, and still persists", async () => { + const db = makeAllocatorDb(); + db.failAllocation = true; + const s = new AppSyncState({ + getDb: () => db as never, + isPostgres: () => true, + dbAssignedVersions: true, + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + const before = Date.now(); + s.recordChange(baseEvent()); + await flush(); + + const events = s.getChangesSince(0).events; + expect(events).toHaveLength(1); + expect(events[0].version).toBeGreaterThanOrEqual(before); + // The legacy best-effort INSERT ran for the fallback event. + expect( + db.log.some( + (q) => + q.sql.includes("INSERT INTO sync_events") && + !q.sql.includes("WITH alloc"), + ), + ).toBe(true); + expect(warn).toHaveBeenCalledTimes(1); + warn.mockRestore(); + }); + + it("a throwing poll listener does not poison the chain", async () => { + const db = makeAllocatorDb(); + const s = new AppSyncState({ + getDb: () => db as never, + isPostgres: () => true, + dbAssignedVersions: true, + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + s.getPollEmitter().on(POLL_CHANGE_EVENT, () => { + throw new Error("listener bug"); + }); + + s.recordChange(baseEvent({ key: "first" })); + await flush(); + s.recordChange(baseEvent({ key: "second" })); + await flush(); + + // Both events reached the buffer despite the throwing listener; later + // events were not silently dropped by a poisoned chain. + const events = s.getChangesSince(0).events; + expect(events.map((e) => e.key)).toEqual(["first", "second"]); + warn.mockRestore(); + }); + + it("fallback reuses the allocating attempt's id so a commit-then-timeout cannot double-persist", async () => { + const db = makeAllocatorDb(); + db.failAllocation = true; + const s = new AppSyncState({ + getDb: () => db as never, + isPostgres: () => true, + dbAssignedVersions: true, + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + s.recordChange(baseEvent()); + await flush(); + + const allocAttempt = db.log.find((q) => q.sql.includes("WITH alloc")); + const legacyInsert = db.log.find( + (q) => + q.sql.includes("INSERT INTO sync_events") && + !q.sql.includes("WITH alloc"), + ); + expect(allocAttempt).toBeTruthy(); + expect(legacyInsert).toBeTruthy(); + // Same durable id on both statements → ON CONFLICT dedupes if the first + // actually committed server-side. + expect(legacyInsert!.args[0]).toBe(allocAttempt!.args[1]); + warn.mockRestore(); + }); + + it("seedVersionFromDb lifts the allocator to the seed's updated_at domain", async () => { + const db = makeAllocatorDb(); + const skewedUpdatedAt = Date.now() + 60_000; + const baseExecute = db.execute.bind(db); + db.execute = async (query: string | { sql: string; args?: unknown[] }) => { + const sql = typeof query === "string" ? query : query.sql; + if (sql.includes("MAX(updated_at)")) { + db.log.push({ + sql, + args: typeof query === "string" ? [] : (query.args ?? []), + }); + return { rows: [{ max_ts: skewedUpdatedAt }], rowsAffected: 0 }; + } + if (sql.includes("UPDATE sync_version SET v = GREATEST")) { + db.log.push({ + sql, + args: typeof query === "string" ? [] : (query.args ?? []), + }); + const floor = Number( + (typeof query === "string" ? [] : (query.args ?? []))[0], + ); + db.state.v = Math.max(db.state.v, floor); + return { rows: [], rowsAffected: 1 }; + } + return baseExecute(query); + }; + const s = new AppSyncState({ + getDb: () => db as never, + isPostgres: () => true, + dbAssignedVersions: true, + }); + + await s.seedVersionFromDb(); + + // The allocator was aligned to the (skew-ahead) seed, so the next + // allocation lands ABOVE the seeded cursor instead of below it. + expect(db.state.v).toBeGreaterThanOrEqual(skewedUpdatedAt); + s.recordChange(baseEvent()); + await flush(); + expect(s.getChangesSince(0).events[0]?.version).toBeGreaterThan( + skewedUpdatedAt, + ); + }); + + it("stays synchronous when sync events are disabled, without a misleading warning", () => { + delete process.env.AGENT_NATIVE_SYNC_EVENTS_ENABLE_IN_TESTS; + const db = makeAllocatorDb(); + const s = new AppSyncState({ + getDb: () => db as never, + isPostgres: () => true, + dbAssignedVersions: true, + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + s.recordChange(baseEvent()); + expect(s.getChangesSince(0).events).toHaveLength(1); + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it("later events keep flowing after a fallback (chain does not wedge)", async () => { + const db = makeAllocatorDb(); + db.failAllocation = true; + const s = new AppSyncState({ + getDb: () => db as never, + isPostgres: () => true, + dbAssignedVersions: true, + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + s.recordChange(baseEvent({ key: "first" })); + await flush(); + db.failAllocation = false; + s.recordChange(baseEvent({ key: "second" })); + await flush(); + + const events = s.getChangesSince(0).events; + expect(events.map((e) => e.key)).toEqual(["first", "second"]); + // Recovery: the second version is DB-allocated and above the fallback one + // (the floor parameter guarantees local monotonicity). + expect(events[1].version).toBeGreaterThan(events[0].version); + warn.mockRestore(); + }); +}); diff --git a/packages/core/src/server/poll.ts b/packages/core/src/server/poll.ts index b04ddeeea6..367f4066a4 100644 --- a/packages/core/src/server/poll.ts +++ b/packages/core/src/server/poll.ts @@ -93,6 +93,42 @@ const ACCESS_CACHE_DENY_TTL_MS = 5_000; const ACCESS_CACHE_MAX = 500; const SCREEN_REFRESH_KEY = "__screen_refresh__"; +/** + * DB-assigned version allocation (see `AppSyncStateOptions.dbAssignedVersions`). + * The allocator is a one-row table advanced with `GREATEST(v + 1, epoch_ms)` — + * the SQL analog of the in-memory `Math.max(version + 1, Date.now())` — inside + * the same autocommit statement as the event insert. The row lock is held to + * the statement's implicit commit, so version order equals commit order across + * every writer, closing the cross-writer clock-skew hole no sequence closes + * (sequences serialize allocation, not commit visibility). Single-statement is + * required: the gateway reaches the app DB through a pooled (transactionless) + * connection. `ON CONFLICT DO UPDATE` (not DO NOTHING) so a deterministic-id + * dedupe loser still gets the winner's version back and never emits a phantom + * version to its own clients. The stored JSON is restamped with the allocated + * version; the read path's column-wins override makes that belt-and-suspenders. + */ +const SEED_SYNC_VERSION_SQL = ` + INSERT INTO sync_version (id, v) + SELECT 1, GREATEST( + COALESCE((SELECT MAX(version) FROM sync_events), 0), + (EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::BIGINT + ) + ON CONFLICT (id) DO NOTHING +`; +const ALLOCATING_INSERT_SQL = ` + WITH alloc AS ( + UPDATE sync_version + SET v = GREATEST(v + 1, (EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::BIGINT, (?)::BIGINT) + WHERE id = 1 + RETURNING v + ) + INSERT INTO sync_events (id, version, event_json, source, type, event_key, owner, org_id, resource_type, resource_id, created_at) + SELECT ?, alloc.v, jsonb_set((?)::jsonb, '{version}', to_jsonb(alloc.v))::text, ?, ?, ?, ?, ?, ?, ?, ? + FROM alloc + ON CONFLICT (id) DO UPDATE SET id = excluded.id + RETURNING version +`; + function timestampValue(value: unknown): number { if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value !== "string") return 0; @@ -341,6 +377,22 @@ export interface AppSyncStateOptions { * across instances. */ deterministicEventIds?: boolean; + /** + * Allocate durable-event versions from the app's Postgres DB (a one-row + * allocator advanced with `GREATEST(v + 1, epoch_ms_now)` inside the insert + * statement) instead of the per-writer in-memory clock counter. + * + * Off by default: a single-writer app's clock counter is already monotonic. + * Hosted realtime has MULTIPLE writers per app DB (the app's serverless + * instances plus gateway instances), where clock skew can assign a LOWER + * version to a LATER event — a client whose cursor passed the higher value + * then filters the later event out forever. DB allocation serializes on the + * allocator row's lock, so version order equals commit order across all + * writers. Versions stay on the epoch-ms scale (existing cursors, the + * detector's timestamp-mixed seed, and lag metrics all assume it). + * Postgres only; ignored on SQLite. + */ + dbAssignedVersions?: boolean; } /** @@ -354,6 +406,13 @@ export class AppSyncState { private readonly isPg: () => boolean; private readonly resolveAccessFn: AccessResolver; private readonly deterministicEventIds: boolean; + private readonly dbAssignedVersions: boolean; + /** Serializes gated recordChange calls so buffer/emit order matches + * allocation order (interleaved allocations could buffer out of order). */ + private recordChain: Promise = Promise.resolve(); + /** Count of DB-allocation failures that fell back to clock versions. */ + private dbVersionFallbacks = 0; + private warnedListenerThrow = false; // Timestamp-aligned versions so all serverless instances produce values in // the same range (seeded from DB, then incremented via Date.now). Plain @@ -415,6 +474,7 @@ export class AppSyncState { this.isPg = options.isPostgres ?? isPostgres; this.resolveAccessFn = options.resolveAccess ?? defaultResolveAccess; this.deterministicEventIds = options.deterministicEventIds ?? false; + this.dbAssignedVersions = options.dbAssignedVersions ?? false; this.pollEmitter.setMaxListeners(0); } @@ -522,6 +582,16 @@ export class AppSyncState { "CREATE INDEX IF NOT EXISTS sync_events_org_version_idx ON sync_events (org_id, version)", guardOptions, ); + if (this.dbAssignedVersions) { + await ensureTableExists( + "sync_version", + "CREATE TABLE IF NOT EXISTS sync_version (id INT PRIMARY KEY, v BIGINT NOT NULL)", + guardOptions, + ); + // Seed at/above both the existing durable max and wall clock, so + // allocated versions never land below live client cursors. + await client.execute(SEED_SYNC_VERSION_SQL); + } return true; } @@ -561,6 +631,7 @@ export class AppSyncState { async persistSyncEvent( event: ChangeEvent, dedupeKey?: string, + presetId?: string, ): Promise { if (!(await this.ensureSyncEventsTable())) return; const client = this.getDb(); @@ -573,7 +644,9 @@ export class AppSyncState { : `INSERT OR IGNORE INTO sync_events (id, version, event_json, source, type, event_key, owner, org_id, resource_type, resource_id, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, args: [ - this.durableEventId(event, dedupeKey), + // presetId lets a gated fallback reuse the allocating attempt's id, + // so a commit-then-timeout can't produce the same event twice. + presetId ?? this.durableEventId(event, dedupeKey), event.version, JSON.stringify(event), event.source, @@ -590,6 +663,69 @@ export class AppSyncState { await this.pruneDurableEvents(client); } + /** + * Persist with a DB-allocated version (see ALLOCATING_INSERT_SQL). Returns + * the allocated version — for a deterministic-id dedupe loser, the winner's + * existing version — or null when persistence is unavailable (caller falls + * back to clock allocation). Unlike `persistSyncEvent`, errors PROPAGATE to + * the caller: silence here would silently drop the event entirely, since the + * buffer/emit are deferred behind this call. + */ + private async persistWithDbAssignedVersion( + event: { source: string; type: string; key?: string; [k: string]: unknown }, + id: string, + ): Promise { + if (!(await this.ensureSyncEventsTable())) return null; + const client = this.getDb(); + const args = [ + // Local monotonicity floor: after a clock fallback, the next allocation + // must not land at/below versions this writer already emitted. + this.version + 1, + id, + // jsonb (unlike the legacy TEXT write) rejects \\u0000 escapes; strip + // them rather than diverting those events to the clock fallback. + JSON.stringify(event).split("\\u0000").join(""), + event.source, + event.type, + event.key ?? null, + (event.owner as string | undefined) ?? null, + (event.orgId as string | undefined) ?? null, + (event.resourceType as string | undefined) ?? null, + (event.resourceId as string | undefined) ?? null, + Date.now(), + ]; + let result = await client.execute({ sql: ALLOCATING_INSERT_SQL, args }); + if (result.rows.length === 0) { + // Allocator row missing (e.g. wiped after ensure): reseed, retry once. + await client.execute(SEED_SYNC_VERSION_SQL).catch(() => {}); + result = await client.execute({ sql: ALLOCATING_INSERT_SQL, args }); + } + const version = timestampValue(result.rows[0]?.version); + await this.pruneDurableEvents(client); + return version > 0 ? version : null; + } + + /** + * Lift the allocator to at least `floor`. Client cursors are max-only, so + * any value that can become a cursor (notably the seed's app-clock + * `updated_at` domain) must never exceed the allocator — a cursor above it + * would filter later, lower-allocated events out permanently. Failure is + * swallowed: alignment degrades to the same soft guarantee as the clock + * fallback. + */ + private async alignVersionAllocator(floor: number): Promise { + if (floor <= 0) return; + try { + if (!(await this.ensureSyncEventsTable())) return; + await this.getDb().execute({ + sql: "UPDATE sync_version SET v = GREATEST(v, (?)::BIGINT) WHERE id = 1", + args: [floor], + }); + } catch { + // Soft guarantee under DB failure, matching the clock fallback. + } + } + async readMaxSyncEventVersion(): Promise { if (!(await this.ensureSyncEventsTable())) return 0; try { @@ -787,14 +923,100 @@ export class AppSyncState { }, opts?: { dedupeKey?: string }, ): void { + if (this.dbAssignedVersions && this.isPg() && !syncEventsDisabled()) { + // No provisional version may reach clients: cursors are max-only, so an + // emitted clock version above a later DB allocation would recreate the + // skew bug. Buffer/emit happen only after the DB returns the version, + // serialized so entries land in allocation order. The catch is a + // backstop: one rejected link must never poison the chain and silently + // drop every later event. + this.recordChain = this.recordChain + .then(() => this.recordWithDbVersion(event, opts?.dedupeKey)) + .catch(() => {}); + return; + } this.version = Math.max(this.version + 1, Date.now()); const entry: ChangeEvent = { ...event, version: this.version }; + this.commitEntry(entry); + void this.persistSyncEvent(entry, opts?.dedupeKey); + } + + /** Buffer + emit. Shared by both version-allocation modes. */ + private commitEntry(entry: ChangeEvent): void { this.buffer.push(entry); if (this.buffer.length > MAX_BUFFER) { this.buffer.splice(0, this.buffer.length - MAX_BUFFER); } this.pollEmitter.emit(POLL_CHANGE_EVENT, entry); - void this.persistSyncEvent(entry, opts?.dedupeKey); + } + + private async recordWithDbVersion( + event: { source: string; type: string; key?: string; [k: string]: unknown }, + dedupeKey?: string, + ): Promise { + // One id for both the allocating attempt AND any fallback persist: if the + // allocating INSERT commits server-side but the call fails (e.g. a query + // timeout after commit), the fallback's ON CONFLICT dedupes against the + // committed row instead of writing the same event twice under two ids. + // Deterministic ids exclude version by design; the random fallback id gets + // a wall-clock prefix instead of its usual version prefix (the prefix is + // debuggability, not semantics — nothing parses row ids). + const id = + this.deterministicEventIds && dedupeKey !== undefined + ? this.durableEventId( + { ...event, version: 0 } as ChangeEvent, + dedupeKey, + ) + : `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; + let version: number | null = null; + try { + version = await this.persistWithDbAssignedVersion(event, id); + } catch { + version = null; + } + if (version == null) { + // Availability fallback: a DB blip must not stall the in-process fast + // path. Clock-allocate and emit; the allocator's GREATEST(v+1, now) + // re-aligns the domain on recovery. The ordering guarantee is soft + // exactly while the DB is unhealthy. + this.dbVersionFallbacks++; + if (this.dbVersionFallbacks === 1) { + console.warn( + "[agent-native] sync version allocation failed; falling back to clock-assigned versions", + ); + } + this.version = Math.max(this.version + 1, Date.now()); + const entry: ChangeEvent = { ...event, version: this.version }; + this.commitEntryForChain(entry); + void this.persistSyncEvent(entry, dedupeKey, id); + return; + } + // A deterministic-id dedupe loser adopts the winner's (possibly older) + // version here — buffer filtering is a full scan, so a non-tail version is + // delivered correctly and the version-keyed combined-read dedupe collapses + // it against the durable row. + this.version = Math.max(this.version, version); + this.commitEntryForChain({ ...event, version }); + } + + /** + * `commitEntry` for the deferred (chained) path: the buffer push must land + * and a throwing pollEmitter listener must not reject the chain — the + * synchronous path propagates such throws to the recordChange caller, but + * here there is no caller to propagate to, only the chain to poison. + */ + private commitEntryForChain(entry: ChangeEvent): void { + try { + this.commitEntry(entry); + } catch (err) { + if (!this.warnedListenerThrow) { + this.warnedListenerThrow = true; + console.warn( + "[agent-native] poll listener threw during deferred emit", + err, + ); + } + } } private recordExtensionChanges( @@ -1079,9 +1301,7 @@ export class AppSyncState { refreshTs = Math.max(refreshTs, timestampValue(row.updated_at)); } - // Seed version — never decrease an already-set value - this.version = Math.max( - this.version, + const seedMax = Math.max( syncEventsTs, appTs, settingsTs, @@ -1089,6 +1309,15 @@ export class AppSyncState { extensionMarkerTs, actionMarkerTs, ); + // The seed mixes app-clock `updated_at` values that can sit AHEAD of the + // allocator (a skew-fast writer's rows). Lift the allocator to the seed + // BEFORE the seed can reach this.version — and through it, client + // cursors — so no later allocation lands below a seeded cursor. + if (this.dbAssignedVersions && this.isPg() && !syncEventsDisabled()) { + await this.alignVersionAllocator(seedMax); + } + // Seed version — never decrease an already-set value + this.version = Math.max(this.version, seedMax); // Set baselines so checkExternalDbChanges detects future writes this.lastAppStateTs = appTs; @@ -1135,9 +1364,20 @@ export class AppSyncState { // second overlapping run that would double-advance the watermarks. if (this.checkPromise) return this.checkPromise; this.lastDbCheck = now; - this.checkPromise = this.doCheckExternalDbChanges().finally(() => { - this.checkPromise = null; - }); + this.checkPromise = this.doCheckExternalDbChanges() + .then(() => { + // Gated: the detector's recordChange calls only SCHEDULED allocations. + // The poll handler awaits this check so detected cross-process writes + // land in the same response — drain the chain here to keep that + // contract (and so a serverless instance frozen after responding + // can't strand them). The chain never rejects (backstopped). + if (this.dbAssignedVersions && this.isPg() && !syncEventsDisabled()) { + return this.recordChain; + } + }) + .finally(() => { + this.checkPromise = null; + }); return this.checkPromise; } @@ -1361,12 +1601,32 @@ export class AppSyncState { let _defaultState: AppSyncState | undefined; +/** + * Hosted-realtime gate for the default instance, env-only. Mirrors the + * fail-closed transport-AND-url pair in `sentry-config.ts`'s + * `resolveRealtimeClientConfig` (not imported — poll.ts is import-cycle + * sensitive). Hosted apps are multi-writer (their own serverless instances + * plus gateway instances share one DB), so THEY must DB-allocate versions too + * — gating only the gateway would leave the app's user-event writes + * clock-versioned and the cross-writer skew hole open. + */ +function hostedRealtimeTransportEnabled(): boolean { + return ( + process.env.AGENT_NATIVE_REALTIME_TRANSPORT?.trim() === "hosted" && + !!process.env.AGENT_NATIVE_REALTIME_GATEWAY_URL?.trim() + ); +} + /** * The process-wide default instance, bound to the global DB. All module-level * exports delegate here so self-hosted apps run exactly one code path. */ export function getDefaultAppSyncState(): AppSyncState { - if (!_defaultState) _defaultState = new AppSyncState(); + if (!_defaultState) { + _defaultState = new AppSyncState({ + dbAssignedVersions: hostedRealtimeTransportEnabled(), + }); + } return _defaultState; } From eca6b4f9b3b841e901ab99bdc7dd2f5b06349a44 Mon Sep 17 00:00:00 2001 From: Korey Kassir Date: Thu, 23 Jul 2026 12:53:04 -0400 Subject: [PATCH 5/9] fix: adjust timeout expectation in runMonitorCheck test for CI consistency --- templates/analytics/server/lib/uptime-monitors.spec.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/analytics/server/lib/uptime-monitors.spec.ts b/templates/analytics/server/lib/uptime-monitors.spec.ts index 99b3f3a4c5..aa9bdb6206 100644 --- a/templates/analytics/server/lib/uptime-monitors.spec.ts +++ b/templates/analytics/server/lib/uptime-monitors.spec.ts @@ -544,7 +544,10 @@ describe("runMonitorCheck timeout budget", () => { kind: "timeout", message: "Timed out after 1000ms", }); - expect(outcome.diagnostics.timings.requestMs).toBeGreaterThanOrEqual(1000); + // Real timers: the abort timer and this Date.now() delta are anchored at + // slightly different instants, so the measured request time can land a few + // ms shy of the nominal 1000ms budget on loaded CI runners. + expect(outcome.diagnostics.timings.requestMs).toBeGreaterThanOrEqual(950); }); }); From 1de8f8c21233ea28a6049a81cc04d04434daae10 Mon Sep 17 00:00:00 2001 From: Korey Kassir Date: Thu, 23 Jul 2026 13:53:41 -0400 Subject: [PATCH 6/9] fix: increase timer step for polling in tests to improve CI performance --- packages/core/src/a2a/client.spec.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/core/src/a2a/client.spec.ts b/packages/core/src/a2a/client.spec.ts index 2558d8f63b..4286353fe8 100644 --- a/packages/core/src/a2a/client.spec.ts +++ b/packages/core/src/a2a/client.spec.ts @@ -186,7 +186,10 @@ describe("A2AClient", () => { init?.method === "POST" && JSON.parse(String(init.body)).method === "tasks/get", ); - while (!hasTaskRead()) await vi.advanceTimersByTimeAsync(1); + // Coarse steps: the first poll lands at pollIntervalMs (1s of fake time), + // and 1ms steps are ~1000 queue drains — enough to blow the real-time test + // timeout on a loaded CI worker. Overshooting the poll instant is harmless. + while (!hasTaskRead()) await vi.advanceTimersByTimeAsync(50); await vi.advanceTimersByTimeAsync(5_000); await assertion; expect(hasTaskRead()).toBe(true); @@ -249,7 +252,8 @@ describe("A2AClient", () => { }, }); - while (taskReads === 0) await vi.advanceTimersByTimeAsync(1); + // Coarse steps for the same CI-load reason as the hung-poll test above. + while (taskReads === 0) await vi.advanceTimersByTimeAsync(50); await vi.advanceTimersByTimeAsync(16_000); await assertion; expect(firstPollSignal?.aborted).toBe(true); From 09b32ef3209fc09c1c7b3fd23e3ddcc923813cb2 Mon Sep 17 00:00:00 2001 From: Korey Kassir Date: Thu, 23 Jul 2026 14:49:04 -0400 Subject: [PATCH 7/9] feat: implement recovery of committed version for failed allocations in AppSyncState --- .../src/server/db-assigned-versions.spec.ts | 65 +++++++++++++++++++ packages/core/src/server/poll.ts | 30 ++++++++- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/packages/core/src/server/db-assigned-versions.spec.ts b/packages/core/src/server/db-assigned-versions.spec.ts index baf333a832..8139bcf099 100644 --- a/packages/core/src/server/db-assigned-versions.spec.ts +++ b/packages/core/src/server/db-assigned-versions.spec.ts @@ -16,6 +16,8 @@ function makeAllocatorDb(shared?: { v: number; ids: Map }) { state, log, failAllocation: false, + /** Simulates commit-then-timeout: the row lands, then the call throws. */ + failAllocationAfterCommit: false, async execute(query: string | { sql: string; args?: unknown[] }) { const sql = typeof query === "string" ? query : query.sql; const args = typeof query === "string" ? [] : (query.args ?? []); @@ -30,6 +32,17 @@ function makeAllocatorDb(shared?: { v: number; ids: Map }) { if (state.v === 0) state.v = Date.now(); return { rows: [], rowsAffected: 1 }; } + if (sql.includes("UPDATE sync_version SET v = GREATEST")) { + state.v = Math.max(state.v, Number(args[0])); + return { rows: [], rowsAffected: 1 }; + } + if (sql.includes("SELECT version FROM sync_events WHERE id")) { + const committed = state.ids.get(String(args[0])); + return { + rows: committed !== undefined ? [{ version: committed }] : [], + rowsAffected: 0, + }; + } if (sql.includes("WITH alloc")) { if (db.failAllocation) throw new Error("neon unavailable"); const floor = Number(args[0]); @@ -41,6 +54,9 @@ function makeAllocatorDb(shared?: { v: number; ids: Map }) { return { rows: [{ version: existing }], rowsAffected: 1 }; } state.ids.set(id, state.v); + if (db.failAllocationAfterCommit) { + throw new Error("timeout after commit"); + } return { rows: [{ version: state.v }], rowsAffected: 1 }; } // Legacy INSERT / DELETE prune / anything else. @@ -214,6 +230,55 @@ describe("dbAssignedVersions", () => { warn.mockRestore(); }); + it("commit-then-timeout recovers the durable row's version instead of clock-falling-back", async () => { + const db = makeAllocatorDb(); + db.failAllocationAfterCommit = true; + const s = new AppSyncState({ + getDb: () => db as never, + isPostgres: () => true, + dbAssignedVersions: true, + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + s.recordChange(baseEvent()); + await flush(); + + const events = s.getChangesSince(0).events; + expect(events).toHaveLength(1); + // Emitted version is the COMMITTED allocator version, not a divergent + // clock value — and no fallback fired. + expect(events[0].version).toBe(db.state.v); + expect(warn).not.toHaveBeenCalled(); + // No second durable write: recovery, not the legacy fallback insert. + expect( + db.log.some( + (q) => + q.sql.includes("INSERT INTO sync_events") && + !q.sql.includes("WITH alloc"), + ), + ).toBe(false); + warn.mockRestore(); + }); + + it("a true allocation failure lifts the allocator to the fallback version before emitting", async () => { + const db = makeAllocatorDb(); + db.failAllocation = true; + const s = new AppSyncState({ + getDb: () => db as never, + isPostgres: () => true, + dbAssignedVersions: true, + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + s.recordChange(baseEvent()); + await flush(); + + const emitted = s.getChangesSince(0).events[0]?.version ?? 0; + expect(emitted).toBeGreaterThan(0); + // The shared allocator was lifted to the fallback value, so other writers' + // next allocations land above the cursors this emit advanced. + expect(db.state.v).toBeGreaterThanOrEqual(emitted); + warn.mockRestore(); + }); + it("a throwing poll listener does not poison the chain", async () => { const db = makeAllocatorDb(); const s = new AppSyncState({ diff --git a/packages/core/src/server/poll.ts b/packages/core/src/server/poll.ts index 367f4066a4..75d73b02fa 100644 --- a/packages/core/src/server/poll.ts +++ b/packages/core/src/server/poll.ts @@ -705,6 +705,21 @@ export class AppSyncState { return version > 0 ? version : null; } + /** Read back the version of an already-committed row (a failed allocating + * call whose statement actually committed). Null when absent/unreachable. */ + private async recoverCommittedVersion(id: string): Promise { + try { + const result = await this.getDb().execute({ + sql: "SELECT version FROM sync_events WHERE id = ?", + args: [id], + }); + const version = timestampValue(result.rows[0]?.version); + return version > 0 ? version : null; + } catch { + return null; + } + } + /** * Lift the allocator to at least `floor`. Client cursors are max-only, so * any value that can become a cursor (notably the seed's app-clock @@ -974,10 +989,20 @@ export class AppSyncState { } catch { version = null; } + if (version == null) { + // The statement may have COMMITTED even though the call failed (e.g. a + // timeout after commit). Recover the durable row's version by the shared + // id so clients see the allocator-assigned value, never a divergent + // clock one. + version = await this.recoverCommittedVersion(id); + } if (version == null) { // Availability fallback: a DB blip must not stall the in-process fast - // path. Clock-allocate and emit; the allocator's GREATEST(v+1, now) - // re-aligns the domain on recovery. The ordering guarantee is soft + // path. Clock-allocate and emit — but first make a best effort to lift + // the allocator to the fallback value, so other writers' next + // allocations land above the cursors this emit will advance (max-only + // cursors above the allocator would filter those events permanently). + // If the DB is down the lift fails too; the ordering guarantee is soft // exactly while the DB is unhealthy. this.dbVersionFallbacks++; if (this.dbVersionFallbacks === 1) { @@ -987,6 +1012,7 @@ export class AppSyncState { } this.version = Math.max(this.version + 1, Date.now()); const entry: ChangeEvent = { ...event, version: this.version }; + await this.alignVersionAllocator(this.version); this.commitEntryForChain(entry); void this.persistSyncEvent(entry, dedupeKey, id); return; From a7dd1d793ea706db969173eb6c78c06d350a9729 Mon Sep 17 00:00:00 2001 From: Korey Kassir Date: Fri, 24 Jul 2026 10:04:00 -0400 Subject: [PATCH 8/9] test: improve timer handling in A2AClient tests for better CI performance --- packages/core/src/a2a/client.spec.ts | 43 ++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/packages/core/src/a2a/client.spec.ts b/packages/core/src/a2a/client.spec.ts index 4286353fe8..5d15f45e93 100644 --- a/packages/core/src/a2a/client.spec.ts +++ b/packages/core/src/a2a/client.spec.ts @@ -9,6 +9,21 @@ import { signA2AToken, } from "./client.js"; +// Captured before fake timers install (which fake setImmediate): the request +// path does real async work outside the mocked fetch (ssrf host checks), so +// fake-time advancing loops must yield REAL event-loop turns or fake time +// races ahead of it on loaded CI runners — aborting requests before the mock +// is ever reached. +const realSetImmediate = setImmediate; +const yieldReal = () => new Promise((r) => realSetImmediate(r)); +async function advanceSettled(vitest: typeof vi, totalMs: number) { + const step = 500; + for (let advanced = 0; advanced < totalMs; advanced += step) { + await vitest.advanceTimersByTimeAsync(Math.min(step, totalMs - advanced)); + await yieldReal(); + } +} + describe("A2AClient", () => { const originalEnv = { ...process.env }; @@ -179,6 +194,9 @@ describe("A2AClient", () => { lastState: "working", timeoutMs: 5_000, }); + // Mark handled: if the promise settles unexpectedly mid-advance, the + // failure must surface at the await below, not as an unhandled rejection. + assertion.catch(() => {}); const hasTaskRead = () => fetchMock.mock.calls.some( @@ -186,11 +204,12 @@ describe("A2AClient", () => { init?.method === "POST" && JSON.parse(String(init.body)).method === "tasks/get", ); - // Coarse steps: the first poll lands at pollIntervalMs (1s of fake time), - // and 1ms steps are ~1000 queue drains — enough to blow the real-time test - // timeout on a loaded CI worker. Overshooting the poll instant is harmless. - while (!hasTaskRead()) await vi.advanceTimersByTimeAsync(50); - await vi.advanceTimersByTimeAsync(5_000); + for (let i = 0; !hasTaskRead(); i++) { + expect(i).toBeLessThan(5_000); + await yieldReal(); + await vi.advanceTimersByTimeAsync(10); + } + await advanceSettled(vi, 5_000); await assertion; expect(hasTaskRead()).toBe(true); expect( @@ -251,10 +270,16 @@ describe("A2AClient", () => { }, }, }); - - // Coarse steps for the same CI-load reason as the hung-poll test above. - while (taskReads === 0) await vi.advanceTimersByTimeAsync(50); - await vi.advanceTimersByTimeAsync(16_000); + // Mark handled: if the promise settles unexpectedly mid-advance, the + // failure must surface at the await below, not as an unhandled rejection. + assertion.catch(() => {}); + + for (let i = 0; taskReads === 0; i++) { + expect(i).toBeLessThan(5_000); + await yieldReal(); + await vi.advanceTimersByTimeAsync(10); + } + await advanceSettled(vi, 16_000); await assertion; expect(firstPollSignal?.aborted).toBe(true); expect(taskReads).toBe(2); From 6678ba5fa47fe3d062bcd7601a9cd603e70a95a2 Mon Sep 17 00:00:00 2001 From: Korey Kassir Date: Fri, 24 Jul 2026 10:52:47 -0400 Subject: [PATCH 9/9] fix: enhance ssrfSafeFetch to block private/internal address requests --- packages/core/src/a2a/client.spec.ts | 29 +++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/core/src/a2a/client.spec.ts b/packages/core/src/a2a/client.spec.ts index c05b45c060..0f59470289 100644 --- a/packages/core/src/a2a/client.spec.ts +++ b/packages/core/src/a2a/client.spec.ts @@ -9,11 +9,30 @@ import { signA2AToken, } from "./client.js"; -// Captured before fake timers install (which fake setImmediate): the request -// path does real async work outside the mocked fetch (ssrf host checks), so -// fake-time advancing loops must yield REAL event-loop turns or fake time -// races ahead of it on loaded CI runners — aborting requests before the mock -// is ever reached. +// ssrfSafeFetch does a REAL node:dns lookup before calling fetch. Under fake +// timers that wall-clock work can take seconds on CI resolvers (agent.test is +// not a real host), so fake time races past request timeouts and deadlines +// before the stubbed fetch is ever reached. Keep the synchronous private-host +// check (the blocking test relies on it; IP literals need no DNS) and skip +// only the DNS phase — full SSRF behavior is covered by url-safety's own spec. +vi.mock("../extensions/url-safety.js", async (importOriginal) => { + const original = + await importOriginal(); + return { + ...original, + ssrfSafeFetch: async (url: string, init?: RequestInit) => { + if (original.isBlockedExtensionUrl(url)) { + throw new Error( + `SSRF blocked: refusing to fetch private/internal address (${url})`, + ); + } + return fetch(url, init); + }, + }; +}); + +// Captured before fake timers install (which fake setImmediate), so waiting +// loops still yield real event-loop turns for any residual real async. const realSetImmediate = setImmediate; const yieldReal = () => new Promise((r) => realSetImmediate(r)); async function advanceSettled(vitest: typeof vi, totalMs: number) {