feat(nextly): one counter every instance reads, so a revocation reaches all of them

.changeset/one-counter-every-instance-reads.md

@@ -0,0 +1,60 @@
+---
+"@nextlyhq/adapter-drizzle": patch
+"@nextlyhq/adapter-mysql": patch
+"@nextlyhq/adapter-postgres": patch
+"@nextlyhq/adapter-sqlite": patch
+"@nextlyhq/admin": patch
+"@nextlyhq/admin-css": patch
+"@nextlyhq/blocks-engine": patch
+"@nextlyhq/blocks-react": patch
+"@nextlyhq/builder": patch
+"@nextlyhq/eslint-config": patch
+"@nextlyhq/eslint-plugin": patch
+"@nextlyhq/module-specifiers": patch
+"@nextlyhq/plugin-form-builder": patch
+"@nextlyhq/plugin-mcp": patch
+"@nextlyhq/plugin-page-builder": patch
+"@nextlyhq/plugin-sdk": patch
+"@nextlyhq/plugin-seo": patch
+"@nextlyhq/prettier-config": patch
+"@nextlyhq/storage-s3": patch
+"@nextlyhq/storage-uploadthing": patch
+"@nextlyhq/storage-vercel-blob": patch
+"@nextlyhq/telemetry": patch
+"@nextlyhq/tsconfig": patch
+"@nextlyhq/ui": patch
+"create-nextly-app": patch
+"nextly": patch
+---
+
+The signal that retires a cached authorization answer is now stored in the
+database, so every instance sees it. It was a counter held in memory, which
+moved only in the process that handled the change: a second instance neither
+saw the move nor had one of its own, and went on serving what it had cached
+until the entry aged out. On the shared tier that meant a revoked grant could
+outlive its revocation by the whole cache lifetime.
+
+Cross-instance revocation now takes effect within about a second. Each instance
+reads the shared counter at most once per second rather than once per check, so
+the cost is one small indexed read per second per instance and not one per
+request. The instance that MADE the change applies it immediately.
+
+Two behaviour changes worth knowing about.
+
+An invalidation naming one user now retires every in-memory answer rather than
+that user's alone. The counter other instances read carries a number and not a
+user id, so a change they can see cannot be narrower than "something in RBAC
+moved", and keeping the scope locally would mean only the instance that made the
+change applied it narrowly. Refilling is a couple of indexed queries and role
+changes are rare; a stale answer costs a grant the install revoked.
+
+A batch of permission writes no longer holds back the in-memory tiers. It never
+existed to: what it saves is the unfiltered rewrite of every stored row, and
+that is still deferred to the end of the batch.
+
+Installations upgraded from an earlier version keep working before they
+reconcile their core tables. The new table arrives through `nextly db:sync`, and
+until it does, every read and write of the counter degrades to the previous
+in-memory behaviour rather than failing the authorization check that asked. The
+degraded state is reported once so an operator can see why cross-instance
+invalidation is not yet in effect.

packages/nextly/src/database/sqlite-core-tables.ts

@@ -1,3 +1,4 @@
+import { RBAC_EPOCH_TABLE } from "../schemas/rbac-epoch/table-name";
 import { STORAGE_FORMAT } from "../schemas/storage-format";
 
 // Raw CREATE TABLE IF NOT EXISTS DDL for all Nextly core SQLite tables.
@@ -238,6 +239,22 @@ export function generateSqliteCoreTableStatements(): string[] {
       ON "user_permission_cache" ("expires_at")`,
     `CREATE INDEX IF NOT EXISTS "upc_user_action_resource_idx"
       ON "user_permission_cache" ("user_id", "action", "resource")`,
+    // The counter every instance reads to decide whether a cached
+    // authorization answer is still current. One row, keyed `global`: the
+    // primary key is what makes a second counter unrepresentable, so nothing
+    // has to keep "there is exactly one" true.
+    //
+    // Reached by an existing installation because this whole bootstrap is
+    // re-run as a reconciliation, and `IF NOT EXISTS` adds only what is absent.
+    // That is why the epoch is a TABLE rather than a column on the rows above:
+    // SQLite skips a `CREATE TABLE` wholesale once the table exists, so a new
+    // column there would never arrive.
+    `CREATE TABLE IF NOT EXISTS "${RBAC_EPOCH_TABLE}" (
+      "id" TEXT PRIMARY KEY,
+      "revision" INTEGER NOT NULL,
+      "generation" TEXT NOT NULL,
+      "updated_at" INTEGER NOT NULL
+    )`,
     `CREATE TABLE IF NOT EXISTS "content_schema_events" (
       "id" INTEGER PRIMARY KEY AUTOINCREMENT,
       "op" TEXT NOT NULL,

packages/nextly/src/domains/auth/services/api-key-service.ts

@@ -69,8 +69,8 @@ import { BaseService } from "../../../services/base-service";
 import {
   isSuperAdmin,
   listRoleSlugsForUserOrRefuse,
-  rbacRevision,
 } from "../../../services/lib/permissions";
+import { refreshEpoch, stampIsCurrent } from "../../../services/lib/rbac-epoch";
 import type { Logger } from "../../../services/shared";
 
 /** The three token types that determine how permissions are resolved at request time. */
@@ -281,7 +281,7 @@ const _apiKeyPermissionsCache = new Map<
      * old set, which the comment in `UserRoleService` said was handled
      * elsewhere and was not.
      */
-    revision: number;
+    revision: string;
   }
 >();
 const _PERMISSIONS_CACHE_TTL_MS = 5 * 60 * 1000;
@@ -748,20 +748,37 @@ export class ApiKeyService extends BaseService {
     keyId: string
   ): Promise<readonly GrantedPermission[]> {
     const cacheKey = `apikey:${keyId}`;
-    const now = Date.now();
     // Read BEFORE the queries below, never after. An invalidation that lands
     // while they are in flight would otherwise be stamped onto the result they
     // return: the rows were read under the old revision and would be filed
     // under the new one, so the next request reuses grants the change was
     // meant to retire, for the whole TTL. Captured here, that entry is already
     // behind when it is written and the next read re-resolves.
-    const resolvedUnder = rbacRevision();
+    //
+    // Refreshed rather than read, which is what makes the comparison below
+    // answer for the INSTALL rather than for this process. These grants are a
+    // copy of the catalogue held for five minutes, and a role revoked on
+    // another instance has to retire them here too.
+    const resolvedUnder = await refreshEpoch();
+
+    // Read AFTER the refresh, not before it. That refresh can wait — on a slow
+    // database, or behind another forced read it queued for — and an age taken
+    // beforehand is the age the entry had when the request started rather than
+    // when it is being served, so an entry that expired during the wait passes
+    // the window one more time.
+    const now = Date.now();
 
     const cached = _apiKeyPermissionsCache.get(cacheKey);
     if (
       cached &&
-      now - cached.cachedAt < _PERMISSIONS_CACHE_TTL_MS &&
-      cached.revision === resolvedUnder
+      // Asked through the shared predicate, not by comparing the stamp. A
+      // match means nothing while this process holds invalidations the shared
+      // row has not accepted: the value being matched against is then one no
+      // other instance has seen, so a revocation made here keeps answering
+      // from this copy for the whole window. The permission tiers ask the same
+      // question and this one was not.
+      stampIsCurrent(cached.revision) &&
+      now - cached.cachedAt < _PERMISSIONS_CACHE_TTL_MS
     ) {
       return cached.grants;
     }

packages/nextly/src/domains/auth/services/permission-cache-service.ts

@@ -288,55 +288,44 @@ export class PermissionCacheService extends BaseService {
    * sees an expired row rather than a missing one.
    */
   async invalidateAll(): Promise<number> {
-    try {
-      const { userPermissionCache } = this.tables;
-      const result = await this.db
-        .update(userPermissionCache)
-        .set({ expiresAt: new Date() });
-      return affectedRowCount(result, this.dialect);
-    } catch (error) {
-      this.logger.error("Failed to invalidate the whole permission cache", {
-        error: String(error),
-      });
-      return 0;
-    }
+    // A failure is RAISED rather than reported as zero. Zero is also what a
+    // successful tombstone over an empty table answers, so swallowing the
+    // error makes the two indistinguishable — and the caller that publishes the
+    // epoch afterwards then announces a retirement that did not happen, which
+    // is the one announcement that must never be made.
+    const { userPermissionCache } = this.tables;
+    const result = await this.db
+      .update(userPermissionCache)
+      .set({ expiresAt: new Date() });
+    return affectedRowCount(result, this.dialect);
   }
 
   async invalidateByUser(userId: string): Promise<number> {
     if (!userId) {
       return 0;
     }
 
-    try {
-      const { userPermissionCache } = this.tables;
+    // Raised rather than reported as zero, for the reason `invalidateAll`
+    // gives: zero is also a successful tombstone that matched no rows.
+    const { userPermissionCache } = this.tables;
 
-      // Write-through invalidation: mark as expired (tombstone) instead of deleting
-      const result = await this.db
-        .update(userPermissionCache)
-        .set({ expiresAt: new Date() })
-        .where(eq(userPermissionCache.userId, userId));
+    // Write-through invalidation: mark as expired (tombstone) instead of deleting
+    const result = await this.db
+      .update(userPermissionCache)
+      .set({ expiresAt: new Date() })
+      .where(eq(userPermissionCache.userId, userId));
 
-      const invalidatedCount = affectedRowCount(result, this.dialect);
-
-      if (process.env.DEBUG_CACHE === "1") {
-        console.log("[cache][dbg] invalidateByUser", {
-          userId,
-          invalidatedCount,
-          method: "tombstone",
-        });
-      }
+    const invalidatedCount = affectedRowCount(result, this.dialect);
 
-      return invalidatedCount;
-    } catch (error) {
-      getAuthLogger()?.log?.("error", {
-        category: "auth",
-        op: "cache",
-        message: "invalidateByUser failed",
+    if (process.env.DEBUG_CACHE === "1") {
+      console.log("[cache][dbg] invalidateByUser", {
         userId,
-        error: String(error),
+        invalidatedCount,
+        method: "tombstone",
       });
-      return 0;
     }
+
+    return invalidatedCount;
   }
 
   /**
@@ -364,7 +353,9 @@ export class PermissionCacheService extends BaseService {
       return 0;
     }
 
-    try {
+    // Raised rather than reported as zero, for the reason `invalidateAll`
+    // gives: zero is also a successful tombstone that matched no rows.
+    {
       const { userPermissionCache } = this.tables;
 
       // Write-through invalidation: mark as expired (tombstone) instead of deleting.
@@ -408,15 +399,6 @@ export class PermissionCacheService extends BaseService {
       }
 
       return invalidatedCount;
-    } catch (error) {
-      getAuthLogger()?.log?.("error", {
-        category: "auth",
-        op: "cache",
-        message: "invalidateByRole failed",
-        roleId,
-        error: String(error),
-      });
-      return 0;
     }
   }
 

packages/nextly/src/domains/auth/services/role-inheritance-service.ts

@@ -96,7 +96,7 @@ export class RoleInheritanceService extends BaseService {
       }
     }
 
-    void invalidatePermissionCache({ roleId: childRoleId });
+    await invalidatePermissionCache({ roleId: childRoleId });
   }
 
   /**
@@ -118,7 +118,7 @@ export class RoleInheritanceService extends BaseService {
         )
       );
 
-    void invalidatePermissionCache({ roleId: childRoleId });
+    await invalidatePermissionCache({ roleId: childRoleId });
   }
 
   /**

packages/nextly/src/domains/auth/services/role-permission-service.ts

@@ -160,7 +160,7 @@ export class RolePermissionService extends BaseService {
       permissionId = newPermId;
     }
 
-    void invalidatePermissionCache({ roleId });
+    await invalidatePermissionCache({ roleId });
   }
 
   /**
@@ -244,7 +244,7 @@ export class RolePermissionService extends BaseService {
         )
       );
 
-    void invalidatePermissionCache({ roleId });
+    await invalidatePermissionCache({ roleId });
   }
 
   /**
@@ -285,7 +285,7 @@ export class RolePermissionService extends BaseService {
       }
     }
 
-    void invalidatePermissionCache({ roleId });
+    await invalidatePermissionCache({ roleId });
 
     return this.listRolePermissions(roleId);
   }

packages/nextly/src/domains/auth/services/role/role-mutation-service.ts

@@ -340,10 +340,11 @@ export class RoleMutationService extends BaseService {
           : // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Drizzle transaction callback type varies by dialect
             await this.db.transaction(async (tx: any) => runMutations(tx));
 
-        // Invalidate cache after successful transaction. `void` marks the
-        // promise as intentionally unawaited - cache invalidation is
-        // fire-and-forget and must not block the create response.
-        void invalidatePermissionCache({ roleId: id });
+        // Awaited, and the ordering is the invariant rather than a preference:
+        // a runtime that freezes after responding can abandon an unawaited
+        // shared write, leaving every other instance on the old epoch with no
+        // sign anything went wrong.
+        await invalidatePermissionCache({ roleId: id });
 
         return {
           id,
@@ -605,7 +606,7 @@ export class RoleMutationService extends BaseService {
           changes.permissionIds !== undefined ||
           changes.childRoleIds !== undefined
         ) {
-          void invalidatePermissionCache({ roleId });
+          await invalidatePermissionCache({ roleId });
         }
 
         return;
@@ -689,8 +690,9 @@ export class RoleMutationService extends BaseService {
         await tx.delete("roles", this.whereEq("id", roleId));
       });
 
-      // Invalidate cache after successful transaction (fire-and-forget).
-      void invalidatePermissionCache({ roleId });
+      // Awaited, for the reason `createRole` gives: an abandoned shared write
+      // leaves the other instances holding answers this delete retired.
+      await invalidatePermissionCache({ roleId });
     } catch (e: unknown) {
       // Re-throw NextlyErrors unchanged. Raw DB errors map via
       // fromDatabaseError, which provides the spec-compliant generic public

packages/nextly/src/domains/auth/services/user-role-service.ts

@@ -115,7 +115,7 @@ export class UserRoleService extends BaseService {
         await insert;
       }
 
-      void invalidatePermissionCache({ userId });
+      await invalidatePermissionCache({ userId });
 
       // Invalidate API key permission caches for this user's read-only and
       // full-access keys — their effective permissions derive from the creator's
@@ -175,7 +175,7 @@ export class UserRoleService extends BaseService {
           )
         );
 
-      void invalidatePermissionCache({ userId });
+      await invalidatePermissionCache({ userId });
 
       // Invalidate API key permission caches for this user's read-only and
       // full-access keys — their effective permissions derive from the creator's

packages/nextly/src/schemas/_dialect-bundles/mysql.ts

@@ -17,6 +17,7 @@ export { users, accounts, sessions } from "../users/mysql";
 // `reconcileCore` hands drizzle-kit.
 export { nextlyFieldGroupLock } from "../field-group-lock/mysql";
 export { nextlyDocumentLock } from "../document-lock/mysql";
+export { nextlyRbacEpoch } from "../rbac-epoch/mysql";
 export { nextlyWidgetLayout } from "../widget-layout/mysql";
 
 export {

packages/nextly/src/schemas/_dialect-bundles/postgres.ts

@@ -32,6 +32,7 @@ export { users, accounts, sessions } from "../users/postgres";
 // `reconcileCore` hands drizzle-kit.
 export { nextlyFieldGroupLock } from "../field-group-lock/postgres";
 export { nextlyDocumentLock } from "../document-lock/postgres";
+export { nextlyRbacEpoch } from "../rbac-epoch/postgres";
 export { nextlyWidgetLayout } from "../widget-layout/postgres";
 
 // Auth tokens.

packages/nextly/src/schemas/_dialect-bundles/sqlite.ts

@@ -17,6 +17,7 @@ export { users, accounts, sessions } from "../users/sqlite";
 // `reconcileCore` hands drizzle-kit.
 export { nextlyFieldGroupLock } from "../field-group-lock/sqlite";
 export { nextlyDocumentLock } from "../document-lock/sqlite";
+export { nextlyRbacEpoch } from "../rbac-epoch/sqlite";
 export { nextlyWidgetLayout } from "../widget-layout/sqlite";
 
 export {