From 8ff05c0ec3823d5c1bc86bb0b60f7f9b29879c54 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 30 Sep 2026 14:01:17 -0400 Subject: [PATCH 01/50] Start pulling cert at 15 days (due to cert update) --- server/lib/traefik/TraefikConfigManager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/lib/traefik/TraefikConfigManager.ts b/server/lib/traefik/TraefikConfigManager.ts index fa742e3359..61381d2b0e 100644 --- a/server/lib/traefik/TraefikConfigManager.ts +++ b/server/lib/traefik/TraefikConfigManager.ts @@ -297,7 +297,7 @@ export class TraefikConfigManager { const secondsUntilExpiry = localState.expiresAt - nowInSeconds; const daysUntilExpiry = secondsUntilExpiry / (60 * 60 * 24); - if (daysUntilExpiry < 45) { + if (daysUntilExpiry < 15) { logger.info( `Fetching certificates due to upcoming expiry for ${domain} (${Math.round(daysUntilExpiry)} days remaining)` ); @@ -314,7 +314,7 @@ export class TraefikConfigManager { const nowInSeconds = Math.floor(Date.now() / 1000); const secondsUntilExpiry = state.expiresAt - nowInSeconds; const daysUntilExpiry = secondsUntilExpiry / (60 * 60 * 24); - if (daysUntilExpiry < 45) { + if (daysUntilExpiry < 15) { logger.info( `Fetching certificates due to upcoming expiry for wildcard cert ${certDomain} (${Math.round(daysUntilExpiry)} days remaining)` ); From 08ae5cc3a31f4e7c3b7401630d2f3cff2feef26b Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Wed, 30 Sep 2026 20:35:06 +0200 Subject: [PATCH 02/50] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/startSchedulers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/startSchedulers.ts b/server/startSchedulers.ts index 4a67495cc0..3f0dcfb1a2 100644 --- a/server/startSchedulers.ts +++ b/server/startSchedulers.ts @@ -12,7 +12,7 @@ export function startSchedulers() { // updates into periodic bulk writes, preventing connection pool exhaustion. startPingAccumulator(); - if (build != "saas") { + if (build !== "saas") { startOlmOfflineChecker(); // this is to handle the offline check for olms startNewtOfflineChecker(); // this is to handle the offline check for newts } From 1aa1ef0249a36aaf1259955fb85e485acad4cb39 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Wed, 30 Sep 2026 21:13:17 +0200 Subject: [PATCH 03/50] =?UTF-8?q?=F0=9F=9A=A7=20wip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/db/pg/schema/schema.ts | 5 ++++- server/db/sqlite/schema/schema.ts | 7 ++++++- src/app/admin/idp/page.tsx | 4 ++-- src/components/AdminIdpTable.tsx | 23 ++++++++++------------- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 145b357566..175ef938c7 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1213,7 +1213,10 @@ export const idp = pgTable("idp", { defaultRoleMapping: varchar("defaultRoleMapping"), defaultOrgMapping: varchar("defaultOrgMapping"), autoProvision: boolean("autoProvision").notNull().default(false), - tags: text("tags") + tags: text("tags"), + directorySyncEnabled: boolean("directorySyncEnabled") + .notNull() + .default(false) }); export const idpOidcConfig = pgTable("idpOidcConfig", { diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index a72e15556a..ce7526fd50 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1520,7 +1520,12 @@ export const idp = sqliteTable("idp", { }) .notNull() .default(false), - tags: text("tags") + tags: text("tags"), + directorySyncEnabled: integer("directorySyncEnabled", { + mode: "boolean" + }) + .notNull() + .default(false) }); // Identity Provider OAuth Configuration diff --git a/src/app/admin/idp/page.tsx b/src/app/admin/idp/page.tsx index 01aac2a1e6..8fda09cd62 100644 --- a/src/app/admin/idp/page.tsx +++ b/src/app/admin/idp/page.tsx @@ -2,7 +2,7 @@ import { internal } from "@app/lib/api"; import { authCookieHeader } from "@app/lib/api/cookies"; import { AxiosResponse } from "axios"; import SettingsSectionTitle from "@app/components/SettingsSectionTitle"; -import IdpTable, { IdpRow } from "@app/components/AdminIdpTable"; +import AdminIdpTable, { IdpRow } from "@app/components/AdminIdpTable"; import { getTranslations } from "next-intl/server"; export default async function IdpPage() { @@ -25,7 +25,7 @@ export default async function IdpPage() { title={t("idpManage")} description={t("idpManageDescription")} /> - + ); } diff --git a/src/components/AdminIdpTable.tsx b/src/components/AdminIdpTable.tsx index 75a7c54543..8dcc28d156 100644 --- a/src/components/AdminIdpTable.tsx +++ b/src/components/AdminIdpTable.tsx @@ -1,26 +1,23 @@ "use client"; -import { ColumnDef } from "@tanstack/react-table"; -import { ExtendedColumnDef } from "@app/components/ui/data-table"; import { IdpDataTable } from "@app/components/AdminIdpDataTable"; -import { Button } from "@app/components/ui/button"; -import { ArrowRight, ArrowUpDown, MoreHorizontal } from "lucide-react"; -import { useState } from "react"; import ConfirmDeleteDialog from "@app/components/ConfirmDeleteDialog"; -import { toast } from "@app/hooks/useToast"; -import { formatAxiosError } from "@app/lib/api"; -import { createApiClient } from "@app/lib/api"; -import { useEnvContext } from "@app/hooks/useEnvContext"; -import { Badge } from "@app/components/ui/badge"; -import { useRouter } from "next/navigation"; +import { Button } from "@app/components/ui/button"; +import { ExtendedColumnDef } from "@app/components/ui/data-table"; import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from "@app/components/ui/dropdown-menu"; -import Link from "next/link"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { ArrowRight, ArrowUpDown, MoreHorizontal } from "lucide-react"; import { useTranslations } from "next-intl"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useState } from "react"; import IdpTypeBadge from "./IdpTypeBadge"; export type IdpRow = { @@ -35,7 +32,7 @@ type Props = { idps: IdpRow[]; }; -export default function IdpTable({ idps }: Props) { +export default function AdminIdpTable({ idps }: Props) { const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); const [selectedIdp, setSelectedIdp] = useState(null); const api = createApiClient(useEnvContext()); From 6616cfa2143905d7205e36052369eb7e978c5396 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Wed, 30 Sep 2026 22:06:13 +0200 Subject: [PATCH 04/50] =?UTF-8?q?=F0=9F=9A=A7=20wip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- messages/en-US.json | 2 + server/db/pg/schema/schema.ts | 2 +- server/db/sqlite/schema/schema.ts | 2 +- server/routers/idp/createOidcIdp.ts | 7 ++++ server/routers/idp/updateOidcIdp.ts | 19 +++++++++ src/app/admin/idp/[idpId]/general/page.tsx | 47 ++++++++++++++++++++-- src/app/admin/idp/create/page.tsx | 39 ++++++++++++++++-- 7 files changed, 108 insertions(+), 10 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index bfb07ef741..28176b2610 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1176,6 +1176,8 @@ "idpAutoProvisionUsers": "Auto Provision Users", "idpAutoProvisionUsersDescription": "When enabled, users will be automatically created in the system upon first login with the ability to map users to roles and organizations.", "idpAutoProvisionConfigureAfterCreate": "You can configure auto provision settings once the identity provider is created.", + "idpDirectorySync": "Directory Sync", + "idpDirectorySyncDescription": "Periodically sync users from the identity provider directory. Requires auto provisioning to be enabled.", "licenseBadge": "EE", "idpType": "Provider Type", "idpTypeDescription": "Select the type of identity provider you want to configure", diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 175ef938c7..b501bea209 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1209,7 +1209,7 @@ export const supporterKey = pgTable("supporterKey", { export const idp = pgTable("idp", { idpId: serial("idpId").primaryKey(), name: varchar("name").notNull(), - type: varchar("type").notNull(), + type: varchar("type").notNull().$type<"oidc" | "google" | "azure">(), defaultRoleMapping: varchar("defaultRoleMapping"), defaultOrgMapping: varchar("defaultOrgMapping"), autoProvision: boolean("autoProvision").notNull().default(false), diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index ce7526fd50..9d799064e0 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1512,7 +1512,7 @@ export const supporterKey = sqliteTable("supporterKey", { export const idp = sqliteTable("idp", { idpId: integer("idpId").primaryKey({ autoIncrement: true }), name: text("name").notNull(), - type: text("type").notNull(), + type: text("type").notNull().$type<"oidc" | "google" | "azure">(), defaultRoleMapping: text("defaultRoleMapping"), defaultOrgMapping: text("defaultOrgMapping"), autoProvision: integer("autoProvision", { diff --git a/server/routers/idp/createOidcIdp.ts b/server/routers/idp/createOidcIdp.ts index 813ab0ea50..6586551def 100644 --- a/server/routers/idp/createOidcIdp.ts +++ b/server/routers/idp/createOidcIdp.ts @@ -26,6 +26,7 @@ const bodySchema = z.strictObject({ namePath: z.string().optional(), scopes: z.string().nonempty(), autoProvision: z.boolean().optional(), + directorySyncEnabled: z.boolean().optional(), tags: z.string().optional(), variant: z.enum(["oidc", "google", "azure"]).optional().default("oidc") }); @@ -93,6 +94,7 @@ export async function createOidcIdp( namePath, name, autoProvision, + directorySyncEnabled, tags, variant } = parsedBody.data; @@ -120,6 +122,11 @@ export async function createOidcIdp( .values({ name, autoProvision, + // directory sync requires auto provisioning and a google/azure idp + directorySyncEnabled: + !!autoProvision && + (variant === "google" || variant === "azure") && + !!directorySyncEnabled, type: "oidc", tags, defaultOrgMapping: `'{{orgId}}'`, diff --git a/server/routers/idp/updateOidcIdp.ts b/server/routers/idp/updateOidcIdp.ts index 69b77770f8..d4fd22ac67 100644 --- a/server/routers/idp/updateOidcIdp.ts +++ b/server/routers/idp/updateOidcIdp.ts @@ -30,6 +30,7 @@ const bodySchema = z.strictObject({ namePath: z.string().optional(), scopes: z.string().optional(), autoProvision: z.boolean().optional(), + directorySyncEnabled: z.boolean().optional(), defaultRoleMapping: z.string().optional(), defaultOrgMapping: z.string().optional(), tags: z.string().optional(), @@ -109,6 +110,7 @@ export async function updateOidcIdp( namePath, name, autoProvision, + directorySyncEnabled, defaultRoleMapping, defaultOrgMapping, tags, @@ -149,10 +151,27 @@ export async function updateOidcIdp( : undefined; const encryptedClientId = clientId ? encrypt(clientId, key) : undefined; + const [existingOidcConfig] = await db + .select({ variant: idpOidcConfig.variant }) + .from(idpOidcConfig) + .where(eq(idpOidcConfig.idpId, idpId)); + + // directory sync requires auto provisioning and a google/azure idp + const effectiveAutoProvision = + autoProvision ?? existingIdp.autoProvision; + const effectiveVariant = variant ?? existingOidcConfig?.variant; + const directorySyncSupported = + effectiveVariant === "google" || effectiveVariant === "azure"; + const effectiveDirectorySyncEnabled = + effectiveAutoProvision && directorySyncSupported + ? directorySyncEnabled + : false; + await db.transaction(async (trx) => { const idpData = { name, autoProvision, + directorySyncEnabled: effectiveDirectorySyncEnabled, defaultRoleMapping, defaultOrgMapping, tags diff --git a/src/app/admin/idp/[idpId]/general/page.tsx b/src/app/admin/idp/[idpId]/general/page.tsx index 8b7ed226f2..dfd543956d 100644 --- a/src/app/admin/idp/[idpId]/general/page.tsx +++ b/src/app/admin/idp/[idpId]/general/page.tsx @@ -74,7 +74,8 @@ export default function GeneralPage() { emailPath: z.string().optional(), namePath: z.string().optional(), scopes: z.string().min(1, { message: t("idpScopeRequired") }), - autoProvision: z.boolean().default(false) + autoProvision: z.boolean().default(false), + directorySyncEnabled: z.boolean().default(false) }); const GoogleFormSchema = z.object({ @@ -83,7 +84,8 @@ export default function GeneralPage() { clientSecret: z .string() .min(1, { message: t("idpClientSecretRequired") }), - autoProvision: z.boolean().default(false) + autoProvision: z.boolean().default(false), + directorySyncEnabled: z.boolean().default(false) }); const AzureFormSchema = z.object({ @@ -93,7 +95,8 @@ export default function GeneralPage() { .string() .min(1, { message: t("idpClientSecretRequired") }), tenantId: z.string().min(1, { message: t("idpTenantIdRequired") }), - autoProvision: z.boolean().default(false) + autoProvision: z.boolean().default(false), + directorySyncEnabled: z.boolean().default(false) }); type OidcFormValues = z.infer; @@ -128,6 +131,7 @@ export default function GeneralPage() { namePath: "name", scopes: "openid profile email", autoProvision: true, + directorySyncEnabled: false, tenantId: "" } }); @@ -166,7 +170,8 @@ export default function GeneralPage() { name: data.idp.name, clientId: data.idpOidcConfig.clientId, clientSecret: data.idpOidcConfig.clientSecret, - autoProvision: data.idp.autoProvision + autoProvision: data.idp.autoProvision, + directorySyncEnabled: data.idp.directorySyncEnabled }; if (idpVariant === "oidc") { @@ -229,6 +234,10 @@ export default function GeneralPage() { clientId: data.clientId, clientSecret: data.clientSecret, autoProvision: data.autoProvision, + directorySyncEnabled: + data.autoProvision && + (variant === "google" || variant === "azure") && + data.directorySyncEnabled, variant }; @@ -433,9 +442,39 @@ export default function GeneralPage() { "autoProvision", checked ); + if (!checked) { + form.setValue( + "directorySyncEnabled", + false + ); + } }} /> + {(variant === "google" || + variant === "azure") && ( +
+ { + form.setValue( + "directorySyncEnabled", + checked + ); + }} + description={t( + "idpDirectorySyncDescription" + )} + /> +
+ )}
{form.watch("autoProvision") && ( diff --git a/src/app/admin/idp/create/page.tsx b/src/app/admin/idp/create/page.tsx index bbe45c87ae..2e72b95a1a 100644 --- a/src/app/admin/idp/create/page.tsx +++ b/src/app/admin/idp/create/page.tsx @@ -68,7 +68,8 @@ export default function Page() { .min(1, { message: t("idpScopeRequired") }) .optional(), tenantId: z.string().optional(), - autoProvision: z.boolean().default(false) + autoProvision: z.boolean().default(false), + directorySyncEnabled: z.boolean().default(false) }); type CreateIdpFormValues = z.infer; @@ -87,7 +88,8 @@ export default function Page() { emailPath: "email", scopes: "openid profile email", tenantId: "", - autoProvision: false + autoProvision: false, + directorySyncEnabled: false } }); @@ -124,6 +126,10 @@ export default function Page() { emailPath: data.emailPath, namePath: data.namePath, autoProvision: data.autoProvision, + directorySyncEnabled: + data.autoProvision && + (data.type === "google" || data.type === "azure") && + data.directorySyncEnabled, scopes: data.scopes, variant: data.type }; @@ -217,7 +223,6 @@ export default function Page() { )} /> - @@ -242,9 +247,35 @@ export default function Page() { defaultChecked={form.getValues("autoProvision")} onCheckedChange={(checked) => { form.setValue("autoProvision", checked); + if (!checked) { + form.setValue( + "directorySyncEnabled", + false + ); + } }} - description={t("idpAutoProvisionConfigureAfterCreate")} + description={t( + "idpAutoProvisionConfigureAfterCreate" + )} /> + {(watchedType === "google" || + watchedType === "azure") && ( + { + form.setValue( + "directorySyncEnabled", + checked + ); + }} + description={t( + "idpDirectorySyncDescription" + )} + /> + )}
From 8c3d0bd7f6d943bae1077e928eed6b28cecc9fd5 Mon Sep 17 00:00:00 2001 From: kdcokenny Date: Wed, 30 Sep 2026 20:27:27 +0000 Subject: [PATCH 05/50] fix: use formatEndpoint for local targets --- server/lib/traefik/loadBalancer.test.ts | 81 +++++++++++++++++++++++++ server/lib/traefik/loadBalancer.ts | 9 +-- 2 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 server/lib/traefik/loadBalancer.test.ts diff --git a/server/lib/traefik/loadBalancer.test.ts b/server/lib/traefik/loadBalancer.test.ts new file mode 100644 index 0000000000..0ffd5201d6 --- /dev/null +++ b/server/lib/traefik/loadBalancer.test.ts @@ -0,0 +1,81 @@ +import "@server/extendZod"; +import { assertEqualsObj } from "@test/assert"; +import { + buildHttpLoadBalancerServers, + buildTcpUdpLoadBalancerServers +} from "./loadBalancer"; +import type { TargetWithSite } from "./types"; + +function makeTarget(ip: string, siteType: string): TargetWithSite { + return { + targetId: 1, + resourceId: 1, + providerId: null, + siteId: 1, + ip, + port: 8080, + internalPort: 9000, + method: "http", + enabled: true, + health: "healthy", + path: null, + pathMatchType: null, + rewritePath: null, + rewritePathType: null, + priority: 100, + mode: "http", + authToken: null, + site: { + siteId: 1, + type: siteType, + subnet: null, + exitNodeId: null, + online: true + } + }; +} + +function runTests() { + const ipCases = [ + { ip: "2001:db8::1", host: "[2001:db8::1]" }, + { ip: "192.0.2.1", host: "192.0.2.1" } + ]; + const cases = [ + ...ipCases, + { ip: "backend.example.com", host: "backend.example.com" } + ]; + + for (const { ip, host } of cases) { + const target = makeTarget(ip, "local"); + assertEqualsObj( + buildHttpLoadBalancerServers([target]), + [{ url: `http://${host}:8080` }], + `Local HTTP target ${ip} should format its endpoint` + ); + assertEqualsObj( + buildTcpUdpLoadBalancerServers([target]), + [{ address: `${host}:8080` }], + `Local TCP/UDP target ${ip} should format its endpoint` + ); + } + + for (const { ip, host } of ipCases) { + const target = makeTarget("backend.example.com", "newt"); + target.site.subnet = `${ip}/${ip.includes(":") ? 128 : 32}`; + + assertEqualsObj( + buildHttpLoadBalancerServers([target]), + [{ url: `http://${host}:9000` }], + "Newt HTTP target should use the subnet endpoint and internal port" + ); + assertEqualsObj( + buildTcpUdpLoadBalancerServers([target]), + [{ address: `${host}:9000` }], + "Newt TCP/UDP target should use the subnet endpoint and internal port" + ); + } + + console.log("All load balancer endpoint tests passed!"); +} + +runTests(); diff --git a/server/lib/traefik/loadBalancer.ts b/server/lib/traefik/loadBalancer.ts index af1454a180..538f6bf074 100644 --- a/server/lib/traefik/loadBalancer.ts +++ b/server/lib/traefik/loadBalancer.ts @@ -1,3 +1,4 @@ +import { formatEndpoint } from "@server/lib/ip"; import { TargetWithSite } from "./types"; /** @@ -48,12 +49,12 @@ export function buildHttpLoadBalancerServers(targets: TargetWithSite[]) { target.site.type === "wireguard" ) { return { - url: `${target.method}://${target.ip}:${target.port}` + url: `${target.method}://${formatEndpoint(target.ip, target.port)}` }; } else if (target.site.type === "newt") { const ip = target.site.subnet!.split("/")[0]; return { - url: `${target.method}://${ip}:${target.internalPort}` + url: `${target.method}://${formatEndpoint(ip, target.internalPort!)}` }; } }) @@ -111,12 +112,12 @@ export function buildTcpUdpLoadBalancerServers(targets: TargetWithSite[]) { target.site.type === "wireguard" ) { return { - address: `${target.ip}:${target.port}` + address: formatEndpoint(target.ip, target.port) }; } else if (target.site.type === "newt") { const ip = target.site.subnet!.split("/")[0]; return { - address: `${ip}:${target.internalPort}` + address: formatEndpoint(ip, target.internalPort!) }; } }); From 938710147a0b09ed3c6f4674ea680ad2ecde4e88 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 1 Oct 2026 09:30:08 -0400 Subject: [PATCH 06/50] Delete token --- server/routers/badger/exchangeSession.ts | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/server/routers/badger/exchangeSession.ts b/server/routers/badger/exchangeSession.ts index 31600dc3e1..c0c77ed8dd 100644 --- a/server/routers/badger/exchangeSession.ts +++ b/server/routers/badger/exchangeSession.ts @@ -4,7 +4,12 @@ import createHttpError from "http-errors"; import { z } from "zod"; import { fromError } from "zod-validation-error"; import logger from "@server/logger"; -import { resourceAccessToken, resources, sessions } from "@server/db"; +import { + resourceAccessToken, + resourceSessions, + resources, + sessions +} from "@server/db"; import { db } from "@server/db"; import { and, eq, inArray, or, sql } from "drizzle-orm"; import { @@ -128,7 +133,18 @@ export async function exchangeSession( ); } - await db.delete(sessions).where(eq(sessions.sessionId, requestToken)); + // Enforce single use: consume the request token. Checking the deleted + // row count makes concurrent exchanges of the same token race safely. + const consumed = await db + .delete(resourceSessions) + .where(eq(resourceSessions.sessionId, requestSession.sessionId)) + .returning(); + + if (consumed.length === 0) { + return next( + createHttpError(HttpCode.UNAUTHORIZED, "Invalid request token") + ); + } const token = generateSessionToken(); From c8f22ca125959eb80c3582db428c32f145563194 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Thu, 1 Oct 2026 14:48:55 -0400 Subject: [PATCH 07/50] fix unlimited license message --- messages/en-US.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index fc15236870..c2d3608f6f 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -591,7 +591,7 @@ "licenseReckeckAll": "Recheck All Keys", "licenseSiteUsage": "Sites Usage", "licenseSiteUsageDecsription": "View the number of sites using this license.", - "licenseNoSiteLimit": "There is no limit on the number of sites using an unlicensed host.", + "licenseNoSiteLimit": "There is no limit on the number of sites for this host.", "licensePurchase": "Purchase License", "licensePurchaseSites": "Purchase Additional Sites", "licenseSitesUsedMax": "{usedSites} of {maxSites} sites used", @@ -600,7 +600,7 @@ "licenseUsageDescription": "View the number of users and sites licensed for this host", "licenseUsageSites": "Sites", "licenseUsageUsers": "Users", - "licenseNoUserLimit": "There is no limit on the number of users using an unlicensed hot", + "licenseNoUserLimit": "There is no limit on the number of users for this host.", "licenseUsersUsedMax": "{usedUsers} of {maxUsers} users used", "licenseUsersUsed": "{count, plural, =0 {# users} one {# user} other {# users}}", "licenseUnlimited": "Unlimited", From 2fc55e9325301cda398e217b4f9deb2bcda80675 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Fri, 2 Oct 2026 03:38:48 +0200 Subject: [PATCH 08/50] =?UTF-8?q?=F0=9F=8F=B7=EF=B8=8F=20update=20types?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/db/pg/schema/schema.ts | 5 ++++- server/db/sqlite/schema/schema.ts | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index b2f539d352..283222db4c 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1224,7 +1224,10 @@ export const idpOidcConfig = pgTable("idpOidcConfig", { idpId: integer("idpId") .notNull() .references(() => idp.idpId, { onDelete: "cascade" }), - variant: varchar("variant").notNull().default("oidc"), + variant: varchar("variant") + .notNull() + .$type<"oidc" | "google" | "azure">() + .default("oidc"), clientId: varchar("clientId").notNull(), clientSecret: varchar("clientSecret").notNull(), authUrl: varchar("authUrl").notNull(), diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 35656b8477..0a3362ff43 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1533,7 +1533,10 @@ export const idpOidcConfig = sqliteTable("idpOidcConfig", { idpOauthConfigId: integer("idpOauthConfigId").primaryKey({ autoIncrement: true }), - variant: text("variant").notNull().default("oidc"), + variant: text("variant") + .notNull() + .$type<"oidc" | "google" | "azure">() + .default("oidc"), idpId: integer("idpId") .notNull() .references(() => idp.idpId, { onDelete: "cascade" }), From c988e631ebace56bd9b06076b714b9e849684f00 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Fri, 2 Oct 2026 03:42:51 +0200 Subject: [PATCH 09/50] =?UTF-8?q?=F0=9F=9A=A7=20WIP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/routers/idp/idpDirectorySync.ts | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 server/routers/idp/idpDirectorySync.ts diff --git a/server/routers/idp/idpDirectorySync.ts b/server/routers/idp/idpDirectorySync.ts new file mode 100644 index 0000000000..b5a3becfad --- /dev/null +++ b/server/routers/idp/idpDirectorySync.ts @@ -0,0 +1,3 @@ +async function fetchIdpDirectoryData(idpId: number) { + // ... +} From a63fa0c88531aba55fd5bb64bb22eba68ecb7174 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Fri, 2 Oct 2026 03:51:15 +0200 Subject: [PATCH 10/50] =?UTF-8?q?=F0=9F=93=9D=20make=20a=20plan=20for=20di?= =?UTF-8?q?rectory=20sync?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DIRECTORY_SYNC_PLAN.md | 207 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 DIRECTORY_SYNC_PLAN.md diff --git a/DIRECTORY_SYNC_PLAN.md b/DIRECTORY_SYNC_PLAN.md new file mode 100644 index 0000000000..4c2fbf21f3 --- /dev/null +++ b/DIRECTORY_SYNC_PLAN.md @@ -0,0 +1,207 @@ +# Plan: IdP directory sync (Microsoft Entra ID and Google Workspace) + +## What already exists + +- `idp.directorySyncEnabled` exists. Create and update only allow it when `autoProvision` is on and the variant is `google` or `azure` ([createOidcIdp.ts](server/routers/idp/createOidcIdp.ts), [updateOidcIdp.ts](server/routers/idp/updateOidcIdp.ts)). Nothing reads the flag yet, and [idpDirectorySync.ts](server/routers/idp/idpDirectorySync.ts) is empty. +- Provisioning only happens at login, in [validateOidcCallback.ts](server/routers/idp/validateOidcCallback.ts): + - `users.username` = JMESPath(`identifierPath`) applied to the ID token claims. + - Org and role mappings are JMESPath expressions over those same claims. + - It only ever removes `userOrgs` rows where `autoProvisioned = true`. +- The Azure tenant ID isn't stored as its own field. The UI re-parses it from `authUrl` ([general/page.tsx](src/app/admin/idp/[idpId]/general/page.tsx)). +- Building blocks for a scheduler: + - [server/startSchedulers.ts](server/startSchedulers.ts) and [server/private/startSchedulers.ts](server/private/startSchedulers.ts) start the existing background jobs. + - [LockManager](server/private/lib/lock.ts) provides a Redis-backed lock that falls back to a local lock, so jobs work with multiple instances. + +## Goals and non-goals + +**Goals** + +- Create and update users in Pangolin without waiting for them to log in. +- Deactivating or removing users who left the directory is **not confirmed yet** (see [Deprovisioning](#4-deprovisioning-not-confirmed)). +- Keep org and role assignments current from group membership. +- Reuse the existing org and role mappings. + +**Non-goals (v1)** + +- SCIM push. +- Webhooks or push notifications. +- Generic OIDC providers. +- Two-way sync. + +## Provider study + +| | Microsoft Entra (done) | Google Workspace (to study) | +| ----------------- | ---------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| API | Microsoft Graph | Admin SDK Directory API (`admin.googleapis.com/admin/directory/v1`) | +| Users | `GET /users`, 999 per page | `users.list?customer=my_customer`, 500 per page | +| Groups / members | `/groups`, `/groups/{id}/members` or `/transitiveMembers` | `groups.list` (200 per page), `members.list?includeDerivedMembership=true` | +| Incremental | ✅ `/users/delta` and `/groups/delta` (with `members@delta`) | ❌ No delta API: full snapshot every run, then diff on our side | +| App-only auth | Client credentials using the **existing** clientId and secret, plus tenant | **Service account + domain-wide delegation** impersonating an admin, so **new credentials**: SA JSON key and admin email | +| Permissions | `User.Read.All` and `GroupMember.Read.All` (application), plus **admin consent** | Scopes `admin.directory.user.readonly`, `admin.directory.group.readonly` and `admin.directory.group.member.readonly`, authorized in the Admin console under **Domain-wide delegation** | +| Throttling | Per app per tenant, `429` + `Retry-After` | Per project quota (believed to be about 2,400 queries/min, to verify), `403 rateLimitExceeded` / `429`, exponential backoff | +| Disabled users | `accountEnabled = false` | `suspended`, `archived` | +| Stable user ID | `id` (= `oid` claim) | `id` (= `sub` claim) | + +### Request counts per sync (Entra) + +With N users, G groups, and up to 999 items per page: + +- Full sync (per group): `ceil(N/999) + ceil(G/999) + Σ ceil(members_g / 999)` ≈ `N/999 + G`. For example, 5,000 users and 200 groups ≈ 206 requests. +- Per user (`/users/{id}/memberOf` for every user): `ceil(N/999) + N`. Avoid this approach. +- Delta sync with no changes: 2 requests (users delta + groups delta). +- `$expand=memberOf` cuts off at about 20 groups per user. Don't rely on it. + +### Google items to verify + +1. That the Google ID token's `sub` really equals the Directory `user.id`. This is believed true, but the matching logic depends on it. +2. Whether to support a second auth option: admin OAuth consent with an offline refresh token and the extra directory scopes on the existing OAuth client. Setup is easier, but sync stops working if that admin leaves. +3. Real quota numbers and how errors look. The Directory API often returns `403` with `reason: rateLimitExceeded` rather than `429`. +4. Group identifiers: group email vs `id`, and which one is friendlier in role mappings. +5. Whether `includeDerivedMembership` is enough for nested groups, or whether we need the Cloud Identity `searchTransitiveGroups` API. +6. The library choice: `google-auth-library`, or signing the JWT-bearer assertion ourselves. Neither `jose` nor any Google or Azure SDK is a dependency today. + +## Key design decisions + +### 1. Matching synced users to users who log in (biggest risk) + +Synced users must get the same `username` that login computes with `identifierPath`. + +- **Azure:** the default `identifierPath` is `sub`. Entra's `sub` is **unique per app and is not the Graph user ID**, so a synced user can't be linked to a logged-in one. Options: + - require `oid` as the identifier when sync is on, or + - map Graph `id` to `oid` and refuse to enable sync if `identifierPath` isn't `oid`, or + - fall back to matching by email. + + Existing Azure IdPs that use `sub` need a migration story. +- **Google:** `sub` = Directory `id`, so the default works (pending verification item 1). + +### 2. Reuse the mappings by building synthetic claims + +For each directory user, build a claims-like object (`{ sub | oid, email, name, groups: [...] }`). Run the existing `identifierPath`, `emailPath`, `namePath`, org mapping and role mapping against it, so admins configure mappings once. + +- For Azure, `groups` should hold group **object IDs**, because that's what the ID token's `groups` claim contains. Mappings then behave the same at login and during sync. +- Google ID tokens contain no groups, so sync becomes the only way to get groups for Google. Decide between emails and IDs (verification item 4). + +### 3. Refactor first + +Pull the provisioning block out of `validateOidcCallback.ts` into a shared function, for example `provisionUserFromClaims(trx, idp, claims, { source: "login" | "sync" })`. Login and sync then share the same logic. This is the main prerequisite. + +### 4. Deprovisioning (not confirmed) + +> ⚠️ Removing or deactivating users during sync has **not been approved**. It's risky: a bad credential, a wrong group filter, a partial API response or a bug in the diff could lock a whole org out at once. Until a decision is made, sync **must not delete, disable or remove anything**. + +**Until a decision is made (v1 default):** + +- Sync only creates and updates users, memberships and roles. +- Users who are missing from the directory, disabled or suspended are only **detected and reported**: logged, counted in the sync run, and possibly listed in the UI as "would be removed". Nothing changes for them in Pangolin. + +**If deprovisioning is approved later, it needs safeguards:** + +- An explicit per-IdP opt-in, off by default. +- It only ever touches `autoProvisioned` memberships. Manually added memberships (`autoProvisioned = false`) are never touched, which matches what login does today. +- Prefer disabling over deleting, with a grace period, for example only acting after a user has been missing for N consecutive syncs. +- A circuit breaker: abort the run if more than X% of the IdP's users, or more than N users, would be removed. +- Never deprovision after an incomplete or failed fetch. That covers a page error mid-run, a throttled run, or an empty response. +- A dry-run mode that shows what would be removed before turning it on. + +**Questions to answer before approving:** + +- Delete the `users` row, disable it, or only remove `autoProvisioned` org memberships? +- Revoke sessions right away? +- What counts as "gone": deleted only, or also disabled/suspended/archived, or also removed from the synced groups? + +### 5. Sync scope + +Add an optional filter for which groups to sync, or only sync users assigned to the app (Azure: `appRoleAssignedTo`). This keeps request counts down for large tenants. + +## Data model (pg and sqlite schemas, plus a migration) + +- **`idpDirectorySyncConfig`**: provider-specific credentials and options. + - Google: SA client email, private key (encrypted), admin email to impersonate, customer ID. + - Both: group filter. + - Azure: an explicit `tenantId` column, instead of parsing `authUrl`. +- **`idpSyncState`**: one row per IdP. Columns: `usersDeltaLink`, `groupsDeltaLink`, `status`, `lastSyncStartedAt`, `lastSyncSucceededAt`, `lastError`, `consecutiveFailures`, `nextSyncAt`. +- **`idpSyncRun`**: optional run history with counts, request count and error. Old rows are pruned. + +Individual Graph or Directory requests and retries are logged, not stored in the DB. Access tokens are cached in memory (or Redis), never stored in the DB. + +## Sync engine + +``` +server/(private/)lib/directorySync/ + providers/azure.ts // token, paging, delta, 429 handling + providers/google.ts // JWT-bearer token, paging, full snapshot + types.ts // DirectoryProvider interface + runSync.ts // claims building, diff, apply via provisionUserFromClaims + scheduler.ts +``` + +- **Provider interface.** Each provider returns either: + - `{ kind: "delta", changedUsers, removedUserIds, membershipChanges, nextState }` (Azure), or + - `{ kind: "snapshot", users, groups, memberships }` (Google, and Azure's first run). + + For a snapshot, the core diffs against the users Pangolin already has for that IdP. Users who are missing, plus Azure `@removed` entries, are only reported for now; see [Deprovisioning](#4-deprovisioning-not-confirmed). +- **Scheduler.** + - A tick runs every minute and selects IdPs where `directorySyncEnabled` is set and `nextSyncAt <= now`. + - It takes the `idp-sync:{idpId}` lock, caps how many IdPs sync at once, and staggers `nextSyncAt`. + - Default interval: 15 to 60 minutes, possibly configurable per IdP. +- **Failure handling.** + - A short `Retry-After` → sleep in memory and retry the same page. + - A long `Retry-After`, or repeated failures → stop the run, set `nextSyncAt`, and don't save delta links. The backoff is `nextSyncAt = now + min(interval * 2^consecutiveFailures, 24h)`, reset on success. + - `410` / `syncStateNotFound` → clear the delta links and do a full sync. +- Save the new delta links only after the whole run succeeds. +- Access tokens are cached in memory until shortly before they expire. + +## API and UI + +- `POST /idp/:idpId/sync`: "Sync now". +- `GET /idp/:idpId/sync-status`: last sync time, status, error, counts. +- The IdP settings page gets: + - Google service account fields. + - Azure tenant ID as a stored field. + - A sync status card. + - Setup instructions: Azure application permissions plus admin consent; Google DWD scopes. +- Clear errors for the usual failures: `403` → "admin consent missing" or "DWD not authorized"; `401` → "bad secret or key". + +## Phases + +1. **Study:** finish the Google verification items above. Confirm `oid` vs `sub` behavior on a test Entra tenant. +2. **Refactor:** extract `provisionUserFromClaims` from the OIDC callback, with no change in behavior. Add tests. +3. **Schema:** add the sync config and state tables and migrations, and store Azure `tenantId` explicitly. +4. **Azure provider:** full sync, then delta. +5. **Scheduler:** locking, backoff, status API, "Sync now". +6. **Google provider:** snapshot plus diff. +7. **UI and docs:** credentials, status card, setup guides. +8. **Deprovisioning, only if approved:** opt-in setting, dry run, grace period, circuit breaker. + +## Open questions + +- Is this an enterprise/licensed feature? If so, the engine belongs in `server/private`. The flag currently sits on the shared `idp` table and routes. +- **Blocking for deprovisioning:** should sync remove or disable users at all? See the questions in [Deprovisioning](#4-deprovisioning-not-confirmed). +- How to handle existing Azure IdPs with `identifierPath = "sub"`: block sync, migrate them, or match by email? +- Which group identifiers to expose in mappings for Google: emails or IDs? +- Does the per-org IdP flow ([server/private/routers/orgIdp](server/private/routers/orgIdp)) need sync too, or only global IdPs? +- Global sync interval, or configurable per IdP? + +## References + +**Microsoft Graph / Entra** + +- List users: https://learn.microsoft.com/en-us/graph/api/user-list +- User delta: https://learn.microsoft.com/en-us/graph/api/user-delta +- Delta query overview: https://learn.microsoft.com/en-us/graph/delta-query-overview +- Group delta: https://learn.microsoft.com/en-us/graph/api/group-delta +- Group members: https://learn.microsoft.com/en-us/graph/api/group-list-members +- Transitive members: https://learn.microsoft.com/en-us/graph/api/group-list-transitivemembers +- App role assignments: https://learn.microsoft.com/en-us/graph/api/serviceprincipal-list-approleassignedto +- Client credentials flow: https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow +- Permissions reference: https://learn.microsoft.com/en-us/graph/permissions-reference +- Admin consent: https://learn.microsoft.com/en-us/entra/identity/enterprise-apps/grant-admin-consent +- Paging: https://learn.microsoft.com/en-us/graph/paging +- Throttling: https://learn.microsoft.com/en-us/graph/throttling +- Throttling limits per service: https://learn.microsoft.com/en-us/graph/throttling-limits +- National clouds: https://learn.microsoft.com/en-us/graph/deployments + +**Google Workspace (to fill in during the study)** + +- Admin SDK Directory API: https://developers.google.com/admin-sdk/directory +- Domain-wide delegation: https://developers.google.com/workspace/guides/create-credentials#optional_set_up_domain-wide_delegation_for_a_service_account From 0841b71d6716d7b8e8ff2575d8ac5407ae948635 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Fri, 2 Oct 2026 23:53:35 +0200 Subject: [PATCH 11/50] =?UTF-8?q?=F0=9F=9A=A7=20Quick'n=20ditry=20test=20c?= =?UTF-8?q?ode=20for=20sync?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DIRECTORY_SYNC_PLAN.md | 22 +- server/private/lib/directorySync/azure.ts | 380 ++++++++++++++++++++++ server/private/lib/directorySync/run.ts | 23 ++ server/routers/idp/idpDirectorySync.ts | 3 - 4 files changed, 417 insertions(+), 11 deletions(-) create mode 100644 server/private/lib/directorySync/azure.ts create mode 100644 server/private/lib/directorySync/run.ts delete mode 100644 server/routers/idp/idpDirectorySync.ts diff --git a/DIRECTORY_SYNC_PLAN.md b/DIRECTORY_SYNC_PLAN.md index 4c2fbf21f3..3951b5c4e5 100644 --- a/DIRECTORY_SYNC_PLAN.md +++ b/DIRECTORY_SYNC_PLAN.md @@ -19,7 +19,7 @@ - Create and update users in Pangolin without waiting for them to log in. - Deactivating or removing users who left the directory is **not confirmed yet** (see [Deprovisioning](#4-deprovisioning-not-confirmed)). - Keep org and role assignments current from group membership. -- Reuse the existing org and role mappings. +- Map directory groups to org roles with an explicit sync-only mapping. **Non-goals (v1)** @@ -74,16 +74,21 @@ Synced users must get the same `username` that login computes with `identifierPa Existing Azure IdPs that use `sub` need a migration story. - **Google:** `sub` = Directory `id`, so the default works (pending verification item 1). -### 2. Reuse the mappings by building synthetic claims +### 2. Sync uses its own group mapping (not the login JMESPath mappings) -For each directory user, build a claims-like object (`{ sub | oid, email, name, groups: [...] }`). Run the existing `identifierPath`, `emailPath`, `namePath`, org mapping and role mapping against it, so admins configure mappings once. +Directory API responses (Graph `user`, Directory `user`) don't look like ID token claims. Faking claims only matches login for a few fields (`oid`, `name`, `upn`, `groups`), and silently diverges for the rest (`sub` is per-app in Entra, `email` is optional, `roles` and custom claims don't exist in Graph). So sync doesn't run the login org/role mappings. -- For Azure, `groups` should hold group **object IDs**, because that's what the ID token's `groups` claim contains. Mappings then behave the same at login and during sync. -- Google ID tokens contain no groups, so sync becomes the only way to get groups for Google. Decide between emails and IDs (verification item 4). +- **Identifier:** fixed rule, not configurable. It must produce the same `users.username` login computes, so sync only runs when `identifierPath` resolves from directory data. Azure: `oid` → Graph `id`, `preferred_username`/`upn` → `userPrincipalName`; `sub` is refused. Google: `sub` → Directory `id`. +- **Email / name:** read directly from directory fields (Azure `mail` / `displayName`). +- **Orgs and roles:** an explicit table `idpGroupMapping (idpId, externalGroupId, orgId, roleId)`. A user gets every role mapped to a group they're a (transitive) member of; an org membership exists wherever they have at least one role. Without full RBAC, only the first mapped role per org is kept. +- Only mapped groups are fetched, which also bounds the request count. +- The admin UI can list directory groups so admins pick from a list instead of pasting IDs. Same model works for Google, where tokens carry no groups at all. + +> ⚠️ **Open: login vs sync ownership.** Login keeps its claim-based mappings and, for `autoProvisioned` orgs, resets roles and removes orgs that don't match ([validateOidcCallback.ts](server/routers/idp/validateOidcCallback.ts)). With two mappings, whichever ran last wins. Options: (a) when sync is on, login skips org/role reconciliation and only authenticates + updates profile; (b) login only adds, never removes or overwrites; (c) accept the flip-flop and tell admins to keep both mappings consistent. ### 3. Refactor first -Pull the provisioning block out of `validateOidcCallback.ts` into a shared function, for example `provisionUserFromClaims(trx, idp, claims, { source: "login" | "sync" })`. Login and sync then share the same logic. This is the main prerequisite. +Pull the "apply" half of the provisioning block out of `validateOidcCallback.ts` into a shared function, for example `applyUserMemberships(trx, idp, user, desired: { orgId, roleIds }[], { source: "login" | "sync" })`: upsert the user, add orgs, re-sync roles on `autoProvisioned` orgs. Login computes `desired` from claims, sync from `idpGroupMapping`; the write path is shared. ### 4. Deprovisioning (not confirmed) @@ -119,6 +124,7 @@ Add an optional filter for which groups to sync, or only sync users assigned to - Google: SA client email, private key (encrypted), admin email to impersonate, customer ID. - Both: group filter. - Azure: an explicit `tenantId` column, instead of parsing `authUrl`. +- **`idpGroupMapping`**: `idpId`, `externalGroupId`, `orgId`, `roleId`. Sync-only org/role mapping (see decision #2). - **`idpSyncState`**: one row per IdP. Columns: `usersDeltaLink`, `groupsDeltaLink`, `status`, `lastSyncStartedAt`, `lastSyncSucceededAt`, `lastError`, `consecutiveFailures`, `nextSyncAt`. - **`idpSyncRun`**: optional run history with counts, request count and error. Old rows are pruned. @@ -131,7 +137,7 @@ server/(private/)lib/directorySync/ providers/azure.ts // token, paging, delta, 429 handling providers/google.ts // JWT-bearer token, paging, full snapshot types.ts // DirectoryProvider interface - runSync.ts // claims building, diff, apply via provisionUserFromClaims + runSync.ts // group mapping resolution, diff, apply scheduler.ts ``` @@ -165,7 +171,7 @@ server/(private/)lib/directorySync/ ## Phases 1. **Study:** finish the Google verification items above. Confirm `oid` vs `sub` behavior on a test Entra tenant. -2. **Refactor:** extract `provisionUserFromClaims` from the OIDC callback, with no change in behavior. Add tests. +2. **Refactor:** extract `applyUserMemberships` from the OIDC callback, with no change in behavior. Add tests. 3. **Schema:** add the sync config and state tables and migrations, and store Azure `tenantId` explicitly. 4. **Azure provider:** full sync, then delta. 5. **Scheduler:** locking, backoff, status API, "Sync now". diff --git a/server/private/lib/directorySync/azure.ts b/server/private/lib/directorySync/azure.ts new file mode 100644 index 0000000000..7c54d4b1c3 --- /dev/null +++ b/server/private/lib/directorySync/azure.ts @@ -0,0 +1,380 @@ +// QUICK & DIRTY: one-shot Entra ID -> Pangolin user sync. +// Full snapshot only (no delta), creates/updates users and memberships, +// never removes anything (see DIRECTORY_SYNC_PLAN.md "Deprovisioning"). +// Org/role assignment comes from an explicit group -> (org, role) mapping, +// NOT the login JMESPath mappings (Graph data != ID token claims). + +import { db, Org } from "@server/db"; +import { + idp, + idpOidcConfig, + idpOrg, + orgs, + roles, + userOrgRoles, + userOrgs, + users +} from "@server/db"; +import { and, eq, inArray } from "drizzle-orm"; +import config from "@server/lib/config"; +import logger from "@server/logger"; +import { decrypt } from "@server/lib/crypto"; +import { generateId } from "@server/auth/sessions/app"; +import { UserType } from "@server/types/UserTypes"; +import { build } from "@server/build"; +import { LimitId } from "@server/lib/billing"; +import { usageService } from "@server/lib/billing/usageService"; +import { calculateUserClientsForOrgs } from "@server/lib/calculateUserClientsForOrgs"; +import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed"; +import { tierMatrix } from "@server/lib/billing/tierMatrix"; +import { assignUserToOrg } from "@server/lib/userOrg"; + +const GRAPH = "https://graph.microsoft.com/v1.0"; + +type GraphUser = { + id: string; + displayName: string | null; + mail: string | null; + userPrincipalName: string | null; + accountEnabled: boolean | null; +}; + +// TODO: move to an `idpGroupMapping` table (idpId, externalGroupId, orgId, roleId) +export type GroupMapping = { + groupId: string; // Entra group object ID + orgId: string; + roleName: string; +}; + +type SyncReport = { + fetchedUsers: number; + fetchedGroups: number; + missingRoles: string[]; + created: number; + updated: number; + skippedDisabled: number; + skippedNoMatch: number; + requests: number; +}; + +async function getAppToken( + tenantId: string, + clientId: string, + clientSecret: string +): Promise { + const res = await fetch( + `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, + { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ + grant_type: "client_credentials", + client_id: clientId, + client_secret: clientSecret, + scope: "https://graph.microsoft.com/.default" + }) + } + ); + const body = await res.json(); + if (!res.ok) { + throw new Error( + `Entra token request failed (${res.status}): ${body.error_description ?? body.error}` + ); + } + return body.access_token; +} + +async function graphGetAll( + token: string, + url: string, + report: SyncReport +): Promise { + const items: T[] = []; + let next: string | undefined = url; + while (next) { + report.requests++; + const res: Response = await fetch(next, { + headers: { Authorization: `Bearer ${token}` } + }); + if (res.status === 429 || res.status === 503) { + const retryAfter = Number(res.headers.get("Retry-After") ?? 5); + logger.warn(`Graph throttled, retrying in ${retryAfter}s`, { + url: next + }); + await new Promise((r) => setTimeout(r, retryAfter * 1000)); + continue; + } + const body: any = await res.json(); + if (!res.ok) { + throw new Error( + `Graph request failed (${res.status}) ${next}: ${body.error?.code} ${body.error?.message}` + ); + } + items.push(...body.value); + next = body["@odata.nextLink"]; + } + return items; +} + +// Graph field that yields the same `username` login computes from the ID token. +const IDENTIFIER_FROM_GRAPH: Record string | null> = { + oid: (u) => u.id, + preferred_username: (u) => u.userPrincipalName, + upn: (u) => u.userPrincipalName +}; + +export async function syncAzureDirectory( + idpId: number, + groupMappings: GroupMapping[] +): Promise { + const report: SyncReport = { + fetchedUsers: 0, + fetchedGroups: 0, + missingRoles: [], + created: 0, + updated: 0, + skippedDisabled: 0, + skippedNoMatch: 0, + requests: 0 + }; + + const [existingIdp] = await db + .select() + .from(idp) + .innerJoin(idpOidcConfig, eq(idpOidcConfig.idpId, idp.idpId)) + .where(and(eq(idp.type, "oidc"), eq(idp.idpId, idpId))); + + if (!existingIdp) throw new Error(`IdP ${idpId} not found`); + if (existingIdp.idpOidcConfig.variant !== "azure") { + throw new Error(`IdP ${idpId} is not an azure IdP`); + } + if (!existingIdp.idp.autoProvision) { + throw new Error(`IdP ${idpId} does not have autoProvision enabled`); + } + if (!existingIdp.idp.directorySyncEnabled) { + logger.warn(`IdP ${idpId} has directorySyncEnabled=false, syncing anyway`); + } + + const identifierPath = existingIdp.idpOidcConfig.identifierPath; + const getIdentifier = IDENTIFIER_FROM_GRAPH[identifierPath]; + if (!getIdentifier) { + // e.g. "sub": Entra's sub is per-app and not exposed by Graph + throw new Error( + `identifierPath "${identifierPath}" can't be resolved from Graph; use one of: ${Object.keys(IDENTIFIER_FROM_GRAPH).join(", ")}` + ); + } + + const tenantId = existingIdp.idpOidcConfig.authUrl.match( + /login\.microsoftonline\.com\/([^/]+)\/oauth2/ + )?.[1]; + if (!tenantId) throw new Error("Could not parse tenant ID from authUrl"); + + const key = config.getRawConfig().server.secret!; + const clientId = decrypt(existingIdp.idpOidcConfig.clientId, key); + const clientSecret = decrypt(existingIdp.idpOidcConfig.clientSecret, key); + + // ---- resolve mappings to (org, roleId) ---- + let allowedOrgs: Org[]; + if (build === "saas") { + const idpOrgs = await db + .select() + .from(idpOrg) + .where(eq(idpOrg.idpId, idpId)) + .innerJoin(orgs, eq(orgs.orgId, idpOrg.orgId)); + allowedOrgs = idpOrgs.map((o) => o.orgs); + } else { + allowedOrgs = await db.select().from(orgs); + } + + const mappedOrgIds = [...new Set(groupMappings.map((m) => m.orgId))]; + const orgRoles = mappedOrgIds.length + ? await db.select().from(roles).where(inArray(roles.orgId, mappedOrgIds)) + : []; + + // groupId -> [{ orgId, roleId }] + const grantsByGroup = new Map(); + for (const m of groupMappings) { + if (!allowedOrgs.some((o) => o.orgId === m.orgId)) { + logger.warn(`Org ${m.orgId} is not available to IdP ${idpId}, ignoring mapping`, m); + continue; + } + const role = orgRoles.find( + (r) => r.orgId === m.orgId && r.name === m.roleName + ); + if (!role) { + report.missingRoles.push(`${m.orgId}/${m.roleName}`); + continue; + } + const list = grantsByGroup.get(m.groupId) ?? []; + list.push({ orgId: m.orgId, roleId: role.roleId }); + grantsByGroup.set(m.groupId, list); + } + + const multiRoleByOrg = new Map(); + for (const orgId of mappedOrgIds) { + multiRoleByOrg.set( + orgId, + await isLicensedOrSubscribed(orgId, tierMatrix.fullRbac) + ); + } + + // ---- fetch directory snapshot (only members of mapped groups) ---- + const token = await getAppToken(tenantId, clientId, clientSecret); + + const graphUsers = await graphGetAll( + token, + `${GRAPH}/users?$select=id,displayName,mail,userPrincipalName,accountEnabled&$top=999`, + report + ); + report.fetchedUsers = graphUsers.length; + + const groupsByUser = new Map(); + for (const groupId of grantsByGroup.keys()) { + const members = await graphGetAll<{ id: string }>( + token, + `${GRAPH}/groups/${groupId}/transitiveMembers/microsoft.graph.user?$select=id&$top=999`, + report + ); + report.fetchedGroups++; + for (const m of members) { + const list = groupsByUser.get(m.id) ?? []; + list.push(groupId); + groupsByUser.set(m.id, list); + } + } + + logger.info( + `Fetched ${graphUsers.length} users and ${report.fetchedGroups} mapped groups from Entra (${report.requests} requests)` + ); + + const touchedOrgIds = new Set(); + const touchedUserIds: string[] = []; + + for (const gu of graphUsers) { + if (gu.accountEnabled === false) { + report.skippedDisabled++; + continue; + } + + const userIdentifier = getIdentifier(gu)?.toLowerCase(); + if (!userIdentifier) { + logger.warn(`No identifier for Entra user ${gu.id}, skipping`); + continue; + } + const email = gu.mail?.toLowerCase() ?? null; + const name = gu.displayName ?? null; + + // orgId -> roleIds, in mapping order + const rolesByOrg = new Map(); + for (const groupId of groupsByUser.get(gu.id) ?? []) { + for (const g of grantsByGroup.get(groupId) ?? []) { + const list = rolesByOrg.get(g.orgId) ?? []; + if (!list.includes(g.roleId)) list.push(g.roleId); + rolesByOrg.set(g.orgId, list); + } + } + for (const [orgId, roleIds] of rolesByOrg) { + if (!multiRoleByOrg.get(orgId)) rolesByOrg.set(orgId, roleIds.slice(0, 1)); + } + + const [existingUser] = await db + .select() + .from(users) + .where( + and( + eq(users.username, userIdentifier), + eq(users.idpId, idpId) + ) + ); + + if (!rolesByOrg.size && !existingUser) { + // not in any mapped group; don't create orphans + report.skippedNoMatch++; + continue; + } + + await db.transaction(async (trx) => { + let userId = existingUser?.userId; + if (!existingUser) { + userId = generateId(15); + await trx.insert(users).values({ + userId, + username: userIdentifier, + email, + name, + type: UserType.OIDC, + idpId, + emailVerified: true, + dateCreated: new Date().toISOString() + }); + report.created++; + } else { + await trx + .update(users) + .set({ email, name }) + .where(eq(users.userId, userId!)); + report.updated++; + } + + const currentUserOrgs = await trx + .select() + .from(userOrgs) + .where(eq(userOrgs.userId, userId!)); + + for (const [orgId, roleIds] of rolesByOrg) { + const current = currentUserOrgs.find((o) => o.orgId === orgId); + + if (!current) { + const org = allowedOrgs.find((o) => o.orgId === orgId)!; + await assignUserToOrg( + org, + { orgId, userId: userId!, autoProvisioned: true }, + roleIds, + trx + ); + touchedOrgIds.add(orgId); + } else if (current.autoProvisioned) { + // re-sync roles 1:1 on auto-provisioned memberships only + await trx + .delete(userOrgRoles) + .where( + and( + eq(userOrgRoles.userId, userId!), + eq(userOrgRoles.orgId, orgId) + ) + ); + await trx.insert(userOrgRoles).values( + roleIds.map((roleId) => ({ + userId: userId!, + orgId, + roleId + })) + ); + } + } + + // NOTE: no removal of stale auto-provisioned orgs — deprovisioning not approved + touchedUserIds.push(userId!); + }); + } + + for (const orgId of touchedOrgIds) { + const count = await db + .select() + .from(userOrgs) + .where(eq(userOrgs.orgId, orgId)); + await usageService.updateCount(orgId, LimitId.USERS, count.length); + } + + for (const userId of touchedUserIds) { + await calculateUserClientsForOrgs(userId).catch((err) => + logger.error("Error calculating user clients after sync", { + userId, + error: err + }) + ); + } + + logger.info("Azure directory sync done", { idpId, ...report }); + return report; +} diff --git a/server/private/lib/directorySync/run.ts b/server/private/lib/directorySync/run.ts new file mode 100644 index 0000000000..33eec52e7f --- /dev/null +++ b/server/private/lib/directorySync/run.ts @@ -0,0 +1,23 @@ +// Usage: npx tsx server/private/lib/directorySync/run.ts +// mappings.json: [{ "groupId": "", "orgId": "acme", "roleName": "Member" }] +import { readFileSync } from "fs"; +import { GroupMapping, syncAzureDirectory } from "./azure"; + +const idpId = Number(process.argv[2]); +const mappingsPath = process.argv[3]; +if (!idpId || !mappingsPath) { + console.error("Usage: run.ts "); + process.exit(1); +} + +const mappings: GroupMapping[] = JSON.parse(readFileSync(mappingsPath, "utf8")); + +syncAzureDirectory(idpId, mappings) + .then((report) => { + console.log(JSON.stringify(report, null, 2)); + process.exit(0); + }) + .catch((err) => { + console.error(err); + process.exit(1); + }); diff --git a/server/routers/idp/idpDirectorySync.ts b/server/routers/idp/idpDirectorySync.ts deleted file mode 100644 index b5a3becfad..0000000000 --- a/server/routers/idp/idpDirectorySync.ts +++ /dev/null @@ -1,3 +0,0 @@ -async function fetchIdpDirectoryData(idpId: number) { - // ... -} From f234b509f55c94285ee2729dfa23f582e9114d9b Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Thu, 8 Oct 2026 21:53:33 +0200 Subject: [PATCH 12/50] =?UTF-8?q?=E2=8F=AA=20Reset=20admin=20pages=20to=20?= =?UTF-8?q?remove=20directory=20sync=20fields?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/routers/idp/createOidcIdp.ts | 7 ---- server/routers/idp/updateOidcIdp.ts | 19 --------- src/app/admin/idp/[idpId]/general/page.tsx | 47 ++-------------------- src/app/admin/idp/create/page.tsx | 39 ++---------------- 4 files changed, 8 insertions(+), 104 deletions(-) diff --git a/server/routers/idp/createOidcIdp.ts b/server/routers/idp/createOidcIdp.ts index 6586551def..813ab0ea50 100644 --- a/server/routers/idp/createOidcIdp.ts +++ b/server/routers/idp/createOidcIdp.ts @@ -26,7 +26,6 @@ const bodySchema = z.strictObject({ namePath: z.string().optional(), scopes: z.string().nonempty(), autoProvision: z.boolean().optional(), - directorySyncEnabled: z.boolean().optional(), tags: z.string().optional(), variant: z.enum(["oidc", "google", "azure"]).optional().default("oidc") }); @@ -94,7 +93,6 @@ export async function createOidcIdp( namePath, name, autoProvision, - directorySyncEnabled, tags, variant } = parsedBody.data; @@ -122,11 +120,6 @@ export async function createOidcIdp( .values({ name, autoProvision, - // directory sync requires auto provisioning and a google/azure idp - directorySyncEnabled: - !!autoProvision && - (variant === "google" || variant === "azure") && - !!directorySyncEnabled, type: "oidc", tags, defaultOrgMapping: `'{{orgId}}'`, diff --git a/server/routers/idp/updateOidcIdp.ts b/server/routers/idp/updateOidcIdp.ts index d4fd22ac67..69b77770f8 100644 --- a/server/routers/idp/updateOidcIdp.ts +++ b/server/routers/idp/updateOidcIdp.ts @@ -30,7 +30,6 @@ const bodySchema = z.strictObject({ namePath: z.string().optional(), scopes: z.string().optional(), autoProvision: z.boolean().optional(), - directorySyncEnabled: z.boolean().optional(), defaultRoleMapping: z.string().optional(), defaultOrgMapping: z.string().optional(), tags: z.string().optional(), @@ -110,7 +109,6 @@ export async function updateOidcIdp( namePath, name, autoProvision, - directorySyncEnabled, defaultRoleMapping, defaultOrgMapping, tags, @@ -151,27 +149,10 @@ export async function updateOidcIdp( : undefined; const encryptedClientId = clientId ? encrypt(clientId, key) : undefined; - const [existingOidcConfig] = await db - .select({ variant: idpOidcConfig.variant }) - .from(idpOidcConfig) - .where(eq(idpOidcConfig.idpId, idpId)); - - // directory sync requires auto provisioning and a google/azure idp - const effectiveAutoProvision = - autoProvision ?? existingIdp.autoProvision; - const effectiveVariant = variant ?? existingOidcConfig?.variant; - const directorySyncSupported = - effectiveVariant === "google" || effectiveVariant === "azure"; - const effectiveDirectorySyncEnabled = - effectiveAutoProvision && directorySyncSupported - ? directorySyncEnabled - : false; - await db.transaction(async (trx) => { const idpData = { name, autoProvision, - directorySyncEnabled: effectiveDirectorySyncEnabled, defaultRoleMapping, defaultOrgMapping, tags diff --git a/src/app/admin/idp/[idpId]/general/page.tsx b/src/app/admin/idp/[idpId]/general/page.tsx index dfd543956d..8b7ed226f2 100644 --- a/src/app/admin/idp/[idpId]/general/page.tsx +++ b/src/app/admin/idp/[idpId]/general/page.tsx @@ -74,8 +74,7 @@ export default function GeneralPage() { emailPath: z.string().optional(), namePath: z.string().optional(), scopes: z.string().min(1, { message: t("idpScopeRequired") }), - autoProvision: z.boolean().default(false), - directorySyncEnabled: z.boolean().default(false) + autoProvision: z.boolean().default(false) }); const GoogleFormSchema = z.object({ @@ -84,8 +83,7 @@ export default function GeneralPage() { clientSecret: z .string() .min(1, { message: t("idpClientSecretRequired") }), - autoProvision: z.boolean().default(false), - directorySyncEnabled: z.boolean().default(false) + autoProvision: z.boolean().default(false) }); const AzureFormSchema = z.object({ @@ -95,8 +93,7 @@ export default function GeneralPage() { .string() .min(1, { message: t("idpClientSecretRequired") }), tenantId: z.string().min(1, { message: t("idpTenantIdRequired") }), - autoProvision: z.boolean().default(false), - directorySyncEnabled: z.boolean().default(false) + autoProvision: z.boolean().default(false) }); type OidcFormValues = z.infer; @@ -131,7 +128,6 @@ export default function GeneralPage() { namePath: "name", scopes: "openid profile email", autoProvision: true, - directorySyncEnabled: false, tenantId: "" } }); @@ -170,8 +166,7 @@ export default function GeneralPage() { name: data.idp.name, clientId: data.idpOidcConfig.clientId, clientSecret: data.idpOidcConfig.clientSecret, - autoProvision: data.idp.autoProvision, - directorySyncEnabled: data.idp.directorySyncEnabled + autoProvision: data.idp.autoProvision }; if (idpVariant === "oidc") { @@ -234,10 +229,6 @@ export default function GeneralPage() { clientId: data.clientId, clientSecret: data.clientSecret, autoProvision: data.autoProvision, - directorySyncEnabled: - data.autoProvision && - (variant === "google" || variant === "azure") && - data.directorySyncEnabled, variant }; @@ -442,39 +433,9 @@ export default function GeneralPage() { "autoProvision", checked ); - if (!checked) { - form.setValue( - "directorySyncEnabled", - false - ); - } }} /> - {(variant === "google" || - variant === "azure") && ( -
- { - form.setValue( - "directorySyncEnabled", - checked - ); - }} - description={t( - "idpDirectorySyncDescription" - )} - /> -
- )}
{form.watch("autoProvision") && ( diff --git a/src/app/admin/idp/create/page.tsx b/src/app/admin/idp/create/page.tsx index 2e72b95a1a..bbe45c87ae 100644 --- a/src/app/admin/idp/create/page.tsx +++ b/src/app/admin/idp/create/page.tsx @@ -68,8 +68,7 @@ export default function Page() { .min(1, { message: t("idpScopeRequired") }) .optional(), tenantId: z.string().optional(), - autoProvision: z.boolean().default(false), - directorySyncEnabled: z.boolean().default(false) + autoProvision: z.boolean().default(false) }); type CreateIdpFormValues = z.infer; @@ -88,8 +87,7 @@ export default function Page() { emailPath: "email", scopes: "openid profile email", tenantId: "", - autoProvision: false, - directorySyncEnabled: false + autoProvision: false } }); @@ -126,10 +124,6 @@ export default function Page() { emailPath: data.emailPath, namePath: data.namePath, autoProvision: data.autoProvision, - directorySyncEnabled: - data.autoProvision && - (data.type === "google" || data.type === "azure") && - data.directorySyncEnabled, scopes: data.scopes, variant: data.type }; @@ -223,6 +217,7 @@ export default function Page() { )} /> + @@ -247,35 +242,9 @@ export default function Page() { defaultChecked={form.getValues("autoProvision")} onCheckedChange={(checked) => { form.setValue("autoProvision", checked); - if (!checked) { - form.setValue( - "directorySyncEnabled", - false - ); - } }} - description={t( - "idpAutoProvisionConfigureAfterCreate" - )} + description={t("idpAutoProvisionConfigureAfterCreate")} /> - {(watchedType === "google" || - watchedType === "azure") && ( - { - form.setValue( - "directorySyncEnabled", - checked - ); - }} - description={t( - "idpDirectorySyncDescription" - )} - /> - )}
From c9ec42a7463afb4d61ebc288c517d73e8c7cd60e Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Thu, 8 Oct 2026 21:57:28 +0200 Subject: [PATCH 13/50] =?UTF-8?q?=F0=9F=9A=A7=20wip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DIRECTORY_SYNC_PLAN.md | 24 ++++++++---- messages/en-US.json | 2 +- server/private/lib/directorySync/azure.ts | 6 ++- .../routers/orgIdp/createOrgOidcIdp.ts | 15 ++++++++ .../routers/orgIdp/updateOrgOidcIdp.ts | 31 +++++++++++++++ .../(private)/idp/[idpId]/general/page.tsx | 37 ++++++++++++++++++ .../settings/(private)/idp/create/page.tsx | 38 ++++++++++++++++++- src/components/AutoProvisionConfigWidget.tsx | 2 +- 8 files changed, 142 insertions(+), 13 deletions(-) diff --git a/DIRECTORY_SYNC_PLAN.md b/DIRECTORY_SYNC_PLAN.md index 3951b5c4e5..a49fa4021b 100644 --- a/DIRECTORY_SYNC_PLAN.md +++ b/DIRECTORY_SYNC_PLAN.md @@ -2,7 +2,11 @@ ## What already exists -- `idp.directorySyncEnabled` exists. Create and update only allow it when `autoProvision` is on and the variant is `google` or `azure` ([createOidcIdp.ts](server/routers/idp/createOidcIdp.ts), [updateOidcIdp.ts](server/routers/idp/updateOidcIdp.ts)). Nothing reads the flag yet, and [idpDirectorySync.ts](server/routers/idp/idpDirectorySync.ts) is empty. +- `idp.directorySyncEnabled` exists. It is mutually exclusive with `autoProvision` (see [decision #2](#2-sync-replaces-auto-provisioning-and-owns-org-and-role-mapping)) and only allowed when the variant is `google` or `azure`. + - Global IdPs: [createOidcIdp.ts](server/routers/idp/createOidcIdp.ts), [updateOidcIdp.ts](server/routers/idp/updateOidcIdp.ts). + - Org IdPs: [createOrgOidcIdp.ts](server/private/routers/orgIdp/createOrgOidcIdp.ts), [updateOrgOidcIdp.ts](server/private/routers/orgIdp/updateOrgOidcIdp.ts). + - Create and update return `400` if both would be on. In the UI, only the org IdP pages expose the toggle (exclusive with auto provisioning). The global admin IdP pages don't show it for now. +- A first Azure sync prototype exists in [directorySync/azure.ts](server/private/lib/directorySync/azure.ts). It refuses to run when `autoProvision` is on. [idpDirectorySync.ts](server/routers/idp/idpDirectorySync.ts) is empty. - Provisioning only happens at login, in [validateOidcCallback.ts](server/routers/idp/validateOidcCallback.ts): - `users.username` = JMESPath(`identifierPath`) applied to the ID token claims. - Org and role mappings are JMESPath expressions over those same claims. @@ -74,7 +78,9 @@ Synced users must get the same `username` that login computes with `identifierPa Existing Azure IdPs that use `sub` need a migration story. - **Google:** `sub` = Directory `id`, so the default works (pending verification item 1). -### 2. Sync uses its own group mapping (not the login JMESPath mappings) +### 2. Sync replaces auto provisioning and owns org and role mapping + +An IdP uses **either** auto provisioning (claim-based JMESPath mappings, applied at login) **or** directory sync (group mapping, applied by the sync job), never both. The two use different role mappings, so running both would make them fight over the same memberships. Directory API responses (Graph `user`, Directory `user`) don't look like ID token claims. Faking claims only matches login for a few fields (`oid`, `name`, `upn`, `groups`), and silently diverges for the rest (`sub` is per-app in Entra, `email` is optional, `roles` and custom claims don't exist in Graph). So sync doesn't run the login org/role mappings. @@ -84,11 +90,15 @@ Directory API responses (Graph `user`, Directory `user`) don't look like ID toke - Only mapped groups are fetched, which also bounds the request count. - The admin UI can list directory groups so admins pick from a list instead of pasting IDs. Same model works for Google, where tokens carry no groups at all. -> ⚠️ **Open: login vs sync ownership.** Login keeps its claim-based mappings and, for `autoProvisioned` orgs, resets roles and removes orgs that don't match ([validateOidcCallback.ts](server/routers/idp/validateOidcCallback.ts)). With two mappings, whichever ran last wins. Options: (a) when sync is on, login skips org/role reconciliation and only authenticates + updates profile; (b) login only adds, never removes or overwrites; (c) accept the flip-flop and tell admins to keep both mappings consistent. +**Login when sync is on.** Since `autoProvision` is off, login takes the non-provisioning branch of [validateOidcCallback.ts](server/routers/idp/validateOidcCallback.ts): + +- It only authenticates. It doesn't create users or touch orgs, roles or profile fields. +- A user who hasn't been synced yet, or has no org membership, is refused with "unprovisioned". Admins can run "Sync now" to fix this. +- Sync is the only writer for users, memberships and roles of that IdP. That includes email and name updates, since login no longer updates them. -### 3. Refactor first +### 3. Shared write path (optional refactor) -Pull the "apply" half of the provisioning block out of `validateOidcCallback.ts` into a shared function, for example `applyUserMemberships(trx, idp, user, desired: { orgId, roleIds }[], { source: "login" | "sync" })`: upsert the user, add orgs, re-sync roles on `autoProvisioned` orgs. Login computes `desired` from claims, sync from `idpGroupMapping`; the write path is shared. +Login and sync no longer run on the same IdP, so sharing code isn't needed for correctness. It's still worth pulling the "apply" half of the provisioning block out of `validateOidcCallback.ts` into a shared function, for example `applyUserMemberships(trx, idp, user, desired: { orgId, roleIds }[])`: upsert the user, add orgs, re-sync roles on `autoProvisioned` orgs. Login computes `desired` from claims, sync from `idpGroupMapping`. Sync writes memberships with `autoProvisioned = true`, so the existing rule that only those rows are ever changed or removed still holds. ### 4. Deprovisioning (not confirmed) @@ -171,7 +181,7 @@ server/(private/)lib/directorySync/ ## Phases 1. **Study:** finish the Google verification items above. Confirm `oid` vs `sub` behavior on a test Entra tenant. -2. **Refactor:** extract `applyUserMemberships` from the OIDC callback, with no change in behavior. Add tests. +2. **Refactor (optional):** extract `applyUserMemberships` from the OIDC callback, with no change in behavior. Add tests. 3. **Schema:** add the sync config and state tables and migrations, and store Azure `tenantId` explicitly. 4. **Azure provider:** full sync, then delta. 5. **Scheduler:** locking, backoff, status API, "Sync now". @@ -185,7 +195,7 @@ server/(private/)lib/directorySync/ - **Blocking for deprovisioning:** should sync remove or disable users at all? See the questions in [Deprovisioning](#4-deprovisioning-not-confirmed). - How to handle existing Azure IdPs with `identifierPath = "sub"`: block sync, migrate them, or match by email? - Which group identifiers to expose in mappings for Google: emails or IDs? -- Does the per-org IdP flow ([server/private/routers/orgIdp](server/private/routers/orgIdp)) need sync too, or only global IdPs? +- Org IdPs ([server/private/routers/orgIdp](server/private/routers/orgIdp)) can already store the flag. For them, should `idpGroupMapping` be limited to the IdP's own org? - Global sync interval, or configurable per IdP? ## References diff --git a/messages/en-US.json b/messages/en-US.json index f06d1d289a..0ec7c4fe8a 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1179,7 +1179,7 @@ "idpAutoProvisionUsersDescription": "When enabled, users will be automatically created in the system upon first login with the ability to map users to roles and organizations.", "idpAutoProvisionConfigureAfterCreate": "You can configure auto provision settings once the identity provider is created.", "idpDirectorySync": "Directory Sync", - "idpDirectorySyncDescription": "Periodically sync users from the identity provider directory. Requires auto provisioning to be enabled.", + "idpDirectorySyncDescription": "Periodically sync users from the identity provider directory. Cannot be enabled together with auto provisioning.", "licenseBadge": "EE", "idpType": "Provider Type", "idpTypeDescription": "Select the type of identity provider you want to configure", diff --git a/server/private/lib/directorySync/azure.ts b/server/private/lib/directorySync/azure.ts index 7c54d4b1c3..8795874685 100644 --- a/server/private/lib/directorySync/azure.ts +++ b/server/private/lib/directorySync/azure.ts @@ -148,8 +148,10 @@ export async function syncAzureDirectory( if (existingIdp.idpOidcConfig.variant !== "azure") { throw new Error(`IdP ${idpId} is not an azure IdP`); } - if (!existingIdp.idp.autoProvision) { - throw new Error(`IdP ${idpId} does not have autoProvision enabled`); + if (existingIdp.idp.autoProvision) { + throw new Error( + `IdP ${idpId} has autoProvision enabled, which is exclusive with directory sync` + ); } if (!existingIdp.idp.directorySyncEnabled) { logger.warn(`IdP ${idpId} has directorySyncEnabled=false, syncing anyway`); diff --git a/server/private/routers/orgIdp/createOrgOidcIdp.ts b/server/private/routers/orgIdp/createOrgOidcIdp.ts index 8b214b8959..dc28e0f133 100644 --- a/server/private/routers/orgIdp/createOrgOidcIdp.ts +++ b/server/private/routers/orgIdp/createOrgOidcIdp.ts @@ -42,6 +42,7 @@ const bodySchema = z.strictObject({ namePath: z.string().optional(), scopes: z.string().nonempty(), autoProvision: z.boolean().optional(), + directorySyncEnabled: z.boolean().optional(), variant: z.enum(["oidc", "google", "azure"]).optional().default("oidc"), roleMapping: z.string().optional(), orgMapping: z.string().nullish(), @@ -122,6 +123,7 @@ export async function createOrgOidcIdp( variant, roleMapping, orgMapping: orgMappingBody, + directorySyncEnabled, tags } = parsedBody.data; @@ -138,6 +140,15 @@ export async function createOrgOidcIdp( } } + if (autoProvision && directorySyncEnabled) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Auto provisioning and directory sync cannot both be enabled" + ) + ); + } + const key = config.getRawConfig().server.secret!; const encryptedSecret = encrypt(clientSecret, key); @@ -150,6 +161,10 @@ export async function createOrgOidcIdp( .values({ name, autoProvision, + // directory sync is only supported for google/azure idps + directorySyncEnabled: + (variant === "google" || variant === "azure") && + !!directorySyncEnabled, type: "oidc", tags }) diff --git a/server/private/routers/orgIdp/updateOrgOidcIdp.ts b/server/private/routers/orgIdp/updateOrgOidcIdp.ts index 9d4698e3c9..acff8d5526 100644 --- a/server/private/routers/orgIdp/updateOrgOidcIdp.ts +++ b/server/private/routers/orgIdp/updateOrgOidcIdp.ts @@ -47,6 +47,7 @@ const bodySchema = z.strictObject({ namePath: z.string().optional(), scopes: z.string().optional(), autoProvision: z.boolean().optional(), + directorySyncEnabled: z.boolean().optional(), roleMapping: z.string().optional(), orgMapping: z.string().nullish(), tags: z.string().optional() @@ -126,6 +127,7 @@ export async function updateOrgOidcIdp( name, roleMapping, orgMapping, + directorySyncEnabled, tags } = parsedBody.data; @@ -181,10 +183,39 @@ export async function updateOrgOidcIdp( : undefined; const encryptedClientId = clientId ? encrypt(clientId, key) : undefined; + const [existingOidcConfig] = await db + .select({ variant: idpOidcConfig.variant }) + .from(idpOidcConfig) + .where(eq(idpOidcConfig.idpId, idpId)); + + // auto provisioning and directory sync are mutually exclusive + const effectiveAutoProvision = + autoProvision ?? existingIdp.autoProvision; + if ( + effectiveAutoProvision && + (directorySyncEnabled ?? existingIdp.directorySyncEnabled) + ) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Auto provisioning and directory sync cannot both be enabled" + ) + ); + } + + // directory sync is only supported for google/azure idps + const directorySyncSupported = + existingOidcConfig?.variant === "google" || + existingOidcConfig?.variant === "azure"; + const effectiveDirectorySyncEnabled = directorySyncSupported + ? directorySyncEnabled + : false; + await db.transaction(async (trx) => { const idpData = { name, autoProvision, + directorySyncEnabled: effectiveDirectorySyncEnabled, tags }; diff --git a/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx b/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx index e7e52b0683..276ba6a6db 100644 --- a/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx +++ b/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx @@ -48,6 +48,7 @@ import AutoProvisionConfigWidget from "@app/components/AutoProvisionConfigWidget import IdpAutoProvisionUsersDescription from "@app/components/IdpAutoProvisionUsersDescription"; import IdpIdentifierChangeDialog from "@app/components/IdpIdentifierChangeDialog"; import { PaidFeaturesAlert } from "@app/components/PaidFeaturesAlert"; +import { SwitchInput } from "@app/components/SwitchInput"; import { tierMatrix } from "@server/lib/billing/tierMatrix"; import { compileRoleMappingExpression, @@ -105,6 +106,7 @@ export default function GeneralPage() { namePath: z.string().nullable().optional(), scopes: z.string().min(1, { message: t("idpScopeRequired") }), autoProvision: z.boolean().default(false), + directorySyncEnabled: z.boolean().default(false), orgMapping: z.string().optional() }); @@ -118,6 +120,7 @@ export default function GeneralPage() { roleMapping: z.string().nullable().optional(), roleId: z.number().nullable().optional(), autoProvision: z.boolean().default(false), + directorySyncEnabled: z.boolean().default(false), orgMapping: z.string().optional() }); @@ -132,6 +135,7 @@ export default function GeneralPage() { roleMapping: z.string().nullable().optional(), roleId: z.number().nullable().optional(), autoProvision: z.boolean().default(false), + directorySyncEnabled: z.boolean().default(false), orgMapping: z.string().optional() }); @@ -168,6 +172,7 @@ export default function GeneralPage() { namePath: "name", scopes: "openid profile email", autoProvision: true, + directorySyncEnabled: false, roleMapping: null, roleId: null, tenantId: "", @@ -220,6 +225,7 @@ export default function GeneralPage() { clientId: data.idpOidcConfig.clientId, clientSecret: data.idpOidcConfig.clientSecret, autoProvision: data.idp.autoProvision, + directorySyncEnabled: data.idp.directorySyncEnabled, roleMapping: roleMapping || null, roleId: null, orgMapping: data.idpOrg?.orgMapping ?? "" @@ -345,6 +351,9 @@ export default function GeneralPage() { clientId: data.clientId, clientSecret: data.clientSecret, autoProvision: data.autoProvision, + directorySyncEnabled: + (variant === "google" || variant === "azure") && + data.directorySyncEnabled, roleMapping: roleMappingExpression, orgMapping: orgMappingTrimmed === "" ? null : orgMappingTrimmed }; @@ -557,6 +566,12 @@ export default function GeneralPage() { autoProvision={form.watch("autoProvision")} onAutoProvisionChange={(checked) => { form.setValue("autoProvision", checked); + if (checked) { + form.setValue( + "directorySyncEnabled", + false + ); + } }} orgId={orgId as string} roleMappingMode={roleMappingMode} @@ -583,6 +598,28 @@ export default function GeneralPage() { name: "orgMapping" }} /> + {(variant === "google" || + variant === "azure") && ( + { + form.setValue( + "directorySyncEnabled", + checked + ); + if (checked) { + form.setValue("autoProvision", false); + } + }} + description={t( + "idpDirectorySyncDescription" + )} + /> + )} diff --git a/src/app/[orgId]/settings/(private)/idp/create/page.tsx b/src/app/[orgId]/settings/(private)/idp/create/page.tsx index 33ef71eee0..8ffc078799 100644 --- a/src/app/[orgId]/settings/(private)/idp/create/page.tsx +++ b/src/app/[orgId]/settings/(private)/idp/create/page.tsx @@ -14,6 +14,7 @@ import { SettingsSectionTitle } from "@app/components/Settings"; import HeaderTitle from "@app/components/SettingsSectionTitle"; +import { SwitchInput } from "@app/components/SwitchInput"; import { OidcIdpProviderTypeSelect } from "@app/components/idp/OidcIdpProviderTypeSelect"; import { Alert, AlertDescription, AlertTitle } from "@app/components/ui/alert"; import { Button } from "@app/components/ui/button"; @@ -92,7 +93,8 @@ export default function Page() { autoProvision: z.boolean().default(false), roleMapping: z.string().nullable().optional(), roleId: z.number().nullable().optional(), - orgMapping: z.string().optional() + orgMapping: z.string().optional(), + directorySyncEnabled: z.boolean().default(false) }); type CreateIdpFormValues = z.infer; @@ -114,7 +116,8 @@ export default function Page() { autoProvision: false, roleMapping: null, roleId: null, - orgMapping: "" + orgMapping: "", + directorySyncEnabled: false } }); @@ -189,6 +192,9 @@ export default function Page() { emailPath: data.emailPath, namePath: data.namePath, autoProvision: data.autoProvision, + directorySyncEnabled: + (data.type === "google" || data.type === "azure") && + data.directorySyncEnabled, roleMapping: roleMappingExpression, scopes: data.scopes, variant: data.type @@ -333,6 +339,12 @@ export default function Page() { "autoProvision", checked ); + if (checked) { + form.setValue( + "directorySyncEnabled", + false + ); + } }} orgId={params.orgId as string} roleMappingMode={roleMappingMode} @@ -365,6 +377,28 @@ export default function Page() { name: "orgMapping" }} /> + {(form.watch("type") === "google" || + form.watch("type") === "azure") && ( + { + form.setValue( + "directorySyncEnabled", + checked + ); + if (checked) { + form.setValue("autoProvision", false); + } + }} + description={t( + "idpDirectorySyncDescription" + )} + /> + )} diff --git a/src/components/AutoProvisionConfigWidget.tsx b/src/components/AutoProvisionConfigWidget.tsx index 4cf9394445..156f74c95a 100644 --- a/src/components/AutoProvisionConfigWidget.tsx +++ b/src/components/AutoProvisionConfigWidget.tsx @@ -86,7 +86,7 @@ export default function AutoProvisionConfigWidget({ From a7f9960a0d9f01f33363cf3ea2c1f767042d64d0 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Thu, 8 Oct 2026 22:21:35 +0200 Subject: [PATCH 14/50] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/db/pg/schema/schema.ts | 4 +- .../(private)/idp/[idpId]/general/page.tsx | 81 +++++++++---------- 2 files changed, 42 insertions(+), 43 deletions(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 283222db4c..66c5127aa3 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1332,7 +1332,9 @@ export const idpOrg = pgTable("idpOrg", { .notNull() .references(() => orgs.orgId, { onDelete: "cascade" }), roleMapping: varchar("roleMapping"), - orgMapping: varchar("orgMapping") + orgMapping: varchar("orgMapping"), + directoryRoleMapping: varchar("directoryRoleMapping"), + directoryOrgMapping: varchar("directoryOrgMapping") }); export const clients = pgTable( diff --git a/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx b/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx index 276ba6a6db..fb9a823b7a 100644 --- a/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx +++ b/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx @@ -1,7 +1,28 @@ "use client"; -import { zodResolver } from "@hookform/resolvers/zod"; -import { z } from "zod"; +import AutoProvisionConfigWidget from "@app/components/AutoProvisionConfigWidget"; +import CopyToClipboard from "@app/components/CopyToClipboard"; +import IdpAutoProvisionUsersDescription from "@app/components/IdpAutoProvisionUsersDescription"; +import IdpIdentifierChangeDialog from "@app/components/IdpIdentifierChangeDialog"; +import IdpTypeBadge from "@app/components/IdpTypeBadge"; +import { + InfoSection, + InfoSectionContent, + InfoSections, + InfoSectionTitle +} from "@app/components/InfoSection"; +import { PaidFeaturesAlert } from "@app/components/PaidFeaturesAlert"; +import { + SettingsContainer, + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionForm, + SettingsSectionGrid, + SettingsSectionHeader, + SettingsSectionTitle +} from "@app/components/Settings"; +import { SwitchInput } from "@app/components/SwitchInput"; import { Button } from "@app/components/ui/button"; import { Form, @@ -13,43 +34,9 @@ import { FormMessage } from "@app/components/ui/form"; import { Input } from "@app/components/ui/input"; -import { useForm } from "react-hook-form"; -import { toast } from "@app/hooks/useToast"; -import { useRouter, useParams, redirect } from "next/navigation"; -import { - SettingsContainer, - SettingsSection, - SettingsSectionHeader, - SettingsSectionTitle, - SettingsSectionDescription, - SettingsSectionBody, - SettingsSectionForm, - SettingsSectionFooter, - SettingsSectionGrid -} from "@app/components/Settings"; -import { formatAxiosError } from "@app/lib/api"; -import { createApiClient } from "@app/lib/api"; import { useEnvContext } from "@app/hooks/useEnvContext"; -import { useState, useEffect } from "react"; -import { Alert, AlertDescription, AlertTitle } from "@app/components/ui/alert"; -import { InfoIcon, ExternalLink } from "lucide-react"; -import { - InfoSection, - InfoSectionContent, - InfoSections, - InfoSectionTitle -} from "@app/components/InfoSection"; -import CopyToClipboard from "@app/components/CopyToClipboard"; -import IdpTypeBadge from "@app/components/IdpTypeBadge"; -import { useTranslations } from "next-intl"; -import { AxiosResponse } from "axios"; -import { ListRolesResponse } from "@server/routers/role"; -import AutoProvisionConfigWidget from "@app/components/AutoProvisionConfigWidget"; -import IdpAutoProvisionUsersDescription from "@app/components/IdpAutoProvisionUsersDescription"; -import IdpIdentifierChangeDialog from "@app/components/IdpIdentifierChangeDialog"; -import { PaidFeaturesAlert } from "@app/components/PaidFeaturesAlert"; -import { SwitchInput } from "@app/components/SwitchInput"; -import { tierMatrix } from "@server/lib/billing/tierMatrix"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; import { compileRoleMappingExpression, createMappingBuilderRule, @@ -58,6 +45,15 @@ import { MappingBuilderRule, RoleMappingMode } from "@app/lib/idpRoleMapping"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { tierMatrix } from "@server/lib/billing/tierMatrix"; +import { ListRolesResponse } from "@server/routers/role"; +import { AxiosResponse } from "axios"; +import { useTranslations } from "next-intl"; +import { useParams, useRouter } from "next/navigation"; +import { useEffect, useState } from "react"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; export default function GeneralPage() { const { env } = useEnvContext(); @@ -143,9 +139,7 @@ export default function GeneralPage() { type GoogleFormValues = z.infer; type AzureFormValues = z.infer; type GeneralFormValues = - | OidcFormValues - | GoogleFormValues - | AzureFormValues; + OidcFormValues | GoogleFormValues | AzureFormValues; // Get the appropriate schema based on variant const getFormSchema = () => { @@ -612,7 +606,10 @@ export default function GeneralPage() { checked ); if (checked) { - form.setValue("autoProvision", false); + form.setValue( + "autoProvision", + false + ); } }} description={t( From a6c21e0a7952ee218c7e567e5a8c8b2d77fc4f7e Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Fri, 9 Oct 2026 20:22:08 +0200 Subject: [PATCH 15/50] =?UTF-8?q?=F0=9F=93=9D=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DIRECTORY_SYNC_PLAN.md | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/DIRECTORY_SYNC_PLAN.md b/DIRECTORY_SYNC_PLAN.md index a49fa4021b..41540e1fee 100644 --- a/DIRECTORY_SYNC_PLAN.md +++ b/DIRECTORY_SYNC_PLAN.md @@ -34,17 +34,17 @@ ## Provider study -| | Microsoft Entra (done) | Google Workspace (to study) | -| ----------------- | ---------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| API | Microsoft Graph | Admin SDK Directory API (`admin.googleapis.com/admin/directory/v1`) | -| Users | `GET /users`, 999 per page | `users.list?customer=my_customer`, 500 per page | -| Groups / members | `/groups`, `/groups/{id}/members` or `/transitiveMembers` | `groups.list` (200 per page), `members.list?includeDerivedMembership=true` | -| Incremental | ✅ `/users/delta` and `/groups/delta` (with `members@delta`) | ❌ No delta API: full snapshot every run, then diff on our side | -| App-only auth | Client credentials using the **existing** clientId and secret, plus tenant | **Service account + domain-wide delegation** impersonating an admin, so **new credentials**: SA JSON key and admin email | -| Permissions | `User.Read.All` and `GroupMember.Read.All` (application), plus **admin consent** | Scopes `admin.directory.user.readonly`, `admin.directory.group.readonly` and `admin.directory.group.member.readonly`, authorized in the Admin console under **Domain-wide delegation** | -| Throttling | Per app per tenant, `429` + `Retry-After` | Per project quota (believed to be about 2,400 queries/min, to verify), `403 rateLimitExceeded` / `429`, exponential backoff | -| Disabled users | `accountEnabled = false` | `suspended`, `archived` | -| Stable user ID | `id` (= `oid` claim) | `id` (= `sub` claim) | +| | Microsoft Entra (done) | Google Workspace (to study) | +| ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| API | Microsoft Graph | Admin SDK Directory API (`admin.googleapis.com/admin/directory/v1`) | +| Users | `GET /users`, 999 per page | `users.list?customer=my_customer`, 500 per page | +| Groups / members | `/groups`, `/groups/{id}/members` or `/transitiveMembers` | `groups.list` (200 per page), `members.list?includeDerivedMembership=true` | +| Incremental | ✅ `/users/delta` and `/groups/delta` (with `members@delta`) | ❌ No delta API: full snapshot every run, then diff on our side | +| App-only auth | Client credentials using the **existing** clientId and secret, plus tenant | **Service account + domain-wide delegation** impersonating an admin, so **new credentials**: SA JSON key and admin email | +| Permissions | `User.Read.All` and `GroupMember.Read.All` (application), plus **admin consent** | Scopes `admin.directory.user.readonly`, `admin.directory.group.readonly` and `admin.directory.group.member.readonly`, authorized in the Admin console under **Domain-wide delegation** | +| Throttling | Per app per tenant, `429` + `Retry-After` | Per project quota (believed to be about 2,400 queries/min, to verify), `403 rateLimitExceeded` / `429`, exponential backoff | +| Disabled users | `accountEnabled = false` | `suspended`, `archived` | +| Stable user ID | `id` (= `oid` claim) | `id` (= `sub` claim) | ### Request counts per sync (Entra) @@ -64,6 +64,17 @@ With N users, G groups, and up to 999 items per page: 5. Whether `includeDerivedMembership` is enough for nested groups, or whether we need the Cloud Identity `searchTransitiveGroups` API. 6. The library choice: `google-auth-library`, or signing the JWT-bearer assertion ourselves. Neither `jose` nor any Google or Azure SDK is a dependency today. +## How other products do it + +- **NetBird:** syncs users and groups. Synced groups play the part that roles play in Pangolin: access controls are set per group. +- **defguard:** works the same way. +- Neither relies much on the IdP's own roles (Entra app roles, Google admin roles). Groups are what gets synced and used. + +**What this means for Pangolin:** + +- **Org IdPs** (one IdP per org): an IdP group can map one-to-one to a Pangolin role in that org, like NetBird and defguard do. +- **Global IdPs** (one IdP shared by several orgs): it's harder. A group has to resolve to an org *and* a role, so a direct group → role mapping isn't enough. Either keep the existing JMESPath org and role mappings, or add a mapping that is scoped per org. + ## Key design decisions ### 1. Matching synced users to users who log in (biggest risk) @@ -195,7 +206,8 @@ server/(private/)lib/directorySync/ - **Blocking for deprovisioning:** should sync remove or disable users at all? See the questions in [Deprovisioning](#4-deprovisioning-not-confirmed). - How to handle existing Azure IdPs with `identifierPath = "sub"`: block sync, migrate them, or match by email? - Which group identifiers to expose in mappings for Google: emails or IDs? -- Org IdPs ([server/private/routers/orgIdp](server/private/routers/orgIdp)) can already store the flag. For them, should `idpGroupMapping` be limited to the IdP's own org? +- Org IdPs ([server/private/routers/orgIdp](server/private/routers/orgIdp)) can already store the flag. For them, should `idpGroupMapping` be limited to the IdP's own org? Org IdPs fit the group = role model most easily (see [How other products do it](#how-other-products-do-it)). +- For global IdPs, how should a synced group resolve to an org and a role? Keep the JMESPath mappings, or add per-org group → role mappings? - Global sync interval, or configurable per IdP? ## References From 7453ea5a9fbc672420d149fe73eb1292ddb7704d Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Sat, 10 Oct 2026 02:03:07 +0200 Subject: [PATCH 16/50] =?UTF-8?q?=F0=9F=9A=A7=20Create=20Directory=20sync?= =?UTF-8?q?=20role=20mapping=20fields?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- messages/en-US.json | 17 + server/db/pg/schema/schema.ts | 4 +- .../routers/orgIdp/updateOrgOidcIdp.ts | 18 +- .../(private)/idp/[idpId]/general/page.tsx | 54 ++- src/components/DirectorySyncConfigWidget.tsx | 77 +++++ .../DirectorySyncRoleMappingFields.tsx | 309 ++++++++++++++++++ src/components/HorizontalTabs.tsx | 15 +- src/lib/idpDirectorySyncRoleMapping.ts | 147 +++++++++ 8 files changed, 626 insertions(+), 15 deletions(-) create mode 100644 src/components/DirectorySyncConfigWidget.tsx create mode 100644 src/components/DirectorySyncRoleMappingFields.tsx create mode 100644 src/lib/idpDirectorySyncRoleMapping.ts diff --git a/messages/en-US.json b/messages/en-US.json index 0ec7c4fe8a..6a7f8fddc1 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1180,6 +1180,23 @@ "idpAutoProvisionConfigureAfterCreate": "You can configure auto provision settings once the identity provider is created.", "idpDirectorySync": "Directory Sync", "idpDirectorySyncDescription": "Periodically sync users from the identity provider directory. Cannot be enabled together with auto provisioning.", + "idpDirectorySyncDeletions": "Sync User Deletions", + "idpDirectorySyncDeletionsDescription": "When enabled, users who are deleted from the directory or no longer match the role mapping are removed from this organization.", + "directorySyncRoleMappingDescription": "Determine how roles are assigned to users synced from the directory.", + "directorySyncRoleMappingModeFixedRoles": "Fixed Roles", + "directorySyncRoleMappingModeGroupMapping": "Group Mapping", + "directorySyncRoleMappingModeAutomatic": "Automatic", + "directorySyncRoleMappingFixedRolesDescription": "Assign the same role set to every synced user.", + "directorySyncRoleMappingGroup": "Directory Group", + "directorySyncRoleMappingGroupPlaceholder": "Group name or ID", + "directorySyncRoleMappingAddGroupRule": "Add Group Mapping", + "directorySyncRoleMappingGroupMappingDescription": "Members of a directory group receive the roles mapped to that group.", + "directorySyncRoleMappingAutomaticDescription": "Each directory group a user belongs to is mapped to the role with the same name.", + "directorySyncRoleMappingCreateMissingRoles": "Create missing roles", + "directorySyncRoleMappingCreateMissingRolesDescription": "When enabled, a role is created for any directory group that doesn't have a matching role yet. Otherwise those groups are skipped.", + "directorySyncRoleMappingGroupMappingSkipNotice": "Only users who belong to at least one of the groups mapped below are synced. Users without a mapped group are not synced.", + "directorySyncRoleMappingAutomaticSkipNotice": "Users who don't belong to any directory group are not synced.", + "directorySyncRoleMappingAutomaticNoCreateSkipNotice": "Users who don't belong to any directory group, or whose groups have no role with the same name, are not synced.", "licenseBadge": "EE", "idpType": "Provider Type", "idpTypeDescription": "Select the type of identity provider you want to configure", diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 66c5127aa3..8ffffa5229 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1334,7 +1334,9 @@ export const idpOrg = pgTable("idpOrg", { roleMapping: varchar("roleMapping"), orgMapping: varchar("orgMapping"), directoryRoleMapping: varchar("directoryRoleMapping"), - directoryOrgMapping: varchar("directoryOrgMapping") + directorySyncDeletions: boolean("directorySyncDeletions") + .notNull() + .default(true) }); export const clients = pgTable( diff --git a/server/private/routers/orgIdp/updateOrgOidcIdp.ts b/server/private/routers/orgIdp/updateOrgOidcIdp.ts index acff8d5526..13ea23311f 100644 --- a/server/private/routers/orgIdp/updateOrgOidcIdp.ts +++ b/server/private/routers/orgIdp/updateOrgOidcIdp.ts @@ -50,6 +50,8 @@ const bodySchema = z.strictObject({ directorySyncEnabled: z.boolean().optional(), roleMapping: z.string().optional(), orgMapping: z.string().nullish(), + directoryRoleMapping: z.string().nullish(), + directorySyncDeletions: z.boolean().optional(), tags: z.string().optional() }); @@ -60,7 +62,6 @@ const UpdateOrgIdpResponseDataSchema = z.object({ idpId: z.number() }); - registry.registerPath({ method: "post", path: "/org/{orgId}/idp/{idpId}/oidc", @@ -81,7 +82,9 @@ registry.registerPath({ description: "Successful response", content: { "application/json": { - schema: createApiResponseSchema(UpdateOrgIdpResponseDataSchema) + schema: createApiResponseSchema( + UpdateOrgIdpResponseDataSchema + ) } } } @@ -128,6 +131,8 @@ export async function updateOrgOidcIdp( roleMapping, orgMapping, directorySyncEnabled, + directoryRoleMapping, + directorySyncDeletions, tags } = parsedBody.data; @@ -255,6 +260,8 @@ export async function updateOrgOidcIdp( const idpOrgPolicyPatch: { roleMapping?: string; orgMapping?: string | null; + directoryRoleMapping?: string | null; + directorySyncDeletions?: boolean; } = {}; if (roleMapping !== undefined) { idpOrgPolicyPatch.roleMapping = roleMapping; @@ -262,6 +269,13 @@ export async function updateOrgOidcIdp( if (orgMapping !== undefined) { idpOrgPolicyPatch.orgMapping = orgMapping; } + if (directoryRoleMapping !== undefined) { + idpOrgPolicyPatch.directoryRoleMapping = directoryRoleMapping; + } + if (directorySyncDeletions !== undefined) { + idpOrgPolicyPatch.directorySyncDeletions = + directorySyncDeletions; + } if (Object.keys(idpOrgPolicyPatch).length > 0) { await trx .update(idpOrg) diff --git a/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx b/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx index fb9a823b7a..0f170c44b4 100644 --- a/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx +++ b/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx @@ -2,6 +2,7 @@ import AutoProvisionConfigWidget from "@app/components/AutoProvisionConfigWidget"; import CopyToClipboard from "@app/components/CopyToClipboard"; +import DirectorySyncConfigWidget from "@app/components/DirectorySyncConfigWidget"; import IdpAutoProvisionUsersDescription from "@app/components/IdpAutoProvisionUsersDescription"; import IdpIdentifierChangeDialog from "@app/components/IdpIdentifierChangeDialog"; import IdpTypeBadge from "@app/components/IdpTypeBadge"; @@ -22,7 +23,6 @@ import { SettingsSectionHeader, SettingsSectionTitle } from "@app/components/Settings"; -import { SwitchInput } from "@app/components/SwitchInput"; import { Button } from "@app/components/ui/button"; import { Form, @@ -45,6 +45,12 @@ import { MappingBuilderRule, RoleMappingMode } from "@app/lib/idpRoleMapping"; +import { + createDefaultDirectorySyncRoleMappingConfig, + DirectorySyncRoleMappingConfig, + parseDirectorySyncRoleMapping, + serializeDirectorySyncRoleMapping +} from "@app/lib/idpDirectorySyncRoleMapping"; import { zodResolver } from "@hookform/resolvers/zod"; import { tierMatrix } from "@server/lib/billing/tierMatrix"; import { ListRolesResponse } from "@server/routers/role"; @@ -72,6 +78,10 @@ export default function GeneralPage() { MappingBuilderRule[] >([createMappingBuilderRule()]); const [rawRoleExpression, setRawRoleExpression] = useState(""); + const [directoryRoleMapping, setDirectoryRoleMapping] = + useState( + createDefaultDirectorySyncRoleMappingConfig + ); const [variant, setVariant] = useState<"oidc" | "google" | "azure">("oidc"); const [originalIdentifierPath, setOriginalIdentifierPath] = useState(""); const [identifierConfirmOpen, setIdentifierConfirmOpen] = useState(false); @@ -103,6 +113,7 @@ export default function GeneralPage() { scopes: z.string().min(1, { message: t("idpScopeRequired") }), autoProvision: z.boolean().default(false), directorySyncEnabled: z.boolean().default(false), + directorySyncDeletions: z.boolean().default(true), orgMapping: z.string().optional() }); @@ -117,6 +128,7 @@ export default function GeneralPage() { roleId: z.number().nullable().optional(), autoProvision: z.boolean().default(false), directorySyncEnabled: z.boolean().default(false), + directorySyncDeletions: z.boolean().default(true), orgMapping: z.string().optional() }); @@ -132,6 +144,7 @@ export default function GeneralPage() { roleId: z.number().nullable().optional(), autoProvision: z.boolean().default(false), directorySyncEnabled: z.boolean().default(false), + directorySyncDeletions: z.boolean().default(true), orgMapping: z.string().optional() }); @@ -167,6 +180,7 @@ export default function GeneralPage() { scopes: "openid profile email", autoProvision: true, directorySyncEnabled: false, + directorySyncDeletions: true, roleMapping: null, roleId: null, tenantId: "", @@ -220,6 +234,8 @@ export default function GeneralPage() { clientSecret: data.idpOidcConfig.clientSecret, autoProvision: data.idp.autoProvision, directorySyncEnabled: data.idp.directorySyncEnabled, + directorySyncDeletions: + data.idpOrg?.directorySyncDeletions ?? true, roleMapping: roleMapping || null, roleId: null, orgMapping: data.idpOrg?.orgMapping ?? "" @@ -254,6 +270,12 @@ export default function GeneralPage() { setRawRoleExpression( detectedRoleMappingConfig.rawExpression ); + + setDirectoryRoleMapping( + parseDirectorySyncRoleMapping( + data.idpOrg.directoryRoleMapping + ) + ); } } catch (e) { toast({ @@ -349,7 +371,10 @@ export default function GeneralPage() { (variant === "google" || variant === "azure") && data.directorySyncEnabled, roleMapping: roleMappingExpression, - orgMapping: orgMappingTrimmed === "" ? null : orgMappingTrimmed + orgMapping: orgMappingTrimmed === "" ? null : orgMappingTrimmed, + directoryRoleMapping: + serializeDirectorySyncRoleMapping(directoryRoleMapping), + directorySyncDeletions: data.directorySyncDeletions }; // Add variant-specific fields @@ -594,13 +619,13 @@ export default function GeneralPage() { /> {(variant === "google" || variant === "azure") && ( - { + onDirectorySyncEnabledChange={( + checked + ) => { form.setValue( "directorySyncEnabled", checked @@ -612,9 +637,20 @@ export default function GeneralPage() { ); } }} - description={t( - "idpDirectorySyncDescription" + orgId={orgId as string} + roleMapping={directoryRoleMapping} + syncDeletions={form.watch( + "directorySyncDeletions" )} + onSyncDeletionsChange={(checked) => + form.setValue( + "directorySyncDeletions", + checked + ) + } + onRoleMappingChange={ + setDirectoryRoleMapping + } /> )} diff --git a/src/components/DirectorySyncConfigWidget.tsx b/src/components/DirectorySyncConfigWidget.tsx new file mode 100644 index 0000000000..7404cd0444 --- /dev/null +++ b/src/components/DirectorySyncConfigWidget.tsx @@ -0,0 +1,77 @@ +"use client"; + +import DirectorySyncRoleMappingFields from "@app/components/DirectorySyncRoleMappingFields"; +import { SwitchInput } from "@app/components/SwitchInput"; +import { DirectorySyncRoleMappingConfig } from "@app/lib/idpDirectorySyncRoleMapping"; +import { useTranslations } from "next-intl"; + +type DirectorySyncConfigWidgetProps = { + directorySyncEnabled: boolean; + onDirectorySyncEnabledChange: (checked: boolean) => void; + roleMapping: DirectorySyncRoleMappingConfig; + onRoleMappingChange: (roleMapping: DirectorySyncRoleMappingConfig) => void; + syncDeletions: boolean; + onSyncDeletionsChange: (checked: boolean) => void; + orgId: string; + showDirectorySyncSwitch?: boolean; + disabled?: boolean; + roleMappingFieldIdPrefix?: string; + directorySyncSwitchId?: string; + syncDeletionsSwitchId?: string; +}; + +export default function DirectorySyncConfigWidget({ + directorySyncEnabled, + onDirectorySyncEnabledChange, + roleMapping, + onRoleMappingChange, + syncDeletions, + onSyncDeletionsChange, + orgId, + showDirectorySyncSwitch = true, + disabled = false, + roleMappingFieldIdPrefix = "org-idp-directory-sync", + directorySyncSwitchId = "directory-sync-toggle", + syncDeletionsSwitchId = "directory-sync-deletions-toggle" +}: DirectorySyncConfigWidgetProps) { + const t = useTranslations(); + + const showRoleMapping = + showDirectorySyncSwitch === false || directorySyncEnabled; + + return ( +
+ {showDirectorySyncSwitch && ( +
+ +
+ )} + + {showRoleMapping && ( +
+ + +
+ )} +
+ ); +} diff --git a/src/components/DirectorySyncRoleMappingFields.tsx b/src/components/DirectorySyncRoleMappingFields.tsx new file mode 100644 index 0000000000..31b8b3d08d --- /dev/null +++ b/src/components/DirectorySyncRoleMappingFields.tsx @@ -0,0 +1,309 @@ +"use client"; + +import { HorizontalTabs } from "@app/components/HorizontalTabs"; +import { Alert, AlertDescription } from "@app/components/ui/alert"; +import { FormDescription, FormLabel } from "@app/components/ui/form"; +import { Button } from "@app/components/ui/button"; +import { CheckboxWithLabel } from "@app/components/ui/checkbox"; +import { Input } from "@app/components/ui/input"; +import { InfoIcon } from "lucide-react"; +import { useTranslations } from "next-intl"; +import { ReactNode, useEffect } from "react"; +import { + createDirectoryGroupRoleRule, + DirectoryGroupRoleRule, + DirectorySyncRoleMappingConfig, + DirectorySyncRoleMappingMode +} from "@app/lib/idpDirectorySyncRoleMapping"; +import { usePaidStatus } from "@app/hooks/usePaidStatus"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { tierMatrix } from "@server/lib/billing/tierMatrix"; +import { build } from "@server/build"; +import { RolesSelector, SelectedRole } from "./roles-selector"; + +export type DirectorySyncRoleMappingFieldsProps = { + value: DirectorySyncRoleMappingConfig; + onChange: (value: DirectorySyncRoleMappingConfig) => void; + orgId: string; + /** Unique prefix for field ids when multiple instances exist on one page. */ + fieldIdPrefix?: string; +}; + +export default function DirectorySyncRoleMappingFields({ + value, + onChange, + orgId, + fieldIdPrefix = "directory-sync-role-mapping" +}: DirectorySyncRoleMappingFieldsProps) { + const t = useTranslations(); + const { env } = useEnvContext(); + const { isPaidUser } = usePaidStatus(); + + const supportsMultipleRolesPerUser = isPaidUser(tierMatrix.fullRbac); + const showSingleRoleDisclaimer = + !env.flags.disableEnterpriseFeatures && + !isPaidUser(tierMatrix.fullRbac); + + const { mode, fixedRoleNames, groupRules, createMissingRoles } = value; + + useEffect(() => { + if (!supportsMultipleRolesPerUser && fixedRoleNames.length > 1) { + onChange({ ...value, fixedRoleNames: [fixedRoleNames[0]] }); + } + }, [supportsMultipleRolesPerUser, fixedRoleNames, value, onChange]); + + const selectRoleNames = ( + nextTags: SelectedRole[], + current: string[] + ): string[] => { + let names = [...new Set(nextTags.map((tag) => tag.text))]; + if (!supportsMultipleRolesPerUser) { + if (names.length === 0 && current.length > 0) { + return [current[current.length - 1]!]; + } + if (names.length > 1) { + names = [names[names.length - 1]!]; + } + } + return names; + }; + + const setGroupRules = (rules: DirectoryGroupRoleRule[]) => + onChange({ + ...value, + groupRules: rules.length ? rules : [createDirectoryGroupRoleRule()] + }); + + const modes: { value: DirectorySyncRoleMappingMode; label: string }[] = [ + { + value: "automatic", + label: t("directorySyncRoleMappingModeAutomatic") + }, + { + value: "groupMapping", + label: t("directorySyncRoleMappingModeGroupMapping") + }, + { + value: "fixedRoles", + label: t("directorySyncRoleMappingModeFixedRoles") + } + ]; + + const groupRulesGridClass = + "md:grid md:grid-cols-[minmax(0,1fr)_minmax(0,1.75fr)_6rem] md:gap-x-3"; + + const panels: Record = { + fixedRoles: ( +
+ ({ + id: name, + text: name + }))} + mapRolesByName + orgId={orgId} + onSelectRoles={(nextTags) => + onChange({ + ...value, + fixedRoleNames: selectRoleNames( + nextTags, + fixedRoleNames + ) + }) + } + /> + + {t("directorySyncRoleMappingFixedRolesDescription")} + +
+ ), + groupMapping: ( +
+
+
+ + {t("directorySyncRoleMappingGroup")} + + + {t("roleMappingAssignRoles")} + + +
+ + {groupRules.map((rule, index) => ( +
+
+ + {t("directorySyncRoleMappingGroup")} + + + setGroupRules( + groupRules.map((row, i) => + i === index + ? { + ...row, + group: e.target.value + } + : row + ) + ) + } + placeholder={t( + "directorySyncRoleMappingGroupPlaceholder" + )} + /> +
+
+ + {t("roleMappingAssignRoles")} + + ({ id: name, text: name }) + )} + buttonText={t("roleMappingAssignRoles")} + mapRolesByName + orgId={orgId} + onSelectRoles={(nextTags) => + setGroupRules( + groupRules.map((row, i) => + i === index + ? { + ...row, + roleNames: + selectRoleNames( + nextTags, + row.roleNames + ) + } + : row + ) + ) + } + /> +
+
+ +
+
+ ))} +
+ + + + {t("directorySyncRoleMappingGroupMappingDescription")} + +
+ ), + automatic: ( +
+ + {t("directorySyncRoleMappingAutomaticDescription")} + + + onChange({ + ...value, + createMissingRoles: checked === true + }) + } + /> + + {t("directorySyncRoleMappingCreateMissingRolesDescription")} + +
+ ) + }; + + const skipNotices: Partial> = { + groupMapping: t("directorySyncRoleMappingGroupMappingSkipNotice"), + automatic: createMissingRoles + ? t("directorySyncRoleMappingAutomaticSkipNotice") + : t("directorySyncRoleMappingAutomaticNoCreateSkipNotice") + }; + + const activeTab = Math.max( + modes.findIndex((option) => option.value === mode), + 0 + ); + + return ( +
+
+ {showSingleRoleDisclaimer && ( + + {build === "saas" + ? t("singleRolePerUserPlanNotice") + : t("singleRolePerUserEditionNotice")} + + )} +
+ + ({ + title: option.label, + href: "#" + }))} + activeTab={activeTab} + onTabChange={(index) => + onChange({ ...value, mode: modes[index]!.value }) + } + > + {modes.map((option) => ( +
+ + {t("directorySyncRoleMappingDescription")} + + + {skipNotices[option.value] && ( + + + + {skipNotices[option.value]} + + + )} + {panels[option.value]} +
+ ))} + +
+ ); +} diff --git a/src/components/HorizontalTabs.tsx b/src/components/HorizontalTabs.tsx index 6c91dbfefd..725e2359b7 100644 --- a/src/components/HorizontalTabs.tsx +++ b/src/components/HorizontalTabs.tsx @@ -24,6 +24,10 @@ interface HorizontalTabsProps { disabled?: boolean; clientSide?: boolean; defaultTab?: number; + /** Client-side only: controlled active tab index. */ + activeTab?: number; + /** Client-side only: called with the clicked tab index. */ + onTabChange?: (index: number) => void; } export function HorizontalTabs({ @@ -31,13 +35,17 @@ export function HorizontalTabs({ items, disabled = false, clientSide = false, - defaultTab = 0 + defaultTab = 0, + activeTab, + onTabChange }: HorizontalTabsProps) { const pathname = usePathname(); const params = useParams(); const { licenseStatus, isUnlocked } = useLicenseStatusContext(); const t = useTranslations(); - const [activeClientTab, setActiveClientTab] = useState(defaultTab); + const [uncontrolledClientTab, setUncontrolledClientTab] = + useState(defaultTab); + const activeClientTab = activeTab ?? uncontrolledClientTab; function hydrateHref(href: string) { return href @@ -75,7 +83,8 @@ export function HorizontalTabs({ type="button" onClick={() => { if (!isDisabled) { - setActiveClientTab(index); + setUncontrolledClientTab(index); + onTabChange?.(index); } }} className={cn( diff --git a/src/lib/idpDirectorySyncRoleMapping.ts b/src/lib/idpDirectorySyncRoleMapping.ts new file mode 100644 index 0000000000..b93a2f20f7 --- /dev/null +++ b/src/lib/idpDirectorySyncRoleMapping.ts @@ -0,0 +1,147 @@ +export type DirectorySyncRoleMappingMode = + "fixedRoles" | "groupMapping" | "automatic"; + +export type DirectoryGroupRoleRule = { + /** Stable React list key; not persisted. */ + id?: string; + /** Directory group name or ID to match. */ + group: string; + roleNames: string[]; +}; + +export type DirectorySyncRoleMappingConfig = { + mode: DirectorySyncRoleMappingMode; + /** `fixedRoles`: roles given to every synced user. */ + fixedRoleNames: string[]; + /** `groupMapping`: explicit directory group → roles rules. */ + groupRules: DirectoryGroupRoleRule[]; + /** `automatic`: create a role named after a group when none exists. */ + createMissingRoles: boolean; +}; + +const DIRECTORY_SYNC_ROLE_MAPPING_VERSION = 1; + +const MODES: DirectorySyncRoleMappingMode[] = [ + "fixedRoles", + "groupMapping", + "automatic" +]; + +function newGroupRoleRuleId(): string { + if (typeof crypto !== "undefined" && "randomUUID" in crypto) { + return crypto.randomUUID(); + } + return `group-rule-${Date.now()}-${Math.random().toString(36).slice(2, 11)}`; +} + +export function createDirectoryGroupRoleRule(): DirectoryGroupRoleRule { + return { + id: newGroupRoleRuleId(), + group: "", + roleNames: [] + }; +} + +export function createDefaultDirectorySyncRoleMappingConfig(): DirectorySyncRoleMappingConfig { + return { + mode: "automatic", + fixedRoleNames: [], + groupRules: [createDirectoryGroupRoleRule()], + createMissingRoles: false + }; +} + +function toStringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((v): v is string => typeof v === "string") + : []; +} + +/** Parse the stored `idpOrg.directoryRoleMapping` value. Invalid values fall back to defaults. */ +export function parseDirectorySyncRoleMapping( + stored: string | null | undefined +): DirectorySyncRoleMappingConfig { + const defaults = createDefaultDirectorySyncRoleMappingConfig(); + if (!stored?.trim()) { + return defaults; + } + + let parsed: unknown; + try { + parsed = JSON.parse(stored); + } catch { + return defaults; + } + if (!parsed || typeof parsed !== "object") { + return defaults; + } + + const raw = parsed as Record; + const mode = MODES.includes(raw.mode as DirectorySyncRoleMappingMode) + ? (raw.mode as DirectorySyncRoleMappingMode) + : defaults.mode; + + const groupRules = Array.isArray(raw.groupRules) + ? raw.groupRules + .filter( + (rule): rule is Record => + !!rule && typeof rule === "object" + ) + .map((rule) => ({ + id: newGroupRoleRuleId(), + group: typeof rule.group === "string" ? rule.group : "", + roleNames: toStringArray(rule.roleNames) + })) + : []; + + return { + mode, + fixedRoleNames: toStringArray(raw.fixedRoleNames), + groupRules: groupRules.length ? groupRules : defaults.groupRules, + createMissingRoles: raw.createMissingRoles === true + }; +} + +/** + * Serialize for `idpOrg.directoryRoleMapping`. Only the active mode's fields are kept. + * Returns null when the active mode has nothing configured. + */ +export function serializeDirectorySyncRoleMapping( + config: DirectorySyncRoleMappingConfig +): string | null { + const base = { + version: DIRECTORY_SYNC_ROLE_MAPPING_VERSION, + mode: config.mode + }; + + switch (config.mode) { + case "fixedRoles": { + const fixedRoleNames = config.fixedRoleNames + .map((name) => name.trim()) + .filter(Boolean); + if (!fixedRoleNames.length) { + return null; + } + return JSON.stringify({ ...base, fixedRoleNames }); + } + case "groupMapping": { + const groupRules = config.groupRules + .map((rule) => ({ + group: rule.group.trim(), + roleNames: rule.roleNames + .map((name) => name.trim()) + .filter(Boolean) + })) + .filter((rule) => rule.group && rule.roleNames.length); + if (!groupRules.length) { + return null; + } + return JSON.stringify({ ...base, groupRules }); + } + case "automatic": + return JSON.stringify({ + ...base, + createMissingRoles: config.createMissingRoles + }); + } +} From fd6050a20469f00ce5be374594824e530db1cfa0 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Sat, 10 Oct 2026 02:06:58 +0200 Subject: [PATCH 17/50] =?UTF-8?q?=F0=9F=92=84=20Put=20directory=20sync=20i?= =?UTF-8?q?nto=20its=20own=20section?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../(private)/idp/[idpId]/general/page.tsx | 35 +++++++++++++++---- src/components/DirectorySyncConfigWidget.tsx | 1 - 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx b/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx index 0f170c44b4..83bb963713 100644 --- a/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx +++ b/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx @@ -617,8 +617,29 @@ export default function GeneralPage() { name: "orgMapping" }} /> - {(variant === "google" || - variant === "azure") && ( + + + + + + {/* Directory Sync Settings */} + {(variant === "google" || variant === "azure") && ( + + + + {t("idpDirectorySync")} + + + {t("idpDirectorySyncDescription")} + + + +
+ - )} - - -
- + + + + + )} {/* Google Configuration */} {variant === "google" && ( diff --git a/src/components/DirectorySyncConfigWidget.tsx b/src/components/DirectorySyncConfigWidget.tsx index 7404cd0444..9085b5d949 100644 --- a/src/components/DirectorySyncConfigWidget.tsx +++ b/src/components/DirectorySyncConfigWidget.tsx @@ -46,7 +46,6 @@ export default function DirectorySyncConfigWidget({ Date: Wed, 14 Oct 2026 06:45:54 +0200 Subject: [PATCH 18/50] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20directory=20sync=20m?= =?UTF-8?q?apping=20on=20create=20page?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../routers/orgIdp/createOrgOidcIdp.ts | 8 +- .../settings/(private)/idp/create/page.tsx | 102 +++++++++++++----- 2 files changed, 85 insertions(+), 25 deletions(-) diff --git a/server/private/routers/orgIdp/createOrgOidcIdp.ts b/server/private/routers/orgIdp/createOrgOidcIdp.ts index dc28e0f133..0ad2093350 100644 --- a/server/private/routers/orgIdp/createOrgOidcIdp.ts +++ b/server/private/routers/orgIdp/createOrgOidcIdp.ts @@ -46,6 +46,8 @@ const bodySchema = z.strictObject({ variant: z.enum(["oidc", "google", "azure"]).optional().default("oidc"), roleMapping: z.string().optional(), orgMapping: z.string().nullish(), + directoryRoleMapping: z.string().nullish(), + directorySyncDeletions: z.boolean().optional(), tags: z.string().optional() }); @@ -124,6 +126,8 @@ export async function createOrgOidcIdp( roleMapping, orgMapping: orgMappingBody, directorySyncEnabled, + directoryRoleMapping, + directorySyncDeletions, tags } = parsedBody.data; @@ -189,7 +193,9 @@ export async function createOrgOidcIdp( idpId: idpRes.idpId, orgId: orgId, roleMapping: roleMapping || null, - orgMapping: orgMappingBody + orgMapping: orgMappingBody, + directoryRoleMapping: directoryRoleMapping || null, + directorySyncDeletions: directorySyncDeletions ?? true }); }); diff --git a/src/app/[orgId]/settings/(private)/idp/create/page.tsx b/src/app/[orgId]/settings/(private)/idp/create/page.tsx index 8ffc078799..e72551ae49 100644 --- a/src/app/[orgId]/settings/(private)/idp/create/page.tsx +++ b/src/app/[orgId]/settings/(private)/idp/create/page.tsx @@ -1,6 +1,7 @@ "use client"; import AutoProvisionConfigWidget from "@app/components/AutoProvisionConfigWidget"; +import DirectorySyncConfigWidget from "@app/components/DirectorySyncConfigWidget"; import IdpAutoProvisionUsersDescription from "@app/components/IdpAutoProvisionUsersDescription"; import { PaidFeaturesAlert } from "@app/components/PaidFeaturesAlert"; import { @@ -14,7 +15,6 @@ import { SettingsSectionTitle } from "@app/components/Settings"; import HeaderTitle from "@app/components/SettingsSectionTitle"; -import { SwitchInput } from "@app/components/SwitchInput"; import { OidcIdpProviderTypeSelect } from "@app/components/idp/OidcIdpProviderTypeSelect"; import { Alert, AlertDescription, AlertTitle } from "@app/components/ui/alert"; import { Button } from "@app/components/ui/button"; @@ -49,6 +49,11 @@ import { MappingBuilderRule, RoleMappingMode } from "@app/lib/idpRoleMapping"; +import { + createDefaultDirectorySyncRoleMappingConfig, + DirectorySyncRoleMappingConfig, + serializeDirectorySyncRoleMapping +} from "@app/lib/idpDirectorySyncRoleMapping"; export default function Page() { const { env } = useEnvContext(); @@ -65,6 +70,10 @@ export default function Page() { MappingBuilderRule[] >([createMappingBuilderRule()]); const [rawRoleExpression, setRawRoleExpression] = useState(""); + const [directoryRoleMapping, setDirectoryRoleMapping] = + useState( + createDefaultDirectorySyncRoleMappingConfig + ); const t = useTranslations(); const { isPaidUser } = usePaidStatus(); @@ -94,7 +103,8 @@ export default function Page() { roleMapping: z.string().nullable().optional(), roleId: z.number().nullable().optional(), orgMapping: z.string().optional(), - directorySyncEnabled: z.boolean().default(false) + directorySyncEnabled: z.boolean().default(false), + directorySyncDeletions: z.boolean().default(true) }); type CreateIdpFormValues = z.infer; @@ -117,7 +127,8 @@ export default function Page() { roleMapping: null, roleId: null, orgMapping: "", - directorySyncEnabled: false + directorySyncEnabled: false, + directorySyncDeletions: true } }); @@ -125,9 +136,9 @@ export default function Page() { useEffect(() => { async function fetchRoles() { const res = await api - .get< - AxiosResponse - >(`/org/${params.orgId}/roles`) + .get>( + `/org/${params.orgId}/roles` + ) .catch((e) => { console.error(e); toast({ @@ -195,6 +206,9 @@ export default function Page() { directorySyncEnabled: (data.type === "google" || data.type === "azure") && data.directorySyncEnabled, + directoryRoleMapping: + serializeDirectorySyncRoleMapping(directoryRoleMapping), + directorySyncDeletions: data.directorySyncDeletions, roleMapping: roleMappingExpression, scopes: data.scopes, variant: data.type @@ -377,32 +391,72 @@ export default function Page() { name: "orgMapping" }} /> - {(form.watch("type") === "google" || - form.watch("type") === "azure") && ( - { + + + + + + {/* Directory Sync Settings */} + {(form.watch("type") === "google" || + form.watch("type") === "azure") && ( + + + + {t("idpDirectorySync")} + + + {t("idpDirectorySyncDescription")} + + + +
+ + { form.setValue( "directorySyncEnabled", checked ); if (checked) { - form.setValue("autoProvision", false); + form.setValue( + "autoProvision", + false + ); } }} - description={t( - "idpDirectorySyncDescription" - )} + orgId={params.orgId as string} + roleMapping={directoryRoleMapping} + onRoleMappingChange={ + setDirectoryRoleMapping + } + syncDeletions={ + form.watch( + "directorySyncDeletions" + ) ?? true + } + onSyncDeletionsChange={(checked) => + form.setValue( + "directorySyncDeletions", + checked + ) + } /> - )} - - -
- + + + + + )} {form.watch("type") === "google" && ( From 28ed49f128dcc046b4227f82e9b011ec95850a8f Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Wed, 14 Oct 2026 07:51:38 +0200 Subject: [PATCH 19/50] =?UTF-8?q?=F0=9F=9A=A7=20wip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/private/lib/directorySync/azure.ts | 709 +++++++++++--------- server/private/lib/directorySync/run.ts | 16 +- server/private/lib/directorySync/runSync.ts | 606 +++++++++++++++++ server/private/lib/directorySync/types.ts | 54 ++ 4 files changed, 1051 insertions(+), 334 deletions(-) create mode 100644 server/private/lib/directorySync/runSync.ts create mode 100644 server/private/lib/directorySync/types.ts diff --git a/server/private/lib/directorySync/azure.ts b/server/private/lib/directorySync/azure.ts index 8795874685..5942e67a6e 100644 --- a/server/private/lib/directorySync/azure.ts +++ b/server/private/lib/directorySync/azure.ts @@ -1,382 +1,443 @@ -// QUICK & DIRTY: one-shot Entra ID -> Pangolin user sync. -// Full snapshot only (no delta), creates/updates users and memberships, -// never removes anything (see DIRECTORY_SYNC_PLAN.md "Deprovisioning"). -// Org/role assignment comes from an explicit group -> (org, role) mapping, -// NOT the login JMESPath mappings (Graph data != ID token claims). +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +// Microsoft Entra ID directory provider (Microsoft Graph, app-only auth). +// +// Required application permissions (with admin consent): User.Read.All, GroupMember.Read.All. +// +// Full fetch: /users/delta + /groups/delta?$select=displayName,members without a token, which +// lists everything and returns delta links for next time. Nested memberships are resolved in +// memory from the direct members. +// +// Incremental fetch: replay both delta links. Group deltas only report direct member changes, +// so every user touched by a change is re-read (/users/{id} + /transitiveMemberOf) to get their +// full current state. Cases that can't be resolved cheaply fall back to a full fetch. -import { db, Org } from "@server/db"; -import { - idp, - idpOidcConfig, - idpOrg, - orgs, - roles, - userOrgRoles, - userOrgs, - users -} from "@server/db"; -import { and, eq, inArray } from "drizzle-orm"; import config from "@server/lib/config"; import logger from "@server/logger"; import { decrypt } from "@server/lib/crypto"; -import { generateId } from "@server/auth/sessions/app"; -import { UserType } from "@server/types/UserTypes"; -import { build } from "@server/build"; -import { LimitId } from "@server/lib/billing"; -import { usageService } from "@server/lib/billing/usageService"; -import { calculateUserClientsForOrgs } from "@server/lib/calculateUserClientsForOrgs"; -import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed"; -import { tierMatrix } from "@server/lib/billing/tierMatrix"; -import { assignUserToOrg } from "@server/lib/userOrg"; +import { + DirectoryCursorExpiredError, + DirectoryFetchContext, + DirectoryFetchResult, + DirectoryGroup, + DirectoryProvider, + DirectoryUser +} from "./types"; const GRAPH = "https://graph.microsoft.com/v1.0"; +const USER_SELECT = "id,displayName,mail,userPrincipalName,accountEnabled"; + +/** Above this many touched users, an incremental run is more expensive than a full one. */ +const MAX_INCREMENTAL_AFFECTED_USERS = 500; +/** Longest Retry-After we sleep through in-process before failing the run. */ +const MAX_RETRY_AFTER_SECONDS = 120; +const MAX_THROTTLE_RETRIES = 5; type GraphUser = { id: string; - displayName: string | null; - mail: string | null; - userPrincipalName: string | null; - accountEnabled: boolean | null; + displayName?: string | null; + mail?: string | null; + userPrincipalName?: string | null; + accountEnabled?: boolean | null; + "@removed"?: { reason: string }; }; -// TODO: move to an `idpGroupMapping` table (idpId, externalGroupId, orgId, roleId) -export type GroupMapping = { - groupId: string; // Entra group object ID - orgId: string; - roleName: string; +type GraphDirectoryObjectRef = { + "@odata.type"?: string; + id: string; + "@removed"?: { reason: string }; }; -type SyncReport = { - fetchedUsers: number; - fetchedGroups: number; - missingRoles: string[]; - created: number; - updated: number; - skippedDisabled: number; - skippedNoMatch: number; - requests: number; +type GraphGroup = { + id: string; + displayName?: string | null; + "members@delta"?: GraphDirectoryObjectRef[]; + "@removed"?: { reason: string }; }; -async function getAppToken( - tenantId: string, - clientId: string, - clientSecret: string -): Promise { - const res = await fetch( - `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, - { - method: "POST", - headers: { "Content-Type": "application/x-www-form-urlencoded" }, - body: new URLSearchParams({ - grant_type: "client_credentials", - client_id: clientId, - client_secret: clientSecret, - scope: "https://graph.microsoft.com/.default" - }) - } - ); - const body = await res.json(); - if (!res.ok) { - throw new Error( - `Entra token request failed (${res.status}): ${body.error_description ?? body.error}` - ); - } - return body.access_token; -} +type AzureCursor = { + usersDeltaLink: string; + groupsDeltaLink: string; +}; -async function graphGetAll( - token: string, - url: string, - report: SyncReport -): Promise { - const items: T[] = []; - let next: string | undefined = url; - while (next) { - report.requests++; - const res: Response = await fetch(next, { - headers: { Authorization: `Bearer ${token}` } - }); - if (res.status === 429 || res.status === 503) { - const retryAfter = Number(res.headers.get("Retry-After") ?? 5); - logger.warn(`Graph throttled, retrying in ${retryAfter}s`, { - url: next - }); - await new Promise((r) => setTimeout(r, retryAfter * 1000)); - continue; - } - const body: any = await res.json(); - if (!res.ok) { - throw new Error( - `Graph request failed (${res.status}) ${next}: ${body.error?.code} ${body.error?.message}` - ); - } - items.push(...body.value); - next = body["@odata.nextLink"]; - } - return items; -} +type GraphPage = { + value: T[]; + "@odata.nextLink"?: string; + "@odata.deltaLink"?: string; +}; // Graph field that yields the same `username` login computes from the ID token. +// `sub` is per-app in Entra and not exposed by Graph, so it can't be synced. const IDENTIFIER_FROM_GRAPH: Record string | null> = { oid: (u) => u.id, - preferred_username: (u) => u.userPrincipalName, - upn: (u) => u.userPrincipalName + preferred_username: (u) => u.userPrincipalName ?? null, + upn: (u) => u.userPrincipalName ?? null, + email: (u) => u.mail ?? null }; -export async function syncAzureDirectory( - idpId: number, - groupMappings: GroupMapping[] -): Promise { - const report: SyncReport = { - fetchedUsers: 0, - fetchedGroups: 0, - missingRoles: [], - created: 0, - updated: 0, - skippedDisabled: 0, - skippedNoMatch: 0, - requests: 0 - }; - - const [existingIdp] = await db - .select() - .from(idp) - .innerJoin(idpOidcConfig, eq(idpOidcConfig.idpId, idp.idpId)) - .where(and(eq(idp.type, "oidc"), eq(idp.idpId, idpId))); - - if (!existingIdp) throw new Error(`IdP ${idpId} not found`); - if (existingIdp.idpOidcConfig.variant !== "azure") { - throw new Error(`IdP ${idpId} is not an azure IdP`); +const tokenCache = new Map(); + +class GraphClient { + constructor( + private readonly getToken: () => Promise, + private readonly onRequest: () => void + ) {} + + /** GET a single Graph URL, sleeping through short throttling. Returns null on 404. */ + async get(url: string): Promise { + for (let attempt = 0; ; attempt++) { + this.onRequest(); + const res = await fetch(url, { + headers: { Authorization: `Bearer ${await this.getToken()}` } + }); + + if (res.status === 429 || res.status === 503) { + const retryAfter = Number(res.headers.get("Retry-After") ?? 5); + if ( + attempt >= MAX_THROTTLE_RETRIES || + retryAfter > MAX_RETRY_AFTER_SECONDS + ) { + throw new Error( + `Graph throttled (${res.status}, Retry-After ${retryAfter}s): ${url}` + ); + } + logger.warn(`Graph throttled, retrying in ${retryAfter}s`, { + url + }); + await new Promise((r) => setTimeout(r, retryAfter * 1000)); + continue; + } + + if (res.status === 404) return null; + + const body: any = await res.json().catch(() => ({})); + + if ( + res.status === 410 || + body.error?.code === "syncStateNotFound" || + body.error?.code === "resyncRequired" + ) { + throw new DirectoryCursorExpiredError( + `Graph delta token expired (${res.status} ${body.error?.code})` + ); + } + if (!res.ok) { + throw new Error( + `Graph request failed (${res.status}) ${url}: ${body.error?.code} ${body.error?.message}` + ); + } + return body as T; + } } - if (existingIdp.idp.autoProvision) { - throw new Error( - `IdP ${idpId} has autoProvision enabled, which is exclusive with directory sync` - ); + + /** Follow @odata.nextLink pages; returns the items and the final delta link, if any. */ + async getAll( + url: string + ): Promise<{ items: T[]; deltaLink: string | null }> { + const items: T[] = []; + let next: string | undefined = url; + while (next) { + const page: GraphPage | null = + await this.get>(next); + if (!page) throw new Error(`Graph returned 404 for ${next}`); + items.push(...page.value); + if (page["@odata.deltaLink"]) { + return { items, deltaLink: page["@odata.deltaLink"] }; + } + next = page["@odata.nextLink"]; + } + return { items, deltaLink: null }; } - if (!existingIdp.idp.directorySyncEnabled) { - logger.warn(`IdP ${idpId} has directorySyncEnabled=false, syncing anyway`); +} + +function isUserRef(ref: GraphDirectoryObjectRef) { + return ref["@odata.type"] === "#microsoft.graph.user"; +} + +function isGroupRef(ref: GraphDirectoryObjectRef) { + return ref["@odata.type"] === "#microsoft.graph.group"; +} + +function parseCursor(cursor: string | null): AzureCursor | null { + if (!cursor) return null; + try { + const parsed = JSON.parse(cursor); + if ( + typeof parsed?.usersDeltaLink === "string" && + typeof parsed?.groupsDeltaLink === "string" + ) { + return parsed; + } + } catch { + // fall through } + logger.warn("Ignoring unreadable Azure directory sync cursor"); + return null; +} - const identifierPath = existingIdp.idpOidcConfig.identifierPath; - const getIdentifier = IDENTIFIER_FROM_GRAPH[identifierPath]; - if (!getIdentifier) { - // e.g. "sub": Entra's sub is per-app and not exposed by Graph - throw new Error( - `identifierPath "${identifierPath}" can't be resolved from Graph; use one of: ${Object.keys(IDENTIFIER_FROM_GRAPH).join(", ")}` +export class AzureDirectoryProvider implements DirectoryProvider { + constructor( + private readonly settings: { + tenantId: string; + /** Encrypted, as stored in `idpOidcConfig`. */ + encryptedClientId: string; + /** Encrypted, as stored in `idpOidcConfig`. */ + encryptedClientSecret: string; + } + ) {} + + /** Parses the tenant ID from an Entra authorize URL. */ + static tenantIdFromAuthUrl(authUrl: string): string | null { + return ( + authUrl.match( + /login\.microsoftonline\.com\/([^/]+)\/oauth2/ + )?.[1] ?? null ); } - const tenantId = existingIdp.idpOidcConfig.authUrl.match( - /login\.microsoftonline\.com\/([^/]+)\/oauth2/ - )?.[1]; - if (!tenantId) throw new Error("Could not parse tenant ID from authUrl"); - - const key = config.getRawConfig().server.secret!; - const clientId = decrypt(existingIdp.idpOidcConfig.clientId, key); - const clientSecret = decrypt(existingIdp.idpOidcConfig.clientSecret, key); - - // ---- resolve mappings to (org, roleId) ---- - let allowedOrgs: Org[]; - if (build === "saas") { - const idpOrgs = await db - .select() - .from(idpOrg) - .where(eq(idpOrg.idpId, idpId)) - .innerJoin(orgs, eq(orgs.orgId, idpOrg.orgId)); - allowedOrgs = idpOrgs.map((o) => o.orgs); - } else { - allowedOrgs = await db.select().from(orgs); + async fetch(ctx: DirectoryFetchContext): Promise { + const getIdentifier = IDENTIFIER_FROM_GRAPH[ctx.identifierPath]; + if (!getIdentifier) { + throw new Error( + `identifierPath "${ctx.identifierPath}" can't be resolved from Graph; use one of: ${Object.keys(IDENTIFIER_FROM_GRAPH).join(", ")}` + ); + } + + const graph = new GraphClient(() => this.getToken(), ctx.onRequest); + const toDirectoryUser = ( + u: GraphUser, + groups: DirectoryGroup[] + ): DirectoryUser => ({ + externalId: u.id, + username: getIdentifier(u)?.toLowerCase() ?? null, + email: u.mail?.toLowerCase() ?? null, + name: u.displayName ?? null, + enabled: u.accountEnabled !== false, + groups + }); + + const cursor = parseCursor(ctx.cursor); + if (cursor) { + const incremental = await this.fetchIncremental( + graph, + cursor, + toDirectoryUser + ); + if (incremental) return incremental; + logger.info( + `Azure incremental sync for IdP ${ctx.idpId} needs a full fetch` + ); + } + return this.fetchFull(graph, toDirectoryUser); } - const mappedOrgIds = [...new Set(groupMappings.map((m) => m.orgId))]; - const orgRoles = mappedOrgIds.length - ? await db.select().from(roles).where(inArray(roles.orgId, mappedOrgIds)) - : []; - - // groupId -> [{ orgId, roleId }] - const grantsByGroup = new Map(); - for (const m of groupMappings) { - if (!allowedOrgs.some((o) => o.orgId === m.orgId)) { - logger.warn(`Org ${m.orgId} is not available to IdP ${idpId}, ignoring mapping`, m); - continue; + private async getToken(): Promise { + const { tenantId, encryptedClientId, encryptedClientSecret } = + this.settings; + const key = config.getRawConfig().server.secret!; + const clientId = decrypt(encryptedClientId, key); + const cacheKey = `${tenantId}:${clientId}`; + + const cached = tokenCache.get(cacheKey); + if (cached && cached.expiresAt > Date.now() + 60_000) { + return cached.token; } - const role = orgRoles.find( - (r) => r.orgId === m.orgId && r.name === m.roleName + + const res = await fetch( + `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, + { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded" + }, + body: new URLSearchParams({ + grant_type: "client_credentials", + client_id: clientId, + client_secret: decrypt(encryptedClientSecret, key), + scope: "https://graph.microsoft.com/.default" + }) + } ); - if (!role) { - report.missingRoles.push(`${m.orgId}/${m.roleName}`); - continue; + const body: any = await res.json().catch(() => ({})); + if (!res.ok) { + throw new Error( + `Entra token request failed (${res.status}): ${body.error_description ?? body.error}` + ); } - const list = grantsByGroup.get(m.groupId) ?? []; - list.push({ orgId: m.orgId, roleId: role.roleId }); - grantsByGroup.set(m.groupId, list); - } - const multiRoleByOrg = new Map(); - for (const orgId of mappedOrgIds) { - multiRoleByOrg.set( - orgId, - await isLicensedOrSubscribed(orgId, tierMatrix.fullRbac) - ); + tokenCache.set(cacheKey, { + token: body.access_token, + expiresAt: Date.now() + Number(body.expires_in ?? 3600) * 1000 + }); + return body.access_token; } - // ---- fetch directory snapshot (only members of mapped groups) ---- - const token = await getAppToken(tenantId, clientId, clientSecret); - - const graphUsers = await graphGetAll( - token, - `${GRAPH}/users?$select=id,displayName,mail,userPrincipalName,accountEnabled&$top=999`, - report - ); - report.fetchedUsers = graphUsers.length; - - const groupsByUser = new Map(); - for (const groupId of grantsByGroup.keys()) { - const members = await graphGetAll<{ id: string }>( - token, - `${GRAPH}/groups/${groupId}/transitiveMembers/microsoft.graph.user?$select=id&$top=999`, - report + private async fetchFull( + graph: GraphClient, + toDirectoryUser: ( + u: GraphUser, + groups: DirectoryGroup[] + ) => DirectoryUser + ): Promise { + const usersRes = await graph.getAll( + `${GRAPH}/users/delta?$select=${USER_SELECT}` ); - report.fetchedGroups++; - for (const m of members) { - const list = groupsByUser.get(m.id) ?? []; - list.push(groupId); - groupsByUser.set(m.id, list); + const groupsRes = await graph.getAll( + `${GRAPH}/groups/delta?$select=displayName,members` + ); + + // A large group can be split over several pages; merge its member chunks. + const groups = new Map(); + // member (user or group) ID -> IDs of groups it directly belongs to + const parentGroups = new Map>(); + for (const g of groupsRes.items) { + if (g["@removed"]) continue; + const existing = groups.get(g.id); + groups.set(g.id, { + id: g.id, + displayName: g.displayName ?? existing?.displayName ?? null + }); + for (const member of g["members@delta"] ?? []) { + if (member["@removed"]) continue; + if (!isUserRef(member) && !isGroupRef(member)) continue; + const parents = parentGroups.get(member.id) ?? new Set(); + parents.add(g.id); + parentGroups.set(member.id, parents); + } } + + const transitiveGroups = (memberId: string): DirectoryGroup[] => { + const seen = new Set(); + const queue = [...(parentGroups.get(memberId) ?? [])]; + while (queue.length) { + const groupId = queue.pop()!; + if (seen.has(groupId)) continue; + seen.add(groupId); + queue.push(...(parentGroups.get(groupId) ?? [])); + } + return [...seen] + .map((id) => groups.get(id)) + .filter((g): g is DirectoryGroup => !!g); + }; + + const users = usersRes.items + .filter((u) => !u["@removed"]) + .map((u) => toDirectoryUser(u, transitiveGroups(u.id))); + + return { + kind: "full", + users, + nextCursor: this.cursorFrom(usersRes.deltaLink, groupsRes.deltaLink) + }; } - logger.info( - `Fetched ${graphUsers.length} users and ${report.fetchedGroups} mapped groups from Entra (${report.requests} requests)` - ); + /** Returns null when a full fetch is required instead. */ + private async fetchIncremental( + graph: GraphClient, + cursor: AzureCursor, + toDirectoryUser: ( + u: GraphUser, + groups: DirectoryGroup[] + ) => DirectoryUser + ): Promise { + const usersRes = await graph.getAll(cursor.usersDeltaLink); + const groupsRes = await graph.getAll( + cursor.groupsDeltaLink + ); - const touchedOrgIds = new Set(); - const touchedUserIds: string[] = []; + const removedExternalIds = new Set(); + const affectedUserIds = new Set(); - for (const gu of graphUsers) { - if (gu.accountEnabled === false) { - report.skippedDisabled++; - continue; + for (const u of usersRes.items) { + if (u["@removed"]) { + removedExternalIds.add(u.id); + } else { + affectedUserIds.add(u.id); + } } - const userIdentifier = getIdentifier(gu)?.toLowerCase(); - if (!userIdentifier) { - logger.warn(`No identifier for Entra user ${gu.id}, skipping`); - continue; + // Groups whose every (transitive) user member must be re-read. + const expandGroupIds = new Set(); + for (const g of groupsRes.items) { + if (g["@removed"]) { + // The former members of a deleted group aren't listed anywhere. + return null; + } + if (g.displayName !== undefined) { + // Possibly renamed: name-based role mappings may change for all members. + expandGroupIds.add(g.id); + } + for (const member of g["members@delta"] ?? []) { + if (isUserRef(member)) { + affectedUserIds.add(member.id); + } else if (isGroupRef(member)) { + expandGroupIds.add(member.id); + } + } } - const email = gu.mail?.toLowerCase() ?? null; - const name = gu.displayName ?? null; - - // orgId -> roleIds, in mapping order - const rolesByOrg = new Map(); - for (const groupId of groupsByUser.get(gu.id) ?? []) { - for (const g of grantsByGroup.get(groupId) ?? []) { - const list = rolesByOrg.get(g.orgId) ?? []; - if (!list.includes(g.roleId)) list.push(g.roleId); - rolesByOrg.set(g.orgId, list); + + for (const groupId of expandGroupIds) { + const members = await graph.getAll<{ id: string }>( + `${GRAPH}/groups/${groupId}/transitiveMembers/microsoft.graph.user?$select=id&$top=999` + ); + for (const m of members.items) affectedUserIds.add(m.id); + if (affectedUserIds.size > MAX_INCREMENTAL_AFFECTED_USERS) { + return null; } } - for (const [orgId, roleIds] of rolesByOrg) { - if (!multiRoleByOrg.get(orgId)) rolesByOrg.set(orgId, roleIds.slice(0, 1)); + if (affectedUserIds.size > MAX_INCREMENTAL_AFFECTED_USERS) { + return null; } - const [existingUser] = await db - .select() - .from(users) - .where( - and( - eq(users.username, userIdentifier), - eq(users.idpId, idpId) + const users: DirectoryUser[] = []; + for (const userId of affectedUserIds) { + if (removedExternalIds.has(userId)) continue; + const u = await graph.get( + `${GRAPH}/users/${userId}?$select=${USER_SELECT}` + ); + if (!u) { + removedExternalIds.add(userId); + continue; + } + const memberOf = await graph.getAll( + `${GRAPH}/users/${userId}/transitiveMemberOf/microsoft.graph.group?$select=id,displayName&$top=999` + ); + users.push( + toDirectoryUser( + u, + memberOf.items.map((g) => ({ + id: g.id, + displayName: g.displayName ?? null + })) ) ); - - if (!rolesByOrg.size && !existingUser) { - // not in any mapped group; don't create orphans - report.skippedNoMatch++; - continue; } - await db.transaction(async (trx) => { - let userId = existingUser?.userId; - if (!existingUser) { - userId = generateId(15); - await trx.insert(users).values({ - userId, - username: userIdentifier, - email, - name, - type: UserType.OIDC, - idpId, - emailVerified: true, - dateCreated: new Date().toISOString() - }); - report.created++; - } else { - await trx - .update(users) - .set({ email, name }) - .where(eq(users.userId, userId!)); - report.updated++; - } - - const currentUserOrgs = await trx - .select() - .from(userOrgs) - .where(eq(userOrgs.userId, userId!)); - - for (const [orgId, roleIds] of rolesByOrg) { - const current = currentUserOrgs.find((o) => o.orgId === orgId); - - if (!current) { - const org = allowedOrgs.find((o) => o.orgId === orgId)!; - await assignUserToOrg( - org, - { orgId, userId: userId!, autoProvisioned: true }, - roleIds, - trx - ); - touchedOrgIds.add(orgId); - } else if (current.autoProvisioned) { - // re-sync roles 1:1 on auto-provisioned memberships only - await trx - .delete(userOrgRoles) - .where( - and( - eq(userOrgRoles.userId, userId!), - eq(userOrgRoles.orgId, orgId) - ) - ); - await trx.insert(userOrgRoles).values( - roleIds.map((roleId) => ({ - userId: userId!, - orgId, - roleId - })) - ); - } - } - - // NOTE: no removal of stale auto-provisioned orgs — deprovisioning not approved - touchedUserIds.push(userId!); - }); - } - - for (const orgId of touchedOrgIds) { - const count = await db - .select() - .from(userOrgs) - .where(eq(userOrgs.orgId, orgId)); - await usageService.updateCount(orgId, LimitId.USERS, count.length); + return { + kind: "incremental", + users, + removedExternalIds: [...removedExternalIds], + nextCursor: this.cursorFrom(usersRes.deltaLink, groupsRes.deltaLink) + }; } - for (const userId of touchedUserIds) { - await calculateUserClientsForOrgs(userId).catch((err) => - logger.error("Error calculating user clients after sync", { - userId, - error: err - }) - ); + private cursorFrom( + usersDeltaLink: string | null, + groupsDeltaLink: string | null + ): string | null { + if (!usersDeltaLink || !groupsDeltaLink) return null; + const cursor: AzureCursor = { usersDeltaLink, groupsDeltaLink }; + return JSON.stringify(cursor); } - - logger.info("Azure directory sync done", { idpId, ...report }); - return report; } diff --git a/server/private/lib/directorySync/run.ts b/server/private/lib/directorySync/run.ts index 33eec52e7f..e144767b52 100644 --- a/server/private/lib/directorySync/run.ts +++ b/server/private/lib/directorySync/run.ts @@ -1,18 +1,14 @@ -// Usage: npx tsx server/private/lib/directorySync/run.ts -// mappings.json: [{ "groupId": "", "orgId": "acme", "roleName": "Member" }] -import { readFileSync } from "fs"; -import { GroupMapping, syncAzureDirectory } from "./azure"; +// Usage: npx tsx server/private/lib/directorySync/run.ts +// Runs one directory sync for the IdP using its saved directory role mapping. +import { runDirectorySync } from "./runSync"; const idpId = Number(process.argv[2]); -const mappingsPath = process.argv[3]; -if (!idpId || !mappingsPath) { - console.error("Usage: run.ts "); +if (!idpId) { + console.error("Usage: run.ts "); process.exit(1); } -const mappings: GroupMapping[] = JSON.parse(readFileSync(mappingsPath, "utf8")); - -syncAzureDirectory(idpId, mappings) +runDirectorySync(idpId) .then((report) => { console.log(JSON.stringify(report, null, 2)); process.exit(0); diff --git a/server/private/lib/directorySync/runSync.ts b/server/private/lib/directorySync/runSync.ts new file mode 100644 index 0000000000..4a4c490c56 --- /dev/null +++ b/server/private/lib/directorySync/runSync.ts @@ -0,0 +1,606 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +// Provider-independent directory sync engine. +// +// 1. Fetch users from the provider (full, or incremental from the stored cursor). +// 2. Resolve each user's roles per org from `idpOrg.directoryRoleMapping`. +// 3. Create/update matching users and their auto-provisioned memberships. +// 4. When `idpOrg.directorySyncDeletions` is on, delete the Pangolin users sync manages that +// were deleted or disabled in the directory, or no longer match any role mapping. +// 5. Save the provider cursor only once everything above succeeded. + +import { + db, + idp, + idpOidcConfig, + idpOrg, + idpSyncState, + idpSyncUser, + Org, + orgs, + roleActions, + roles, + userOrgRoles, + userOrgs, + users +} from "@server/db"; +import { and, eq } from "drizzle-orm"; +import logger from "@server/logger"; +import { generateId } from "@server/auth/sessions/app"; +import { UserType } from "@server/types/UserTypes"; +import { LimitId } from "@server/lib/billing"; +import { usageService } from "@server/lib/billing/usageService"; +import { tierMatrix } from "@server/lib/billing/tierMatrix"; +import { calculateUserClientsForOrgs } from "@server/lib/calculateUserClientsForOrgs"; +import { assignUserToOrg, removeUserFromOrg } from "@server/lib/userOrg"; +import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed"; +import { defaultRoleAllowedActions } from "@server/routers/role/createRole"; +import { lockManager } from "@server/private/lib/lock"; +import { + DirectorySyncRoleMappingConfig, + parseDirectorySyncRoleMapping +} from "@app/lib/idpDirectorySyncRoleMapping"; +import { AzureDirectoryProvider } from "./azure"; +import { + DirectoryCursorExpiredError, + DirectoryFetchResult, + DirectoryProvider, + DirectoryUser +} from "./types"; + +const LOCK_TTL_MS = 30 * 60 * 1000; +/** Refuse a run that would delete more than this share of the users sync manages... */ +const MAX_DELETION_RATIO = 0.5; +/** ...unless it deletes at most this many users. */ +const DELETION_RATIO_MIN_USERS = 10; + +export type DirectorySyncReport = { + fetchKind: DirectoryFetchResult["kind"] | null; + requests: number; + fetchedUsers: number; + created: number; + updated: number; + deleted: number; + membershipsRemoved: number; + skippedNoMatch: number; + skippedNoIdentifier: number; + createdRoles: string[]; + missingRoles: string[]; +}; + +type OrgMapping = { + org: Org; + config: DirectorySyncRoleMappingConfig; + syncDeletions: boolean; + multiRole: boolean; + /** Lower-cased role name -> role. */ + rolesByName: Map; +}; + +export async function runDirectorySync( + idpId: number +): Promise { + return lockManager.withLock( + `idp-sync:${idpId}`, + () => runLocked(idpId), + LOCK_TTL_MS + ); +} + +async function runLocked(idpId: number): Promise { + const report: DirectorySyncReport = { + fetchKind: null, + requests: 0, + fetchedUsers: 0, + created: 0, + updated: 0, + deleted: 0, + membershipsRemoved: 0, + skippedNoMatch: 0, + skippedNoIdentifier: 0, + createdRoles: [], + missingRoles: [] + }; + + const [existingIdp] = await db + .select() + .from(idp) + .innerJoin(idpOidcConfig, eq(idpOidcConfig.idpId, idp.idpId)) + .where(eq(idp.idpId, idpId)); + + if (!existingIdp) throw new Error(`IdP ${idpId} not found`); + if (!existingIdp.idp.directorySyncEnabled) { + throw new Error(`Directory sync is not enabled for IdP ${idpId}`); + } + if (existingIdp.idp.autoProvision) { + throw new Error( + `IdP ${idpId} has autoProvision enabled, which is exclusive with directory sync` + ); + } + + const provider = createProvider(existingIdp.idpOidcConfig); + + const [state] = await db + .select() + .from(idpSyncState) + .where(eq(idpSyncState.idpId, idpId)); + + const startedAt = Date.now(); + await db + .insert(idpSyncState) + .values({ idpId, lastSyncStartedAt: startedAt }) + .onConflictDoUpdate({ + target: idpSyncState.idpId, + set: { lastSyncStartedAt: startedAt } + }); + + try { + const fetchWith = (cursor: string | null) => + provider.fetch({ + idpId, + identifierPath: existingIdp.idpOidcConfig.identifierPath, + cursor, + onRequest: () => report.requests++ + }); + + let result: DirectoryFetchResult; + try { + result = await fetchWith(state?.cursor ?? null); + } catch (err) { + if (!(err instanceof DirectoryCursorExpiredError)) throw err; + logger.info( + `Directory sync cursor expired for IdP ${idpId}, running a full sync` + ); + result = await fetchWith(null); + } + + report.fetchKind = result.kind; + report.fetchedUsers = result.users.length; + + await applyResult(idpId, result, report); + + await db + .update(idpSyncState) + .set({ + cursor: result.nextCursor, + lastSyncSucceededAt: Date.now(), + lastError: null, + consecutiveFailures: 0 + }) + .where(eq(idpSyncState.idpId, idpId)); + } catch (err) { + await db + .update(idpSyncState) + .set({ + lastError: err instanceof Error ? err.message : String(err), + consecutiveFailures: (state?.consecutiveFailures ?? 0) + 1 + }) + .where(eq(idpSyncState.idpId, idpId)); + throw err; + } + + logger.info("Directory sync done", { idpId, ...report }); + return report; +} + +function createProvider( + oidcConfig: typeof idpOidcConfig.$inferSelect +): DirectoryProvider { + switch (oidcConfig.variant) { + case "azure": { + const tenantId = AzureDirectoryProvider.tenantIdFromAuthUrl( + oidcConfig.authUrl + ); + if (!tenantId) { + throw new Error("Could not parse Azure tenant ID from authUrl"); + } + return new AzureDirectoryProvider({ + tenantId, + encryptedClientId: oidcConfig.clientId, + encryptedClientSecret: oidcConfig.clientSecret + }); + } + default: + throw new Error( + `Directory sync is not supported for variant "${oidcConfig.variant}"` + ); + } +} + +async function loadOrgMappings(idpId: number): Promise { + const rows = await db + .select() + .from(idpOrg) + .innerJoin(orgs, eq(orgs.orgId, idpOrg.orgId)) + .where(eq(idpOrg.idpId, idpId)); + + const mappings: OrgMapping[] = []; + for (const row of rows) { + if (!row.idpOrg.directoryRoleMapping) { + logger.warn( + `IdP ${idpId} has no directory role mapping for org ${row.orgs.orgId}, skipping that org` + ); + continue; + } + const orgRoles = await db + .select() + .from(roles) + .where(eq(roles.orgId, row.orgs.orgId)); + + mappings.push({ + org: row.orgs, + config: parseDirectorySyncRoleMapping( + row.idpOrg.directoryRoleMapping + ), + syncDeletions: row.idpOrg.directorySyncDeletions, + multiRole: await isLicensedOrSubscribed( + row.orgs.orgId, + tierMatrix.fullRbac + ), + rolesByName: new Map( + orgRoles.map((r) => [ + r.name.toLowerCase(), + { roleId: r.roleId, isAdmin: !!r.isAdmin } + ]) + ) + }); + } + return mappings; +} + +/** Role IDs the user should have in this org, in mapping order. Empty = no match. */ +async function resolveRoleIds( + mapping: OrgMapping, + user: DirectoryUser, + report: DirectorySyncReport +): Promise { + const { config, org } = mapping; + const roleIds: number[] = []; + const addRole = (roleId: number) => { + if (!roleIds.includes(roleId)) roleIds.push(roleId); + }; + const addByName = (roleName: string) => { + const role = mapping.rolesByName.get(roleName.toLowerCase()); + if (role) { + addRole(role.roleId); + } else if (!report.missingRoles.includes(`${org.orgId}/${roleName}`)) { + report.missingRoles.push(`${org.orgId}/${roleName}`); + } + }; + + switch (config.mode) { + case "fixedRoles": + config.fixedRoleNames.forEach(addByName); + break; + case "groupMapping": + for (const rule of config.groupRules) { + const match = rule.group.trim().toLowerCase(); + const isMember = user.groups.some( + (g) => + g.id.toLowerCase() === match || + g.displayName?.toLowerCase() === match + ); + if (isMember) rule.roleNames.forEach(addByName); + } + break; + case "automatic": + for (const group of user.groups) { + const name = group.displayName?.trim(); + if (!name) continue; + const role = mapping.rolesByName.get(name.toLowerCase()); + if (role) { + // Never hand out admin roles implicitly from a group name. + if (!role.isAdmin) addRole(role.roleId); + } else if (config.createMissingRoles) { + addRole(await createRole(mapping, name, report)); + } else if ( + !report.missingRoles.includes(`${org.orgId}/${name}`) + ) { + report.missingRoles.push(`${org.orgId}/${name}`); + } + } + break; + } + + return mapping.multiRole ? roleIds : roleIds.slice(0, 1); +} + +async function createRole( + mapping: OrgMapping, + name: string, + report: DirectorySyncReport +): Promise { + const orgId = mapping.org.orgId; + const roleId = await db.transaction(async (trx) => { + const [role] = await trx + .insert(roles) + .values({ + orgId, + name, + description: "Created by directory sync" + }) + .returning(); + await trx.insert(roleActions).values( + defaultRoleAllowedActions.map((actionId) => ({ + roleId: role.roleId, + actionId, + orgId + })) + ); + return role.roleId; + }); + + mapping.rolesByName.set(name.toLowerCase(), { roleId, isAdmin: false }); + report.createdRoles.push(`${orgId}/${name}`); + return roleId; +} + +async function applyResult( + idpId: number, + result: DirectoryFetchResult, + report: DirectorySyncReport +) { + const mappings = await loadOrgMappings(idpId); + if (!mappings.length) { + throw new Error( + `IdP ${idpId} has no org with a directory role mapping, nothing to sync` + ); + } + // Deleting a user removes it from every org, so every org has to allow it. + const deletionsEnabled = mappings.every((m) => m.syncDeletions); + + const links = await db + .select() + .from(idpSyncUser) + .where(eq(idpSyncUser.idpId, idpId)); + const userIdByExternalId = new Map( + links.map((l) => [l.externalId, l.userId]) + ); + + // ---- plan: who to upsert, who to delete ---- + const toUpsert: { + user: DirectoryUser; + rolesByOrg: Map; + }[] = []; + const toDelete = new Set(); // external IDs + + if (result.kind === "incremental") { + result.removedExternalIds.forEach((id) => toDelete.add(id)); + } else { + const seen = new Set(result.users.map((u) => u.externalId)); + for (const externalId of userIdByExternalId.keys()) { + if (!seen.has(externalId)) toDelete.add(externalId); + } + } + + for (const user of result.users) { + if (!user.enabled) { + toDelete.add(user.externalId); + continue; + } + if (!user.username) { + logger.warn( + `No identifier for directory user ${user.externalId}, skipping` + ); + report.skippedNoIdentifier++; + continue; + } + + const rolesByOrg = new Map(); + for (const mapping of mappings) { + const roleIds = await resolveRoleIds(mapping, user, report); + if (roleIds.length) rolesByOrg.set(mapping.org.orgId, roleIds); + } + + if (!rolesByOrg.size) { + report.skippedNoMatch++; + toDelete.add(user.externalId); + continue; + } + toUpsert.push({ user, rolesByOrg }); + } + + // Only users sync manages can be deleted. + const deleteUserIds = deletionsEnabled + ? [...toDelete] + .map((externalId) => userIdByExternalId.get(externalId)) + .filter((userId): userId is string => !!userId) + : []; + + if ( + deleteUserIds.length > DELETION_RATIO_MIN_USERS && + deleteUserIds.length > userIdByExternalId.size * MAX_DELETION_RATIO + ) { + throw new Error( + `Refusing to delete ${deleteUserIds.length} of ${userIdByExternalId.size} synced users in one run` + ); + } + + // ---- apply ---- + const touchedOrgIds = new Set(); + const touchedUserIds = new Set(); + + for (const { user, rolesByOrg } of toUpsert) { + const userId = await upsertUser( + idpId, + user, + userIdByExternalId.get(user.externalId), + report + ); + touchedUserIds.add(userId); + + await db.transaction(async (trx) => { + const currentUserOrgs = await trx + .select() + .from(userOrgs) + .where(eq(userOrgs.userId, userId)); + + for (const mapping of mappings) { + const orgId = mapping.org.orgId; + const roleIds = rolesByOrg.get(orgId) ?? []; + const current = currentUserOrgs.find((o) => o.orgId === orgId); + + if (!current) { + if (!roleIds.length) continue; + await assignUserToOrg( + mapping.org, + { orgId, userId, autoProvisioned: true }, + roleIds, + trx + ); + touchedOrgIds.add(orgId); + continue; + } + + // Memberships added by hand are never changed by sync. + if (!current.autoProvisioned) continue; + + if (!roleIds.length) { + if (mapping.syncDeletions) { + await removeUserFromOrg(mapping.org, userId, trx); + report.membershipsRemoved++; + touchedOrgIds.add(orgId); + } + continue; + } + + await trx + .delete(userOrgRoles) + .where( + and( + eq(userOrgRoles.userId, userId), + eq(userOrgRoles.orgId, orgId) + ) + ); + await trx + .insert(userOrgRoles) + .values( + roleIds.map((roleId) => ({ userId, orgId, roleId })) + ); + } + }); + } + + for (const userId of deleteUserIds) { + const [user] = await db + .select() + .from(users) + .where(and(eq(users.userId, userId), eq(users.idpId, idpId))); + if (!user) continue; + if (user.serverAdmin) { + logger.warn( + `Directory sync won't delete server admin ${user.username}` + ); + continue; + } + + const memberships = await db + .select() + .from(userOrgs) + .where(eq(userOrgs.userId, userId)); + memberships.forEach((m) => touchedOrgIds.add(m.orgId)); + + // Cascades to sessions, memberships and the idpSyncUser link. + await db.delete(users).where(eq(users.userId, userId)); + touchedUserIds.add(userId); + report.deleted++; + } + + for (const orgId of touchedOrgIds) { + const members = await db + .select() + .from(userOrgs) + .where(eq(userOrgs.orgId, orgId)); + await usageService.updateCount(orgId, LimitId.USERS, members.length); + } + + for (const userId of touchedUserIds) { + await calculateUserClientsForOrgs(userId).catch((err) => + logger.error("Error calculating user clients after sync", { + userId, + error: err + }) + ); + } +} + +/** + * Create or update the Pangolin user for a directory user and make sure it is linked. + * An existing user of this IdP with the same username (e.g. created by an earlier login) + * is adopted instead of duplicated. + */ +async function upsertUser( + idpId: number, + user: DirectoryUser, + linkedUserId: string | undefined, + report: DirectorySyncReport +): Promise { + const username = user.username!; + + return db.transaction(async (trx) => { + let [existing] = linkedUserId + ? await trx + .select() + .from(users) + .where( + and( + eq(users.userId, linkedUserId), + eq(users.idpId, idpId) + ) + ) + : []; + if (!existing) { + [existing] = await trx + .select() + .from(users) + .where( + and(eq(users.username, username), eq(users.idpId, idpId)) + ); + } + + let userId: string; + if (existing) { + userId = existing.userId; + await trx + .update(users) + .set({ username, email: user.email, name: user.name }) + .where(eq(users.userId, userId)); + report.updated++; + } else { + userId = generateId(15); + await trx.insert(users).values({ + userId, + username, + email: user.email, + name: user.name, + type: UserType.OIDC, + idpId, + emailVerified: true, + dateCreated: new Date().toISOString() + }); + report.created++; + } + + if (linkedUserId !== userId) { + await trx + .insert(idpSyncUser) + .values({ idpId, externalId: user.externalId, userId }) + .onConflictDoUpdate({ + target: [idpSyncUser.idpId, idpSyncUser.externalId], + set: { userId } + }); + } + return userId; + }); +} diff --git a/server/private/lib/directorySync/types.ts b/server/private/lib/directorySync/types.ts new file mode 100644 index 0000000000..627094ef51 --- /dev/null +++ b/server/private/lib/directorySync/types.ts @@ -0,0 +1,54 @@ +// Provider-neutral shapes used by the directory sync engine (runSync.ts). +// Each provider (Azure, Google, ...) turns its API into these. + +export type DirectoryGroup = { + /** Stable provider group ID. */ + id: string; + displayName: string | null; +}; + +export type DirectoryUser = { + /** Stable provider user ID (Entra object ID, Google directory ID). */ + externalId: string; + /** Value that login computes as `users.username` (already resolved from `identifierPath`). */ + username: string | null; + email: string | null; + name: string | null; + /** False when the account is disabled or suspended in the directory. */ + enabled: boolean; + /** Every group the user belongs to, nested memberships included. */ + groups: DirectoryGroup[]; +}; + +export type DirectoryFetchResult = + | { + /** Every user in the directory. Users sync knows about that are missing were deleted. */ + kind: "full"; + users: DirectoryUser[]; + nextCursor: string | null; + } + | { + /** Only users that changed since the cursor, with their full current state. */ + kind: "incremental"; + users: DirectoryUser[]; + /** External IDs of users deleted from the directory. */ + removedExternalIds: string[]; + nextCursor: string | null; + }; + +export type DirectoryFetchContext = { + idpId: number; + /** `idpOidcConfig.identifierPath`, used to compute `DirectoryUser.username`. */ + identifierPath: string; + /** Opaque provider state from the previous successful run. Null = full fetch. */ + cursor: string | null; + /** Called once per HTTP request, for the run report. */ + onRequest: () => void; +}; + +export interface DirectoryProvider { + fetch(ctx: DirectoryFetchContext): Promise; +} + +/** Thrown by a provider when the stored cursor can't be used anymore; the engine retries with a full fetch. */ +export class DirectoryCursorExpiredError extends Error {} From 6e8158cf777001475b55a79f471adc627601b984 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Wed, 14 Oct 2026 07:52:14 +0200 Subject: [PATCH 20/50] =?UTF-8?q?=F0=9F=97=83=EF=B8=8F=20add=20idp=20sync?= =?UTF-8?q?=20state=20tables?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/db/pg/schema/schema.ts | 29 ++++++++++++++++++++++++ server/db/sqlite/schema/schema.ts | 37 ++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 8ffffa5229..c59c0bd73c 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1339,6 +1339,35 @@ export const idpOrg = pgTable("idpOrg", { .default(true) }); +// Runtime state of the directory sync job, one row per IdP. Deleting the row +// forces a full sync on the next run. +export const idpSyncState = pgTable("idpSyncState", { + idpId: integer("idpId") + .primaryKey() + .references(() => idp.idpId, { onDelete: "cascade" }), + // Provider-defined opaque JSON (e.g. Azure delta links). Null = full sync. + cursor: text("cursor"), + lastSyncStartedAt: bigint("lastSyncStartedAt", { mode: "number" }), + lastSyncSucceededAt: bigint("lastSyncSucceededAt", { mode: "number" }), + lastError: text("lastError"), + consecutiveFailures: integer("consecutiveFailures").notNull().default(0) +}); + +// Links a directory user (stable provider ID) to the Pangolin user sync manages. +export const idpSyncUser = pgTable( + "idpSyncUser", + { + idpId: integer("idpId") + .notNull() + .references(() => idp.idpId, { onDelete: "cascade" }), + externalId: varchar("externalId").notNull(), + userId: varchar("userId") + .notNull() + .references(() => users.userId, { onDelete: "cascade" }) + }, + (t) => [primaryKey({ columns: [t.idpId, t.externalId] })] +); + export const clients = pgTable( "clients", { diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 0a3362ff43..55ccaa87ba 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1646,9 +1646,44 @@ export const idpOrg = sqliteTable("idpOrg", { .notNull() .references(() => orgs.orgId, { onDelete: "cascade" }), roleMapping: text("roleMapping"), - orgMapping: text("orgMapping") + orgMapping: text("orgMapping"), + directoryRoleMapping: text("directoryRoleMapping"), + directorySyncDeletions: integer("directorySyncDeletions", { + mode: "boolean" + }) + .notNull() + .default(true) +}); + +// Runtime state of the directory sync job, one row per IdP. Deleting the row +// forces a full sync on the next run. +export const idpSyncState = sqliteTable("idpSyncState", { + idpId: integer("idpId") + .primaryKey() + .references(() => idp.idpId, { onDelete: "cascade" }), + // Provider-defined opaque JSON (e.g. Azure delta links). Null = full sync. + cursor: text("cursor"), + lastSyncStartedAt: integer("lastSyncStartedAt"), + lastSyncSucceededAt: integer("lastSyncSucceededAt"), + lastError: text("lastError"), + consecutiveFailures: integer("consecutiveFailures").notNull().default(0) }); +// Links a directory user (stable provider ID) to the Pangolin user sync manages. +export const idpSyncUser = sqliteTable( + "idpSyncUser", + { + idpId: integer("idpId") + .notNull() + .references(() => idp.idpId, { onDelete: "cascade" }), + externalId: text("externalId").notNull(), + userId: text("userId") + .notNull() + .references(() => users.userId, { onDelete: "cascade" }) + }, + (t) => [primaryKey({ columns: [t.idpId, t.externalId] })] +); + // Blueprint runs export const blueprints = sqliteTable("blueprints", { blueprintId: integer("blueprintId").primaryKey({ From c22622410f806cc3f41f73df19239cb2d87ef922 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Wed, 14 Oct 2026 10:10:57 +0200 Subject: [PATCH 21/50] =?UTF-8?q?=F0=9F=9A=A7=20wip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/private/lib/directorySync/_azure.ts | 443 ++++++++++++++ server/private/lib/directorySync/azure.ts | 536 ++++++----------- server/private/lib/directorySync/run.ts | 27 +- server/private/lib/directorySync/runSync.ts | 606 -------------------- server/private/lib/directorySync/types.ts | 17 +- 5 files changed, 640 insertions(+), 989 deletions(-) create mode 100644 server/private/lib/directorySync/_azure.ts delete mode 100644 server/private/lib/directorySync/runSync.ts diff --git a/server/private/lib/directorySync/_azure.ts b/server/private/lib/directorySync/_azure.ts new file mode 100644 index 0000000000..cf870cba2a --- /dev/null +++ b/server/private/lib/directorySync/_azure.ts @@ -0,0 +1,443 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +// Microsoft Entra ID directory provider (Microsoft Graph, app-only auth). +// +// Required application permissions (with admin consent): User.Read.All, GroupMember.Read.All. +// +// Full fetch: /users/delta + /groups/delta?$select=displayName,members without a token, which +// lists everything and returns delta links for next time. Nested memberships are resolved in +// memory from the direct members. +// +// Incremental fetch: replay both delta links. Group deltas only report direct member changes, +// so every user touched by a change is re-read (/users/{id} + /transitiveMemberOf) to get their +// full current state. Cases that can't be resolved cheaply fall back to a full fetch. + +import config from "@server/lib/config"; +import logger from "@server/logger"; +import { decrypt } from "@server/lib/crypto"; +import { + DirectoryCursorExpiredError, + DirectoryFetchContext, + DirectoryFetchResult, + DirectoryGroup, + DirectorySyncProvider, + DirectoryUser +} from "./types"; + +const GRAPH = "https://graph.microsoft.com/v1.0"; +const USER_SELECT = "id,displayName,mail,userPrincipalName,accountEnabled"; + +/** Above this many touched users, an incremental run is more expensive than a full one. */ +const MAX_INCREMENTAL_AFFECTED_USERS = 500; +/** Longest Retry-After we sleep through in-process before failing the run. */ +const MAX_RETRY_AFTER_SECONDS = 120; +const MAX_THROTTLE_RETRIES = 5; + +type GraphUser = { + id: string; + displayName?: string | null; + mail?: string | null; + userPrincipalName?: string | null; + accountEnabled?: boolean | null; + "@removed"?: { reason: string }; +}; + +type GraphDirectoryObjectRef = { + "@odata.type"?: string; + id: string; + "@removed"?: { reason: string }; +}; + +type GraphGroup = { + id: string; + displayName?: string | null; + "members@delta"?: GraphDirectoryObjectRef[]; + "@removed"?: { reason: string }; +}; + +type AzureCursor = { + usersDeltaLink: string; + groupsDeltaLink: string; +}; + +type GraphPage = { + value: T[]; + "@odata.nextLink"?: string; + "@odata.deltaLink"?: string; +}; + +// Graph field that yields the same `username` login computes from the ID token. +// `sub` is per-app in Entra and not exposed by Graph, so it can't be synced. +const IDENTIFIER_FROM_GRAPH: Record string | null> = { + oid: (u) => u.id, + preferred_username: (u) => u.userPrincipalName ?? null, + upn: (u) => u.userPrincipalName ?? null, + email: (u) => u.mail ?? null +}; + +const tokenCache = new Map(); + +class GraphClient { + constructor( + private readonly getToken: () => Promise, + private readonly onRequest: () => void + ) {} + + /** GET a single Graph URL, sleeping through short throttling. Returns null on 404. */ + async get(url: string): Promise { + for (let attempt = 0; ; attempt++) { + this.onRequest(); + const res = await fetch(url, { + headers: { Authorization: `Bearer ${await this.getToken()}` } + }); + + if (res.status === 429 || res.status === 503) { + const retryAfter = Number(res.headers.get("Retry-After") ?? 5); + if ( + attempt >= MAX_THROTTLE_RETRIES || + retryAfter > MAX_RETRY_AFTER_SECONDS + ) { + throw new Error( + `Graph throttled (${res.status}, Retry-After ${retryAfter}s): ${url}` + ); + } + logger.warn(`Graph throttled, retrying in ${retryAfter}s`, { + url + }); + await new Promise((r) => setTimeout(r, retryAfter * 1000)); + continue; + } + + if (res.status === 404) return null; + + const body: any = await res.json().catch(() => ({})); + + if ( + res.status === 410 || + body.error?.code === "syncStateNotFound" || + body.error?.code === "resyncRequired" + ) { + throw new DirectoryCursorExpiredError( + `Graph delta token expired (${res.status} ${body.error?.code})` + ); + } + if (!res.ok) { + throw new Error( + `Graph request failed (${res.status}) ${url}: ${body.error?.code} ${body.error?.message}` + ); + } + return body as T; + } + } + + /** Follow @odata.nextLink pages; returns the items and the final delta link, if any. */ + async getAll( + url: string + ): Promise<{ items: T[]; deltaLink: string | null }> { + const items: T[] = []; + let next: string | undefined = url; + while (next) { + const page: GraphPage | null = + await this.get>(next); + if (!page) throw new Error(`Graph returned 404 for ${next}`); + items.push(...page.value); + if (page["@odata.deltaLink"]) { + return { items, deltaLink: page["@odata.deltaLink"] }; + } + next = page["@odata.nextLink"]; + } + return { items, deltaLink: null }; + } +} + +function isUserRef(ref: GraphDirectoryObjectRef) { + return ref["@odata.type"] === "#microsoft.graph.user"; +} + +function isGroupRef(ref: GraphDirectoryObjectRef) { + return ref["@odata.type"] === "#microsoft.graph.group"; +} + +function parseCursor(cursor: string | null): AzureCursor | null { + if (!cursor) return null; + try { + const parsed = JSON.parse(cursor); + if ( + typeof parsed?.usersDeltaLink === "string" && + typeof parsed?.groupsDeltaLink === "string" + ) { + return parsed; + } + } catch { + // fall through + } + logger.warn("Ignoring unreadable Azure directory sync cursor"); + return null; +} + +export class AzureDirectoryProvider implements DirectorySyncProvider { + constructor( + private readonly settings: { + tenantId: string; + /** Encrypted, as stored in `idpOidcConfig`. */ + encryptedClientId: string; + /** Encrypted, as stored in `idpOidcConfig`. */ + encryptedClientSecret: string; + } + ) {} + + /** Parses the tenant ID from an Entra authorize URL. */ + static tenantIdFromAuthUrl(authUrl: string): string | null { + return ( + authUrl.match( + /login\.microsoftonline\.com\/([^/]+)\/oauth2/ + )?.[1] ?? null + ); + } + + async fetch(ctx: DirectoryFetchContext): Promise { + const getIdentifier = IDENTIFIER_FROM_GRAPH[ctx.identifierPath]; + if (!getIdentifier) { + throw new Error( + `identifierPath "${ctx.identifierPath}" can't be resolved from Graph; use one of: ${Object.keys(IDENTIFIER_FROM_GRAPH).join(", ")}` + ); + } + + const graph = new GraphClient(() => this.getToken(), ctx.onRequest); + const toDirectoryUser = ( + u: GraphUser, + groups: DirectoryGroup[] + ): DirectoryUser => ({ + externalId: u.id, + username: getIdentifier(u)?.toLowerCase() ?? null, + email: u.mail?.toLowerCase() ?? null, + name: u.displayName ?? null, + enabled: u.accountEnabled !== false, + groups + }); + + const cursor = parseCursor(ctx.cursor); + if (cursor) { + const incremental = await this.fetchIncremental( + graph, + cursor, + toDirectoryUser + ); + if (incremental) return incremental; + logger.info( + `Azure incremental sync for IdP ${ctx.idpId} needs a full fetch` + ); + } + return this.fetchFull(graph, toDirectoryUser); + } + + private async getToken(): Promise { + const { tenantId, encryptedClientId, encryptedClientSecret } = + this.settings; + const key = config.getRawConfig().server.secret!; + const clientId = decrypt(encryptedClientId, key); + const cacheKey = `${tenantId}:${clientId}`; + + const cached = tokenCache.get(cacheKey); + if (cached && cached.expiresAt > Date.now() + 60_000) { + return cached.token; + } + + const res = await fetch( + `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, + { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded" + }, + body: new URLSearchParams({ + grant_type: "client_credentials", + client_id: clientId, + client_secret: decrypt(encryptedClientSecret, key), + scope: "https://graph.microsoft.com/.default" + }) + } + ); + const body: any = await res.json().catch(() => ({})); + if (!res.ok) { + throw new Error( + `Entra token request failed (${res.status}): ${body.error_description ?? body.error}` + ); + } + + tokenCache.set(cacheKey, { + token: body.access_token, + expiresAt: Date.now() + Number(body.expires_in ?? 3600) * 1000 + }); + return body.access_token; + } + + private async fetchFull( + graph: GraphClient, + toDirectoryUser: ( + u: GraphUser, + groups: DirectoryGroup[] + ) => DirectoryUser + ): Promise { + const usersRes = await graph.getAll( + `${GRAPH}/users/delta?$select=${USER_SELECT}` + ); + const groupsRes = await graph.getAll( + `${GRAPH}/groups/delta?$select=displayName,members` + ); + + // A large group can be split over several pages; merge its member chunks. + const groups = new Map(); + // member (user or group) ID -> IDs of groups it directly belongs to + const parentGroups = new Map>(); + for (const g of groupsRes.items) { + if (g["@removed"]) continue; + const existing = groups.get(g.id); + groups.set(g.id, { + id: g.id, + displayName: g.displayName ?? existing?.displayName ?? null + }); + for (const member of g["members@delta"] ?? []) { + if (member["@removed"]) continue; + if (!isUserRef(member) && !isGroupRef(member)) continue; + const parents = parentGroups.get(member.id) ?? new Set(); + parents.add(g.id); + parentGroups.set(member.id, parents); + } + } + + const transitiveGroups = (memberId: string): DirectoryGroup[] => { + const seen = new Set(); + const queue = [...(parentGroups.get(memberId) ?? [])]; + while (queue.length) { + const groupId = queue.pop()!; + if (seen.has(groupId)) continue; + seen.add(groupId); + queue.push(...(parentGroups.get(groupId) ?? [])); + } + return [...seen] + .map((id) => groups.get(id)) + .filter((g): g is DirectoryGroup => !!g); + }; + + const users = usersRes.items + .filter((u) => !u["@removed"]) + .map((u) => toDirectoryUser(u, transitiveGroups(u.id))); + + return { + kind: "full", + users, + nextCursor: this.cursorFrom(usersRes.deltaLink, groupsRes.deltaLink) + }; + } + + /** Returns null when a full fetch is required instead. */ + private async fetchIncremental( + graph: GraphClient, + cursor: AzureCursor, + toDirectoryUser: ( + u: GraphUser, + groups: DirectoryGroup[] + ) => DirectoryUser + ): Promise { + const usersRes = await graph.getAll(cursor.usersDeltaLink); + const groupsRes = await graph.getAll( + cursor.groupsDeltaLink + ); + + const removedExternalIds = new Set(); + const affectedUserIds = new Set(); + + for (const u of usersRes.items) { + if (u["@removed"]) { + removedExternalIds.add(u.id); + } else { + affectedUserIds.add(u.id); + } + } + + // Groups whose every (transitive) user member must be re-read. + const expandGroupIds = new Set(); + for (const g of groupsRes.items) { + if (g["@removed"]) { + // The former members of a deleted group aren't listed anywhere. + return null; + } + if (g.displayName !== undefined) { + // Possibly renamed: name-based role mappings may change for all members. + expandGroupIds.add(g.id); + } + for (const member of g["members@delta"] ?? []) { + if (isUserRef(member)) { + affectedUserIds.add(member.id); + } else if (isGroupRef(member)) { + expandGroupIds.add(member.id); + } + } + } + + for (const groupId of expandGroupIds) { + const members = await graph.getAll<{ id: string }>( + `${GRAPH}/groups/${groupId}/transitiveMembers/microsoft.graph.user?$select=id&$top=999` + ); + for (const m of members.items) affectedUserIds.add(m.id); + if (affectedUserIds.size > MAX_INCREMENTAL_AFFECTED_USERS) { + return null; + } + } + if (affectedUserIds.size > MAX_INCREMENTAL_AFFECTED_USERS) { + return null; + } + + const users: DirectoryUser[] = []; + for (const userId of affectedUserIds) { + if (removedExternalIds.has(userId)) continue; + const u = await graph.get( + `${GRAPH}/users/${userId}?$select=${USER_SELECT}` + ); + if (!u) { + removedExternalIds.add(userId); + continue; + } + const memberOf = await graph.getAll( + `${GRAPH}/users/${userId}/transitiveMemberOf/microsoft.graph.group?$select=id,displayName&$top=999` + ); + users.push( + toDirectoryUser( + u, + memberOf.items.map((g) => ({ + id: g.id, + displayName: g.displayName ?? null + })) + ) + ); + } + + return { + kind: "incremental", + users, + removedExternalIds: [...removedExternalIds], + nextCursor: this.cursorFrom(usersRes.deltaLink, groupsRes.deltaLink) + }; + } + + private cursorFrom( + usersDeltaLink: string | null, + groupsDeltaLink: string | null + ): string | null { + if (!usersDeltaLink || !groupsDeltaLink) return null; + const cursor: AzureCursor = { usersDeltaLink, groupsDeltaLink }; + return JSON.stringify(cursor); + } +} diff --git a/server/private/lib/directorySync/azure.ts b/server/private/lib/directorySync/azure.ts index 5942e67a6e..ca5529f9bc 100644 --- a/server/private/lib/directorySync/azure.ts +++ b/server/private/lib/directorySync/azure.ts @@ -11,433 +11,217 @@ * This file is not licensed under the AGPLv3. */ -// Microsoft Entra ID directory provider (Microsoft Graph, app-only auth). -// -// Required application permissions (with admin consent): User.Read.All, GroupMember.Read.All. -// -// Full fetch: /users/delta + /groups/delta?$select=displayName,members without a token, which -// lists everything and returns delta links for next time. Nested memberships are resolved in -// memory from the direct members. -// -// Incremental fetch: replay both delta links. Group deltas only report direct member changes, -// so every user touched by a change is re-read (/users/{id} + /transitiveMemberOf) to get their -// full current state. Cases that can't be resolved cheaply fall back to a full fetch. - -import config from "@server/lib/config"; -import logger from "@server/logger"; -import { decrypt } from "@server/lib/crypto"; -import { - DirectoryCursorExpiredError, +import { db, idp, idpOidcConfig, idpOrg } from "@server/db"; +import { eq, and } from "drizzle-orm"; +import type { DirectoryFetchContext, DirectoryFetchResult, - DirectoryGroup, - DirectoryProvider, - DirectoryUser + DirectorySyncProvider } from "./types"; +import { decrypt } from "@server/lib/crypto"; +import config from "@server/lib/config"; +import cache from "@server/lib/cache"; +import z from "zod"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { parseDirectorySyncRoleMapping } from "@app/lib/idpDirectorySyncRoleMapping"; + +async function ensureValidIdp(idpId: number) { + const [idpRes] = await db + .select() + .from(idp) + .where(eq(idp.idpId, idpId)) + .innerJoin(idpOidcConfig, eq(idpOidcConfig.idpId, idp.idpId)) + .innerJoin(idpOrg, and(eq(idpOrg.idpId, idp.idpId))) + .limit(1); + + if (!idpRes) { + throw new Error( + `Directory sync: IdP ${idpId} not found, or it has no OIDC config or org mapping` + ); + } -const GRAPH = "https://graph.microsoft.com/v1.0"; -const USER_SELECT = "id,displayName,mail,userPrincipalName,accountEnabled"; - -/** Above this many touched users, an incremental run is more expensive than a full one. */ -const MAX_INCREMENTAL_AFFECTED_USERS = 500; -/** Longest Retry-After we sleep through in-process before failing the run. */ -const MAX_RETRY_AFTER_SECONDS = 120; -const MAX_THROTTLE_RETRIES = 5; - -type GraphUser = { - id: string; - displayName?: string | null; - mail?: string | null; - userPrincipalName?: string | null; - accountEnabled?: boolean | null; - "@removed"?: { reason: string }; -}; - -type GraphDirectoryObjectRef = { - "@odata.type"?: string; - id: string; - "@removed"?: { reason: string }; -}; - -type GraphGroup = { - id: string; - displayName?: string | null; - "members@delta"?: GraphDirectoryObjectRef[]; - "@removed"?: { reason: string }; -}; - -type AzureCursor = { - usersDeltaLink: string; - groupsDeltaLink: string; -}; - -type GraphPage = { - value: T[]; - "@odata.nextLink"?: string; - "@odata.deltaLink"?: string; -}; + if (idpRes.idpOidcConfig.variant !== "azure") { + throw new Error( + `Directory sync: IdP ${idpId} has variant "${idpRes.idpOidcConfig.variant}", expected "azure"` + ); + } -// Graph field that yields the same `username` login computes from the ID token. -// `sub` is per-app in Entra and not exposed by Graph, so it can't be synced. -const IDENTIFIER_FROM_GRAPH: Record string | null> = { - oid: (u) => u.id, - preferred_username: (u) => u.userPrincipalName ?? null, - upn: (u) => u.userPrincipalName ?? null, - email: (u) => u.mail ?? null -}; + if (!idpRes.idp.directorySyncEnabled) { + throw new Error( + `Directory sync: directory sync is not enabled on IdP ${idpId}` + ); + } -const tokenCache = new Map(); + if (idpRes.idp.autoProvision) { + throw new Error( + `Directory sync: IdP ${idpId} has both directory sync and auto-provisioning enabled; only one can be active` + ); + } + return idpRes; +} -class GraphClient { - constructor( - private readonly getToken: () => Promise, - private readonly onRequest: () => void - ) {} +export class AzureDirectorySyncProvider implements DirectorySyncProvider { + #TOKEN_CACHE_KEY_PREFIX = "azureDirectorySyncToken"; + #microsoftGraphBaseUrl = "https://graph.microsoft.com/v1.0"; - /** GET a single Graph URL, sleeping through short throttling. Returns null on 404. */ - async get(url: string): Promise { - for (let attempt = 0; ; attempt++) { - this.onRequest(); - const res = await fetch(url, { - headers: { Authorization: `Bearer ${await this.getToken()}` } - }); + async fetch(ctx: DirectoryFetchContext): Promise { + // first type of fetch: `full` + // 0. fetch idp and check that it is valid + const idpRes = await ensureValidIdp(ctx.idpId); + + // 1. get access token (and cache) + const token = await this.#getToken(idpRes); + const directorySyncMapping = parseDirectorySyncRoleMapping( + idpRes.idpOrg.directoryRoleMapping + ); - if (res.status === 429 || res.status === 503) { - const retryAfter = Number(res.headers.get("Retry-After") ?? 5); - if ( - attempt >= MAX_THROTTLE_RETRIES || - retryAfter > MAX_RETRY_AFTER_SECONDS - ) { - throw new Error( - `Graph throttled (${res.status}, Retry-After ${retryAfter}s): ${url}` - ); - } - logger.warn(`Graph throttled, retrying in ${retryAfter}s`, { - url - }); - await new Promise((r) => setTimeout(r, retryAfter * 1000)); - continue; - } + if (directorySyncMapping.mode === "fixedRoles") { + // .. + } - if (res.status === 404) return null; + // if fixed roles + // 2. get all users + // if create or map groups + // 2. get list of all groups + // 3. get group members ? - const body: any = await res.json().catch(() => ({})); + console.dir({ + token + }); - if ( - res.status === 410 || - body.error?.code === "syncStateNotFound" || - body.error?.code === "resyncRequired" - ) { - throw new DirectoryCursorExpiredError( - `Graph delta token expired (${res.status} ${body.error?.code})` - ); - } - if (!res.ok) { - throw new Error( - `Graph request failed (${res.status}) ${url}: ${body.error?.code} ${body.error?.message}` - ); - } - return body as T; - } + return { + kind: "full", + users: [], + nextCursor: null + }; } - /** Follow @odata.nextLink pages; returns the items and the final delta link, if any. */ - async getAll( - url: string - ): Promise<{ items: T[]; deltaLink: string | null }> { - const items: T[] = []; - let next: string | undefined = url; - while (next) { - const page: GraphPage | null = - await this.get>(next); - if (!page) throw new Error(`Graph returned 404 for ${next}`); - items.push(...page.value); - if (page["@odata.deltaLink"]) { - return { items, deltaLink: page["@odata.deltaLink"] }; - } - next = page["@odata.nextLink"]; - } - return { items, deltaLink: null }; - } -} + async #getToken( + idpRes: Awaited> + ): Promise { + const cacheKey = `${this.#TOKEN_CACHE_KEY_PREFIX}:${idpRes.idp.idpId}`; + const cached = await cache.get(cacheKey); -function isUserRef(ref: GraphDirectoryObjectRef) { - return ref["@odata.type"] === "#microsoft.graph.user"; -} + if (cached) { + logger.info("Returning cached access token"); + return cached; + } -function isGroupRef(ref: GraphDirectoryObjectRef) { - return ref["@odata.type"] === "#microsoft.graph.group"; -} + logger.info("Fetching fresh access token"); + const key = config.getRawConfig().server.secret!; -function parseCursor(cursor: string | null): AzureCursor | null { - if (!cursor) return null; - try { - const parsed = JSON.parse(cursor); - if ( - typeof parsed?.usersDeltaLink === "string" && - typeof parsed?.groupsDeltaLink === "string" - ) { - return parsed; - } - } catch { - // fall through - } - logger.warn("Ignoring unreadable Azure directory sync cursor"); - return null; -} + let clientId = ""; + let clientSecret = ""; -export class AzureDirectoryProvider implements DirectoryProvider { - constructor( - private readonly settings: { - tenantId: string; - /** Encrypted, as stored in `idpOidcConfig`. */ - encryptedClientId: string; - /** Encrypted, as stored in `idpOidcConfig`. */ - encryptedClientSecret: string; + if (idpRes.idp.type === "oidc") { + clientSecret = decrypt(idpRes.idpOidcConfig.clientSecret, key); + clientId = decrypt(idpRes.idpOidcConfig.clientId, key); } - ) {} - /** Parses the tenant ID from an Entra authorize URL. */ - static tenantIdFromAuthUrl(authUrl: string): string | null { - return ( - authUrl.match( - /login\.microsoftonline\.com\/([^/]+)\/oauth2/ - )?.[1] ?? null - ); - } + const res = await fetch(idpRes.idpOidcConfig.tokenUrl, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded" + }, + body: new URLSearchParams({ + grant_type: "client_credentials", + client_id: clientId, + client_secret: clientSecret, + scope: "https://graph.microsoft.com/.default" + }) + }); - async fetch(ctx: DirectoryFetchContext): Promise { - const getIdentifier = IDENTIFIER_FROM_GRAPH[ctx.identifierPath]; - if (!getIdentifier) { + const body: any = await res.json().catch(() => ({})); + if (!res.ok) { + const reason = + body.error_description ?? + body.error ?? + res.statusText ?? + "no error details in response"; throw new Error( - `identifierPath "${ctx.identifierPath}" can't be resolved from Graph; use one of: ${Object.keys(IDENTIFIER_FROM_GRAPH).join(", ")}` + `Directory sync: Entra token request for IdP ${idpRes.idp.idpId} failed (HTTP ${res.status}): ${reason}` ); } - const graph = new GraphClient(() => this.getToken(), ctx.onRequest); - const toDirectoryUser = ( - u: GraphUser, - groups: DirectoryGroup[] - ): DirectoryUser => ({ - externalId: u.id, - username: getIdentifier(u)?.toLowerCase() ?? null, - email: u.mail?.toLowerCase() ?? null, - name: u.displayName ?? null, - enabled: u.accountEnabled !== false, - groups + const tokenBodySchema = z.object({ + expires_in: z.int(), + access_token: z.string() }); - const cursor = parseCursor(ctx.cursor); - if (cursor) { - const incremental = await this.fetchIncremental( - graph, - cursor, - toDirectoryUser + try { + const tokenResponse = tokenBodySchema.parse(body); + await cache.set( + cacheKey, + tokenResponse.access_token, + tokenResponse.expires_in - 30 // keep the response 30 seconds less than the token to make sure to always have a fresh token ); - if (incremental) return incremental; + logger.info( - `Azure incremental sync for IdP ${ctx.idpId} needs a full fetch` + `Got new access token (expires in: ${tokenResponse.expires_in}s)` + ); + + return tokenResponse.access_token; + } catch (e) { + throw new Error( + `Directory sync: unexpected Entra token response for IdP ${idpRes.idp.idpId}: ${fromError(e).toString()}`, + { cause: e } ); } - return this.fetchFull(graph, toDirectoryUser); } - private async getToken(): Promise { - const { tenantId, encryptedClientId, encryptedClientSecret } = - this.settings; - const key = config.getRawConfig().server.secret!; - const clientId = decrypt(encryptedClientId, key); - const cacheKey = `${tenantId}:${clientId}`; - - const cached = tokenCache.get(cacheKey); - if (cached && cached.expiresAt > Date.now() + 60_000) { - return cached.token; - } + async #fetchAllUsers(token: string) { + //... + const userSelectQuery = + "id,displayName,mail,userPrincipalName,accountEnabled"; const res = await fetch( - `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, + `${this.#microsoftGraphBaseUrl}/users?$select=${userSelectQuery}`, { - method: "POST", headers: { - "Content-Type": "application/x-www-form-urlencoded" - }, - body: new URLSearchParams({ - grant_type: "client_credentials", - client_id: clientId, - client_secret: decrypt(encryptedClientSecret, key), - scope: "https://graph.microsoft.com/.default" - }) + Authorization: `Bearer ${token}` + } } ); + const body: any = await res.json().catch(() => ({})); if (!res.ok) { + const reason = + body.error_description ?? + body.error ?? + res.statusText ?? + "no error details in response"; throw new Error( - `Entra token request failed (${res.status}): ${body.error_description ?? body.error}` + `Directory sync: Entra fetch user failed (HTTP ${res.status}): ${reason}` // todo: improve ); } - tokenCache.set(cacheKey, { - token: body.access_token, - expiresAt: Date.now() + Number(body.expires_in ?? 3600) * 1000 + const userBodySchema = z.object({ + value: z.array( + z.object({ + mail: z.email(), + userPrincipalName: z.string(), + id: z.uuidv4() + }) + ) }); - return body.access_token; - } - - private async fetchFull( - graph: GraphClient, - toDirectoryUser: ( - u: GraphUser, - groups: DirectoryGroup[] - ) => DirectoryUser - ): Promise { - const usersRes = await graph.getAll( - `${GRAPH}/users/delta?$select=${USER_SELECT}` - ); - const groupsRes = await graph.getAll( - `${GRAPH}/groups/delta?$select=displayName,members` - ); - // A large group can be split over several pages; merge its member chunks. - const groups = new Map(); - // member (user or group) ID -> IDs of groups it directly belongs to - const parentGroups = new Map>(); - for (const g of groupsRes.items) { - if (g["@removed"]) continue; - const existing = groups.get(g.id); - groups.set(g.id, { - id: g.id, - displayName: g.displayName ?? existing?.displayName ?? null - }); - for (const member of g["members@delta"] ?? []) { - if (member["@removed"]) continue; - if (!isUserRef(member) && !isGroupRef(member)) continue; - const parents = parentGroups.get(member.id) ?? new Set(); - parents.add(g.id); - parentGroups.set(member.id, parents); - } - } - - const transitiveGroups = (memberId: string): DirectoryGroup[] => { - const seen = new Set(); - const queue = [...(parentGroups.get(memberId) ?? [])]; - while (queue.length) { - const groupId = queue.pop()!; - if (seen.has(groupId)) continue; - seen.add(groupId); - queue.push(...(parentGroups.get(groupId) ?? [])); - } - return [...seen] - .map((id) => groups.get(id)) - .filter((g): g is DirectoryGroup => !!g); - }; - - const users = usersRes.items - .filter((u) => !u["@removed"]) - .map((u) => toDirectoryUser(u, transitiveGroups(u.id))); - - return { - kind: "full", - users, - nextCursor: this.cursorFrom(usersRes.deltaLink, groupsRes.deltaLink) - }; - } + try { + const tokenResponse = userBodySchema.parse(body); - /** Returns null when a full fetch is required instead. */ - private async fetchIncremental( - graph: GraphClient, - cursor: AzureCursor, - toDirectoryUser: ( - u: GraphUser, - groups: DirectoryGroup[] - ) => DirectoryUser - ): Promise { - const usersRes = await graph.getAll(cursor.usersDeltaLink); - const groupsRes = await graph.getAll( - cursor.groupsDeltaLink - ); - - const removedExternalIds = new Set(); - const affectedUserIds = new Set(); - - for (const u of usersRes.items) { - if (u["@removed"]) { - removedExternalIds.add(u.id); - } else { - affectedUserIds.add(u.id); - } - } - - // Groups whose every (transitive) user member must be re-read. - const expandGroupIds = new Set(); - for (const g of groupsRes.items) { - if (g["@removed"]) { - // The former members of a deleted group aren't listed anywhere. - return null; - } - if (g.displayName !== undefined) { - // Possibly renamed: name-based role mappings may change for all members. - expandGroupIds.add(g.id); - } - for (const member of g["members@delta"] ?? []) { - if (isUserRef(member)) { - affectedUserIds.add(member.id); - } else if (isGroupRef(member)) { - expandGroupIds.add(member.id); - } - } - } - - for (const groupId of expandGroupIds) { - const members = await graph.getAll<{ id: string }>( - `${GRAPH}/groups/${groupId}/transitiveMembers/microsoft.graph.user?$select=id&$top=999` - ); - for (const m of members.items) affectedUserIds.add(m.id); - if (affectedUserIds.size > MAX_INCREMENTAL_AFFECTED_USERS) { - return null; - } - } - if (affectedUserIds.size > MAX_INCREMENTAL_AFFECTED_USERS) { - return null; - } - - const users: DirectoryUser[] = []; - for (const userId of affectedUserIds) { - if (removedExternalIds.has(userId)) continue; - const u = await graph.get( - `${GRAPH}/users/${userId}?$select=${USER_SELECT}` - ); - if (!u) { - removedExternalIds.add(userId); - continue; - } - const memberOf = await graph.getAll( - `${GRAPH}/users/${userId}/transitiveMemberOf/microsoft.graph.group?$select=id,displayName&$top=999` - ); - users.push( - toDirectoryUser( - u, - memberOf.items.map((g) => ({ - id: g.id, - displayName: g.displayName ?? null - })) - ) + return tokenResponse; + } catch (e) { + throw new Error( + `Directory sync: `, // todo: improve + { cause: e } ); } - - return { - kind: "incremental", - users, - removedExternalIds: [...removedExternalIds], - nextCursor: this.cursorFrom(usersRes.deltaLink, groupsRes.deltaLink) - }; } - private cursorFrom( - usersDeltaLink: string | null, - groupsDeltaLink: string | null - ): string | null { - if (!usersDeltaLink || !groupsDeltaLink) return null; - const cursor: AzureCursor = { usersDeltaLink, groupsDeltaLink }; - return JSON.stringify(cursor); + async #fetchAllGroups(token: string) { + //... + } + async #fetchGroupMembers(token: string) { + //... } } diff --git a/server/private/lib/directorySync/run.ts b/server/private/lib/directorySync/run.ts index e144767b52..9892e2bd99 100644 --- a/server/private/lib/directorySync/run.ts +++ b/server/private/lib/directorySync/run.ts @@ -1,6 +1,6 @@ -// Usage: npx tsx server/private/lib/directorySync/run.ts -// Runs one directory sync for the IdP using its saved directory role mapping. -import { runDirectorySync } from "./runSync"; +// // Usage: npx tsx server/private/lib/directorySync/run.ts +// // Runs one directory sync for the IdP using its saved directory role mapping. +import { AzureDirectorySyncProvider } from "./azure"; const idpId = Number(process.argv[2]); if (!idpId) { @@ -8,12 +8,29 @@ if (!idpId) { process.exit(1); } -runDirectorySync(idpId) +const provider = new AzureDirectorySyncProvider(); + +provider + .fetch({ + idpId: idpId, + identifierPath: "email", + cursor: null + }) .then((report) => { - console.log(JSON.stringify(report, null, 2)); + console.dir({ report }, { depth: null }); process.exit(0); }) .catch((err) => { console.error(err); process.exit(1); }); + +// runDirectorySync(idpId) +// .then((report) => { +// console.log(JSON.stringify(report, null, 2)); +// process.exit(0); +// }) +// .catch((err) => { +// console.error(err); +// process.exit(1); +// }); diff --git a/server/private/lib/directorySync/runSync.ts b/server/private/lib/directorySync/runSync.ts deleted file mode 100644 index 4a4c490c56..0000000000 --- a/server/private/lib/directorySync/runSync.ts +++ /dev/null @@ -1,606 +0,0 @@ -/* - * This file is part of a proprietary work. - * - * Copyright (c) 2025-2026 Fossorial, Inc. - * All rights reserved. - * - * This file is licensed under the Fossorial Commercial License. - * You may not use this file except in compliance with the License. - * Unauthorized use, copying, modification, or distribution is strictly prohibited. - * - * This file is not licensed under the AGPLv3. - */ - -// Provider-independent directory sync engine. -// -// 1. Fetch users from the provider (full, or incremental from the stored cursor). -// 2. Resolve each user's roles per org from `idpOrg.directoryRoleMapping`. -// 3. Create/update matching users and their auto-provisioned memberships. -// 4. When `idpOrg.directorySyncDeletions` is on, delete the Pangolin users sync manages that -// were deleted or disabled in the directory, or no longer match any role mapping. -// 5. Save the provider cursor only once everything above succeeded. - -import { - db, - idp, - idpOidcConfig, - idpOrg, - idpSyncState, - idpSyncUser, - Org, - orgs, - roleActions, - roles, - userOrgRoles, - userOrgs, - users -} from "@server/db"; -import { and, eq } from "drizzle-orm"; -import logger from "@server/logger"; -import { generateId } from "@server/auth/sessions/app"; -import { UserType } from "@server/types/UserTypes"; -import { LimitId } from "@server/lib/billing"; -import { usageService } from "@server/lib/billing/usageService"; -import { tierMatrix } from "@server/lib/billing/tierMatrix"; -import { calculateUserClientsForOrgs } from "@server/lib/calculateUserClientsForOrgs"; -import { assignUserToOrg, removeUserFromOrg } from "@server/lib/userOrg"; -import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed"; -import { defaultRoleAllowedActions } from "@server/routers/role/createRole"; -import { lockManager } from "@server/private/lib/lock"; -import { - DirectorySyncRoleMappingConfig, - parseDirectorySyncRoleMapping -} from "@app/lib/idpDirectorySyncRoleMapping"; -import { AzureDirectoryProvider } from "./azure"; -import { - DirectoryCursorExpiredError, - DirectoryFetchResult, - DirectoryProvider, - DirectoryUser -} from "./types"; - -const LOCK_TTL_MS = 30 * 60 * 1000; -/** Refuse a run that would delete more than this share of the users sync manages... */ -const MAX_DELETION_RATIO = 0.5; -/** ...unless it deletes at most this many users. */ -const DELETION_RATIO_MIN_USERS = 10; - -export type DirectorySyncReport = { - fetchKind: DirectoryFetchResult["kind"] | null; - requests: number; - fetchedUsers: number; - created: number; - updated: number; - deleted: number; - membershipsRemoved: number; - skippedNoMatch: number; - skippedNoIdentifier: number; - createdRoles: string[]; - missingRoles: string[]; -}; - -type OrgMapping = { - org: Org; - config: DirectorySyncRoleMappingConfig; - syncDeletions: boolean; - multiRole: boolean; - /** Lower-cased role name -> role. */ - rolesByName: Map; -}; - -export async function runDirectorySync( - idpId: number -): Promise { - return lockManager.withLock( - `idp-sync:${idpId}`, - () => runLocked(idpId), - LOCK_TTL_MS - ); -} - -async function runLocked(idpId: number): Promise { - const report: DirectorySyncReport = { - fetchKind: null, - requests: 0, - fetchedUsers: 0, - created: 0, - updated: 0, - deleted: 0, - membershipsRemoved: 0, - skippedNoMatch: 0, - skippedNoIdentifier: 0, - createdRoles: [], - missingRoles: [] - }; - - const [existingIdp] = await db - .select() - .from(idp) - .innerJoin(idpOidcConfig, eq(idpOidcConfig.idpId, idp.idpId)) - .where(eq(idp.idpId, idpId)); - - if (!existingIdp) throw new Error(`IdP ${idpId} not found`); - if (!existingIdp.idp.directorySyncEnabled) { - throw new Error(`Directory sync is not enabled for IdP ${idpId}`); - } - if (existingIdp.idp.autoProvision) { - throw new Error( - `IdP ${idpId} has autoProvision enabled, which is exclusive with directory sync` - ); - } - - const provider = createProvider(existingIdp.idpOidcConfig); - - const [state] = await db - .select() - .from(idpSyncState) - .where(eq(idpSyncState.idpId, idpId)); - - const startedAt = Date.now(); - await db - .insert(idpSyncState) - .values({ idpId, lastSyncStartedAt: startedAt }) - .onConflictDoUpdate({ - target: idpSyncState.idpId, - set: { lastSyncStartedAt: startedAt } - }); - - try { - const fetchWith = (cursor: string | null) => - provider.fetch({ - idpId, - identifierPath: existingIdp.idpOidcConfig.identifierPath, - cursor, - onRequest: () => report.requests++ - }); - - let result: DirectoryFetchResult; - try { - result = await fetchWith(state?.cursor ?? null); - } catch (err) { - if (!(err instanceof DirectoryCursorExpiredError)) throw err; - logger.info( - `Directory sync cursor expired for IdP ${idpId}, running a full sync` - ); - result = await fetchWith(null); - } - - report.fetchKind = result.kind; - report.fetchedUsers = result.users.length; - - await applyResult(idpId, result, report); - - await db - .update(idpSyncState) - .set({ - cursor: result.nextCursor, - lastSyncSucceededAt: Date.now(), - lastError: null, - consecutiveFailures: 0 - }) - .where(eq(idpSyncState.idpId, idpId)); - } catch (err) { - await db - .update(idpSyncState) - .set({ - lastError: err instanceof Error ? err.message : String(err), - consecutiveFailures: (state?.consecutiveFailures ?? 0) + 1 - }) - .where(eq(idpSyncState.idpId, idpId)); - throw err; - } - - logger.info("Directory sync done", { idpId, ...report }); - return report; -} - -function createProvider( - oidcConfig: typeof idpOidcConfig.$inferSelect -): DirectoryProvider { - switch (oidcConfig.variant) { - case "azure": { - const tenantId = AzureDirectoryProvider.tenantIdFromAuthUrl( - oidcConfig.authUrl - ); - if (!tenantId) { - throw new Error("Could not parse Azure tenant ID from authUrl"); - } - return new AzureDirectoryProvider({ - tenantId, - encryptedClientId: oidcConfig.clientId, - encryptedClientSecret: oidcConfig.clientSecret - }); - } - default: - throw new Error( - `Directory sync is not supported for variant "${oidcConfig.variant}"` - ); - } -} - -async function loadOrgMappings(idpId: number): Promise { - const rows = await db - .select() - .from(idpOrg) - .innerJoin(orgs, eq(orgs.orgId, idpOrg.orgId)) - .where(eq(idpOrg.idpId, idpId)); - - const mappings: OrgMapping[] = []; - for (const row of rows) { - if (!row.idpOrg.directoryRoleMapping) { - logger.warn( - `IdP ${idpId} has no directory role mapping for org ${row.orgs.orgId}, skipping that org` - ); - continue; - } - const orgRoles = await db - .select() - .from(roles) - .where(eq(roles.orgId, row.orgs.orgId)); - - mappings.push({ - org: row.orgs, - config: parseDirectorySyncRoleMapping( - row.idpOrg.directoryRoleMapping - ), - syncDeletions: row.idpOrg.directorySyncDeletions, - multiRole: await isLicensedOrSubscribed( - row.orgs.orgId, - tierMatrix.fullRbac - ), - rolesByName: new Map( - orgRoles.map((r) => [ - r.name.toLowerCase(), - { roleId: r.roleId, isAdmin: !!r.isAdmin } - ]) - ) - }); - } - return mappings; -} - -/** Role IDs the user should have in this org, in mapping order. Empty = no match. */ -async function resolveRoleIds( - mapping: OrgMapping, - user: DirectoryUser, - report: DirectorySyncReport -): Promise { - const { config, org } = mapping; - const roleIds: number[] = []; - const addRole = (roleId: number) => { - if (!roleIds.includes(roleId)) roleIds.push(roleId); - }; - const addByName = (roleName: string) => { - const role = mapping.rolesByName.get(roleName.toLowerCase()); - if (role) { - addRole(role.roleId); - } else if (!report.missingRoles.includes(`${org.orgId}/${roleName}`)) { - report.missingRoles.push(`${org.orgId}/${roleName}`); - } - }; - - switch (config.mode) { - case "fixedRoles": - config.fixedRoleNames.forEach(addByName); - break; - case "groupMapping": - for (const rule of config.groupRules) { - const match = rule.group.trim().toLowerCase(); - const isMember = user.groups.some( - (g) => - g.id.toLowerCase() === match || - g.displayName?.toLowerCase() === match - ); - if (isMember) rule.roleNames.forEach(addByName); - } - break; - case "automatic": - for (const group of user.groups) { - const name = group.displayName?.trim(); - if (!name) continue; - const role = mapping.rolesByName.get(name.toLowerCase()); - if (role) { - // Never hand out admin roles implicitly from a group name. - if (!role.isAdmin) addRole(role.roleId); - } else if (config.createMissingRoles) { - addRole(await createRole(mapping, name, report)); - } else if ( - !report.missingRoles.includes(`${org.orgId}/${name}`) - ) { - report.missingRoles.push(`${org.orgId}/${name}`); - } - } - break; - } - - return mapping.multiRole ? roleIds : roleIds.slice(0, 1); -} - -async function createRole( - mapping: OrgMapping, - name: string, - report: DirectorySyncReport -): Promise { - const orgId = mapping.org.orgId; - const roleId = await db.transaction(async (trx) => { - const [role] = await trx - .insert(roles) - .values({ - orgId, - name, - description: "Created by directory sync" - }) - .returning(); - await trx.insert(roleActions).values( - defaultRoleAllowedActions.map((actionId) => ({ - roleId: role.roleId, - actionId, - orgId - })) - ); - return role.roleId; - }); - - mapping.rolesByName.set(name.toLowerCase(), { roleId, isAdmin: false }); - report.createdRoles.push(`${orgId}/${name}`); - return roleId; -} - -async function applyResult( - idpId: number, - result: DirectoryFetchResult, - report: DirectorySyncReport -) { - const mappings = await loadOrgMappings(idpId); - if (!mappings.length) { - throw new Error( - `IdP ${idpId} has no org with a directory role mapping, nothing to sync` - ); - } - // Deleting a user removes it from every org, so every org has to allow it. - const deletionsEnabled = mappings.every((m) => m.syncDeletions); - - const links = await db - .select() - .from(idpSyncUser) - .where(eq(idpSyncUser.idpId, idpId)); - const userIdByExternalId = new Map( - links.map((l) => [l.externalId, l.userId]) - ); - - // ---- plan: who to upsert, who to delete ---- - const toUpsert: { - user: DirectoryUser; - rolesByOrg: Map; - }[] = []; - const toDelete = new Set(); // external IDs - - if (result.kind === "incremental") { - result.removedExternalIds.forEach((id) => toDelete.add(id)); - } else { - const seen = new Set(result.users.map((u) => u.externalId)); - for (const externalId of userIdByExternalId.keys()) { - if (!seen.has(externalId)) toDelete.add(externalId); - } - } - - for (const user of result.users) { - if (!user.enabled) { - toDelete.add(user.externalId); - continue; - } - if (!user.username) { - logger.warn( - `No identifier for directory user ${user.externalId}, skipping` - ); - report.skippedNoIdentifier++; - continue; - } - - const rolesByOrg = new Map(); - for (const mapping of mappings) { - const roleIds = await resolveRoleIds(mapping, user, report); - if (roleIds.length) rolesByOrg.set(mapping.org.orgId, roleIds); - } - - if (!rolesByOrg.size) { - report.skippedNoMatch++; - toDelete.add(user.externalId); - continue; - } - toUpsert.push({ user, rolesByOrg }); - } - - // Only users sync manages can be deleted. - const deleteUserIds = deletionsEnabled - ? [...toDelete] - .map((externalId) => userIdByExternalId.get(externalId)) - .filter((userId): userId is string => !!userId) - : []; - - if ( - deleteUserIds.length > DELETION_RATIO_MIN_USERS && - deleteUserIds.length > userIdByExternalId.size * MAX_DELETION_RATIO - ) { - throw new Error( - `Refusing to delete ${deleteUserIds.length} of ${userIdByExternalId.size} synced users in one run` - ); - } - - // ---- apply ---- - const touchedOrgIds = new Set(); - const touchedUserIds = new Set(); - - for (const { user, rolesByOrg } of toUpsert) { - const userId = await upsertUser( - idpId, - user, - userIdByExternalId.get(user.externalId), - report - ); - touchedUserIds.add(userId); - - await db.transaction(async (trx) => { - const currentUserOrgs = await trx - .select() - .from(userOrgs) - .where(eq(userOrgs.userId, userId)); - - for (const mapping of mappings) { - const orgId = mapping.org.orgId; - const roleIds = rolesByOrg.get(orgId) ?? []; - const current = currentUserOrgs.find((o) => o.orgId === orgId); - - if (!current) { - if (!roleIds.length) continue; - await assignUserToOrg( - mapping.org, - { orgId, userId, autoProvisioned: true }, - roleIds, - trx - ); - touchedOrgIds.add(orgId); - continue; - } - - // Memberships added by hand are never changed by sync. - if (!current.autoProvisioned) continue; - - if (!roleIds.length) { - if (mapping.syncDeletions) { - await removeUserFromOrg(mapping.org, userId, trx); - report.membershipsRemoved++; - touchedOrgIds.add(orgId); - } - continue; - } - - await trx - .delete(userOrgRoles) - .where( - and( - eq(userOrgRoles.userId, userId), - eq(userOrgRoles.orgId, orgId) - ) - ); - await trx - .insert(userOrgRoles) - .values( - roleIds.map((roleId) => ({ userId, orgId, roleId })) - ); - } - }); - } - - for (const userId of deleteUserIds) { - const [user] = await db - .select() - .from(users) - .where(and(eq(users.userId, userId), eq(users.idpId, idpId))); - if (!user) continue; - if (user.serverAdmin) { - logger.warn( - `Directory sync won't delete server admin ${user.username}` - ); - continue; - } - - const memberships = await db - .select() - .from(userOrgs) - .where(eq(userOrgs.userId, userId)); - memberships.forEach((m) => touchedOrgIds.add(m.orgId)); - - // Cascades to sessions, memberships and the idpSyncUser link. - await db.delete(users).where(eq(users.userId, userId)); - touchedUserIds.add(userId); - report.deleted++; - } - - for (const orgId of touchedOrgIds) { - const members = await db - .select() - .from(userOrgs) - .where(eq(userOrgs.orgId, orgId)); - await usageService.updateCount(orgId, LimitId.USERS, members.length); - } - - for (const userId of touchedUserIds) { - await calculateUserClientsForOrgs(userId).catch((err) => - logger.error("Error calculating user clients after sync", { - userId, - error: err - }) - ); - } -} - -/** - * Create or update the Pangolin user for a directory user and make sure it is linked. - * An existing user of this IdP with the same username (e.g. created by an earlier login) - * is adopted instead of duplicated. - */ -async function upsertUser( - idpId: number, - user: DirectoryUser, - linkedUserId: string | undefined, - report: DirectorySyncReport -): Promise { - const username = user.username!; - - return db.transaction(async (trx) => { - let [existing] = linkedUserId - ? await trx - .select() - .from(users) - .where( - and( - eq(users.userId, linkedUserId), - eq(users.idpId, idpId) - ) - ) - : []; - if (!existing) { - [existing] = await trx - .select() - .from(users) - .where( - and(eq(users.username, username), eq(users.idpId, idpId)) - ); - } - - let userId: string; - if (existing) { - userId = existing.userId; - await trx - .update(users) - .set({ username, email: user.email, name: user.name }) - .where(eq(users.userId, userId)); - report.updated++; - } else { - userId = generateId(15); - await trx.insert(users).values({ - userId, - username, - email: user.email, - name: user.name, - type: UserType.OIDC, - idpId, - emailVerified: true, - dateCreated: new Date().toISOString() - }); - report.created++; - } - - if (linkedUserId !== userId) { - await trx - .insert(idpSyncUser) - .values({ idpId, externalId: user.externalId, userId }) - .onConflictDoUpdate({ - target: [idpSyncUser.idpId, idpSyncUser.externalId], - set: { userId } - }); - } - return userId; - }); -} diff --git a/server/private/lib/directorySync/types.ts b/server/private/lib/directorySync/types.ts index 627094ef51..455c557940 100644 --- a/server/private/lib/directorySync/types.ts +++ b/server/private/lib/directorySync/types.ts @@ -1,3 +1,16 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + // Provider-neutral shapes used by the directory sync engine (runSync.ts). // Each provider (Azure, Google, ...) turns its API into these. @@ -43,10 +56,10 @@ export type DirectoryFetchContext = { /** Opaque provider state from the previous successful run. Null = full fetch. */ cursor: string | null; /** Called once per HTTP request, for the run report. */ - onRequest: () => void; + // onRequest: () => void; }; -export interface DirectoryProvider { +export interface DirectorySyncProvider { fetch(ctx: DirectoryFetchContext): Promise; } From ec350fb8664d1645607ae8ee81a442321730da23 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Wed, 14 Oct 2026 10:31:29 +0200 Subject: [PATCH 22/50] =?UTF-8?q?=F0=9F=9A=A7=20directory=20sync=20cleaner?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/private/lib/directorySync/azure.ts | 29 +++++++++++------------ 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/server/private/lib/directorySync/azure.ts b/server/private/lib/directorySync/azure.ts index ca5529f9bc..87dbe4290f 100644 --- a/server/private/lib/directorySync/azure.ts +++ b/server/private/lib/directorySync/azure.ts @@ -61,6 +61,7 @@ async function ensureValidIdp(idpId: number) { return idpRes; } +type IdpDbResponse = Awaited>; export class AzureDirectorySyncProvider implements DirectorySyncProvider { #TOKEN_CACHE_KEY_PREFIX = "azureDirectorySyncToken"; #microsoftGraphBaseUrl = "https://graph.microsoft.com/v1.0"; @@ -77,7 +78,11 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { ); if (directorySyncMapping.mode === "fixedRoles") { - // .. + const users = await this.#fetchAllUsers(token, idpRes); + + logger.info({ + users + }); } // if fixed roles @@ -86,10 +91,6 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { // 2. get list of all groups // 3. get group members ? - console.dir({ - token - }); - return { kind: "full", users: [], @@ -97,9 +98,7 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { }; } - async #getToken( - idpRes: Awaited> - ): Promise { + async #getToken(idpRes: IdpDbResponse): Promise { const cacheKey = `${this.#TOKEN_CACHE_KEY_PREFIX}:${idpRes.idp.idpId}`; const cached = await cache.get(cacheKey); @@ -170,7 +169,7 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { } } - async #fetchAllUsers(token: string) { + async #fetchAllUsers(token: string, idpRes: IdpDbResponse) { //... const userSelectQuery = "id,displayName,mail,userPrincipalName,accountEnabled"; @@ -199,17 +198,17 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { const userBodySchema = z.object({ value: z.array( z.object({ - mail: z.email(), - userPrincipalName: z.string(), - id: z.uuidv4() + id: z.uuid(), + displayName: z.string(), + mail: z.email().optional(), + accountEnabled: z.boolean(), + userPrincipalName: z.string() }) ) }); try { - const tokenResponse = userBodySchema.parse(body); - - return tokenResponse; + return userBodySchema.parse(body); } catch (e) { throw new Error( `Directory sync: `, // todo: improve From 7b4c93c99bb86c4c1ce54bbdb449cf2ad69eea68 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Thu, 15 Oct 2026 17:55:52 +0200 Subject: [PATCH 23/50] =?UTF-8?q?=F0=9F=9A=A7=20fetch=20all=20azure=20user?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/private/lib/directorySync/azure.ts | 378 +++++++++++++----- .../private/lib/directorySync/syncWorkflow.ts | 0 2 files changed, 288 insertions(+), 90 deletions(-) create mode 100644 server/private/lib/directorySync/syncWorkflow.ts diff --git a/server/private/lib/directorySync/azure.ts b/server/private/lib/directorySync/azure.ts index 87dbe4290f..7333202861 100644 --- a/server/private/lib/directorySync/azure.ts +++ b/server/private/lib/directorySync/azure.ts @@ -16,7 +16,8 @@ import { eq, and } from "drizzle-orm"; import type { DirectoryFetchContext, DirectoryFetchResult, - DirectorySyncProvider + DirectorySyncProvider, + DirectoryUser } from "./types"; import { decrypt } from "@server/lib/crypto"; import config from "@server/lib/config"; @@ -26,40 +27,61 @@ import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import { parseDirectorySyncRoleMapping } from "@app/lib/idpDirectorySyncRoleMapping"; -async function ensureValidIdp(idpId: number) { - const [idpRes] = await db - .select() - .from(idp) - .where(eq(idp.idpId, idpId)) - .innerJoin(idpOidcConfig, eq(idpOidcConfig.idpId, idp.idpId)) - .innerJoin(idpOrg, and(eq(idpOrg.idpId, idp.idpId))) - .limit(1); +const removedSchema = z.object({ reason: z.string() }); - if (!idpRes) { - throw new Error( - `Directory sync: IdP ${idpId} not found, or it has no OIDC config or org mapping` - ); - } +const userBodySchema = z.object({ + "@odata.nextLink": z.url().optional(), + "@odata.deltaLink": z.url().optional(), + value: z.array( + z.union([ + z.object({ + id: z.uuid(), + displayName: z.string().optional(), + mail: z.email().nullable().optional(), + accountEnabled: z.boolean().optional(), + userPrincipalName: z.string().optional(), + otherMails: z.array(z.email()).optional(), + "@removed": removedSchema.optional() + }) + ]) + ) +}); - if (idpRes.idpOidcConfig.variant !== "azure") { - throw new Error( - `Directory sync: IdP ${idpId} has variant "${idpRes.idpOidcConfig.variant}", expected "azure"` - ); - } +const groupBodySchema = z.object({ + "@odata.nextLink": z.url().optional(), + "@odata.deltaLink": z.url().optional(), + value: z.array( + z.union([ + z.object({ + id: z.uuid(), + "@removed": removedSchema + }), + z.object({ + id: z.uuid(), + displayName: z.string(), + // omitted when the group has no member (changes) + "members@delta": z + .array( + z.object({ + id: z.uuid(), + "@removed": removedSchema.optional() + }) + ) + .default([]) + }) + ]) + ) +}); - if (!idpRes.idp.directorySyncEnabled) { - throw new Error( - `Directory sync: directory sync is not enabled on IdP ${idpId}` - ); - } +const tokenBodySchema = z.object({ + expires_in: z.int(), + access_token: z.string() +}); - if (idpRes.idp.autoProvision) { - throw new Error( - `Directory sync: IdP ${idpId} has both directory sync and auto-provisioning enabled; only one can be active` - ); - } - return idpRes; -} +type MicrosoftGraphUser = z.output["value"][number]; +type MicrosoftGraphGroup = z.output["value"][number]; + +type ExistingGraphUser = Required>; type IdpDbResponse = Awaited>; export class AzureDirectorySyncProvider implements DirectorySyncProvider { @@ -67,33 +89,81 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { #microsoftGraphBaseUrl = "https://graph.microsoft.com/v1.0"; async fetch(ctx: DirectoryFetchContext): Promise { + let directoryUsers: Record = {}; + const removedExternalIds: string[] = []; // first type of fetch: `full` // 0. fetch idp and check that it is valid const idpRes = await ensureValidIdp(ctx.idpId); // 1. get access token (and cache) const token = await this.#getToken(idpRes); - const directorySyncMapping = parseDirectorySyncRoleMapping( - idpRes.idpOrg.directoryRoleMapping - ); - if (directorySyncMapping.mode === "fixedRoles") { - const users = await this.#fetchAllUsers(token, idpRes); + // 1. get users & groups + const [userResponse, groupResponse] = await Promise.all([ + this.#fetchAllUsers(token, idpRes.idp.idpId), + this.#fetchAllGroups(token, idpRes.idp.idpId) + ]); - logger.info({ - users - }); + for (let user of userResponse.users) { + if ("@removed" in user) { + removedExternalIds.push(user.id); + continue; + } + const existingUser = user as unknown as ExistingGraphUser; + + let email = existingUser.mail ?? existingUser.otherMails[0]; + + if (!email) { + // uPn is formatted with `#EXT#` at the end, we remove that + const extSuffixRegex = /(\#EXT\#.*$)/; + const uPnWithoutEXT = existingUser.userPrincipalName.replaceAll( + extSuffixRegex, + "" + ); + + // split uPn with `_` (Entra `_` replaces `@`) + let splitted = uPnWithoutEXT.split("_"); + if (splitted.length === 2) { + email = splitted.join("@"); + } + } + + if (!email) { + logger.warn( + `Directory sync: skipping Entra user ${existingUser.userPrincipalName} (id: ${existingUser.id}) for IdP ${idpRes.idp.idpId}: no email found in \`mail\`, \`otherMails\` or \`userPrincipalName\`` + ); + } else { + directoryUsers[existingUser.id] = { + email, + externalId: existingUser.id, + enabled: existingUser.accountEnabled, + name: existingUser.displayName, + username: email, + groups: [] + }; + } } - // if fixed roles - // 2. get all users - // if create or map groups - // 2. get list of all groups - // 3. get group members ? + for (const group of groupResponse.groups) { + if ("@removed" in group) { + continue; + } + + for (const member of group["members@delta"]) { + const user = directoryUsers[member.id]; + if (user) { + user.groups.push({ + displayName: group.displayName, + id: group.id + }); + } + } + } return { - kind: "full", - users: [], + kind: "incremental", + users: Object.entries(directoryUsers).map(([_, user]) => user), + removedExternalIds, nextCursor: null }; } @@ -143,11 +213,6 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { ); } - const tokenBodySchema = z.object({ - expires_in: z.int(), - access_token: z.string() - }); - try { const tokenResponse = tokenBodySchema.parse(body); await cache.set( @@ -169,58 +234,191 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { } } - async #fetchAllUsers(token: string, idpRes: IdpDbResponse) { - //... - const userSelectQuery = - "id,displayName,mail,userPrincipalName,accountEnabled"; + async #fetchAllUsers(token: string, idpId: number) { + // we need to follow `@odata.nextLink` as-is until it is absent, + // the last page returns `@odata.deltaLink` instead + // (microsoft graph returns a max of 100 users per page) + const sp = new URLSearchParams({ + $select: + "id,displayName,mail,userPrincipalName,accountEnabled,otherMails" + }); + let nextUrl: string | undefined = + `${this.#microsoftGraphBaseUrl}/users/delta?${sp.toString()}`; + let deltaLink: string | null = null; + const allUsers: MicrosoftGraphUser[] = []; - const res = await fetch( - `${this.#microsoftGraphBaseUrl}/users?$select=${userSelectQuery}`, - { + while (nextUrl) { + logger.info(`Fetching URL: ${nextUrl}`); + const res = await fetch(nextUrl, { headers: { Authorization: `Bearer ${token}` } + }); + + const body: any = await res.json().catch(() => ({})); + if (!res.ok) { + const reason = + body.error?.message ?? + body.error?.code ?? + res.statusText ?? + "no error details in response"; + throw new Error( + `Directory sync: Entra users request for IdP ${idpId} failed (HTTP ${res.status}): ${reason}` + ); } - ); - const body: any = await res.json().catch(() => ({})); - if (!res.ok) { - const reason = - body.error_description ?? - body.error ?? - res.statusText ?? - "no error details in response"; - throw new Error( - `Directory sync: Entra fetch user failed (HTTP ${res.status}): ${reason}` // todo: improve - ); + try { + const response = userBodySchema.parse(body); + + allUsers.push(...response.value); + nextUrl = response["@odata.nextLink"]; + deltaLink = response["@odata.deltaLink"] ?? null; + } catch (e) { + console.dir( + { + body + }, + { depth: null } + ); + throw new Error( + `Directory sync: unexpected Entra users response for IdP ${idpId}: ${fromError(e).toString()}`, + { cause: e } + ); + } } - const userBodySchema = z.object({ - value: z.array( - z.object({ - id: z.uuid(), - displayName: z.string(), - mail: z.email().optional(), - accountEnabled: z.boolean(), - userPrincipalName: z.string() - }) - ) + logger.info(`Fetched ${allUsers.length} users from Microsoft Graph`); + return { + deltaLink, + users: allUsers + }; + } + + async #fetchAllGroups(token: string, idpId: number) { + // we need to follow `@odata.nextLink` as-is until it is absent, + // the last page returns `@odata.deltaLink` instead + // (microsoft graph returns a max of 100 groups per page) + const sp = new URLSearchParams({ + $select: "id,displayName,members" }); + let nextUrl: string | undefined = + `${this.#microsoftGraphBaseUrl}/groups/delta?${sp.toString()}`; + let deltaLink: string | null = null; + const allGroups: MicrosoftGraphGroup[] = []; - try { - return userBodySchema.parse(body); - } catch (e) { - throw new Error( - `Directory sync: `, // todo: improve - { cause: e } - ); + while (nextUrl) { + logger.info(`Fetching URL: ${nextUrl}`); + + const res = await fetch(nextUrl, { + headers: { + Authorization: `Bearer ${token}` + } + }); + + const body: any = await res.json().catch(() => ({})); + if (!res.ok) { + const reason = + body.error?.message ?? + body.error?.code ?? + res.statusText ?? + "no error details in response"; + throw new Error( + `Directory sync: Entra groups request for IdP ${idpId} failed (HTTP ${res.status}): ${reason}` + ); + } + + try { + const response = groupBodySchema.parse(body); + + allGroups.push(...response.value); + nextUrl = response["@odata.nextLink"]; + deltaLink = response["@odata.deltaLink"] ?? null; + } catch (e) { + console.dir( + { + body + }, + { depth: null } + ); + throw new Error( + `Directory sync: unexpected Entra groups response for IdP ${idpId}: ${fromError(e).toString()}`, + { cause: e } + ); + } } + + logger.info(`Fetched ${allGroups.length} groups from Microsoft Graph`); + return { + groups: allGroups, + deltaLink + }; } + // async #fetchGroupMembers(token: string, groupId: string) { + // const userSelect = "id,displayName"; + + // const res = await fetch( + // `${this.#microsoftGraphBaseUrl}/groups/${groupId}/transitiveMembers?$select=${userSelect}`, + // { + // headers: { + // Authorization: `Bearer ${token}` + // } + // } + // ); + + // const body: any = await res.json().catch(() => ({})); + // if (!res.ok) { + // const reason = + // body.error_description ?? + // body.error ?? + // res.statusText ?? + // "no error details in response"; + // throw new Error( + // `Directory sync: Entra fetch user failed (HTTP ${res.status}): ${reason}` // todo: improve + // ); + // } - async #fetchAllGroups(token: string) { - //... + // try { + // return groupBodySchema.parse(body).value; + // } catch (e) { + // throw new Error( + // `Directory sync: `, // todo: improve + // { cause: e } + // ); + // } + // } +} + +async function ensureValidIdp(idpId: number) { + const [idpRes] = await db + .select() + .from(idp) + .where(eq(idp.idpId, idpId)) + .innerJoin(idpOidcConfig, eq(idpOidcConfig.idpId, idp.idpId)) + .innerJoin(idpOrg, and(eq(idpOrg.idpId, idp.idpId))) + .limit(1); + + if (!idpRes) { + throw new Error( + `Directory sync: IdP ${idpId} not found, or it has no OIDC config or org mapping` + ); + } + + if (idpRes.idpOidcConfig.variant !== "azure") { + throw new Error( + `Directory sync: IdP ${idpId} has variant "${idpRes.idpOidcConfig.variant}", expected "azure"` + ); } - async #fetchGroupMembers(token: string) { - //... + + if (!idpRes.idp.directorySyncEnabled) { + throw new Error( + `Directory sync: directory sync is not enabled on IdP ${idpId}` + ); } + + if (idpRes.idp.autoProvision) { + throw new Error( + `Directory sync: IdP ${idpId} has both directory sync and auto-provisioning enabled; only one can be active` + ); + } + return idpRes; } diff --git a/server/private/lib/directorySync/syncWorkflow.ts b/server/private/lib/directorySync/syncWorkflow.ts new file mode 100644 index 0000000000..e69de29bb2 From bb06041182e8ede1e56abe85e382d4aa0152dd65 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Thu, 15 Oct 2026 18:40:28 +0200 Subject: [PATCH 24/50] =?UTF-8?q?=F0=9F=9A=A7=20sync=20working=20=3F=3F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/private/lib/directorySync/azure.ts | 240 ++++++++++-------- server/private/lib/directorySync/run.ts | 45 ++-- .../private/lib/directorySync/syncWorkflow.ts | 68 +++++ server/private/lib/directorySync/types.ts | 22 +- 4 files changed, 231 insertions(+), 144 deletions(-) diff --git a/server/private/lib/directorySync/azure.ts b/server/private/lib/directorySync/azure.ts index 7333202861..3d3910f7b8 100644 --- a/server/private/lib/directorySync/azure.ts +++ b/server/private/lib/directorySync/azure.ts @@ -25,51 +25,43 @@ import cache from "@server/lib/cache"; import z from "zod"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; -import { parseDirectorySyncRoleMapping } from "@app/lib/idpDirectorySyncRoleMapping"; - -const removedSchema = z.object({ reason: z.string() }); const userBodySchema = z.object({ "@odata.nextLink": z.url().optional(), - "@odata.deltaLink": z.url().optional(), value: z.array( - z.union([ - z.object({ - id: z.uuid(), - displayName: z.string().optional(), - mail: z.email().nullable().optional(), - accountEnabled: z.boolean().optional(), - userPrincipalName: z.string().optional(), - otherMails: z.array(z.email()).optional(), - "@removed": removedSchema.optional() - }) - ]) + z.object({ + id: z.uuid(), + mail: z.email().nullish(), + displayName: z.string().nullish(), + accountEnabled: z.boolean().nullish(), + userPrincipalName: z.string().nullish(), + otherMails: z.array(z.email()).nullish() + }) ) }); +// we only do initial delta rounds for groups (no stored `deltaLink`), which return the +// current state of the directory and never contain `@removed` entries const groupBodySchema = z.object({ "@odata.nextLink": z.url().optional(), "@odata.deltaLink": z.url().optional(), value: z.array( - z.union([ - z.object({ - id: z.uuid(), - "@removed": removedSchema - }), - z.object({ - id: z.uuid(), - displayName: z.string(), - // omitted when the group has no member (changes) - "members@delta": z - .array( - z.object({ - id: z.uuid(), - "@removed": removedSchema.optional() - }) - ) - .default([]) - }) - ]) + z.object({ + id: z.uuid(), + // large groups can be returned several times with their members split + // across the entries, the repeated entries may not have `displayName` + displayName: z.string().nullish(), + // omitted when the group has no members + "members@delta": z + .array( + z.object({ + // user, group, device, servicePrincipal, orgContact... + "@odata.type": z.string(), + id: z.uuid() + }) + ) + .default([]) + }) ) }); @@ -81,7 +73,13 @@ const tokenBodySchema = z.object({ type MicrosoftGraphUser = z.output["value"][number]; type MicrosoftGraphGroup = z.output["value"][number]; -type ExistingGraphUser = Required>; +type GraphGroupNode = { + displayName: string | null; + /** Groups that have this group as a direct member. */ + parentGroupIds: Set; + /** Users that are direct members of this group. */ + userIds: Set; +}; type IdpDbResponse = Awaited>; export class AzureDirectorySyncProvider implements DirectorySyncProvider { @@ -89,34 +87,26 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { #microsoftGraphBaseUrl = "https://graph.microsoft.com/v1.0"; async fetch(ctx: DirectoryFetchContext): Promise { - let directoryUsers: Record = {}; - const removedExternalIds: string[] = []; - // first type of fetch: `full` + const directoryUsers: Record = {}; // 0. fetch idp and check that it is valid const idpRes = await ensureValidIdp(ctx.idpId); // 1. get access token (and cache) const token = await this.#getToken(idpRes); - // 1. get users & groups + // 2. get users & groups const [userResponse, groupResponse] = await Promise.all([ this.#fetchAllUsers(token, idpRes.idp.idpId), this.#fetchAllGroups(token, idpRes.idp.idpId) ]); - for (let user of userResponse.users) { - if ("@removed" in user) { - removedExternalIds.push(user.id); - continue; - } - const existingUser = user as unknown as ExistingGraphUser; - - let email = existingUser.mail ?? existingUser.otherMails[0]; + for (const user of userResponse.users) { + let email = user.mail ?? user.otherMails?.[0]; - if (!email) { + if (!email && user.userPrincipalName) { // uPn is formatted with `#EXT#` at the end, we remove that const extSuffixRegex = /(\#EXT\#.*$)/; - const uPnWithoutEXT = existingUser.userPrincipalName.replaceAll( + const uPnWithoutEXT = user.userPrincipalName.replaceAll( extSuffixRegex, "" ); @@ -130,40 +120,111 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { if (!email) { logger.warn( - `Directory sync: skipping Entra user ${existingUser.userPrincipalName} (id: ${existingUser.id}) for IdP ${idpRes.idp.idpId}: no email found in \`mail\`, \`otherMails\` or \`userPrincipalName\`` + `Directory sync: skipping Entra user ${user.userPrincipalName} (id: ${user.id}) for IdP ${idpRes.idp.idpId}: no email found in \`mail\`, \`otherMails\` or \`userPrincipalName\`` ); } else { - directoryUsers[existingUser.id] = { + directoryUsers[user.id] = { email, - externalId: existingUser.id, - enabled: existingUser.accountEnabled, - name: existingUser.displayName, + externalId: user.id, + enabled: user.accountEnabled ?? true, + name: user.displayName ?? null, username: email, groups: [] }; } } + // 3. build the group tree, merging the repeated entries of large groups + const groupNodes = new Map(); + const getGroupNode = (groupId: string) => { + let node = groupNodes.get(groupId); + if (!node) { + node = { + displayName: null, + parentGroupIds: new Set(), + userIds: new Set() + }; + groupNodes.set(groupId, node); + } + return node; + }; + for (const group of groupResponse.groups) { - if ("@removed" in group) { - continue; + const node = getGroupNode(group.id); + if (group.displayName) { + node.displayName = group.displayName; } for (const member of group["members@delta"]) { - const user = directoryUsers[member.id]; - if (user) { - user.groups.push({ - displayName: group.displayName, - id: group.id - }); + if (member["@odata.type"] === "#microsoft.graph.user") { + node.userIds.add(member.id); + } else if (member["@odata.type"] === "#microsoft.graph.group") { + getGroupNode(member.id).parentGroupIds.add(group.id); + } + } + } + + // 4. expand nested memberships: a user is a member of its direct groups + // and of every group above them + const ancestorsCache = new Map>(); + const getGroupWithAncestors = ( + groupId: string, + visiting = new Set() + ): Set => { + const cached = ancestorsCache.get(groupId); + if (cached) { + return cached; + } + + const result = new Set([groupId]); + visiting.add(groupId); + for (const parentId of groupNodes.get(groupId)?.parentGroupIds ?? + []) { + // guard against cycles + if (visiting.has(parentId)) { + continue; + } + for (const ancestorId of getGroupWithAncestors( + parentId, + visiting + )) { + result.add(ancestorId); + } + } + visiting.delete(groupId); + + ancestorsCache.set(groupId, result); + return result; + }; + + const userGroupIds = new Map>(); + for (const [groupId, node] of groupNodes) { + for (const userId of node.userIds) { + if (!directoryUsers[userId]) { + continue; + } + + let groupIds = userGroupIds.get(userId); + if (!groupIds) { + groupIds = new Set(); + userGroupIds.set(userId, groupIds); + } + for (const id of getGroupWithAncestors(groupId)) { + groupIds.add(id); } } } + for (const [userId, groupIds] of userGroupIds) { + directoryUsers[userId].groups = [...groupIds].map((id) => ({ + id, + displayName: groupNodes.get(id)?.displayName ?? null + })); + } + return { - kind: "incremental", - users: Object.entries(directoryUsers).map(([_, user]) => user), - removedExternalIds, + kind: "full", + users: Object.values(directoryUsers), nextCursor: null }; } @@ -235,16 +296,15 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { } async #fetchAllUsers(token: string, idpId: number) { - // we need to follow `@odata.nextLink` as-is until it is absent, - // the last page returns `@odata.deltaLink` instead - // (microsoft graph returns a max of 100 users per page) + // we need to follow `@odata.nextLink` as-is until it is absent + // (microsoft graph returns a max of 999 users per page) const sp = new URLSearchParams({ $select: - "id,displayName,mail,userPrincipalName,accountEnabled,otherMails" + "id,displayName,mail,userPrincipalName,accountEnabled,otherMails", + $top: "999" }); let nextUrl: string | undefined = - `${this.#microsoftGraphBaseUrl}/users/delta?${sp.toString()}`; - let deltaLink: string | null = null; + `${this.#microsoftGraphBaseUrl}/users?${sp.toString()}`; const allUsers: MicrosoftGraphUser[] = []; while (nextUrl) { @@ -272,7 +332,6 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { allUsers.push(...response.value); nextUrl = response["@odata.nextLink"]; - deltaLink = response["@odata.deltaLink"] ?? null; } catch (e) { console.dir( { @@ -289,7 +348,6 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { logger.info(`Fetched ${allUsers.length} users from Microsoft Graph`); return { - deltaLink, users: allUsers }; } @@ -298,6 +356,8 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { // we need to follow `@odata.nextLink` as-is until it is absent, // the last page returns `@odata.deltaLink` instead // (microsoft graph returns a max of 100 groups per page) + const MAX_PAGES = 3; + let pagesFound = 0; const sp = new URLSearchParams({ $select: "id,displayName,members" }); @@ -327,6 +387,7 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { ); } + pagesFound += 1; try { const response = groupBodySchema.parse(body); @@ -353,39 +414,6 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { deltaLink }; } - // async #fetchGroupMembers(token: string, groupId: string) { - // const userSelect = "id,displayName"; - - // const res = await fetch( - // `${this.#microsoftGraphBaseUrl}/groups/${groupId}/transitiveMembers?$select=${userSelect}`, - // { - // headers: { - // Authorization: `Bearer ${token}` - // } - // } - // ); - - // const body: any = await res.json().catch(() => ({})); - // if (!res.ok) { - // const reason = - // body.error_description ?? - // body.error ?? - // res.statusText ?? - // "no error details in response"; - // throw new Error( - // `Directory sync: Entra fetch user failed (HTTP ${res.status}): ${reason}` // todo: improve - // ); - // } - - // try { - // return groupBodySchema.parse(body).value; - // } catch (e) { - // throw new Error( - // `Directory sync: `, // todo: improve - // { cause: e } - // ); - // } - // } } async function ensureValidIdp(idpId: number) { diff --git a/server/private/lib/directorySync/run.ts b/server/private/lib/directorySync/run.ts index 9892e2bd99..7d904b36ae 100644 --- a/server/private/lib/directorySync/run.ts +++ b/server/private/lib/directorySync/run.ts @@ -1,29 +1,32 @@ // // Usage: npx tsx server/private/lib/directorySync/run.ts // // Runs one directory sync for the IdP using its saved directory role mapping. -import { AzureDirectorySyncProvider } from "./azure"; +// import { AzureDirectorySyncProvider } from "./azure"; +import { runIdpDirectorySync } from "./syncWorkflow"; -const idpId = Number(process.argv[2]); -if (!idpId) { - console.error("Usage: run.ts "); - process.exit(1); -} +await runIdpDirectorySync(); -const provider = new AzureDirectorySyncProvider(); +// const idpId = Number(process.argv[2]); +// if (!idpId) { +// console.error("Usage: run.ts "); +// process.exit(1); +// } -provider - .fetch({ - idpId: idpId, - identifierPath: "email", - cursor: null - }) - .then((report) => { - console.dir({ report }, { depth: null }); - process.exit(0); - }) - .catch((err) => { - console.error(err); - process.exit(1); - }); +// const provider = new AzureDirectorySyncProvider(); + +// provider +// .fetch({ +// idpId: idpId, +// identifierPath: "email", +// cursor: null +// }) +// .then((report) => { +// console.dir({ report }, { depth: null }); +// process.exit(0); +// }) +// .catch((err) => { +// console.error(err); +// process.exit(1); +// }); // runDirectorySync(idpId) // .then((report) => { diff --git a/server/private/lib/directorySync/syncWorkflow.ts b/server/private/lib/directorySync/syncWorkflow.ts index e69de29bb2..7724c924df 100644 --- a/server/private/lib/directorySync/syncWorkflow.ts +++ b/server/private/lib/directorySync/syncWorkflow.ts @@ -0,0 +1,68 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ +import { db, idp, idpOidcConfig, idpOrg } from "@server/db"; + +import { durationToMs } from "@app/lib/durationToMs"; +import { and, eq, inArray } from "drizzle-orm"; +import type { DirectorySyncProvider, DirectoryFetchContext } from "./types"; +import { AzureDirectorySyncProvider } from "./azure"; + +// Track if the offline checker interval is running +let directorySyncInterval: NodeJS.Timeout | null = null; +const DIRECTORY_SYNC_RUN_INTERVAL = durationToMs(5, "minutes"); + +export async function runIdpDirectorySync() { + // to sync idps, we need + const idpsToSync = await db + .select() + .from(idp) + .innerJoin(idpOidcConfig, eq(idpOidcConfig.idpId, idp.idpId)) + .innerJoin(idpOrg, and(eq(idpOrg.idpId, idp.idpId))) + .where( + and( + inArray(idpOidcConfig.variant, ["azure", "google"]), + eq(idp.directorySyncEnabled, true), + eq(idp.autoProvision, false) + ) + ); + + const azureSyncProvider = new AzureDirectorySyncProvider(); + const fetchPromises = idpsToSync + .map(({ idp, idpOidcConfig }) => { + if (idpOidcConfig.variant === "azure") { + return syncIdp(azureSyncProvider, { + idpId: idp.idpId, + identifierPath: idpOidcConfig.identifierPath, + cursor: null + }); + } + return null; + }) + .filter((p) => p != null); + + await Promise.allSettled(fetchPromises); +} + +async function syncIdp( + provider: DirectorySyncProvider, + ctx: DirectoryFetchContext +) { + const result = await provider.fetch(ctx); + + // ... + console.dir({ + result + }); + + return result; +} diff --git a/server/private/lib/directorySync/types.ts b/server/private/lib/directorySync/types.ts index 455c557940..01aa901b93 100644 --- a/server/private/lib/directorySync/types.ts +++ b/server/private/lib/directorySync/types.ts @@ -33,21 +33,11 @@ export type DirectoryUser = { groups: DirectoryGroup[]; }; -export type DirectoryFetchResult = - | { - /** Every user in the directory. Users sync knows about that are missing were deleted. */ - kind: "full"; - users: DirectoryUser[]; - nextCursor: string | null; - } - | { - /** Only users that changed since the cursor, with their full current state. */ - kind: "incremental"; - users: DirectoryUser[]; - /** External IDs of users deleted from the directory. */ - removedExternalIds: string[]; - nextCursor: string | null; - }; +export type DirectoryFetchResult = { + kind: "full"; + users: DirectoryUser[]; + nextCursor: string | null; +}; export type DirectoryFetchContext = { idpId: number; @@ -55,8 +45,6 @@ export type DirectoryFetchContext = { identifierPath: string; /** Opaque provider state from the previous successful run. Null = full fetch. */ cursor: string | null; - /** Called once per HTTP request, for the run report. */ - // onRequest: () => void; }; export interface DirectorySyncProvider { From c518326e7fe253b3c160faf10ab04237514ce010 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Thu, 15 Oct 2026 21:26:51 +0200 Subject: [PATCH 25/50] =?UTF-8?q?=F0=9F=9A=A7=20working=20on=20inserting/u?= =?UTF-8?q?pdating=20users=20from=20synched=20external=20idps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/private/lib/directorySync/azure.ts | 12 +++- server/private/lib/directorySync/run.ts | 1 + .../private/lib/directorySync/syncWorkflow.ts | 61 ++++++++++++++++--- server/private/lib/directorySync/types.ts | 11 ++-- 4 files changed, 71 insertions(+), 14 deletions(-) diff --git a/server/private/lib/directorySync/azure.ts b/server/private/lib/directorySync/azure.ts index 3d3910f7b8..1ed90eaa69 100644 --- a/server/private/lib/directorySync/azure.ts +++ b/server/private/lib/directorySync/azure.ts @@ -223,9 +223,8 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { } return { - kind: "full", - users: Object.values(directoryUsers), - nextCursor: null + users: Object.values(directoryUsers) + // nextCursor: null }; } @@ -393,6 +392,13 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { allGroups.push(...response.value); nextUrl = response["@odata.nextLink"]; + + // Sometimes, microsoft graph will send empty pages with a `@odata.nextLink` even + // though they are the last page, in that case, we stop the loop because this shouldn't happen + // DO NOT ASK ME WHY THIS IS THE CASE, IDK ¯\_(ツ)_/¯ + if (response.value.length === 0) { + nextUrl = undefined; + } deltaLink = response["@odata.deltaLink"] ?? null; } catch (e) { console.dir( diff --git a/server/private/lib/directorySync/run.ts b/server/private/lib/directorySync/run.ts index 7d904b36ae..987649bb6c 100644 --- a/server/private/lib/directorySync/run.ts +++ b/server/private/lib/directorySync/run.ts @@ -4,6 +4,7 @@ import { runIdpDirectorySync } from "./syncWorkflow"; await runIdpDirectorySync(); +process.exit(0); // const idpId = Number(process.argv[2]); // if (!idpId) { diff --git a/server/private/lib/directorySync/syncWorkflow.ts b/server/private/lib/directorySync/syncWorkflow.ts index 7724c924df..441fe693be 100644 --- a/server/private/lib/directorySync/syncWorkflow.ts +++ b/server/private/lib/directorySync/syncWorkflow.ts @@ -10,12 +10,26 @@ * * This file is not licensed under the AGPLv3. */ -import { db, idp, idpOidcConfig, idpOrg } from "@server/db"; +import { + db, + idp, + idpOidcConfig, + idpOrg, + idpSyncUser, + users, + type User +} from "@server/db"; import { durationToMs } from "@app/lib/durationToMs"; -import { and, eq, inArray } from "drizzle-orm"; +import { and, eq, inArray, type InferInsertModel } from "drizzle-orm"; import type { DirectorySyncProvider, DirectoryFetchContext } from "./types"; import { AzureDirectorySyncProvider } from "./azure"; +import { parseDirectorySyncRoleMapping } from "@app/lib/idpDirectorySyncRoleMapping"; +import { generateId } from "@cli/commands/setAdminCredentials"; +import { UserType } from "@server/types/UserTypes"; + +type InsertUser = InferInsertModel; +type UpdateUser = Omit; // Track if the offline checker interval is running let directorySyncInterval: NodeJS.Timeout | null = null; @@ -38,12 +52,15 @@ export async function runIdpDirectorySync() { const azureSyncProvider = new AzureDirectorySyncProvider(); const fetchPromises = idpsToSync - .map(({ idp, idpOidcConfig }) => { + .map(({ idp, idpOidcConfig, idpOrg }) => { if (idpOidcConfig.variant === "azure") { return syncIdp(azureSyncProvider, { idpId: idp.idpId, + orgId: idpOrg.orgId, identifierPath: idpOidcConfig.identifierPath, - cursor: null + roleMapping: parseDirectorySyncRoleMapping( + idpOrg.directoryRoleMapping + ) }); } return null; @@ -59,10 +76,40 @@ async function syncIdp( ) { const result = await provider.fetch(ctx); - // ... - console.dir({ - result + const existingUsers = await db + .select() + .from(users) + .where(eq(users.idpId, ctx.idpId)); + + const usersToInsert: InsertUser[] = []; + + for (const user of result.users) { + const userId = generateId(15); + usersToInsert.push({ + userId, + username: user.username, + email: user.email, + name: user.name, + type: UserType.OIDC, + idpId: ctx.idpId, + emailVerified: true, // OIDC users are always verified + dateCreated: new Date().toISOString() + }); + } + + // 1st case: the users don't exist + await db.transaction(async (trx) => { + trx.insert(users).values(usersToInsert); }); + // ... + // console.dir( + // { + // result + // }, + // { depth: null } + // ); + // await db.select().from(idp) + return result; } diff --git a/server/private/lib/directorySync/types.ts b/server/private/lib/directorySync/types.ts index 01aa901b93..30b9b0b0de 100644 --- a/server/private/lib/directorySync/types.ts +++ b/server/private/lib/directorySync/types.ts @@ -11,6 +11,8 @@ * This file is not licensed under the AGPLv3. */ +import type { DirectorySyncRoleMappingConfig } from "@app/lib/idpDirectorySyncRoleMapping"; + // Provider-neutral shapes used by the directory sync engine (runSync.ts). // Each provider (Azure, Google, ...) turns its API into these. @@ -24,7 +26,7 @@ export type DirectoryUser = { /** Stable provider user ID (Entra object ID, Google directory ID). */ externalId: string; /** Value that login computes as `users.username` (already resolved from `identifierPath`). */ - username: string | null; + username: string; email: string | null; name: string | null; /** False when the account is disabled or suspended in the directory. */ @@ -34,17 +36,18 @@ export type DirectoryUser = { }; export type DirectoryFetchResult = { - kind: "full"; users: DirectoryUser[]; - nextCursor: string | null; + // nextCursor: string | null; }; export type DirectoryFetchContext = { idpId: number; + orgId: string; + roleMapping: DirectorySyncRoleMappingConfig; /** `idpOidcConfig.identifierPath`, used to compute `DirectoryUser.username`. */ identifierPath: string; /** Opaque provider state from the previous successful run. Null = full fetch. */ - cursor: string | null; + // cursor: string | null; }; export interface DirectorySyncProvider { From 3e32b610e0c8c9df3ad136ba28fabb0884043bdb Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Thu, 15 Oct 2026 23:19:28 +0200 Subject: [PATCH 26/50] =?UTF-8?q?=F0=9F=9A=A7=20insert=20users?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../private/lib/directorySync/syncWorkflow.ts | 76 ++++++++++++++----- 1 file changed, 59 insertions(+), 17 deletions(-) diff --git a/server/private/lib/directorySync/syncWorkflow.ts b/server/private/lib/directorySync/syncWorkflow.ts index 441fe693be..a6f911d351 100644 --- a/server/private/lib/directorySync/syncWorkflow.ts +++ b/server/private/lib/directorySync/syncWorkflow.ts @@ -16,20 +16,40 @@ import { idpOidcConfig, idpOrg, idpSyncUser, + orgs, + roleActions, + roles, + userOrgRoles, + userOrgs, users, type User } from "@server/db"; import { durationToMs } from "@app/lib/durationToMs"; import { and, eq, inArray, type InferInsertModel } from "drizzle-orm"; -import type { DirectorySyncProvider, DirectoryFetchContext } from "./types"; +import type { + DirectorySyncProvider, + DirectoryFetchContext, + DirectoryUser +} from "./types"; import { AzureDirectorySyncProvider } from "./azure"; -import { parseDirectorySyncRoleMapping } from "@app/lib/idpDirectorySyncRoleMapping"; -import { generateId } from "@cli/commands/setAdminCredentials"; +import { + parseDirectorySyncRoleMapping, + type DirectorySyncRoleMappingConfig +} from "@app/lib/idpDirectorySyncRoleMapping"; +import { generateId } from "@server/auth/sessions/app"; import { UserType } from "@server/types/UserTypes"; +import { assignUserToOrg, removeUserFromOrg } from "@server/lib/userOrg"; +import { calculateUserClientsForOrgs } from "@server/lib/calculateUserClientsForOrgs"; +import { usageService } from "@server/lib/billing/usageService"; +import { LimitId } from "@server/lib/billing"; +import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed"; +import { tierMatrix } from "@server/lib/billing/tierMatrix"; +import { defaultRoleAllowedActions } from "@server/routers/role/createRole"; +import logger from "@server/logger"; type InsertUser = InferInsertModel; -type UpdateUser = Omit; +type UpdateUser = Pick; // Track if the offline checker interval is running let directorySyncInterval: NodeJS.Timeout | null = null; @@ -81,25 +101,47 @@ async function syncIdp( .from(users) .where(eq(users.idpId, ctx.idpId)); + const map = new Map(); + for (const u of existingUsers) { + map.set(u.username, u); + } + const usersToInsert: InsertUser[] = []; + const usersToUpdate: UpdateUser[] = []; + const deletedUserIds: string[] = []; for (const user of result.users) { - const userId = generateId(15); - usersToInsert.push({ - userId, - username: user.username, - email: user.email, - name: user.name, - type: UserType.OIDC, - idpId: ctx.idpId, - emailVerified: true, // OIDC users are always verified - dateCreated: new Date().toISOString() - }); + const existing = map.get(user.username); + if (existing) { + usersToUpdate.push({ + username: user.username, + email: user.email, + name: user.name, + userId: existing.userId + }); + } else { + const userId = generateId(15); + usersToInsert.push({ + userId, + username: user.username, + email: user.email, + name: user.name, + type: UserType.OIDC, + idpId: ctx.idpId, + emailVerified: true, // OIDC users are always verified + dateCreated: new Date().toISOString() + }); + } } - // 1st case: the users don't exist await db.transaction(async (trx) => { - trx.insert(users).values(usersToInsert); + await trx.insert(users).values(usersToInsert); + + await Promise.all( + usersToUpdate.map((u) => + trx.update(users).set(u).where(eq(users.userId, u.userId)) + ) + ); }); // ... From e411e5bd2ea5227bdd9012855933452d5af0fda7 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Fri, 16 Oct 2026 21:54:59 +0200 Subject: [PATCH 27/50] =?UTF-8?q?=F0=9F=92=A1=20Add=20comments=20for=20the?= =?UTF-8?q?=20global=20idps=20sync=20workflows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../private/lib/directorySync/syncWorkflow.ts | 59 ++++++++----------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/server/private/lib/directorySync/syncWorkflow.ts b/server/private/lib/directorySync/syncWorkflow.ts index a6f911d351..d093134c10 100644 --- a/server/private/lib/directorySync/syncWorkflow.ts +++ b/server/private/lib/directorySync/syncWorkflow.ts @@ -10,43 +10,15 @@ * * This file is not licensed under the AGPLv3. */ -import { - db, - idp, - idpOidcConfig, - idpOrg, - idpSyncUser, - orgs, - roleActions, - roles, - userOrgRoles, - userOrgs, - users, - type User -} from "@server/db"; +import { db, idp, idpOidcConfig, idpOrg, users } from "@server/db"; import { durationToMs } from "@app/lib/durationToMs"; -import { and, eq, inArray, type InferInsertModel } from "drizzle-orm"; -import type { - DirectorySyncProvider, - DirectoryFetchContext, - DirectoryUser -} from "./types"; -import { AzureDirectorySyncProvider } from "./azure"; -import { - parseDirectorySyncRoleMapping, - type DirectorySyncRoleMappingConfig -} from "@app/lib/idpDirectorySyncRoleMapping"; +import { parseDirectorySyncRoleMapping } from "@app/lib/idpDirectorySyncRoleMapping"; import { generateId } from "@server/auth/sessions/app"; import { UserType } from "@server/types/UserTypes"; -import { assignUserToOrg, removeUserFromOrg } from "@server/lib/userOrg"; -import { calculateUserClientsForOrgs } from "@server/lib/calculateUserClientsForOrgs"; -import { usageService } from "@server/lib/billing/usageService"; -import { LimitId } from "@server/lib/billing"; -import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed"; -import { tierMatrix } from "@server/lib/billing/tierMatrix"; -import { defaultRoleAllowedActions } from "@server/routers/role/createRole"; -import logger from "@server/logger"; +import { and, eq, inArray, type InferInsertModel } from "drizzle-orm"; +import { AzureDirectorySyncProvider } from "./azure"; +import type { DirectoryFetchContext, DirectorySyncProvider } from "./types"; type InsertUser = InferInsertModel; type UpdateUser = Pick; @@ -94,6 +66,27 @@ async function syncIdp( provider: DirectorySyncProvider, ctx: DirectoryFetchContext ) { + // Note + // username is unique only within IDPs or within internal ? + // we can have two users with the same username, one in an IDP and another `internal` + // Which means: a user can have two accounts w/ the same username (email in azure for ex) + // and each account can have different roles within the same Org + + // ... + // Later: we might need to change the `identifierPath` for azure/google idps to `sub` (for ex) + // which is an immutable external ID that survives username/email changes + // but first we need to check if that ID is the same returned by Entra/google workspace + // if it's not the same ID, we might kepp `idpSyncUser` otherwise, we may need to see how + // we can maybe get rid of that table (or just not need it anymore) + + // ... + // For global idps, we will rely on org policies + // we will not store a default mapping for directory sync + // if the admin wants to add the synched IDP to an org, they can create an org policy + // & set the role mapping to `automatic` (for ex) + // for global idps, if there is not an org policy for a specific org, the user is not added to that org + + // 1. Fetch const result = await provider.fetch(ctx); const existingUsers = await db From 0fbf68bcd2d05dc3e43f2c364d77f69ba457233c Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Wed, 21 Oct 2026 18:02:10 +0200 Subject: [PATCH 28/50] =?UTF-8?q?=F0=9F=9A=A7=20wip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/db/pg/schema/schema.ts | 1 + server/private/lib/directorySync/run.ts | 9 +- .../private/lib/directorySync/syncWorkflow.ts | 273 +++++++++++++++--- 3 files changed, 240 insertions(+), 43 deletions(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index c59c0bd73c..5d79460229 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -663,6 +663,7 @@ export const users = pgTable("user", { idpId: integer("idpId").references(() => idp.idpId, { onDelete: "cascade" }), + externalUserId: varchar("externalUserId"), // user's ID in the external IdP (e.g. Azure object id) passwordHash: varchar("passwordHash"), twoFactorEnabled: boolean("twoFactorEnabled").notNull().default(false), twoFactorSetupRequested: boolean("twoFactorSetupRequested").default(false), diff --git a/server/private/lib/directorySync/run.ts b/server/private/lib/directorySync/run.ts index 987649bb6c..9b978839b6 100644 --- a/server/private/lib/directorySync/run.ts +++ b/server/private/lib/directorySync/run.ts @@ -3,7 +3,14 @@ // import { AzureDirectorySyncProvider } from "./azure"; import { runIdpDirectorySync } from "./syncWorkflow"; -await runIdpDirectorySync(); +await runIdpDirectorySync().then((changes) => { + console.dir( + { + changes + }, + { depth: null } + ); +}); process.exit(0); // const idpId = Number(process.argv[2]); diff --git a/server/private/lib/directorySync/syncWorkflow.ts b/server/private/lib/directorySync/syncWorkflow.ts index d093134c10..21bf387d34 100644 --- a/server/private/lib/directorySync/syncWorkflow.ts +++ b/server/private/lib/directorySync/syncWorkflow.ts @@ -10,19 +10,58 @@ * * This file is not licensed under the AGPLv3. */ -import { db, idp, idpOidcConfig, idpOrg, users } from "@server/db"; +import { + db, + idp, + idpOidcConfig, + idpOrg, + roles, + userOrgs, + users, + type Role +} from "@server/db"; import { durationToMs } from "@app/lib/durationToMs"; -import { parseDirectorySyncRoleMapping } from "@app/lib/idpDirectorySyncRoleMapping"; +import { + parseDirectorySyncRoleMapping, + type DirectorySyncRoleMappingConfig +} from "@app/lib/idpDirectorySyncRoleMapping"; import { generateId } from "@server/auth/sessions/app"; import { UserType } from "@server/types/UserTypes"; -import { and, eq, inArray, type InferInsertModel } from "drizzle-orm"; +import { + and, + eq, + inArray, + type InferInsertModel, + type InferSelectModel +} from "drizzle-orm"; import { AzureDirectorySyncProvider } from "./azure"; -import type { DirectoryFetchContext, DirectorySyncProvider } from "./types"; +import type { + DirectoryFetchContext, + DirectorySyncProvider, + DirectoryUser +} from "./types"; type InsertUser = InferInsertModel; type UpdateUser = Pick; +type SyncedUser = { + user: T; + /** Roles the user should have in `orgId` after the sync. */ + roleIds: number[]; +}; + +// the planned changes for one idpOrg, computed first and +// applied in a single transaction at the end +export type IdpOrgSyncPlan = { + idpId: number; + orgId: string; + usersToCreate: SyncedUser[]; + usersToUpdate: SyncedUser[]; + /** Users still in Pangolin but gone from the directory (or no longer matching any role); removed from the org, not deleted. */ + userIdsToRemoveFromOrg: string[]; +}; + // Track if the offline checker interval is running let directorySyncInterval: NodeJS.Timeout | null = null; const DIRECTORY_SYNC_RUN_INTERVAL = durationToMs(5, "minutes"); @@ -59,7 +98,7 @@ export async function runIdpDirectorySync() { }) .filter((p) => p != null); - await Promise.allSettled(fetchPromises); + return await Promise.allSettled(fetchPromises); } async function syncIdp( @@ -89,53 +128,136 @@ async function syncIdp( // 1. Fetch const result = await provider.fetch(ctx); + /** + * user => + * user (has roles within) - org + * + * algorithm: + + fetch users for org: + - if user is new to pangolin and idp: + - if fixed roles: + -> create new user in DB + - if group mapping: + - check that the user groups match at least one of the mapped groups: + -> create new user in DB + - if automatic groups: + - if create: + -> create new user in DB + - if not create: + - check that group names matches: + -> create new user in DB + + -> Add user to Org with the correct roles + + - if user is not new to pangolin and idp: + - if fixed roles or automatic groups with create : + - if user doesn't belongs to org: + -> add user to org with the correct roles + - else: + -> update user roles in org + - if group mapping matches any role: + -> update user roles in org + - if automatic groups without create still matches groups: + -> update user roles in org + - else: + -> remove user from org + + - if user exists in pangolin & IDP but not in remote users: + -> remove user from org + +for each remote user: + roleIds = resolveRoles(user, ctx.roleMapping) // fixed / group mapping / automatic + // [] = user doesn't qualify for this org + if user is disabled: roleIds = [] + + pangolinUser = match by externalUserId, then username + if no pangolinUser: + if roleIds empty: skip + create user + else: + update profile (name, email, username) // user-level, always safe + + membership = userOrg for (pangolinUser, orgId) + if no membership: + if roleIds non-empty: add to org with roleIds, flag = true + else if membership.flag is true: + if roleIds non-empty: set roles to roleIds + else: remove from org + else (manual membership): + skip // admin-owned, don't touch + +for each membership with flag = true whose user is not in the remote set: + remove from org // user row stays (may be in other orgs) + + + */ + const existingUsers = await db .select() .from(users) .where(eq(users.idpId, ctx.idpId)); - const map = new Map(); - for (const u of existingUsers) { - map.set(u.username, u); - } + const existingUserOrgs = await db + .select() + .from(users) + .innerJoin(userOrgs, eq(userOrgs.userId, users.userId)) + .where(and(eq(users.idpId, ctx.idpId), eq(userOrgs.orgId, ctx.orgId))); - const usersToInsert: InsertUser[] = []; - const usersToUpdate: UpdateUser[] = []; - const deletedUserIds: string[] = []; - - for (const user of result.users) { - const existing = map.get(user.username); - if (existing) { - usersToUpdate.push({ - username: user.username, - email: user.email, - name: user.name, - userId: existing.userId - }); + const existingPangolinUsers = new Map< + string, + (typeof existingUsers)[number] + >(); + for (const u of existingUsers) { + // rely on the external ID in the tenant if it exists, else use the username + if (u.externalUserId) { + existingPangolinUsers.set(u.externalUserId, u); } else { - const userId = generateId(15); - usersToInsert.push({ - userId, - username: user.username, - email: user.email, - name: user.name, - type: UserType.OIDC, - idpId: ctx.idpId, - emailVerified: true, // OIDC users are always verified - dateCreated: new Date().toISOString() - }); + existingPangolinUsers.set(u.username, u); } } - await db.transaction(async (trx) => { - await trx.insert(users).values(usersToInsert); + const syncPlan: IdpOrgSyncPlan = { + idpId: ctx.idpId, + orgId: ctx.orgId, + usersToCreate: [], + usersToUpdate: [], + userIdsToRemoveFromOrg: [] + }; - await Promise.all( - usersToUpdate.map((u) => - trx.update(users).set(u).where(eq(users.userId, u.userId)) - ) - ); - }); + // for (const user of result.users) { + // const existing = map.get(user.externalId) ?? map.get(user.username); + // if (existing) { + // usersToUpdate.push({ + // username: user.username, + // email: user.email, + // name: user.name, + // userId: existing.userId + // }); + // } else { + // const userId = generateId(15); + // usersToInsert.push({ + // userId, + // username: user.username, + // email: user.email, + // name: user.name, + // type: UserType.OIDC, + // idpId: ctx.idpId, + // emailVerified: true, // OIDC users are always verified + // dateCreated: new Date().toISOString() + // }); + // } + // } + + // await db.transaction(async (trx) => { + // await trx.insert(users).values(usersToInsert); + + // await Promise.all( + // usersToUpdate.map((u) => + // trx.update(users).set(u).where(eq(users.userId, u.userId)) + // ) + // ); + // }); // ... // console.dir( @@ -146,5 +268,72 @@ async function syncIdp( // ); // await db.select().from(idp) - return result; + return syncPlan; +} + +async function resolveUserRoles( + user: DirectoryUser, + roleMapping: DirectorySyncRoleMappingConfig, + orgId: string +) { + if (!user.enabled) return []; + + let matchedRoles: Role[] = []; + + switch (roleMapping.mode) { + case "fixedRoles": { + if (roleMapping.fixedRoleNames.length > 0) { + matchedRoles = await db + .select() + .from(roles) + .where( + and( + eq(roles.orgId, orgId), + inArray(roles.name, roleMapping.fixedRoleNames) + ) + ); + } + break; + } + case "groupMapping": { + const roleNames: string[] = []; + const groupMapping = new Map( + roleMapping.groupRules.map((rule) => [ + rule.group, + rule.roleNames + ]) + ); + + for (const group of user.groups) { + let roles: string[] = []; + if (group.displayName) { + roles = + groupMapping.get(group.displayName) ?? + groupMapping.get(group.id) ?? + []; + } else { + roles = + groupMapping.get(group.displayName ?? group.id) ?? []; + } + roleNames.push(...roles); + } + + if (roleNames.length > 0) { + matchedRoles = await db + .select() + .from(roles) + .where( + and( + eq(roles.orgId, orgId), + inArray(roles.name, roleNames) + ) + ); + } + break; + } + case "automatic": + break; + } + + return matchedRoles.map((role) => role.roleId); } From a789e36589b3181c6770335bff9efe7bc5aeb194 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Thu, 22 Oct 2026 19:42:59 +0200 Subject: [PATCH 29/50] =?UTF-8?q?=F0=9F=9A=A7=20Implement=20sync=20plan=20?= =?UTF-8?q?in=20sync=20workflow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/db/pg/schema/schema.ts | 1 + .../private/lib/directorySync/syncWorkflow.ts | 163 ++++++++++++++---- 2 files changed, 135 insertions(+), 29 deletions(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 5d79460229..21b1e23a08 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -737,6 +737,7 @@ export const userOrgs = pgTable( .notNull(), isOwner: boolean("isOwner").notNull().default(false), autoProvisioned: boolean("autoProvisioned").default(false), + directorySynched: boolean("directorySynched").default(false), pamUsername: varchar("pamUsername") // cleaned username for ssh and such }, (t) => [ diff --git a/server/private/lib/directorySync/syncWorkflow.ts b/server/private/lib/directorySync/syncWorkflow.ts index 21bf387d34..04a5cf1574 100644 --- a/server/private/lib/directorySync/syncWorkflow.ts +++ b/server/private/lib/directorySync/syncWorkflow.ts @@ -43,12 +43,17 @@ import type { } from "./types"; type InsertUser = InferInsertModel; -type UpdateUser = Pick; +type UpdateUser = Pick< + InsertUser, + "username" | "email" | "name" | "userId" | "externalUserId" +>; type SyncedUser = { user: T; + /** What to do with the user's membership in `orgId`. */ + membershipAction: "add" | "updateRoles" | "remove" | "unchanged"; /** Roles the user should have in `orgId` after the sync. */ - roleIds: number[]; + roles: Array<{ id: number; name: string }>; }; // the planned changes for one idpOrg, computed first and @@ -60,6 +65,8 @@ export type IdpOrgSyncPlan = { usersToUpdate: SyncedUser[]; /** Users still in Pangolin but gone from the directory (or no longer matching any role); removed from the org, not deleted. */ userIdsToRemoveFromOrg: string[]; + /** Directory users not provisioned because they don't match any role in the org. */ + skippedUsers: DirectoryUser[]; }; // Track if the offline checker interval is running @@ -85,14 +92,15 @@ export async function runIdpDirectorySync() { const fetchPromises = idpsToSync .map(({ idp, idpOidcConfig, idpOrg }) => { if (idpOidcConfig.variant === "azure") { - return syncIdp(azureSyncProvider, { + const ctx = { idpId: idp.idpId, orgId: idpOrg.orgId, identifierPath: idpOidcConfig.identifierPath, roleMapping: parseDirectorySyncRoleMapping( idpOrg.directoryRoleMapping ) - }); + } satisfies DirectoryFetchContext; + return syncIdp(azureSyncProvider, ctx); } return null; }) @@ -217,37 +225,100 @@ for each membership with flag = true whose user is not in the remote set: } } + const existingPangolinUserOrgs = new Map< + string, + (typeof existingUserOrgs)[number] + >(); + for (const u of existingUserOrgs) { + existingPangolinUserOrgs.set(u.user.userId, u); + } + const syncPlan: IdpOrgSyncPlan = { idpId: ctx.idpId, orgId: ctx.orgId, usersToCreate: [], usersToUpdate: [], - userIdsToRemoveFromOrg: [] + userIdsToRemoveFromOrg: [], + skippedUsers: [] }; - // for (const user of result.users) { - // const existing = map.get(user.externalId) ?? map.get(user.username); - // if (existing) { - // usersToUpdate.push({ - // username: user.username, - // email: user.email, - // name: user.name, - // userId: existing.userId - // }); - // } else { - // const userId = generateId(15); - // usersToInsert.push({ - // userId, - // username: user.username, - // email: user.email, - // name: user.name, - // type: UserType.OIDC, - // idpId: ctx.idpId, - // emailVerified: true, // OIDC users are always verified - // dateCreated: new Date().toISOString() - // }); - // } - // } + for (const user of result.users) { + const roles = await resolveUserRoles(user, ctx.roleMapping, ctx.orgId); + + const pangolinUser = + existingPangolinUsers.get(user.externalId) ?? + existingPangolinUsers.get(user.username); + + // this is a new user + if (!pangolinUser) { + // the user doesn't match any group/role on the org + if (roles.length === 0) { + syncPlan.skippedUsers.push(user); + continue; + } + + syncPlan.usersToCreate.push({ + user: { + userId: generateId(15), + username: user.username, + externalUserId: user.externalId, + email: user.email, + name: user.name, + type: UserType.OIDC, + idpId: ctx.idpId, + emailVerified: true, // OIDC users are always verified + dateCreated: new Date().toISOString() + }, + roles, + membershipAction: "add" + }); + } else { + const membership = existingPangolinUserOrgs.get( + pangolinUser.userId + ); + + let membershipAction: SyncedUser["membershipAction"] = + "updateRoles"; + + if (!membership) { + if (roles.length === 0) { + // skip users which don't belong to the org + membershipAction = "unchanged"; + syncPlan.skippedUsers.push(user); + } else { + membershipAction = "add"; + } + } else if ( + membership.userOrgs.autoProvisioned || + membership.userOrgs.directorySynched + ) { + if (membership.userOrgs.isOwner) { + // skip owner membership + syncPlan.skippedUsers.push(user); + membershipAction = "unchanged"; + } else if (roles.length === 0) { + syncPlan.userIdsToRemoveFromOrg.push(pangolinUser.userId); + membershipAction = "remove"; + } + } else { + // skip manual memberships + syncPlan.skippedUsers.push(user); + membershipAction = "unchanged"; + } + + syncPlan.usersToUpdate.push({ + user: { + userId: pangolinUser.userId, + username: user.username, + email: user.email, + name: user.name, + externalUserId: user.externalId + }, + roles, + membershipAction + }); + } + } // await db.transaction(async (trx) => { // await trx.insert(users).values(usersToInsert); @@ -332,8 +403,42 @@ async function resolveUserRoles( break; } case "automatic": + const roleNames = user.groups + .map((g) => g.displayName) + .filter((name) => name !== undefined && name !== null); + + if (roleNames.length > 0) { + matchedRoles = await db + .select() + .from(roles) + .where( + and( + eq(roles.orgId, orgId), + inArray(roles.name, roleNames) + ) + ); + } + + if (roleMapping.createMissingRoles) { + const missingRoles = new Set( + matchedRoles.map((r) => r.name) + ).difference(new Set(roleNames)); + + const result = await db + .insert(roles) + .values( + [...missingRoles].map((role) => ({ + name: role, + orgId: orgId + })) + ) + .returning(); + + matchedRoles.push(...result); + } + break; } - return matchedRoles.map((role) => role.roleId); + return matchedRoles.map((role) => ({ id: role.roleId, name: role.name })); } From 9ab54173176f581b3315645be09ae1380bb67b64 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Thu, 22 Oct 2026 22:44:55 +0200 Subject: [PATCH 30/50] =?UTF-8?q?=E2=9C=A8=20finish=20directory=20sync=20(?= =?UTF-8?q?but=20there=20is=20still=20a=20bug=3F)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/db/pg/schema/schema.ts | 60 ++-- server/db/sqlite/schema/schema.ts | 77 +++-- .../private/lib/directorySync/syncWorkflow.ts | 307 +++++++++++++----- 3 files changed, 311 insertions(+), 133 deletions(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 21b1e23a08..0008934f58 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -654,29 +654,40 @@ export const userSiteResources = pgTable( ] ); -export const users = pgTable("user", { - userId: varchar("id").primaryKey(), - email: varchar("email"), - username: varchar("username").notNull(), - name: varchar("name"), - type: varchar("type").notNull(), // "internal", "oidc" - idpId: integer("idpId").references(() => idp.idpId, { - onDelete: "cascade" - }), - externalUserId: varchar("externalUserId"), // user's ID in the external IdP (e.g. Azure object id) - passwordHash: varchar("passwordHash"), - twoFactorEnabled: boolean("twoFactorEnabled").notNull().default(false), - twoFactorSetupRequested: boolean("twoFactorSetupRequested").default(false), - twoFactorSecret: varchar("twoFactorSecret"), - emailVerified: boolean("emailVerified").notNull().default(false), - dateCreated: varchar("dateCreated").notNull(), - termsAcceptedTimestamp: varchar("termsAcceptedTimestamp"), - termsVersion: varchar("termsVersion"), - marketingEmailConsent: boolean("marketingEmailConsent").default(false), - serverAdmin: boolean("serverAdmin").notNull().default(false), - lastPasswordChange: bigint("lastPasswordChange", { mode: "number" }), - locale: varchar("locale") -}); +export const users = pgTable( + "user", + { + userId: varchar("id").primaryKey(), + email: varchar("email"), + username: varchar("username").notNull(), + name: varchar("name"), + type: varchar("type").notNull(), // "internal", "oidc" + idpId: integer("idpId").references(() => idp.idpId, { + onDelete: "cascade" + }), + externalUserId: varchar("externalUserId"), // user's ID in the external IdP (e.g. Azure object id) + passwordHash: varchar("passwordHash"), + twoFactorEnabled: boolean("twoFactorEnabled").notNull().default(false), + twoFactorSetupRequested: boolean("twoFactorSetupRequested").default( + false + ), + twoFactorSecret: varchar("twoFactorSecret"), + emailVerified: boolean("emailVerified").notNull().default(false), + dateCreated: varchar("dateCreated").notNull(), + termsAcceptedTimestamp: varchar("termsAcceptedTimestamp"), + termsVersion: varchar("termsVersion"), + marketingEmailConsent: boolean("marketingEmailConsent").default(false), + serverAdmin: boolean("serverAdmin").notNull().default(false), + lastPasswordChange: bigint("lastPasswordChange", { mode: "number" }), + locale: varchar("locale") + }, + (t) => [ + uniqueIndex("idx_user_idpId_externalUserId").on( + t.idpId, + t.externalUserId + ) + ] +); export const newts = pgTable( "newt", @@ -742,7 +753,8 @@ export const userOrgs = pgTable( }, (t) => [ index("idx_userOrgs_userid").on(t.userId), - index("idx_userOrgs_orgid").on(t.orgId) + index("idx_userOrgs_orgid").on(t.orgId), + unique().on(t.userId, t.orgId) ] ); diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 55ccaa87ba..a290e4f913 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -632,38 +632,48 @@ export const userSiteResources = sqliteTable("userSiteResources", { .references(() => siteResources.siteResourceId, { onDelete: "cascade" }) }); -export const users = sqliteTable("user", { - userId: text("id").primaryKey(), - email: text("email"), - username: text("username").notNull(), - name: text("name"), - type: text("type").notNull(), // "internal", "oidc" - idpId: integer("idpId").references(() => idp.idpId, { - onDelete: "cascade" - }), - passwordHash: text("passwordHash"), - twoFactorEnabled: integer("twoFactorEnabled", { mode: "boolean" }) - .notNull() - .default(false), - twoFactorSetupRequested: integer("twoFactorSetupRequested", { - mode: "boolean" - }).default(false), - twoFactorSecret: text("twoFactorSecret"), - emailVerified: integer("emailVerified", { mode: "boolean" }) - .notNull() - .default(false), - dateCreated: text("dateCreated").notNull(), - termsAcceptedTimestamp: text("termsAcceptedTimestamp"), - termsVersion: text("termsVersion"), - marketingEmailConsent: integer("marketingEmailConsent", { - mode: "boolean" - }).default(false), - serverAdmin: integer("serverAdmin", { mode: "boolean" }) - .notNull() - .default(false), - lastPasswordChange: integer("lastPasswordChange"), - locale: text("locale") -}); +export const users = sqliteTable( + "user", + { + userId: text("id").primaryKey(), + email: text("email"), + username: text("username").notNull(), + name: text("name"), + type: text("type").notNull(), // "internal", "oidc" + idpId: integer("idpId").references(() => idp.idpId, { + onDelete: "cascade" + }), + externalUserId: text("externalUserId"), // user's ID in the external IdP (e.g. Azure object id) + passwordHash: text("passwordHash"), + twoFactorEnabled: integer("twoFactorEnabled", { mode: "boolean" }) + .notNull() + .default(false), + twoFactorSetupRequested: integer("twoFactorSetupRequested", { + mode: "boolean" + }).default(false), + twoFactorSecret: text("twoFactorSecret"), + emailVerified: integer("emailVerified", { mode: "boolean" }) + .notNull() + .default(false), + dateCreated: text("dateCreated").notNull(), + termsAcceptedTimestamp: text("termsAcceptedTimestamp"), + termsVersion: text("termsVersion"), + marketingEmailConsent: integer("marketingEmailConsent", { + mode: "boolean" + }).default(false), + serverAdmin: integer("serverAdmin", { mode: "boolean" }) + .notNull() + .default(false), + lastPasswordChange: integer("lastPasswordChange"), + locale: text("locale") + }, + (t) => [ + uniqueIndex("idx_user_idpId_externalUserId").on( + t.idpId, + t.externalUserId + ) + ] +); export const securityKeys = sqliteTable("webauthnCredentials", { credentialId: text("credentialId").primaryKey(), @@ -1019,6 +1029,9 @@ export const userOrgs = sqliteTable( autoProvisioned: integer("autoProvisioned", { mode: "boolean" }).default(false), + directorySynched: integer("directorySynched", { + mode: "boolean" + }).default(false), pamUsername: text("pamUsername") // cleaned username for ssh and such }, (table) => [ diff --git a/server/private/lib/directorySync/syncWorkflow.ts b/server/private/lib/directorySync/syncWorkflow.ts index 04a5cf1574..a4bf3e8ef5 100644 --- a/server/private/lib/directorySync/syncWorkflow.ts +++ b/server/private/lib/directorySync/syncWorkflow.ts @@ -10,12 +10,21 @@ * * This file is not licensed under the AGPLv3. */ + +import { extendZodWithOpenApi } from "@asteasolutions/zod-to-openapi"; +import { z } from "zod"; + +// This is important to make sure schemas imported from other files accept `z.openapi()` as a function +extendZodWithOpenApi(z); + import { db, idp, idpOidcConfig, idpOrg, + orgs, roles, + userOrgRoles, userOrgs, users, type Role @@ -27,14 +36,11 @@ import { type DirectorySyncRoleMappingConfig } from "@app/lib/idpDirectorySyncRoleMapping"; import { generateId } from "@server/auth/sessions/app"; +import { calculateUserClientsForOrgs } from "@server/lib/calculateUserClientsForOrgs"; +import { assignUserToOrg, removeUserFromOrg } from "@server/lib/userOrg"; +import logger from "@server/logger"; import { UserType } from "@server/types/UserTypes"; -import { - and, - eq, - inArray, - type InferInsertModel, - type InferSelectModel -} from "drizzle-orm"; +import { and, eq, inArray, sql, type InferInsertModel } from "drizzle-orm"; import { AzureDirectorySyncProvider } from "./azure"; import type { DirectoryFetchContext, @@ -48,12 +54,27 @@ type UpdateUser = Pick< "username" | "email" | "name" | "userId" | "externalUserId" >; -type SyncedUser = { - user: T; - /** What to do with the user's membership in `orgId`. */ - membershipAction: "add" | "updateRoles" | "remove" | "unchanged"; - /** Roles the user should have in `orgId` after the sync. */ - roles: Array<{ id: number; name: string }>; +type SyncedRole = { id: number; name: string }; + +/** What happens to one user: their Pangolin profile and their membership in `orgId` are decided separately. */ +export type UserSyncEntry = { + /** `null` when the user is in Pangolin but gone from the directory. */ + directoryUser: DirectoryUser | null; + user: + | { operation: "create"; data: InsertUser } + | { operation: "update"; data: UpdateUser } + | { operation: "unchanged"; userId: string } + /** New directory user not provisioned in Pangolin. */ + | { operation: "skip" }; + + membership: + | { operation: "add" | "updateRoles"; roles: SyncedRole[] } + /** Removed from the org, the user itself is not deleted. */ + | { operation: "remove" } + | { + operation: "unchanged"; + reason: "noMatchingRoles" | "manualMembership" | "owner"; + }; }; // the planned changes for one idpOrg, computed first and @@ -61,12 +82,7 @@ type SyncedUser = { export type IdpOrgSyncPlan = { idpId: number; orgId: string; - usersToCreate: SyncedUser[]; - usersToUpdate: SyncedUser[]; - /** Users still in Pangolin but gone from the directory (or no longer matching any role); removed from the org, not deleted. */ - userIdsToRemoveFromOrg: string[]; - /** Directory users not provisioned because they don't match any role in the org. */ - skippedUsers: DirectoryUser[]; + entries: UserSyncEntry[]; }; // Track if the offline checker interval is running @@ -236,10 +252,7 @@ for each membership with flag = true whose user is not in the remote set: const syncPlan: IdpOrgSyncPlan = { idpId: ctx.idpId, orgId: ctx.orgId, - usersToCreate: [], - usersToUpdate: [], - userIdsToRemoveFromOrg: [], - skippedUsers: [] + entries: [] }; for (const user of result.users) { @@ -253,95 +266,235 @@ for each membership with flag = true whose user is not in the remote set: if (!pangolinUser) { // the user doesn't match any group/role on the org if (roles.length === 0) { - syncPlan.skippedUsers.push(user); + syncPlan.entries.push({ + directoryUser: user, + user: { operation: "skip" }, + membership: { + operation: "unchanged", + reason: "noMatchingRoles" + } + }); continue; } - syncPlan.usersToCreate.push({ + syncPlan.entries.push({ + directoryUser: user, user: { - userId: generateId(15), - username: user.username, - externalUserId: user.externalId, - email: user.email, - name: user.name, - type: UserType.OIDC, - idpId: ctx.idpId, - emailVerified: true, // OIDC users are always verified - dateCreated: new Date().toISOString() + operation: "create", + data: { + userId: generateId(15), + username: user.username, + externalUserId: user.externalId, + email: user.email, + name: user.name, + type: UserType.OIDC, + idpId: ctx.idpId, + emailVerified: true, // OIDC users are always verified + dateCreated: new Date().toISOString() + } }, - roles, - membershipAction: "add" + membership: { operation: "add", roles } }); } else { const membership = existingPangolinUserOrgs.get( pangolinUser.userId ); - let membershipAction: SyncedUser["membershipAction"] = - "updateRoles"; + let membershipOperation: UserSyncEntry["membership"]; if (!membership) { - if (roles.length === 0) { - // skip users which don't belong to the org - membershipAction = "unchanged"; - syncPlan.skippedUsers.push(user); - } else { - membershipAction = "add"; - } + membershipOperation = + roles.length === 0 + ? // skip users which don't belong to the org + { operation: "unchanged", reason: "noMatchingRoles" } + : { operation: "add", roles }; } else if ( membership.userOrgs.autoProvisioned || membership.userOrgs.directorySynched ) { if (membership.userOrgs.isOwner) { - // skip owner membership - syncPlan.skippedUsers.push(user); - membershipAction = "unchanged"; + membershipOperation = { + operation: "unchanged", + reason: "owner" + }; } else if (roles.length === 0) { - syncPlan.userIdsToRemoveFromOrg.push(pangolinUser.userId); - membershipAction = "remove"; + membershipOperation = { operation: "remove" }; + } else { + membershipOperation = { operation: "updateRoles", roles }; } } else { - // skip manual memberships - syncPlan.skippedUsers.push(user); - membershipAction = "unchanged"; + membershipOperation = { + operation: "unchanged", + reason: "manualMembership" + }; } - syncPlan.usersToUpdate.push({ + syncPlan.entries.push({ + directoryUser: user, user: { - userId: pangolinUser.userId, - username: user.username, - email: user.email, - name: user.name, - externalUserId: user.externalId + operation: "update", + data: { + userId: pangolinUser.userId, + username: user.username, + email: user.email, + name: user.name, + externalUserId: user.externalId + } }, - roles, - membershipAction + membership: membershipOperation }); } } - // await db.transaction(async (trx) => { - // await trx.insert(users).values(usersToInsert); + // users still in the org through sync/provisioning but gone from the directory + const directoryUserIds = new Set( + syncPlan.entries.flatMap((e) => + e.user.operation === "update" ? [e.user.data.userId] : [] + ) + ); + for (const { user, userOrgs: membership } of existingUserOrgs) { + if (directoryUserIds.has(user.userId)) continue; + if (!membership.autoProvisioned && !membership.directorySynched) + continue; + if (membership.isOwner) continue; + + syncPlan.entries.push({ + directoryUser: null, + user: { operation: "unchanged", userId: user.userId }, + membership: { operation: "remove" } + }); + } + + const [org] = await db.select().from(orgs).where(eq(orgs.orgId, ctx.orgId)); + if (!org) { + throw new Error(`Org ${ctx.orgId} not found for directory sync`); + } - // await Promise.all( - // usersToUpdate.map((u) => - // trx.update(users).set(u).where(eq(users.userId, u.userId)) - // ) - // ); - // }); + // the upsert can return an existing row's userId instead of the generated one + // if a user is synched in multiple orgs, the generated `userId` will be + // different when creating the same user, but the `externalId` will be the same + const userIdsByExternalId = new Map(); - // ... - // console.dir( - // { - // result - // }, - // { depth: null } - // ); - // await db.select().from(idp) + await db.transaction(async (trx) => { + // create new users + // upsert so the same directory user synced into several orgs isn't created twice + const usersToUpsert = syncPlan.entries.flatMap((e) => + e.user.operation === "create" ? [e.user.data] : [] + ); + if (usersToUpsert.length > 0) { + const upsertedUsers = await trx + .insert(users) + .values(usersToUpsert) + .onConflictDoUpdate({ + target: [users.idpId, users.externalUserId], + set: { + username: sql`excluded.username`, + email: sql`excluded.email`, + name: sql`excluded.name` + } + }) + .returning({ + userId: users.userId, + externalUserId: users.externalUserId + }); + + for (const u of upsertedUsers) { + userIdsByExternalId.set(u.externalUserId!, u.userId); + } + } + + // update existing users by userId, this also backfills `externalUserId` + // for users previously matched by username + const usersToUpdate = syncPlan.entries.flatMap((e) => + e.user.operation === "update" ? [e.user.data] : [] + ); + await Promise.all( + usersToUpdate.map((u) => + trx + .update(users) + .set(u) + .where( + and( + eq(users.userId, u.userId), + eq(users.idpId, ctx.idpId) + ) + ) + ) + ); + + // apply memberships sequentially, `assignUserToOrg` and `removeUserFromOrg` + // read the user's other orgs for billing + for (const entry of syncPlan.entries) { + const userId = getEntryUserId(entry, userIdsByExternalId); + if (!userId) continue; + + switch (entry.membership.operation) { + case "add": + await assignUserToOrg( + org, + { userId, orgId: ctx.orgId, directorySynched: true }, + entry.membership.roles.map((r) => r.id), + trx + ); + break; + case "updateRoles": + // remove previous roles + await trx + .delete(userOrgRoles) + .where( + and( + eq(userOrgRoles.userId, userId), + eq(userOrgRoles.orgId, ctx.orgId) + ) + ); + // add new roles + await trx.insert(userOrgRoles).values( + entry.membership.roles.map((r) => ({ + userId, + orgId: ctx.orgId, + roleId: r.id + })) + ); + break; + case "remove": + await removeUserFromOrg(org, userId, trx); + break; + } + } + }); + + for (const entry of syncPlan.entries) { + if (entry.membership.operation === "unchanged") continue; + const userId = getEntryUserId(entry, userIdsByExternalId); + if (!userId) continue; + + calculateUserClientsForOrgs(userId).catch((error) => { + logger.error( + "Error calculating user clients after directory sync", + { error } + ); + }); + } return syncPlan; } +function getEntryUserId( + entry: UserSyncEntry, + userIdsByExternalId: Map +) { + switch (entry.user.operation) { + case "create": + return userIdsByExternalId.get(entry.user.data.externalUserId!); + case "update": + return entry.user.data.userId; + case "unchanged": + return entry.user.userId; + case "skip": + return undefined; + } +} + async function resolveUserRoles( user: DirectoryUser, roleMapping: DirectorySyncRoleMappingConfig, From bbeecc6b4ec6ed8f4f225eb60b6519d3552862ba Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Thu, 22 Oct 2026 22:52:05 +0200 Subject: [PATCH 31/50] =?UTF-8?q?=E2=9C=A8=20Directory=20sync=20works=20!!?= =?UTF-8?q?!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/private/lib/directorySync/run.ts | 2 ++ server/private/lib/directorySync/syncWorkflow.ts | 6 ------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/server/private/lib/directorySync/run.ts b/server/private/lib/directorySync/run.ts index 9b978839b6..feef24e676 100644 --- a/server/private/lib/directorySync/run.ts +++ b/server/private/lib/directorySync/run.ts @@ -1,6 +1,8 @@ // // Usage: npx tsx server/private/lib/directorySync/run.ts // // Runs one directory sync for the IdP using its saved directory role mapping. // import { AzureDirectorySyncProvider } from "./azure"; +// must be imported first, before any module that calls `z.openapi()` +import "@server/extendZod"; import { runIdpDirectorySync } from "./syncWorkflow"; await runIdpDirectorySync().then((changes) => { diff --git a/server/private/lib/directorySync/syncWorkflow.ts b/server/private/lib/directorySync/syncWorkflow.ts index a4bf3e8ef5..3a3cc8bdb2 100644 --- a/server/private/lib/directorySync/syncWorkflow.ts +++ b/server/private/lib/directorySync/syncWorkflow.ts @@ -11,12 +11,6 @@ * This file is not licensed under the AGPLv3. */ -import { extendZodWithOpenApi } from "@asteasolutions/zod-to-openapi"; -import { z } from "zod"; - -// This is important to make sure schemas imported from other files accept `z.openapi()` as a function -extendZodWithOpenApi(z); - import { db, idp, From 61657c9d2cfc9a006f1794b37e31c8a2f8bec9c4 Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Fri, 23 Oct 2026 18:53:07 +0200 Subject: [PATCH 32/50] =?UTF-8?q?=F0=9F=9A=A7=20add=20dry=20run=20to=20syn?= =?UTF-8?q?c=20workflow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/private/lib/directorySync/run.ts | 5 +- .../private/lib/directorySync/syncWorkflow.ts | 50 +++++++++++++++---- 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/server/private/lib/directorySync/run.ts b/server/private/lib/directorySync/run.ts index feef24e676..697bffaa59 100644 --- a/server/private/lib/directorySync/run.ts +++ b/server/private/lib/directorySync/run.ts @@ -5,7 +5,10 @@ import "@server/extendZod"; import { runIdpDirectorySync } from "./syncWorkflow"; -await runIdpDirectorySync().then((changes) => { +// pass `--dry-run` to only compute the sync plans, without applying them +const dryRun = process.argv.includes("--dry-run"); + +await runIdpDirectorySync(dryRun).then((changes) => { console.dir( { changes diff --git a/server/private/lib/directorySync/syncWorkflow.ts b/server/private/lib/directorySync/syncWorkflow.ts index 3a3cc8bdb2..de2ec89759 100644 --- a/server/private/lib/directorySync/syncWorkflow.ts +++ b/server/private/lib/directorySync/syncWorkflow.ts @@ -83,7 +83,7 @@ export type IdpOrgSyncPlan = { let directorySyncInterval: NodeJS.Timeout | null = null; const DIRECTORY_SYNC_RUN_INTERVAL = durationToMs(5, "minutes"); -export async function runIdpDirectorySync() { +export async function runIdpDirectorySync(dryRun: boolean = false) { // to sync idps, we need const idpsToSync = await db .select() @@ -110,7 +110,7 @@ export async function runIdpDirectorySync() { idpOrg.directoryRoleMapping ) } satisfies DirectoryFetchContext; - return syncIdp(azureSyncProvider, ctx); + return syncIdp(azureSyncProvider, ctx, dryRun); } return null; }) @@ -121,7 +121,8 @@ export async function runIdpDirectorySync() { async function syncIdp( provider: DirectorySyncProvider, - ctx: DirectoryFetchContext + ctx: DirectoryFetchContext, + dryRun: boolean ) { // Note // username is unique only within IDPs or within internal ? @@ -250,7 +251,12 @@ for each membership with flag = true whose user is not in the remote set: }; for (const user of result.users) { - const roles = await resolveUserRoles(user, ctx.roleMapping, ctx.orgId); + const roles = await resolveUserRoles( + user, + ctx.roleMapping, + ctx.orgId, + dryRun + ); const pangolinUser = existingPangolinUsers.get(user.externalId) ?? @@ -348,9 +354,11 @@ for each membership with flag = true whose user is not in the remote set: ); for (const { user, userOrgs: membership } of existingUserOrgs) { if (directoryUserIds.has(user.userId)) continue; - if (!membership.autoProvisioned && !membership.directorySynched) + if ( + membership.isOwner || + (!membership.autoProvisioned && !membership.directorySynched) + ) continue; - if (membership.isOwner) continue; syncPlan.entries.push({ directoryUser: null, @@ -364,6 +372,10 @@ for each membership with flag = true whose user is not in the remote set: throw new Error(`Org ${ctx.orgId} not found for directory sync`); } + if (dryRun) { + return syncPlan; + } + // the upsert can return an existing row's userId instead of the generated one // if a user is synched in multiple orgs, the generated `userId` will be // different when creating the same user, but the `externalId` will be the same @@ -492,8 +504,9 @@ function getEntryUserId( async function resolveUserRoles( user: DirectoryUser, roleMapping: DirectorySyncRoleMappingConfig, - orgId: string -) { + orgId: string, + dryRun: boolean +): Promise { if (!user.enabled) return []; let matchedRoles: Role[] = []; @@ -567,9 +580,24 @@ async function resolveUserRoles( } if (roleMapping.createMissingRoles) { - const missingRoles = new Set( - matchedRoles.map((r) => r.name) - ).difference(new Set(roleNames)); + const missingRoles = new Set(roleNames).difference( + new Set(matchedRoles.map((r) => r.name)) + ); + + if (missingRoles.size === 0) { + break; + } + + if (dryRun) { + // the roles would be created when applying the plan, they don't have an id yet + return [ + ...matchedRoles.map((role) => ({ + id: role.roleId, + name: role.name + })), + ...[...missingRoles].map((name) => ({ id: -1, name })) + ]; + } const result = await db .insert(roles) From 1e4661672e709c7cfb9b954a647d7ea5f484c5ad Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Fri, 23 Oct 2026 18:54:03 +0200 Subject: [PATCH 33/50] =?UTF-8?q?=F0=9F=90=9B=20handle=20`@removed`=20in?= =?UTF-8?q?=20returned=20delta?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/private/lib/directorySync/azure.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/server/private/lib/directorySync/azure.ts b/server/private/lib/directorySync/azure.ts index 1ed90eaa69..314939d0b3 100644 --- a/server/private/lib/directorySync/azure.ts +++ b/server/private/lib/directorySync/azure.ts @@ -40,8 +40,10 @@ const userBodySchema = z.object({ ) }); -// we only do initial delta rounds for groups (no stored `deltaLink`), which return the -// current state of the directory and never contain `@removed` entries +// entries flagged as removed (e.g. soft-deleted groups or removed members), these +// can show up even in initial delta rounds (no stored `deltaLink`) +const removedSchema = z.object({ reason: z.string() }).optional(); + const groupBodySchema = z.object({ "@odata.nextLink": z.url().optional(), "@odata.deltaLink": z.url().optional(), @@ -51,13 +53,15 @@ const groupBodySchema = z.object({ // large groups can be returned several times with their members split // across the entries, the repeated entries may not have `displayName` displayName: z.string().nullish(), + "@removed": removedSchema, // omitted when the group has no members "members@delta": z .array( z.object({ // user, group, device, servicePrincipal, orgContact... "@odata.type": z.string(), - id: z.uuid() + id: z.uuid(), + "@removed": removedSchema }) ) .default([]) @@ -150,12 +154,20 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { }; for (const group of groupResponse.groups) { + if (group["@removed"]) { + continue; + } + const node = getGroupNode(group.id); if (group.displayName) { node.displayName = group.displayName; } for (const member of group["members@delta"]) { + if (member["@removed"]) { + continue; + } + if (member["@odata.type"] === "#microsoft.graph.user") { node.userIds.add(member.id); } else if (member["@odata.type"] === "#microsoft.graph.group") { From 8c12e60f8014709315c0194e1e25494308ee079f Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Fri, 23 Oct 2026 22:16:50 +0200 Subject: [PATCH 34/50] =?UTF-8?q?=F0=9F=9A=A7=20WIP:=20google?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/private/lib/directorySync/azure.ts | 39 +---------------- .../lib/directorySync/ensureValidIdp.ts | 42 +++++++++++++++++++ server/private/lib/directorySync/google.ts | 16 +++++++ 3 files changed, 60 insertions(+), 37 deletions(-) create mode 100644 server/private/lib/directorySync/ensureValidIdp.ts create mode 100644 server/private/lib/directorySync/google.ts diff --git a/server/private/lib/directorySync/azure.ts b/server/private/lib/directorySync/azure.ts index 314939d0b3..e86e173b4e 100644 --- a/server/private/lib/directorySync/azure.ts +++ b/server/private/lib/directorySync/azure.ts @@ -25,6 +25,7 @@ import cache from "@server/lib/cache"; import z from "zod"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; +import { ensureValidIdp, type IdpDbResponse } from "./ensureValidIdp"; const userBodySchema = z.object({ "@odata.nextLink": z.url().optional(), @@ -85,7 +86,6 @@ type GraphGroupNode = { userIds: Set; }; -type IdpDbResponse = Awaited>; export class AzureDirectorySyncProvider implements DirectorySyncProvider { #TOKEN_CACHE_KEY_PREFIX = "azureDirectorySyncToken"; #microsoftGraphBaseUrl = "https://graph.microsoft.com/v1.0"; @@ -93,7 +93,7 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { async fetch(ctx: DirectoryFetchContext): Promise { const directoryUsers: Record = {}; // 0. fetch idp and check that it is valid - const idpRes = await ensureValidIdp(ctx.idpId); + const idpRes = await ensureValidIdp(ctx.idpId, "azure"); // 1. get access token (and cache) const token = await this.#getToken(idpRes); @@ -433,38 +433,3 @@ export class AzureDirectorySyncProvider implements DirectorySyncProvider { }; } } - -async function ensureValidIdp(idpId: number) { - const [idpRes] = await db - .select() - .from(idp) - .where(eq(idp.idpId, idpId)) - .innerJoin(idpOidcConfig, eq(idpOidcConfig.idpId, idp.idpId)) - .innerJoin(idpOrg, and(eq(idpOrg.idpId, idp.idpId))) - .limit(1); - - if (!idpRes) { - throw new Error( - `Directory sync: IdP ${idpId} not found, or it has no OIDC config or org mapping` - ); - } - - if (idpRes.idpOidcConfig.variant !== "azure") { - throw new Error( - `Directory sync: IdP ${idpId} has variant "${idpRes.idpOidcConfig.variant}", expected "azure"` - ); - } - - if (!idpRes.idp.directorySyncEnabled) { - throw new Error( - `Directory sync: directory sync is not enabled on IdP ${idpId}` - ); - } - - if (idpRes.idp.autoProvision) { - throw new Error( - `Directory sync: IdP ${idpId} has both directory sync and auto-provisioning enabled; only one can be active` - ); - } - return idpRes; -} diff --git a/server/private/lib/directorySync/ensureValidIdp.ts b/server/private/lib/directorySync/ensureValidIdp.ts new file mode 100644 index 0000000000..40aaeead8d --- /dev/null +++ b/server/private/lib/directorySync/ensureValidIdp.ts @@ -0,0 +1,42 @@ +import { db, idp, idpOidcConfig, idpOrg, type IdpOidcConfig } from "@server/db"; +import { eq, and } from "drizzle-orm"; + +export async function ensureValidIdp( + idpId: number, + variant: IdpOidcConfig["variant"] +) { + const [idpRes] = await db + .select() + .from(idp) + .where(eq(idp.idpId, idpId)) + .innerJoin(idpOidcConfig, eq(idpOidcConfig.idpId, idp.idpId)) + .innerJoin(idpOrg, and(eq(idpOrg.idpId, idp.idpId))) + .limit(1); + + if (!idpRes) { + throw new Error( + `Directory sync: IdP ${idpId} not found, or it has no OIDC config or org mapping` + ); + } + + if (idpRes.idpOidcConfig.variant !== variant) { + throw new Error( + `Directory sync: IdP ${idpId} has variant "${idpRes.idpOidcConfig.variant}", expected "${variant}"` + ); + } + + if (!idpRes.idp.directorySyncEnabled) { + throw new Error( + `Directory sync: directory sync is not enabled on IdP ${idpId}` + ); + } + + if (idpRes.idp.autoProvision) { + throw new Error( + `Directory sync: IdP ${idpId} has both directory sync and auto-provisioning enabled; only one can be active` + ); + } + return idpRes; +} + +export type IdpDbResponse = Awaited>; diff --git a/server/private/lib/directorySync/google.ts b/server/private/lib/directorySync/google.ts new file mode 100644 index 0000000000..a6de54ac96 --- /dev/null +++ b/server/private/lib/directorySync/google.ts @@ -0,0 +1,16 @@ +import type { + DirectoryFetchContext, + DirectoryFetchResult, + DirectorySyncProvider, + DirectoryUser +} from "./types"; +import { ensureValidIdp } from "./ensureValidIdp"; + +export class AzureDirectorySyncProvider implements DirectorySyncProvider { + async fetch(ctx: DirectoryFetchContext): Promise { + const directoryUsers: Record = {}; + // 0. fetch idp and check that it is valid + const idpRes = await ensureValidIdp(ctx.idpId, "google"); + throw new Error("Method not implemented."); + } +} From e4c9c8b2c4529b5fed7fd6b9db46e2bf84311eae Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Wed, 28 Oct 2026 18:42:22 +0100 Subject: [PATCH 35/50] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20directory=20sync=20c?= =?UTF-8?q?onfig?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/db/pg/schema/schema.ts | 34 +- server/db/sqlite/schema/schema.ts | 34 +- server/private/lib/directorySync/_azure.ts | 443 --------------------- 3 files changed, 8 insertions(+), 503 deletions(-) delete mode 100644 server/private/lib/directorySync/_azure.ts diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 0008934f58..d54e0cdeee 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1230,7 +1230,10 @@ export const idp = pgTable("idp", { tags: text("tags"), directorySyncEnabled: boolean("directorySyncEnabled") .notNull() - .default(false) + .default(false), + // Provider-defined JSON, encrypted (e.g. Google service account key + admin email). + // Null = the provider reuses the OIDC client credentials (Azure). + directorySyncConfig: text("directorySyncConfig") }); export const idpOidcConfig = pgTable("idpOidcConfig", { @@ -1353,35 +1356,6 @@ export const idpOrg = pgTable("idpOrg", { .default(true) }); -// Runtime state of the directory sync job, one row per IdP. Deleting the row -// forces a full sync on the next run. -export const idpSyncState = pgTable("idpSyncState", { - idpId: integer("idpId") - .primaryKey() - .references(() => idp.idpId, { onDelete: "cascade" }), - // Provider-defined opaque JSON (e.g. Azure delta links). Null = full sync. - cursor: text("cursor"), - lastSyncStartedAt: bigint("lastSyncStartedAt", { mode: "number" }), - lastSyncSucceededAt: bigint("lastSyncSucceededAt", { mode: "number" }), - lastError: text("lastError"), - consecutiveFailures: integer("consecutiveFailures").notNull().default(0) -}); - -// Links a directory user (stable provider ID) to the Pangolin user sync manages. -export const idpSyncUser = pgTable( - "idpSyncUser", - { - idpId: integer("idpId") - .notNull() - .references(() => idp.idpId, { onDelete: "cascade" }), - externalId: varchar("externalId").notNull(), - userId: varchar("userId") - .notNull() - .references(() => users.userId, { onDelete: "cascade" }) - }, - (t) => [primaryKey({ columns: [t.idpId, t.externalId] })] -); - export const clients = pgTable( "clients", { diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index a290e4f913..0de8cb1905 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1538,7 +1538,10 @@ export const idp = sqliteTable("idp", { mode: "boolean" }) .notNull() - .default(false) + .default(false), + // Provider-defined JSON, encrypted (e.g. Google service account key + admin email). + // Null = the provider reuses the OIDC client credentials (Azure). + directorySyncConfig: text("directorySyncConfig") }); // Identity Provider OAuth Configuration @@ -1668,35 +1671,6 @@ export const idpOrg = sqliteTable("idpOrg", { .default(true) }); -// Runtime state of the directory sync job, one row per IdP. Deleting the row -// forces a full sync on the next run. -export const idpSyncState = sqliteTable("idpSyncState", { - idpId: integer("idpId") - .primaryKey() - .references(() => idp.idpId, { onDelete: "cascade" }), - // Provider-defined opaque JSON (e.g. Azure delta links). Null = full sync. - cursor: text("cursor"), - lastSyncStartedAt: integer("lastSyncStartedAt"), - lastSyncSucceededAt: integer("lastSyncSucceededAt"), - lastError: text("lastError"), - consecutiveFailures: integer("consecutiveFailures").notNull().default(0) -}); - -// Links a directory user (stable provider ID) to the Pangolin user sync manages. -export const idpSyncUser = sqliteTable( - "idpSyncUser", - { - idpId: integer("idpId") - .notNull() - .references(() => idp.idpId, { onDelete: "cascade" }), - externalId: text("externalId").notNull(), - userId: text("userId") - .notNull() - .references(() => users.userId, { onDelete: "cascade" }) - }, - (t) => [primaryKey({ columns: [t.idpId, t.externalId] })] -); - // Blueprint runs export const blueprints = sqliteTable("blueprints", { blueprintId: integer("blueprintId").primaryKey({ diff --git a/server/private/lib/directorySync/_azure.ts b/server/private/lib/directorySync/_azure.ts deleted file mode 100644 index cf870cba2a..0000000000 --- a/server/private/lib/directorySync/_azure.ts +++ /dev/null @@ -1,443 +0,0 @@ -/* - * This file is part of a proprietary work. - * - * Copyright (c) 2025-2026 Fossorial, Inc. - * All rights reserved. - * - * This file is licensed under the Fossorial Commercial License. - * You may not use this file except in compliance with the License. - * Unauthorized use, copying, modification, or distribution is strictly prohibited. - * - * This file is not licensed under the AGPLv3. - */ - -// Microsoft Entra ID directory provider (Microsoft Graph, app-only auth). -// -// Required application permissions (with admin consent): User.Read.All, GroupMember.Read.All. -// -// Full fetch: /users/delta + /groups/delta?$select=displayName,members without a token, which -// lists everything and returns delta links for next time. Nested memberships are resolved in -// memory from the direct members. -// -// Incremental fetch: replay both delta links. Group deltas only report direct member changes, -// so every user touched by a change is re-read (/users/{id} + /transitiveMemberOf) to get their -// full current state. Cases that can't be resolved cheaply fall back to a full fetch. - -import config from "@server/lib/config"; -import logger from "@server/logger"; -import { decrypt } from "@server/lib/crypto"; -import { - DirectoryCursorExpiredError, - DirectoryFetchContext, - DirectoryFetchResult, - DirectoryGroup, - DirectorySyncProvider, - DirectoryUser -} from "./types"; - -const GRAPH = "https://graph.microsoft.com/v1.0"; -const USER_SELECT = "id,displayName,mail,userPrincipalName,accountEnabled"; - -/** Above this many touched users, an incremental run is more expensive than a full one. */ -const MAX_INCREMENTAL_AFFECTED_USERS = 500; -/** Longest Retry-After we sleep through in-process before failing the run. */ -const MAX_RETRY_AFTER_SECONDS = 120; -const MAX_THROTTLE_RETRIES = 5; - -type GraphUser = { - id: string; - displayName?: string | null; - mail?: string | null; - userPrincipalName?: string | null; - accountEnabled?: boolean | null; - "@removed"?: { reason: string }; -}; - -type GraphDirectoryObjectRef = { - "@odata.type"?: string; - id: string; - "@removed"?: { reason: string }; -}; - -type GraphGroup = { - id: string; - displayName?: string | null; - "members@delta"?: GraphDirectoryObjectRef[]; - "@removed"?: { reason: string }; -}; - -type AzureCursor = { - usersDeltaLink: string; - groupsDeltaLink: string; -}; - -type GraphPage = { - value: T[]; - "@odata.nextLink"?: string; - "@odata.deltaLink"?: string; -}; - -// Graph field that yields the same `username` login computes from the ID token. -// `sub` is per-app in Entra and not exposed by Graph, so it can't be synced. -const IDENTIFIER_FROM_GRAPH: Record string | null> = { - oid: (u) => u.id, - preferred_username: (u) => u.userPrincipalName ?? null, - upn: (u) => u.userPrincipalName ?? null, - email: (u) => u.mail ?? null -}; - -const tokenCache = new Map(); - -class GraphClient { - constructor( - private readonly getToken: () => Promise, - private readonly onRequest: () => void - ) {} - - /** GET a single Graph URL, sleeping through short throttling. Returns null on 404. */ - async get(url: string): Promise { - for (let attempt = 0; ; attempt++) { - this.onRequest(); - const res = await fetch(url, { - headers: { Authorization: `Bearer ${await this.getToken()}` } - }); - - if (res.status === 429 || res.status === 503) { - const retryAfter = Number(res.headers.get("Retry-After") ?? 5); - if ( - attempt >= MAX_THROTTLE_RETRIES || - retryAfter > MAX_RETRY_AFTER_SECONDS - ) { - throw new Error( - `Graph throttled (${res.status}, Retry-After ${retryAfter}s): ${url}` - ); - } - logger.warn(`Graph throttled, retrying in ${retryAfter}s`, { - url - }); - await new Promise((r) => setTimeout(r, retryAfter * 1000)); - continue; - } - - if (res.status === 404) return null; - - const body: any = await res.json().catch(() => ({})); - - if ( - res.status === 410 || - body.error?.code === "syncStateNotFound" || - body.error?.code === "resyncRequired" - ) { - throw new DirectoryCursorExpiredError( - `Graph delta token expired (${res.status} ${body.error?.code})` - ); - } - if (!res.ok) { - throw new Error( - `Graph request failed (${res.status}) ${url}: ${body.error?.code} ${body.error?.message}` - ); - } - return body as T; - } - } - - /** Follow @odata.nextLink pages; returns the items and the final delta link, if any. */ - async getAll( - url: string - ): Promise<{ items: T[]; deltaLink: string | null }> { - const items: T[] = []; - let next: string | undefined = url; - while (next) { - const page: GraphPage | null = - await this.get>(next); - if (!page) throw new Error(`Graph returned 404 for ${next}`); - items.push(...page.value); - if (page["@odata.deltaLink"]) { - return { items, deltaLink: page["@odata.deltaLink"] }; - } - next = page["@odata.nextLink"]; - } - return { items, deltaLink: null }; - } -} - -function isUserRef(ref: GraphDirectoryObjectRef) { - return ref["@odata.type"] === "#microsoft.graph.user"; -} - -function isGroupRef(ref: GraphDirectoryObjectRef) { - return ref["@odata.type"] === "#microsoft.graph.group"; -} - -function parseCursor(cursor: string | null): AzureCursor | null { - if (!cursor) return null; - try { - const parsed = JSON.parse(cursor); - if ( - typeof parsed?.usersDeltaLink === "string" && - typeof parsed?.groupsDeltaLink === "string" - ) { - return parsed; - } - } catch { - // fall through - } - logger.warn("Ignoring unreadable Azure directory sync cursor"); - return null; -} - -export class AzureDirectoryProvider implements DirectorySyncProvider { - constructor( - private readonly settings: { - tenantId: string; - /** Encrypted, as stored in `idpOidcConfig`. */ - encryptedClientId: string; - /** Encrypted, as stored in `idpOidcConfig`. */ - encryptedClientSecret: string; - } - ) {} - - /** Parses the tenant ID from an Entra authorize URL. */ - static tenantIdFromAuthUrl(authUrl: string): string | null { - return ( - authUrl.match( - /login\.microsoftonline\.com\/([^/]+)\/oauth2/ - )?.[1] ?? null - ); - } - - async fetch(ctx: DirectoryFetchContext): Promise { - const getIdentifier = IDENTIFIER_FROM_GRAPH[ctx.identifierPath]; - if (!getIdentifier) { - throw new Error( - `identifierPath "${ctx.identifierPath}" can't be resolved from Graph; use one of: ${Object.keys(IDENTIFIER_FROM_GRAPH).join(", ")}` - ); - } - - const graph = new GraphClient(() => this.getToken(), ctx.onRequest); - const toDirectoryUser = ( - u: GraphUser, - groups: DirectoryGroup[] - ): DirectoryUser => ({ - externalId: u.id, - username: getIdentifier(u)?.toLowerCase() ?? null, - email: u.mail?.toLowerCase() ?? null, - name: u.displayName ?? null, - enabled: u.accountEnabled !== false, - groups - }); - - const cursor = parseCursor(ctx.cursor); - if (cursor) { - const incremental = await this.fetchIncremental( - graph, - cursor, - toDirectoryUser - ); - if (incremental) return incremental; - logger.info( - `Azure incremental sync for IdP ${ctx.idpId} needs a full fetch` - ); - } - return this.fetchFull(graph, toDirectoryUser); - } - - private async getToken(): Promise { - const { tenantId, encryptedClientId, encryptedClientSecret } = - this.settings; - const key = config.getRawConfig().server.secret!; - const clientId = decrypt(encryptedClientId, key); - const cacheKey = `${tenantId}:${clientId}`; - - const cached = tokenCache.get(cacheKey); - if (cached && cached.expiresAt > Date.now() + 60_000) { - return cached.token; - } - - const res = await fetch( - `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, - { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded" - }, - body: new URLSearchParams({ - grant_type: "client_credentials", - client_id: clientId, - client_secret: decrypt(encryptedClientSecret, key), - scope: "https://graph.microsoft.com/.default" - }) - } - ); - const body: any = await res.json().catch(() => ({})); - if (!res.ok) { - throw new Error( - `Entra token request failed (${res.status}): ${body.error_description ?? body.error}` - ); - } - - tokenCache.set(cacheKey, { - token: body.access_token, - expiresAt: Date.now() + Number(body.expires_in ?? 3600) * 1000 - }); - return body.access_token; - } - - private async fetchFull( - graph: GraphClient, - toDirectoryUser: ( - u: GraphUser, - groups: DirectoryGroup[] - ) => DirectoryUser - ): Promise { - const usersRes = await graph.getAll( - `${GRAPH}/users/delta?$select=${USER_SELECT}` - ); - const groupsRes = await graph.getAll( - `${GRAPH}/groups/delta?$select=displayName,members` - ); - - // A large group can be split over several pages; merge its member chunks. - const groups = new Map(); - // member (user or group) ID -> IDs of groups it directly belongs to - const parentGroups = new Map>(); - for (const g of groupsRes.items) { - if (g["@removed"]) continue; - const existing = groups.get(g.id); - groups.set(g.id, { - id: g.id, - displayName: g.displayName ?? existing?.displayName ?? null - }); - for (const member of g["members@delta"] ?? []) { - if (member["@removed"]) continue; - if (!isUserRef(member) && !isGroupRef(member)) continue; - const parents = parentGroups.get(member.id) ?? new Set(); - parents.add(g.id); - parentGroups.set(member.id, parents); - } - } - - const transitiveGroups = (memberId: string): DirectoryGroup[] => { - const seen = new Set(); - const queue = [...(parentGroups.get(memberId) ?? [])]; - while (queue.length) { - const groupId = queue.pop()!; - if (seen.has(groupId)) continue; - seen.add(groupId); - queue.push(...(parentGroups.get(groupId) ?? [])); - } - return [...seen] - .map((id) => groups.get(id)) - .filter((g): g is DirectoryGroup => !!g); - }; - - const users = usersRes.items - .filter((u) => !u["@removed"]) - .map((u) => toDirectoryUser(u, transitiveGroups(u.id))); - - return { - kind: "full", - users, - nextCursor: this.cursorFrom(usersRes.deltaLink, groupsRes.deltaLink) - }; - } - - /** Returns null when a full fetch is required instead. */ - private async fetchIncremental( - graph: GraphClient, - cursor: AzureCursor, - toDirectoryUser: ( - u: GraphUser, - groups: DirectoryGroup[] - ) => DirectoryUser - ): Promise { - const usersRes = await graph.getAll(cursor.usersDeltaLink); - const groupsRes = await graph.getAll( - cursor.groupsDeltaLink - ); - - const removedExternalIds = new Set(); - const affectedUserIds = new Set(); - - for (const u of usersRes.items) { - if (u["@removed"]) { - removedExternalIds.add(u.id); - } else { - affectedUserIds.add(u.id); - } - } - - // Groups whose every (transitive) user member must be re-read. - const expandGroupIds = new Set(); - for (const g of groupsRes.items) { - if (g["@removed"]) { - // The former members of a deleted group aren't listed anywhere. - return null; - } - if (g.displayName !== undefined) { - // Possibly renamed: name-based role mappings may change for all members. - expandGroupIds.add(g.id); - } - for (const member of g["members@delta"] ?? []) { - if (isUserRef(member)) { - affectedUserIds.add(member.id); - } else if (isGroupRef(member)) { - expandGroupIds.add(member.id); - } - } - } - - for (const groupId of expandGroupIds) { - const members = await graph.getAll<{ id: string }>( - `${GRAPH}/groups/${groupId}/transitiveMembers/microsoft.graph.user?$select=id&$top=999` - ); - for (const m of members.items) affectedUserIds.add(m.id); - if (affectedUserIds.size > MAX_INCREMENTAL_AFFECTED_USERS) { - return null; - } - } - if (affectedUserIds.size > MAX_INCREMENTAL_AFFECTED_USERS) { - return null; - } - - const users: DirectoryUser[] = []; - for (const userId of affectedUserIds) { - if (removedExternalIds.has(userId)) continue; - const u = await graph.get( - `${GRAPH}/users/${userId}?$select=${USER_SELECT}` - ); - if (!u) { - removedExternalIds.add(userId); - continue; - } - const memberOf = await graph.getAll( - `${GRAPH}/users/${userId}/transitiveMemberOf/microsoft.graph.group?$select=id,displayName&$top=999` - ); - users.push( - toDirectoryUser( - u, - memberOf.items.map((g) => ({ - id: g.id, - displayName: g.displayName ?? null - })) - ) - ); - } - - return { - kind: "incremental", - users, - removedExternalIds: [...removedExternalIds], - nextCursor: this.cursorFrom(usersRes.deltaLink, groupsRes.deltaLink) - }; - } - - private cursorFrom( - usersDeltaLink: string | null, - groupsDeltaLink: string | null - ): string | null { - if (!usersDeltaLink || !groupsDeltaLink) return null; - const cursor: AzureCursor = { usersDeltaLink, groupsDeltaLink }; - return JSON.stringify(cursor); - } -} From 7e8c9e4fe371b0585d20836874346936834d0d5a Mon Sep 17 00:00:00 2001 From: Fred KISSIE Date: Thu, 29 Oct 2026 18:37:10 +0100 Subject: [PATCH 36/50] =?UTF-8?q?=F0=9F=9A=A7=20directory=20sync=20fields?= =?UTF-8?q?=20for=20google=20idp?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- messages/en-US.json | 10 + server/private/lib/directorySync/config.ts | 98 +++++++++ .../lib/directorySync/ensureValidIdp.ts | 8 + server/private/lib/directorySync/google.ts | 53 +++++ .../routers/orgIdp/createOrgOidcIdp.ts | 30 ++- server/private/routers/orgIdp/getOrgIdp.ts | 11 + .../routers/orgIdp/updateOrgOidcIdp.ts | 23 ++ server/routers/idp/getIdp.ts | 3 + server/routers/orgIdp/types.ts | 5 +- .../(private)/idp/[idpId]/general/page.tsx | 53 ++++- .../settings/(private)/idp/create/page.tsx | 30 ++- .../GoogleDirectorySyncConfigFields.tsx | 207 ++++++++++++++++++ src/lib/idpDirectorySyncConfig.ts | 69 ++++++ 13 files changed, 589 insertions(+), 11 deletions(-) create mode 100644 server/private/lib/directorySync/config.ts create mode 100644 src/components/GoogleDirectorySyncConfigFields.tsx create mode 100644 src/lib/idpDirectorySyncConfig.ts diff --git a/messages/en-US.json b/messages/en-US.json index 6a7f8fddc1..0ba717bd7b 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1182,6 +1182,16 @@ "idpDirectorySyncDescription": "Periodically sync users from the identity provider directory. Cannot be enabled together with auto provisioning.", "idpDirectorySyncDeletions": "Sync User Deletions", "idpDirectorySyncDeletionsDescription": "When enabled, users who are deleted from the directory or no longer match the role mapping are removed from this organization.", + "idpGoogleServiceAccountKey": "Service Account Key", + "idpGoogleServiceAccountKeyDescription": "JSON key of a service account with read access to the Admin SDK Directory API (users and groups).", + "idpGoogleServiceAccountKeyConfigured": "A key for {email} is stored. Leave empty to keep it.", + "idpGoogleServiceAccountKeyUpload": "Upload JSON Key", + "idpGoogleServiceAccountKeyRequired": "A service account key is required to enable directory sync", + "idpGoogleServiceAccountKeyInvalid": "Invalid service account key, paste the full JSON key file downloaded from Google Cloud", + "idpGoogleAdminEmail": "Admin Email (Optional)", + "idpGoogleAdminEmailDescription": "Workspace admin the service account impersonates through domain-wide delegation. Leave empty if the service account has an admin role assigned.", + "idpGoogleCustomerId": "Customer ID (Optional)", + "idpGoogleCustomerIdDescription": "Google Workspace customer ID (e.g. C01abc23d). Defaults to the customer of the admin when empty.", "directorySyncRoleMappingDescription": "Determine how roles are assigned to users synced from the directory.", "directorySyncRoleMappingModeFixedRoles": "Fixed Roles", "directorySyncRoleMappingModeGroupMapping": "Group Mapping", diff --git a/server/private/lib/directorySync/config.ts b/server/private/lib/directorySync/config.ts new file mode 100644 index 0000000000..7825a429f5 --- /dev/null +++ b/server/private/lib/directorySync/config.ts @@ -0,0 +1,98 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +import { decrypt, encrypt } from "@server/lib/crypto"; +import config from "@server/lib/config"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { + directorySyncConfigSchemas, + variantRequiresDirectorySyncConfig, + type DirectorySyncVariant +} from "@app/lib/idpDirectorySyncConfig"; + +export function decryptDirectorySyncConfig( + encrypted: string | null +): Record | null { + if (!encrypted) return null; + try { + return JSON.parse( + decrypt(encrypted, config.getRawConfig().server.secret!) + ); + } catch (e) { + logger.error("Directory sync: failed to decrypt stored config", e); + return null; + } +} + +type ResolveResult = + // `value` undefined = leave the stored column untouched + { success: true; value: string | undefined } | { success: false; error: string }; + +/** + * Validates the directory sync config sent for `variant` and returns the encrypted + * value to store. Secrets omitted from `incoming` (e.g. the Google service account + * key on update) are kept from the stored config. + */ +export function resolveDirectorySyncConfig({ + variant, + incoming, + existingEncrypted, + directorySyncEnabled +}: { + variant: DirectorySyncVariant; + incoming: Record | undefined; + existingEncrypted: string | null; + directorySyncEnabled: boolean; +}): ResolveResult { + if (!variantRequiresDirectorySyncConfig(variant)) { + return { success: true, value: undefined }; + } + + const existing = decryptDirectorySyncConfig(existingEncrypted); + const merged = incoming + ? { + ...incoming, + serviceAccountKey: + incoming.serviceAccountKey ?? existing?.serviceAccountKey + } + : existing; + + if (!merged?.serviceAccountKey) { + if (directorySyncEnabled) { + return { + success: false, + error: "A service account key is required to enable directory sync" + }; + } + // nothing worth storing yet + return { success: true, value: undefined }; + } + + if (!incoming) { + return { success: true, value: undefined }; + } + + const parsed = directorySyncConfigSchemas[variant].safeParse(merged); + if (!parsed.success) { + return { success: false, error: fromError(parsed.error).toString() }; + } + + return { + success: true, + value: encrypt( + JSON.stringify(parsed.data), + config.getRawConfig().server.secret! + ) + }; +} diff --git a/server/private/lib/directorySync/ensureValidIdp.ts b/server/private/lib/directorySync/ensureValidIdp.ts index 40aaeead8d..c3f0c6c7d9 100644 --- a/server/private/lib/directorySync/ensureValidIdp.ts +++ b/server/private/lib/directorySync/ensureValidIdp.ts @@ -36,6 +36,14 @@ export async function ensureValidIdp( `Directory sync: IdP ${idpId} has both directory sync and auto-provisioning enabled; only one can be active` ); } + + if ( + idpRes.idpOidcConfig.variant === "google" && + !idpRes.idp.directorySyncConfig + ) { + throw new Error(`Directory sync: Google IdP ${idpId} has no directory sync config; a service account is required`); + } + return idpRes; } diff --git a/server/private/lib/directorySync/google.ts b/server/private/lib/directorySync/google.ts index a6de54ac96..3ccbceee77 100644 --- a/server/private/lib/directorySync/google.ts +++ b/server/private/lib/directorySync/google.ts @@ -6,11 +6,64 @@ import type { } from "./types"; import { ensureValidIdp } from "./ensureValidIdp"; +import { createSign } from "node:crypto"; + +type ServiceAccountKey = { + client_email: string; + private_key: string; + token_uri: string; // https://oauth2.googleapis.com/token +}; + +const SCOPES = [ + "https://www.googleapis.com/auth/admin.directory.user.readonly", + "https://www.googleapis.com/auth/admin.directory.group.readonly", + "https://www.googleapis.com/auth/admin.directory.group.member.readonly" +].join(" "); + +function b64url(input: string | Buffer) { + return Buffer.from(input).toString("base64url"); +} + export class AzureDirectorySyncProvider implements DirectorySyncProvider { + #TOKEN_CACHE_KEY_PREFIX = "googleDirectorySyncToken"; + #googleAPIBaseUrl = "https://admin.googleapis.com/admin/directory/v1"; + async fetch(ctx: DirectoryFetchContext): Promise { const directoryUsers: Record = {}; // 0. fetch idp and check that it is valid const idpRes = await ensureValidIdp(ctx.idpId, "google"); throw new Error("Method not implemented."); } + + async #getGoogleAccessToken(key: ServiceAccountKey, adminEmail?: string) { + const now = Math.floor(Date.now() / 1000); + const header = b64url(JSON.stringify({ alg: "RS256", typ: "JWT" })); + const claims = b64url( + JSON.stringify({ + iss: key.client_email, + sub: adminEmail, // only with domain-wide delegation + scope: SCOPES, + aud: key.token_uri, + iat: now, + exp: now + 3600 // max 1h + }) + ); + const signature = createSign("RSA-SHA256") + .update(`${header}.${claims}`) + .sign(key.private_key, "base64url"); + + const res = await fetch(key.token_uri, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ + grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer", + assertion: `${header}.${claims}.${signature}` + }) + }); + if (!res.ok) throw new Error(`Google token error: ${await res.text()}`); + return (await res.json()) as { + access_token: string; + expires_in: number; + }; + } } diff --git a/server/private/routers/orgIdp/createOrgOidcIdp.ts b/server/private/routers/orgIdp/createOrgOidcIdp.ts index 0ad2093350..4865b3cf69 100644 --- a/server/private/routers/orgIdp/createOrgOidcIdp.ts +++ b/server/private/routers/orgIdp/createOrgOidcIdp.ts @@ -28,6 +28,7 @@ import { CreateOrgIdpResponse } from "@server/routers/orgIdp/types"; import { isSubscribed } from "#private/lib/isSubscribed"; import { tierMatrix } from "@server/lib/billing/tierMatrix"; import { build } from "@server/build"; +import { resolveDirectorySyncConfig } from "#private/lib/directorySync/config"; const paramsSchema = z.strictObject({ orgId: z.string().nonempty() }); @@ -48,6 +49,8 @@ const bodySchema = z.strictObject({ orgMapping: z.string().nullish(), directoryRoleMapping: z.string().nullish(), directorySyncDeletions: z.boolean().optional(), + // provider-specific, validated against the variant's schema + directorySyncConfig: z.record(z.string(), z.unknown()).optional(), tags: z.string().optional() }); @@ -128,6 +131,7 @@ export async function createOrgOidcIdp( directorySyncEnabled, directoryRoleMapping, directorySyncDeletions, + directorySyncConfig, tags } = parsedBody.data; @@ -153,6 +157,26 @@ export async function createOrgOidcIdp( ); } + // directory sync is only supported for google/azure idps + const effectiveDirectorySyncEnabled = + (variant === "google" || variant === "azure") && + !!directorySyncEnabled; + + const resolvedDirectorySyncConfig = resolveDirectorySyncConfig({ + variant, + incoming: directorySyncConfig, + existingEncrypted: null, + directorySyncEnabled: effectiveDirectorySyncEnabled + }); + if (!resolvedDirectorySyncConfig.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + resolvedDirectorySyncConfig.error + ) + ); + } + const key = config.getRawConfig().server.secret!; const encryptedSecret = encrypt(clientSecret, key); @@ -165,10 +189,8 @@ export async function createOrgOidcIdp( .values({ name, autoProvision, - // directory sync is only supported for google/azure idps - directorySyncEnabled: - (variant === "google" || variant === "azure") && - !!directorySyncEnabled, + directorySyncEnabled: effectiveDirectorySyncEnabled, + directorySyncConfig: resolvedDirectorySyncConfig.value, type: "oidc", tags }) diff --git a/server/private/routers/orgIdp/getOrgIdp.ts b/server/private/routers/orgIdp/getOrgIdp.ts index e22929d466..ea91798994 100644 --- a/server/private/routers/orgIdp/getOrgIdp.ts +++ b/server/private/routers/orgIdp/getOrgIdp.ts @@ -26,6 +26,8 @@ import config from "@server/lib/config"; import { decrypt } from "@server/lib/crypto"; import { generateOidcRedirectUrl } from "@server/lib/idp/generateRedirectUrl"; import { GetOrgIdpResponse } from "@server/routers/orgIdp/types"; +import { redactDirectorySyncConfig } from "@app/lib/idpDirectorySyncConfig"; +import { decryptDirectorySyncConfig } from "#private/lib/directorySync/config"; const paramsSchema = z .object({ @@ -113,9 +115,18 @@ export async function getOrgIdp( orgId ); + // never send the stored secrets (e.g. service account private key) back + const directorySyncConfig = idpRes.idpOidcConfig + ? redactDirectorySyncConfig( + idpRes.idpOidcConfig.variant, + decryptDirectorySyncConfig(idpRes.idp.directorySyncConfig) + ) + : null; + return response(res, { data: { ...idpRes, + idp: { ...idpRes.idp, directorySyncConfig }, redirectUrl }, success: true, diff --git a/server/private/routers/orgIdp/updateOrgOidcIdp.ts b/server/private/routers/orgIdp/updateOrgOidcIdp.ts index 13ea23311f..96968f20fd 100644 --- a/server/private/routers/orgIdp/updateOrgOidcIdp.ts +++ b/server/private/routers/orgIdp/updateOrgOidcIdp.ts @@ -28,6 +28,7 @@ import config from "@server/lib/config"; import { isSubscribed } from "#private/lib/isSubscribed"; import { tierMatrix } from "@server/lib/billing/tierMatrix"; import { build } from "@server/build"; +import { resolveDirectorySyncConfig } from "#private/lib/directorySync/config"; const paramsSchema = z .object({ @@ -52,6 +53,8 @@ const bodySchema = z.strictObject({ orgMapping: z.string().nullish(), directoryRoleMapping: z.string().nullish(), directorySyncDeletions: z.boolean().optional(), + // provider-specific, omitted secrets are kept from the stored config + directorySyncConfig: z.record(z.string(), z.unknown()).optional(), tags: z.string().optional() }); @@ -133,6 +136,7 @@ export async function updateOrgOidcIdp( directorySyncEnabled, directoryRoleMapping, directorySyncDeletions, + directorySyncConfig, tags } = parsedBody.data; @@ -216,11 +220,30 @@ export async function updateOrgOidcIdp( ? directorySyncEnabled : false; + const resolvedDirectorySyncConfig = resolveDirectorySyncConfig({ + variant: existingOidcConfig?.variant ?? "oidc", + incoming: directorySyncConfig, + existingEncrypted: existingIdp.directorySyncConfig, + directorySyncEnabled: !!( + effectiveDirectorySyncEnabled ?? + existingIdp.directorySyncEnabled + ) + }); + if (!resolvedDirectorySyncConfig.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + resolvedDirectorySyncConfig.error + ) + ); + } + await db.transaction(async (trx) => { const idpData = { name, autoProvision, directorySyncEnabled: effectiveDirectorySyncEnabled, + directorySyncConfig: resolvedDirectorySyncConfig.value, tags }; diff --git a/server/routers/idp/getIdp.ts b/server/routers/idp/getIdp.ts index 10aa71c8bc..48bcfb1dd0 100644 --- a/server/routers/idp/getIdp.ts +++ b/server/routers/idp/getIdp.ts @@ -90,6 +90,9 @@ export async function getIdp( idpRes.idpOidcConfig!.clientId = decrypt(clientId, key); } + // directory sync credentials are only managed through org IdPs + idpRes.idp.directorySyncConfig = null; + return response(res, { data: idpRes, success: true, diff --git a/server/routers/orgIdp/types.ts b/server/routers/orgIdp/types.ts index 40dbb2cf49..270e0b8630 100644 --- a/server/routers/orgIdp/types.ts +++ b/server/routers/orgIdp/types.ts @@ -1,4 +1,5 @@ import { Idp, IdpOidcConfig } from "@server/db"; +import type { RedactedDirectorySyncConfig } from "@app/lib/idpDirectorySyncConfig"; export type CreateOrgIdpResponse = { idpId: number; @@ -6,7 +7,9 @@ export type CreateOrgIdpResponse = { }; export type GetOrgIdpResponse = { - idp: Idp; + idp: Omit & { + directorySyncConfig: RedactedDirectorySyncConfig | null; + }; idpOidcConfig: IdpOidcConfig | null; redirectUrl: string; }; diff --git a/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx b/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx index 83bb963713..0747426e3e 100644 --- a/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx +++ b/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx @@ -3,6 +3,11 @@ import AutoProvisionConfigWidget from "@app/components/AutoProvisionConfigWidget"; import CopyToClipboard from "@app/components/CopyToClipboard"; import DirectorySyncConfigWidget from "@app/components/DirectorySyncConfigWidget"; +import GoogleDirectorySyncConfigFields, { + buildGoogleDirectorySyncConfig, + googleDirectorySyncFormFields, + refineGoogleDirectorySyncFields +} from "@app/components/GoogleDirectorySyncConfigFields"; import IdpAutoProvisionUsersDescription from "@app/components/IdpAutoProvisionUsersDescription"; import IdpIdentifierChangeDialog from "@app/components/IdpIdentifierChangeDialog"; import IdpTypeBadge from "@app/components/IdpTypeBadge"; @@ -83,6 +88,8 @@ export default function GeneralPage() { createDefaultDirectorySyncRoleMappingConfig ); const [variant, setVariant] = useState<"oidc" | "google" | "azure">("oidc"); + const [configuredServiceAccountEmail, setConfiguredServiceAccountEmail] = + useState(); const [originalIdentifierPath, setOriginalIdentifierPath] = useState(""); const [identifierConfirmOpen, setIdentifierConfirmOpen] = useState(false); const [pendingPayload, setPendingPayload] = useState + refineGoogleDirectorySyncFields(data, ctx, { + // the stored key is kept when the field is left empty + keyRequired: !configuredServiceAccountEmail, + t + }) + ); // Azure form schema (simplified with tenant ID) const AzureFormSchema = z.object({ @@ -184,7 +198,10 @@ export default function GeneralPage() { roleMapping: null, roleId: null, tenantId: "", - orgMapping: "" + orgMapping: "", + googleServiceAccountKey: "", + googleAdminEmail: "", + googleCustomerId: "" } }); @@ -253,6 +270,14 @@ export default function GeneralPage() { formData.scopes = data.idpOidcConfig.scopes; } else if (idpVariant === "azure") { formData.tenantId = tenantId; + } else if (idpVariant === "google") { + const syncConfig = data.idp.directorySyncConfig; + formData.googleServiceAccountKey = ""; + formData.googleAdminEmail = syncConfig?.adminEmail ?? ""; + formData.googleCustomerId = syncConfig?.customerId ?? ""; + setConfiguredServiceAccountEmail( + syncConfig?.serviceAccountEmail + ); } form.reset(formData); @@ -412,7 +437,10 @@ export default function GeneralPage() { identifierPath: "email", emailPath: "email", namePath: "name", - scopes: "openid profile email" + scopes: "openid profile email", + directorySyncConfig: buildGoogleDirectorySyncConfig( + data as GoogleFormValues + ) }; } @@ -449,6 +477,15 @@ export default function GeneralPage() { if (typeof payload.identifierPath === "string") { setOriginalIdentifierPath(payload.identifierPath); } + const syncConfig = payload.directorySyncConfig as + | ReturnType + | undefined; + if (syncConfig?.serviceAccountKey) { + setConfiguredServiceAccountEmail( + syncConfig.serviceAccountKey.client_email + ); + form.setValue("googleServiceAccountKey", ""); + } toast({ title: t("success"), description: t("idpUpdatedDescription") @@ -673,6 +710,14 @@ export default function GeneralPage() { setDirectoryRoleMapping } /> + {variant === "google" && + form.watch("directorySyncEnabled") && ( + + )} diff --git a/src/app/[orgId]/settings/(private)/idp/create/page.tsx b/src/app/[orgId]/settings/(private)/idp/create/page.tsx index e72551ae49..0ab197fe07 100644 --- a/src/app/[orgId]/settings/(private)/idp/create/page.tsx +++ b/src/app/[orgId]/settings/(private)/idp/create/page.tsx @@ -2,6 +2,11 @@ import AutoProvisionConfigWidget from "@app/components/AutoProvisionConfigWidget"; import DirectorySyncConfigWidget from "@app/components/DirectorySyncConfigWidget"; +import GoogleDirectorySyncConfigFields, { + buildGoogleDirectorySyncConfig, + googleDirectorySyncFormFields, + refineGoogleDirectorySyncFields +} from "@app/components/GoogleDirectorySyncConfigFields"; import IdpAutoProvisionUsersDescription from "@app/components/IdpAutoProvisionUsersDescription"; import { PaidFeaturesAlert } from "@app/components/PaidFeaturesAlert"; import { @@ -104,7 +109,15 @@ export default function Page() { roleId: z.number().nullable().optional(), orgMapping: z.string().optional(), directorySyncEnabled: z.boolean().default(false), - directorySyncDeletions: z.boolean().default(true) + directorySyncDeletions: z.boolean().default(true), + ...googleDirectorySyncFormFields + }).superRefine((data, ctx) => { + if (data.type === "google") { + refineGoogleDirectorySyncFields(data, ctx, { + keyRequired: true, + t + }); + } }); type CreateIdpFormValues = z.infer; @@ -128,7 +141,10 @@ export default function Page() { roleId: null, orgMapping: "", directorySyncEnabled: false, - directorySyncDeletions: true + directorySyncDeletions: true, + googleServiceAccountKey: "", + googleAdminEmail: "", + googleCustomerId: "" } }); @@ -213,6 +229,10 @@ export default function Page() { scopes: data.scopes, variant: data.type }; + if (data.type === "google") { + payload.directorySyncConfig = + buildGoogleDirectorySyncConfig(data); + } const trimmedOrgMapping = data.orgMapping?.trim(); if (trimmedOrgMapping) { payload.orgMapping = trimmedOrgMapping; @@ -452,6 +472,12 @@ export default function Page() { ) } /> + {form.watch("type") === "google" && + form.watch( + "directorySyncEnabled" + ) && ( + + )} diff --git a/src/components/GoogleDirectorySyncConfigFields.tsx b/src/components/GoogleDirectorySyncConfigFields.tsx new file mode 100644 index 0000000000..e3eb2f264f --- /dev/null +++ b/src/components/GoogleDirectorySyncConfigFields.tsx @@ -0,0 +1,207 @@ +"use client"; + +import { Button } from "@app/components/ui/button"; +import { + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage +} from "@app/components/ui/form"; +import { Input } from "@app/components/ui/input"; +import { Textarea } from "@app/components/ui/textarea"; +import { parseGoogleServiceAccountKey } from "@app/lib/idpDirectorySyncConfig"; +import { useTranslations } from "next-intl"; +import { useRef } from "react"; +import { useFormContext } from "react-hook-form"; +import { z, type email } from "zod"; + +export type GoogleDirectorySyncFormValues = { + googleServiceAccountKey?: string; + googleAdminEmail?: string; + googleCustomerId?: string; +}; + +export const googleDirectorySyncFormFields = { + googleServiceAccountKey: z.string().optional(), + googleAdminEmail: z.string().optional(), + googleCustomerId: z.string().optional() +}; + +/** `superRefine` body for forms including `googleDirectorySyncFormFields`. */ +export function refineGoogleDirectorySyncFields( + data: GoogleDirectorySyncFormValues & { directorySyncEnabled?: boolean }, + ctx: z.RefinementCtx, + { + keyRequired, + t + }: { keyRequired: boolean; t: ReturnType } +) { + const rawKey = data.googleServiceAccountKey?.trim(); + if (rawKey) { + if (!parseGoogleServiceAccountKey(rawKey)) { + ctx.addIssue({ + code: "custom", + path: ["googleServiceAccountKey"], + message: t("idpGoogleServiceAccountKeyInvalid") + }); + } + } else if (data.directorySyncEnabled && keyRequired) { + ctx.addIssue({ + code: "custom", + path: ["googleServiceAccountKey"], + message: t("idpGoogleServiceAccountKeyRequired") + }); + } + + const adminEmail = data.googleAdminEmail?.trim(); + if (adminEmail && !z.email().safeParse(adminEmail).success) { + ctx.addIssue({ + code: "custom", + path: ["googleAdminEmail"], + message: t("emailInvalid") + }); + } +} + +/** API `directorySyncConfig` payload, an omitted key keeps the stored one on update. */ +export function buildGoogleDirectorySyncConfig( + data: GoogleDirectorySyncFormValues +) { + const rawKey = data.googleServiceAccountKey?.trim(); + return { + serviceAccountKey: rawKey + ? (parseGoogleServiceAccountKey(rawKey) ?? undefined) + : undefined, + adminEmail: data.googleAdminEmail?.trim() || undefined, + customerId: data.googleCustomerId?.trim() || undefined + }; +} + +type GoogleDirectorySyncConfigFieldsProps = { + /** Service account of the stored key, the key field becomes optional when set. */ + configuredServiceAccountEmail?: string; +}; + +// Must be rendered inside the `
` of a form with `GoogleDirectorySyncFormValues` fields. +export default function GoogleDirectorySyncConfigFields({ + configuredServiceAccountEmail +}: GoogleDirectorySyncConfigFieldsProps) { + const t = useTranslations(); + const form = useFormContext(); + const fileInputRef = useRef(null); + + return ( +
+ ( + +
+ + {t("idpGoogleServiceAccountKey")} + + + { + const file = e.target.files?.[0]; + e.target.value = ""; + if (!file) return; + form.setValue( + "googleServiceAccountKey", + await file.text(), + { shouldValidate: true } + ); + }} + /> +
+ + {configuredServiceAccountEmail && ( + + {t.rich( + "idpGoogleServiceAccountKeyConfigured", + { + email: configuredServiceAccountEmail, + em: (children) => ( + + {children} + + ) + } + )} + + )} + + +