diff --git a/DIRECTORY_SYNC_PLAN.md b/DIRECTORY_SYNC_PLAN.md new file mode 100644 index 0000000000..41540e1fee --- /dev/null +++ b/DIRECTORY_SYNC_PLAN.md @@ -0,0 +1,235 @@ +# Plan: IdP directory sync (Microsoft Entra ID and Google Workspace) + +## What already exists + +- `idp.directorySyncEnabled` exists. It is mutually exclusive with `autoProvision` (see [decision #2](#2-sync-replaces-auto-provisioning-and-owns-org-and-role-mapping)) and only allowed when the variant is `google` or `azure`. + - Global IdPs: [createOidcIdp.ts](server/routers/idp/createOidcIdp.ts), [updateOidcIdp.ts](server/routers/idp/updateOidcIdp.ts). + - Org IdPs: [createOrgOidcIdp.ts](server/private/routers/orgIdp/createOrgOidcIdp.ts), [updateOrgOidcIdp.ts](server/private/routers/orgIdp/updateOrgOidcIdp.ts). + - Create and update return `400` if both would be on. In the UI, only the org IdP pages expose the toggle (exclusive with auto provisioning). The global admin IdP pages don't show it for now. +- A first Azure sync prototype exists in [directorySync/azure.ts](server/private/lib/directorySync/azure.ts). It refuses to run when `autoProvision` is on. [idpDirectorySync.ts](server/routers/idp/idpDirectorySync.ts) is empty. +- Provisioning only happens at login, in [validateOidcCallback.ts](server/routers/idp/validateOidcCallback.ts): + - `users.username` = JMESPath(`identifierPath`) applied to the ID token claims. + - Org and role mappings are JMESPath expressions over those same claims. + - It only ever removes `userOrgs` rows where `autoProvisioned = true`. +- The Azure tenant ID isn't stored as its own field. The UI re-parses it from `authUrl` ([general/page.tsx](src/app/admin/idp/[idpId]/general/page.tsx)). +- Building blocks for a scheduler: + - [server/startSchedulers.ts](server/startSchedulers.ts) and [server/private/startSchedulers.ts](server/private/startSchedulers.ts) start the existing background jobs. + - [LockManager](server/private/lib/lock.ts) provides a Redis-backed lock that falls back to a local lock, so jobs work with multiple instances. + +## Goals and non-goals + +**Goals** + +- Create and update users in Pangolin without waiting for them to log in. +- Deactivating or removing users who left the directory is **not confirmed yet** (see [Deprovisioning](#4-deprovisioning-not-confirmed)). +- Keep org and role assignments current from group membership. +- Map directory groups to org roles with an explicit sync-only mapping. + +**Non-goals (v1)** + +- SCIM push. +- Webhooks or push notifications. +- Generic OIDC providers. +- Two-way sync. + +## Provider study + +| | Microsoft Entra (done) | Google Workspace (to study) | +| ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| API | Microsoft Graph | Admin SDK Directory API (`admin.googleapis.com/admin/directory/v1`) | +| Users | `GET /users`, 999 per page | `users.list?customer=my_customer`, 500 per page | +| Groups / members | `/groups`, `/groups/{id}/members` or `/transitiveMembers` | `groups.list` (200 per page), `members.list?includeDerivedMembership=true` | +| Incremental | ✅ `/users/delta` and `/groups/delta` (with `members@delta`) | ❌ No delta API: full snapshot every run, then diff on our side | +| App-only auth | Client credentials using the **existing** clientId and secret, plus tenant | **Service account + domain-wide delegation** impersonating an admin, so **new credentials**: SA JSON key and admin email | +| Permissions | `User.Read.All` and `GroupMember.Read.All` (application), plus **admin consent** | Scopes `admin.directory.user.readonly`, `admin.directory.group.readonly` and `admin.directory.group.member.readonly`, authorized in the Admin console under **Domain-wide delegation** | +| Throttling | Per app per tenant, `429` + `Retry-After` | Per project quota (believed to be about 2,400 queries/min, to verify), `403 rateLimitExceeded` / `429`, exponential backoff | +| Disabled users | `accountEnabled = false` | `suspended`, `archived` | +| Stable user ID | `id` (= `oid` claim) | `id` (= `sub` claim) | + +### Request counts per sync (Entra) + +With N users, G groups, and up to 999 items per page: + +- Full sync (per group): `ceil(N/999) + ceil(G/999) + Σ ceil(members_g / 999)` ≈ `N/999 + G`. For example, 5,000 users and 200 groups ≈ 206 requests. +- Per user (`/users/{id}/memberOf` for every user): `ceil(N/999) + N`. Avoid this approach. +- Delta sync with no changes: 2 requests (users delta + groups delta). +- `$expand=memberOf` cuts off at about 20 groups per user. Don't rely on it. + +### Google items to verify + +1. That the Google ID token's `sub` really equals the Directory `user.id`. This is believed true, but the matching logic depends on it. +2. Whether to support a second auth option: admin OAuth consent with an offline refresh token and the extra directory scopes on the existing OAuth client. Setup is easier, but sync stops working if that admin leaves. +3. Real quota numbers and how errors look. The Directory API often returns `403` with `reason: rateLimitExceeded` rather than `429`. +4. Group identifiers: group email vs `id`, and which one is friendlier in role mappings. +5. Whether `includeDerivedMembership` is enough for nested groups, or whether we need the Cloud Identity `searchTransitiveGroups` API. +6. The library choice: `google-auth-library`, or signing the JWT-bearer assertion ourselves. Neither `jose` nor any Google or Azure SDK is a dependency today. + +## How other products do it + +- **NetBird:** syncs users and groups. Synced groups play the part that roles play in Pangolin: access controls are set per group. +- **defguard:** works the same way. +- Neither relies much on the IdP's own roles (Entra app roles, Google admin roles). Groups are what gets synced and used. + +**What this means for Pangolin:** + +- **Org IdPs** (one IdP per org): an IdP group can map one-to-one to a Pangolin role in that org, like NetBird and defguard do. +- **Global IdPs** (one IdP shared by several orgs): it's harder. A group has to resolve to an org *and* a role, so a direct group → role mapping isn't enough. Either keep the existing JMESPath org and role mappings, or add a mapping that is scoped per org. + +## Key design decisions + +### 1. Matching synced users to users who log in (biggest risk) + +Synced users must get the same `username` that login computes with `identifierPath`. + +- **Azure:** the default `identifierPath` is `sub`. Entra's `sub` is **unique per app and is not the Graph user ID**, so a synced user can't be linked to a logged-in one. Options: + - require `oid` as the identifier when sync is on, or + - map Graph `id` to `oid` and refuse to enable sync if `identifierPath` isn't `oid`, or + - fall back to matching by email. + + Existing Azure IdPs that use `sub` need a migration story. +- **Google:** `sub` = Directory `id`, so the default works (pending verification item 1). + +### 2. Sync replaces auto provisioning and owns org and role mapping + +An IdP uses **either** auto provisioning (claim-based JMESPath mappings, applied at login) **or** directory sync (group mapping, applied by the sync job), never both. The two use different role mappings, so running both would make them fight over the same memberships. + +Directory API responses (Graph `user`, Directory `user`) don't look like ID token claims. Faking claims only matches login for a few fields (`oid`, `name`, `upn`, `groups`), and silently diverges for the rest (`sub` is per-app in Entra, `email` is optional, `roles` and custom claims don't exist in Graph). So sync doesn't run the login org/role mappings. + +- **Identifier:** fixed rule, not configurable. It must produce the same `users.username` login computes, so sync only runs when `identifierPath` resolves from directory data. Azure: `oid` → Graph `id`, `preferred_username`/`upn` → `userPrincipalName`; `sub` is refused. Google: `sub` → Directory `id`. +- **Email / name:** read directly from directory fields (Azure `mail` / `displayName`). +- **Orgs and roles:** an explicit table `idpGroupMapping (idpId, externalGroupId, orgId, roleId)`. A user gets every role mapped to a group they're a (transitive) member of; an org membership exists wherever they have at least one role. Without full RBAC, only the first mapped role per org is kept. +- Only mapped groups are fetched, which also bounds the request count. +- The admin UI can list directory groups so admins pick from a list instead of pasting IDs. Same model works for Google, where tokens carry no groups at all. + +**Login when sync is on.** Since `autoProvision` is off, login takes the non-provisioning branch of [validateOidcCallback.ts](server/routers/idp/validateOidcCallback.ts): + +- It only authenticates. It doesn't create users or touch orgs, roles or profile fields. +- A user who hasn't been synced yet, or has no org membership, is refused with "unprovisioned". Admins can run "Sync now" to fix this. +- Sync is the only writer for users, memberships and roles of that IdP. That includes email and name updates, since login no longer updates them. + +### 3. Shared write path (optional refactor) + +Login and sync no longer run on the same IdP, so sharing code isn't needed for correctness. It's still worth pulling the "apply" half of the provisioning block out of `validateOidcCallback.ts` into a shared function, for example `applyUserMemberships(trx, idp, user, desired: { orgId, roleIds }[])`: upsert the user, add orgs, re-sync roles on `autoProvisioned` orgs. Login computes `desired` from claims, sync from `idpGroupMapping`. Sync writes memberships with `autoProvisioned = true`, so the existing rule that only those rows are ever changed or removed still holds. + +### 4. Deprovisioning (not confirmed) + +> ⚠️ Removing or deactivating users during sync has **not been approved**. It's risky: a bad credential, a wrong group filter, a partial API response or a bug in the diff could lock a whole org out at once. Until a decision is made, sync **must not delete, disable or remove anything**. + +**Until a decision is made (v1 default):** + +- Sync only creates and updates users, memberships and roles. +- Users who are missing from the directory, disabled or suspended are only **detected and reported**: logged, counted in the sync run, and possibly listed in the UI as "would be removed". Nothing changes for them in Pangolin. + +**If deprovisioning is approved later, it needs safeguards:** + +- An explicit per-IdP opt-in, off by default. +- It only ever touches `autoProvisioned` memberships. Manually added memberships (`autoProvisioned = false`) are never touched, which matches what login does today. +- Prefer disabling over deleting, with a grace period, for example only acting after a user has been missing for N consecutive syncs. +- A circuit breaker: abort the run if more than X% of the IdP's users, or more than N users, would be removed. +- Never deprovision after an incomplete or failed fetch. That covers a page error mid-run, a throttled run, or an empty response. +- A dry-run mode that shows what would be removed before turning it on. + +**Questions to answer before approving:** + +- Delete the `users` row, disable it, or only remove `autoProvisioned` org memberships? +- Revoke sessions right away? +- What counts as "gone": deleted only, or also disabled/suspended/archived, or also removed from the synced groups? + +### 5. Sync scope + +Add an optional filter for which groups to sync, or only sync users assigned to the app (Azure: `appRoleAssignedTo`). This keeps request counts down for large tenants. + +## Data model (pg and sqlite schemas, plus a migration) + +- **`idpDirectorySyncConfig`**: provider-specific credentials and options. + - Google: SA client email, private key (encrypted), admin email to impersonate, customer ID. + - Both: group filter. + - Azure: an explicit `tenantId` column, instead of parsing `authUrl`. +- **`idpGroupMapping`**: `idpId`, `externalGroupId`, `orgId`, `roleId`. Sync-only org/role mapping (see decision #2). +- **`idpSyncState`**: one row per IdP. Columns: `usersDeltaLink`, `groupsDeltaLink`, `status`, `lastSyncStartedAt`, `lastSyncSucceededAt`, `lastError`, `consecutiveFailures`, `nextSyncAt`. +- **`idpSyncRun`**: optional run history with counts, request count and error. Old rows are pruned. + +Individual Graph or Directory requests and retries are logged, not stored in the DB. Access tokens are cached in memory (or Redis), never stored in the DB. + +## Sync engine + +``` +server/(private/)lib/directorySync/ + providers/azure.ts // token, paging, delta, 429 handling + providers/google.ts // JWT-bearer token, paging, full snapshot + types.ts // DirectoryProvider interface + runSync.ts // group mapping resolution, diff, apply + scheduler.ts +``` + +- **Provider interface.** Each provider returns either: + - `{ kind: "delta", changedUsers, removedUserIds, membershipChanges, nextState }` (Azure), or + - `{ kind: "snapshot", users, groups, memberships }` (Google, and Azure's first run). + + For a snapshot, the core diffs against the users Pangolin already has for that IdP. Users who are missing, plus Azure `@removed` entries, are only reported for now; see [Deprovisioning](#4-deprovisioning-not-confirmed). +- **Scheduler.** + - A tick runs every minute and selects IdPs where `directorySyncEnabled` is set and `nextSyncAt <= now`. + - It takes the `idp-sync:{idpId}` lock, caps how many IdPs sync at once, and staggers `nextSyncAt`. + - Default interval: 15 to 60 minutes, possibly configurable per IdP. +- **Failure handling.** + - A short `Retry-After` → sleep in memory and retry the same page. + - A long `Retry-After`, or repeated failures → stop the run, set `nextSyncAt`, and don't save delta links. The backoff is `nextSyncAt = now + min(interval * 2^consecutiveFailures, 24h)`, reset on success. + - `410` / `syncStateNotFound` → clear the delta links and do a full sync. +- Save the new delta links only after the whole run succeeds. +- Access tokens are cached in memory until shortly before they expire. + +## API and UI + +- `POST /idp/:idpId/sync`: "Sync now". +- `GET /idp/:idpId/sync-status`: last sync time, status, error, counts. +- The IdP settings page gets: + - Google service account fields. + - Azure tenant ID as a stored field. + - A sync status card. + - Setup instructions: Azure application permissions plus admin consent; Google DWD scopes. +- Clear errors for the usual failures: `403` → "admin consent missing" or "DWD not authorized"; `401` → "bad secret or key". + +## Phases + +1. **Study:** finish the Google verification items above. Confirm `oid` vs `sub` behavior on a test Entra tenant. +2. **Refactor (optional):** extract `applyUserMemberships` from the OIDC callback, with no change in behavior. Add tests. +3. **Schema:** add the sync config and state tables and migrations, and store Azure `tenantId` explicitly. +4. **Azure provider:** full sync, then delta. +5. **Scheduler:** locking, backoff, status API, "Sync now". +6. **Google provider:** snapshot plus diff. +7. **UI and docs:** credentials, status card, setup guides. +8. **Deprovisioning, only if approved:** opt-in setting, dry run, grace period, circuit breaker. + +## Open questions + +- Is this an enterprise/licensed feature? If so, the engine belongs in `server/private`. The flag currently sits on the shared `idp` table and routes. +- **Blocking for deprovisioning:** should sync remove or disable users at all? See the questions in [Deprovisioning](#4-deprovisioning-not-confirmed). +- How to handle existing Azure IdPs with `identifierPath = "sub"`: block sync, migrate them, or match by email? +- Which group identifiers to expose in mappings for Google: emails or IDs? +- Org IdPs ([server/private/routers/orgIdp](server/private/routers/orgIdp)) can already store the flag. For them, should `idpGroupMapping` be limited to the IdP's own org? Org IdPs fit the group = role model most easily (see [How other products do it](#how-other-products-do-it)). +- For global IdPs, how should a synced group resolve to an org and a role? Keep the JMESPath mappings, or add per-org group → role mappings? +- Global sync interval, or configurable per IdP? + +## References + +**Microsoft Graph / Entra** + +- List users: https://learn.microsoft.com/en-us/graph/api/user-list +- User delta: https://learn.microsoft.com/en-us/graph/api/user-delta +- Delta query overview: https://learn.microsoft.com/en-us/graph/delta-query-overview +- Group delta: https://learn.microsoft.com/en-us/graph/api/group-delta +- Group members: https://learn.microsoft.com/en-us/graph/api/group-list-members +- Transitive members: https://learn.microsoft.com/en-us/graph/api/group-list-transitivemembers +- App role assignments: https://learn.microsoft.com/en-us/graph/api/serviceprincipal-list-approleassignedto +- Client credentials flow: https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow +- Permissions reference: https://learn.microsoft.com/en-us/graph/permissions-reference +- Admin consent: https://learn.microsoft.com/en-us/entra/identity/enterprise-apps/grant-admin-consent +- Paging: https://learn.microsoft.com/en-us/graph/paging +- Throttling: https://learn.microsoft.com/en-us/graph/throttling +- Throttling limits per service: https://learn.microsoft.com/en-us/graph/throttling-limits +- National clouds: https://learn.microsoft.com/en-us/graph/deployments + +**Google Workspace (to fill in during the study)** + +- Admin SDK Directory API: https://developers.google.com/admin-sdk/directory +- Domain-wide delegation: https://developers.google.com/workspace/guides/create-credentials#optional_set_up_domain-wide_delegation_for_a_service_account diff --git a/messages/en-US.json b/messages/en-US.json index fc15236870..6a7f8fddc1 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -591,7 +591,7 @@ "licenseReckeckAll": "Recheck All Keys", "licenseSiteUsage": "Sites Usage", "licenseSiteUsageDecsription": "View the number of sites using this license.", - "licenseNoSiteLimit": "There is no limit on the number of sites using an unlicensed host.", + "licenseNoSiteLimit": "There is no limit on the number of sites for this host.", "licensePurchase": "Purchase License", "licensePurchaseSites": "Purchase Additional Sites", "licenseSitesUsedMax": "{usedSites} of {maxSites} sites used", @@ -600,7 +600,7 @@ "licenseUsageDescription": "View the number of users and sites licensed for this host", "licenseUsageSites": "Sites", "licenseUsageUsers": "Users", - "licenseNoUserLimit": "There is no limit on the number of users using an unlicensed hot", + "licenseNoUserLimit": "There is no limit on the number of users for this host.", "licenseUsersUsedMax": "{usedUsers} of {maxUsers} users used", "licenseUsersUsed": "{count, plural, =0 {# users} one {# user} other {# users}}", "licenseUnlimited": "Unlimited", @@ -1178,6 +1178,25 @@ "idpAutoProvisionUsers": "Auto Provision Users", "idpAutoProvisionUsersDescription": "When enabled, users will be automatically created in the system upon first login with the ability to map users to roles and organizations.", "idpAutoProvisionConfigureAfterCreate": "You can configure auto provision settings once the identity provider is created.", + "idpDirectorySync": "Directory Sync", + "idpDirectorySyncDescription": "Periodically sync users from the identity provider directory. Cannot be enabled together with auto provisioning.", + "idpDirectorySyncDeletions": "Sync User Deletions", + "idpDirectorySyncDeletionsDescription": "When enabled, users who are deleted from the directory or no longer match the role mapping are removed from this organization.", + "directorySyncRoleMappingDescription": "Determine how roles are assigned to users synced from the directory.", + "directorySyncRoleMappingModeFixedRoles": "Fixed Roles", + "directorySyncRoleMappingModeGroupMapping": "Group Mapping", + "directorySyncRoleMappingModeAutomatic": "Automatic", + "directorySyncRoleMappingFixedRolesDescription": "Assign the same role set to every synced user.", + "directorySyncRoleMappingGroup": "Directory Group", + "directorySyncRoleMappingGroupPlaceholder": "Group name or ID", + "directorySyncRoleMappingAddGroupRule": "Add Group Mapping", + "directorySyncRoleMappingGroupMappingDescription": "Members of a directory group receive the roles mapped to that group.", + "directorySyncRoleMappingAutomaticDescription": "Each directory group a user belongs to is mapped to the role with the same name.", + "directorySyncRoleMappingCreateMissingRoles": "Create missing roles", + "directorySyncRoleMappingCreateMissingRolesDescription": "When enabled, a role is created for any directory group that doesn't have a matching role yet. Otherwise those groups are skipped.", + "directorySyncRoleMappingGroupMappingSkipNotice": "Only users who belong to at least one of the groups mapped below are synced. Users without a mapped group are not synced.", + "directorySyncRoleMappingAutomaticSkipNotice": "Users who don't belong to any directory group are not synced.", + "directorySyncRoleMappingAutomaticNoCreateSkipNotice": "Users who don't belong to any directory group, or whose groups have no role with the same name, are not synced.", "licenseBadge": "EE", "idpType": "Provider Type", "idpTypeDescription": "Select the type of identity provider you want to configure", diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index a4210ab8e8..0008934f58 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -654,28 +654,40 @@ export const userSiteResources = pgTable( ] ); -export const users = pgTable("user", { - userId: varchar("id").primaryKey(), - email: varchar("email"), - username: varchar("username").notNull(), - name: varchar("name"), - type: varchar("type").notNull(), // "internal", "oidc" - idpId: integer("idpId").references(() => idp.idpId, { - onDelete: "cascade" - }), - passwordHash: varchar("passwordHash"), - twoFactorEnabled: boolean("twoFactorEnabled").notNull().default(false), - twoFactorSetupRequested: boolean("twoFactorSetupRequested").default(false), - twoFactorSecret: varchar("twoFactorSecret"), - emailVerified: boolean("emailVerified").notNull().default(false), - dateCreated: varchar("dateCreated").notNull(), - termsAcceptedTimestamp: varchar("termsAcceptedTimestamp"), - termsVersion: varchar("termsVersion"), - marketingEmailConsent: boolean("marketingEmailConsent").default(false), - serverAdmin: boolean("serverAdmin").notNull().default(false), - lastPasswordChange: bigint("lastPasswordChange", { mode: "number" }), - locale: varchar("locale") -}); +export const users = pgTable( + "user", + { + userId: varchar("id").primaryKey(), + email: varchar("email"), + username: varchar("username").notNull(), + name: varchar("name"), + type: varchar("type").notNull(), // "internal", "oidc" + idpId: integer("idpId").references(() => idp.idpId, { + onDelete: "cascade" + }), + externalUserId: varchar("externalUserId"), // user's ID in the external IdP (e.g. Azure object id) + passwordHash: varchar("passwordHash"), + twoFactorEnabled: boolean("twoFactorEnabled").notNull().default(false), + twoFactorSetupRequested: boolean("twoFactorSetupRequested").default( + false + ), + twoFactorSecret: varchar("twoFactorSecret"), + emailVerified: boolean("emailVerified").notNull().default(false), + dateCreated: varchar("dateCreated").notNull(), + termsAcceptedTimestamp: varchar("termsAcceptedTimestamp"), + termsVersion: varchar("termsVersion"), + marketingEmailConsent: boolean("marketingEmailConsent").default(false), + serverAdmin: boolean("serverAdmin").notNull().default(false), + lastPasswordChange: bigint("lastPasswordChange", { mode: "number" }), + locale: varchar("locale") + }, + (t) => [ + uniqueIndex("idx_user_idpId_externalUserId").on( + t.idpId, + t.externalUserId + ) + ] +); export const newts = pgTable( "newt", @@ -736,11 +748,13 @@ export const userOrgs = pgTable( .notNull(), isOwner: boolean("isOwner").notNull().default(false), autoProvisioned: boolean("autoProvisioned").default(false), + directorySynched: boolean("directorySynched").default(false), pamUsername: varchar("pamUsername") // cleaned username for ssh and such }, (t) => [ index("idx_userOrgs_userid").on(t.userId), - index("idx_userOrgs_orgid").on(t.orgId) + index("idx_userOrgs_orgid").on(t.orgId), + unique().on(t.userId, t.orgId) ] ); @@ -1209,11 +1223,14 @@ export const supporterKey = pgTable("supporterKey", { export const idp = pgTable("idp", { idpId: serial("idpId").primaryKey(), name: varchar("name").notNull(), - type: varchar("type").notNull(), + type: varchar("type").notNull().$type<"oidc" | "google" | "azure">(), defaultRoleMapping: varchar("defaultRoleMapping"), defaultOrgMapping: varchar("defaultOrgMapping"), autoProvision: boolean("autoProvision").notNull().default(false), - tags: text("tags") + tags: text("tags"), + directorySyncEnabled: boolean("directorySyncEnabled") + .notNull() + .default(false) }); export const idpOidcConfig = pgTable("idpOidcConfig", { @@ -1221,7 +1238,10 @@ export const idpOidcConfig = pgTable("idpOidcConfig", { idpId: integer("idpId") .notNull() .references(() => idp.idpId, { onDelete: "cascade" }), - variant: varchar("variant").notNull().default("oidc"), + variant: varchar("variant") + .notNull() + .$type<"oidc" | "google" | "azure">() + .default("oidc"), clientId: varchar("clientId").notNull(), clientSecret: varchar("clientSecret").notNull(), authUrl: varchar("authUrl").notNull(), @@ -1326,9 +1346,42 @@ export const idpOrg = pgTable("idpOrg", { .notNull() .references(() => orgs.orgId, { onDelete: "cascade" }), roleMapping: varchar("roleMapping"), - orgMapping: varchar("orgMapping") + orgMapping: varchar("orgMapping"), + directoryRoleMapping: varchar("directoryRoleMapping"), + directorySyncDeletions: boolean("directorySyncDeletions") + .notNull() + .default(true) +}); + +// Runtime state of the directory sync job, one row per IdP. Deleting the row +// forces a full sync on the next run. +export const idpSyncState = pgTable("idpSyncState", { + idpId: integer("idpId") + .primaryKey() + .references(() => idp.idpId, { onDelete: "cascade" }), + // Provider-defined opaque JSON (e.g. Azure delta links). Null = full sync. + cursor: text("cursor"), + lastSyncStartedAt: bigint("lastSyncStartedAt", { mode: "number" }), + lastSyncSucceededAt: bigint("lastSyncSucceededAt", { mode: "number" }), + lastError: text("lastError"), + consecutiveFailures: integer("consecutiveFailures").notNull().default(0) }); +// Links a directory user (stable provider ID) to the Pangolin user sync manages. +export const idpSyncUser = pgTable( + "idpSyncUser", + { + idpId: integer("idpId") + .notNull() + .references(() => idp.idpId, { onDelete: "cascade" }), + externalId: varchar("externalId").notNull(), + userId: varchar("userId") + .notNull() + .references(() => users.userId, { onDelete: "cascade" }) + }, + (t) => [primaryKey({ columns: [t.idpId, t.externalId] })] +); + export const clients = pgTable( "clients", { diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index a2c2339aa7..a290e4f913 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -632,38 +632,48 @@ export const userSiteResources = sqliteTable("userSiteResources", { .references(() => siteResources.siteResourceId, { onDelete: "cascade" }) }); -export const users = sqliteTable("user", { - userId: text("id").primaryKey(), - email: text("email"), - username: text("username").notNull(), - name: text("name"), - type: text("type").notNull(), // "internal", "oidc" - idpId: integer("idpId").references(() => idp.idpId, { - onDelete: "cascade" - }), - passwordHash: text("passwordHash"), - twoFactorEnabled: integer("twoFactorEnabled", { mode: "boolean" }) - .notNull() - .default(false), - twoFactorSetupRequested: integer("twoFactorSetupRequested", { - mode: "boolean" - }).default(false), - twoFactorSecret: text("twoFactorSecret"), - emailVerified: integer("emailVerified", { mode: "boolean" }) - .notNull() - .default(false), - dateCreated: text("dateCreated").notNull(), - termsAcceptedTimestamp: text("termsAcceptedTimestamp"), - termsVersion: text("termsVersion"), - marketingEmailConsent: integer("marketingEmailConsent", { - mode: "boolean" - }).default(false), - serverAdmin: integer("serverAdmin", { mode: "boolean" }) - .notNull() - .default(false), - lastPasswordChange: integer("lastPasswordChange"), - locale: text("locale") -}); +export const users = sqliteTable( + "user", + { + userId: text("id").primaryKey(), + email: text("email"), + username: text("username").notNull(), + name: text("name"), + type: text("type").notNull(), // "internal", "oidc" + idpId: integer("idpId").references(() => idp.idpId, { + onDelete: "cascade" + }), + externalUserId: text("externalUserId"), // user's ID in the external IdP (e.g. Azure object id) + passwordHash: text("passwordHash"), + twoFactorEnabled: integer("twoFactorEnabled", { mode: "boolean" }) + .notNull() + .default(false), + twoFactorSetupRequested: integer("twoFactorSetupRequested", { + mode: "boolean" + }).default(false), + twoFactorSecret: text("twoFactorSecret"), + emailVerified: integer("emailVerified", { mode: "boolean" }) + .notNull() + .default(false), + dateCreated: text("dateCreated").notNull(), + termsAcceptedTimestamp: text("termsAcceptedTimestamp"), + termsVersion: text("termsVersion"), + marketingEmailConsent: integer("marketingEmailConsent", { + mode: "boolean" + }).default(false), + serverAdmin: integer("serverAdmin", { mode: "boolean" }) + .notNull() + .default(false), + lastPasswordChange: integer("lastPasswordChange"), + locale: text("locale") + }, + (t) => [ + uniqueIndex("idx_user_idpId_externalUserId").on( + t.idpId, + t.externalUserId + ) + ] +); export const securityKeys = sqliteTable("webauthnCredentials", { credentialId: text("credentialId").primaryKey(), @@ -1019,6 +1029,9 @@ export const userOrgs = sqliteTable( autoProvisioned: integer("autoProvisioned", { mode: "boolean" }).default(false), + directorySynched: integer("directorySynched", { + mode: "boolean" + }).default(false), pamUsername: text("pamUsername") // cleaned username for ssh and such }, (table) => [ @@ -1512,7 +1525,7 @@ export const supporterKey = sqliteTable("supporterKey", { export const idp = sqliteTable("idp", { idpId: integer("idpId").primaryKey({ autoIncrement: true }), name: text("name").notNull(), - type: text("type").notNull(), + type: text("type").notNull().$type<"oidc" | "google" | "azure">(), defaultRoleMapping: text("defaultRoleMapping"), defaultOrgMapping: text("defaultOrgMapping"), autoProvision: integer("autoProvision", { @@ -1520,7 +1533,12 @@ export const idp = sqliteTable("idp", { }) .notNull() .default(false), - tags: text("tags") + tags: text("tags"), + directorySyncEnabled: integer("directorySyncEnabled", { + mode: "boolean" + }) + .notNull() + .default(false) }); // Identity Provider OAuth Configuration @@ -1528,7 +1546,10 @@ export const idpOidcConfig = sqliteTable("idpOidcConfig", { idpOauthConfigId: integer("idpOauthConfigId").primaryKey({ autoIncrement: true }), - variant: text("variant").notNull().default("oidc"), + variant: text("variant") + .notNull() + .$type<"oidc" | "google" | "azure">() + .default("oidc"), idpId: integer("idpId") .notNull() .references(() => idp.idpId, { onDelete: "cascade" }), @@ -1638,9 +1659,44 @@ export const idpOrg = sqliteTable("idpOrg", { .notNull() .references(() => orgs.orgId, { onDelete: "cascade" }), roleMapping: text("roleMapping"), - orgMapping: text("orgMapping") + orgMapping: text("orgMapping"), + directoryRoleMapping: text("directoryRoleMapping"), + directorySyncDeletions: integer("directorySyncDeletions", { + mode: "boolean" + }) + .notNull() + .default(true) }); +// Runtime state of the directory sync job, one row per IdP. Deleting the row +// forces a full sync on the next run. +export const idpSyncState = sqliteTable("idpSyncState", { + idpId: integer("idpId") + .primaryKey() + .references(() => idp.idpId, { onDelete: "cascade" }), + // Provider-defined opaque JSON (e.g. Azure delta links). Null = full sync. + cursor: text("cursor"), + lastSyncStartedAt: integer("lastSyncStartedAt"), + lastSyncSucceededAt: integer("lastSyncSucceededAt"), + lastError: text("lastError"), + consecutiveFailures: integer("consecutiveFailures").notNull().default(0) +}); + +// Links a directory user (stable provider ID) to the Pangolin user sync manages. +export const idpSyncUser = sqliteTable( + "idpSyncUser", + { + idpId: integer("idpId") + .notNull() + .references(() => idp.idpId, { onDelete: "cascade" }), + externalId: text("externalId").notNull(), + userId: text("userId") + .notNull() + .references(() => users.userId, { onDelete: "cascade" }) + }, + (t) => [primaryKey({ columns: [t.idpId, t.externalId] })] +); + // Blueprint runs export const blueprints = sqliteTable("blueprints", { blueprintId: integer("blueprintId").primaryKey({ diff --git a/server/lib/traefik/TraefikConfigManager.ts b/server/lib/traefik/TraefikConfigManager.ts index fa742e3359..61381d2b0e 100644 --- a/server/lib/traefik/TraefikConfigManager.ts +++ b/server/lib/traefik/TraefikConfigManager.ts @@ -297,7 +297,7 @@ export class TraefikConfigManager { const secondsUntilExpiry = localState.expiresAt - nowInSeconds; const daysUntilExpiry = secondsUntilExpiry / (60 * 60 * 24); - if (daysUntilExpiry < 45) { + if (daysUntilExpiry < 15) { logger.info( `Fetching certificates due to upcoming expiry for ${domain} (${Math.round(daysUntilExpiry)} days remaining)` ); @@ -314,7 +314,7 @@ export class TraefikConfigManager { const nowInSeconds = Math.floor(Date.now() / 1000); const secondsUntilExpiry = state.expiresAt - nowInSeconds; const daysUntilExpiry = secondsUntilExpiry / (60 * 60 * 24); - if (daysUntilExpiry < 45) { + if (daysUntilExpiry < 15) { logger.info( `Fetching certificates due to upcoming expiry for wildcard cert ${certDomain} (${Math.round(daysUntilExpiry)} days remaining)` ); diff --git a/server/lib/traefik/loadBalancer.test.ts b/server/lib/traefik/loadBalancer.test.ts new file mode 100644 index 0000000000..0ffd5201d6 --- /dev/null +++ b/server/lib/traefik/loadBalancer.test.ts @@ -0,0 +1,81 @@ +import "@server/extendZod"; +import { assertEqualsObj } from "@test/assert"; +import { + buildHttpLoadBalancerServers, + buildTcpUdpLoadBalancerServers +} from "./loadBalancer"; +import type { TargetWithSite } from "./types"; + +function makeTarget(ip: string, siteType: string): TargetWithSite { + return { + targetId: 1, + resourceId: 1, + providerId: null, + siteId: 1, + ip, + port: 8080, + internalPort: 9000, + method: "http", + enabled: true, + health: "healthy", + path: null, + pathMatchType: null, + rewritePath: null, + rewritePathType: null, + priority: 100, + mode: "http", + authToken: null, + site: { + siteId: 1, + type: siteType, + subnet: null, + exitNodeId: null, + online: true + } + }; +} + +function runTests() { + const ipCases = [ + { ip: "2001:db8::1", host: "[2001:db8::1]" }, + { ip: "192.0.2.1", host: "192.0.2.1" } + ]; + const cases = [ + ...ipCases, + { ip: "backend.example.com", host: "backend.example.com" } + ]; + + for (const { ip, host } of cases) { + const target = makeTarget(ip, "local"); + assertEqualsObj( + buildHttpLoadBalancerServers([target]), + [{ url: `http://${host}:8080` }], + `Local HTTP target ${ip} should format its endpoint` + ); + assertEqualsObj( + buildTcpUdpLoadBalancerServers([target]), + [{ address: `${host}:8080` }], + `Local TCP/UDP target ${ip} should format its endpoint` + ); + } + + for (const { ip, host } of ipCases) { + const target = makeTarget("backend.example.com", "newt"); + target.site.subnet = `${ip}/${ip.includes(":") ? 128 : 32}`; + + assertEqualsObj( + buildHttpLoadBalancerServers([target]), + [{ url: `http://${host}:9000` }], + "Newt HTTP target should use the subnet endpoint and internal port" + ); + assertEqualsObj( + buildTcpUdpLoadBalancerServers([target]), + [{ address: `${host}:9000` }], + "Newt TCP/UDP target should use the subnet endpoint and internal port" + ); + } + + console.log("All load balancer endpoint tests passed!"); +} + +runTests(); diff --git a/server/lib/traefik/loadBalancer.ts b/server/lib/traefik/loadBalancer.ts index af1454a180..538f6bf074 100644 --- a/server/lib/traefik/loadBalancer.ts +++ b/server/lib/traefik/loadBalancer.ts @@ -1,3 +1,4 @@ +import { formatEndpoint } from "@server/lib/ip"; import { TargetWithSite } from "./types"; /** @@ -48,12 +49,12 @@ export function buildHttpLoadBalancerServers(targets: TargetWithSite[]) { target.site.type === "wireguard" ) { return { - url: `${target.method}://${target.ip}:${target.port}` + url: `${target.method}://${formatEndpoint(target.ip, target.port)}` }; } else if (target.site.type === "newt") { const ip = target.site.subnet!.split("/")[0]; return { - url: `${target.method}://${ip}:${target.internalPort}` + url: `${target.method}://${formatEndpoint(ip, target.internalPort!)}` }; } }) @@ -111,12 +112,12 @@ export function buildTcpUdpLoadBalancerServers(targets: TargetWithSite[]) { target.site.type === "wireguard" ) { return { - address: `${target.ip}:${target.port}` + address: formatEndpoint(target.ip, target.port) }; } else if (target.site.type === "newt") { const ip = target.site.subnet!.split("/")[0]; return { - address: `${ip}:${target.internalPort}` + address: formatEndpoint(ip, target.internalPort!) }; } }); diff --git a/server/private/lib/directorySync/_azure.ts b/server/private/lib/directorySync/_azure.ts new file mode 100644 index 0000000000..cf870cba2a --- /dev/null +++ b/server/private/lib/directorySync/_azure.ts @@ -0,0 +1,443 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +// Microsoft Entra ID directory provider (Microsoft Graph, app-only auth). +// +// Required application permissions (with admin consent): User.Read.All, GroupMember.Read.All. +// +// Full fetch: /users/delta + /groups/delta?$select=displayName,members without a token, which +// lists everything and returns delta links for next time. Nested memberships are resolved in +// memory from the direct members. +// +// Incremental fetch: replay both delta links. Group deltas only report direct member changes, +// so every user touched by a change is re-read (/users/{id} + /transitiveMemberOf) to get their +// full current state. Cases that can't be resolved cheaply fall back to a full fetch. + +import config from "@server/lib/config"; +import logger from "@server/logger"; +import { decrypt } from "@server/lib/crypto"; +import { + DirectoryCursorExpiredError, + DirectoryFetchContext, + DirectoryFetchResult, + DirectoryGroup, + DirectorySyncProvider, + DirectoryUser +} from "./types"; + +const GRAPH = "https://graph.microsoft.com/v1.0"; +const USER_SELECT = "id,displayName,mail,userPrincipalName,accountEnabled"; + +/** Above this many touched users, an incremental run is more expensive than a full one. */ +const MAX_INCREMENTAL_AFFECTED_USERS = 500; +/** Longest Retry-After we sleep through in-process before failing the run. */ +const MAX_RETRY_AFTER_SECONDS = 120; +const MAX_THROTTLE_RETRIES = 5; + +type GraphUser = { + id: string; + displayName?: string | null; + mail?: string | null; + userPrincipalName?: string | null; + accountEnabled?: boolean | null; + "@removed"?: { reason: string }; +}; + +type GraphDirectoryObjectRef = { + "@odata.type"?: string; + id: string; + "@removed"?: { reason: string }; +}; + +type GraphGroup = { + id: string; + displayName?: string | null; + "members@delta"?: GraphDirectoryObjectRef[]; + "@removed"?: { reason: string }; +}; + +type AzureCursor = { + usersDeltaLink: string; + groupsDeltaLink: string; +}; + +type GraphPage = { + value: T[]; + "@odata.nextLink"?: string; + "@odata.deltaLink"?: string; +}; + +// Graph field that yields the same `username` login computes from the ID token. +// `sub` is per-app in Entra and not exposed by Graph, so it can't be synced. +const IDENTIFIER_FROM_GRAPH: Record string | null> = { + oid: (u) => u.id, + preferred_username: (u) => u.userPrincipalName ?? null, + upn: (u) => u.userPrincipalName ?? null, + email: (u) => u.mail ?? null +}; + +const tokenCache = new Map(); + +class GraphClient { + constructor( + private readonly getToken: () => Promise, + private readonly onRequest: () => void + ) {} + + /** GET a single Graph URL, sleeping through short throttling. Returns null on 404. */ + async get(url: string): Promise { + for (let attempt = 0; ; attempt++) { + this.onRequest(); + const res = await fetch(url, { + headers: { Authorization: `Bearer ${await this.getToken()}` } + }); + + if (res.status === 429 || res.status === 503) { + const retryAfter = Number(res.headers.get("Retry-After") ?? 5); + if ( + attempt >= MAX_THROTTLE_RETRIES || + retryAfter > MAX_RETRY_AFTER_SECONDS + ) { + throw new Error( + `Graph throttled (${res.status}, Retry-After ${retryAfter}s): ${url}` + ); + } + logger.warn(`Graph throttled, retrying in ${retryAfter}s`, { + url + }); + await new Promise((r) => setTimeout(r, retryAfter * 1000)); + continue; + } + + if (res.status === 404) return null; + + const body: any = await res.json().catch(() => ({})); + + if ( + res.status === 410 || + body.error?.code === "syncStateNotFound" || + body.error?.code === "resyncRequired" + ) { + throw new DirectoryCursorExpiredError( + `Graph delta token expired (${res.status} ${body.error?.code})` + ); + } + if (!res.ok) { + throw new Error( + `Graph request failed (${res.status}) ${url}: ${body.error?.code} ${body.error?.message}` + ); + } + return body as T; + } + } + + /** Follow @odata.nextLink pages; returns the items and the final delta link, if any. */ + async getAll( + url: string + ): Promise<{ items: T[]; deltaLink: string | null }> { + const items: T[] = []; + let next: string | undefined = url; + while (next) { + const page: GraphPage | null = + await this.get>(next); + if (!page) throw new Error(`Graph returned 404 for ${next}`); + items.push(...page.value); + if (page["@odata.deltaLink"]) { + return { items, deltaLink: page["@odata.deltaLink"] }; + } + next = page["@odata.nextLink"]; + } + return { items, deltaLink: null }; + } +} + +function isUserRef(ref: GraphDirectoryObjectRef) { + return ref["@odata.type"] === "#microsoft.graph.user"; +} + +function isGroupRef(ref: GraphDirectoryObjectRef) { + return ref["@odata.type"] === "#microsoft.graph.group"; +} + +function parseCursor(cursor: string | null): AzureCursor | null { + if (!cursor) return null; + try { + const parsed = JSON.parse(cursor); + if ( + typeof parsed?.usersDeltaLink === "string" && + typeof parsed?.groupsDeltaLink === "string" + ) { + return parsed; + } + } catch { + // fall through + } + logger.warn("Ignoring unreadable Azure directory sync cursor"); + return null; +} + +export class AzureDirectoryProvider implements DirectorySyncProvider { + constructor( + private readonly settings: { + tenantId: string; + /** Encrypted, as stored in `idpOidcConfig`. */ + encryptedClientId: string; + /** Encrypted, as stored in `idpOidcConfig`. */ + encryptedClientSecret: string; + } + ) {} + + /** Parses the tenant ID from an Entra authorize URL. */ + static tenantIdFromAuthUrl(authUrl: string): string | null { + return ( + authUrl.match( + /login\.microsoftonline\.com\/([^/]+)\/oauth2/ + )?.[1] ?? null + ); + } + + async fetch(ctx: DirectoryFetchContext): Promise { + const getIdentifier = IDENTIFIER_FROM_GRAPH[ctx.identifierPath]; + if (!getIdentifier) { + throw new Error( + `identifierPath "${ctx.identifierPath}" can't be resolved from Graph; use one of: ${Object.keys(IDENTIFIER_FROM_GRAPH).join(", ")}` + ); + } + + const graph = new GraphClient(() => this.getToken(), ctx.onRequest); + const toDirectoryUser = ( + u: GraphUser, + groups: DirectoryGroup[] + ): DirectoryUser => ({ + externalId: u.id, + username: getIdentifier(u)?.toLowerCase() ?? null, + email: u.mail?.toLowerCase() ?? null, + name: u.displayName ?? null, + enabled: u.accountEnabled !== false, + groups + }); + + const cursor = parseCursor(ctx.cursor); + if (cursor) { + const incremental = await this.fetchIncremental( + graph, + cursor, + toDirectoryUser + ); + if (incremental) return incremental; + logger.info( + `Azure incremental sync for IdP ${ctx.idpId} needs a full fetch` + ); + } + return this.fetchFull(graph, toDirectoryUser); + } + + private async getToken(): Promise { + const { tenantId, encryptedClientId, encryptedClientSecret } = + this.settings; + const key = config.getRawConfig().server.secret!; + const clientId = decrypt(encryptedClientId, key); + const cacheKey = `${tenantId}:${clientId}`; + + const cached = tokenCache.get(cacheKey); + if (cached && cached.expiresAt > Date.now() + 60_000) { + return cached.token; + } + + const res = await fetch( + `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, + { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded" + }, + body: new URLSearchParams({ + grant_type: "client_credentials", + client_id: clientId, + client_secret: decrypt(encryptedClientSecret, key), + scope: "https://graph.microsoft.com/.default" + }) + } + ); + const body: any = await res.json().catch(() => ({})); + if (!res.ok) { + throw new Error( + `Entra token request failed (${res.status}): ${body.error_description ?? body.error}` + ); + } + + tokenCache.set(cacheKey, { + token: body.access_token, + expiresAt: Date.now() + Number(body.expires_in ?? 3600) * 1000 + }); + return body.access_token; + } + + private async fetchFull( + graph: GraphClient, + toDirectoryUser: ( + u: GraphUser, + groups: DirectoryGroup[] + ) => DirectoryUser + ): Promise { + const usersRes = await graph.getAll( + `${GRAPH}/users/delta?$select=${USER_SELECT}` + ); + const groupsRes = await graph.getAll( + `${GRAPH}/groups/delta?$select=displayName,members` + ); + + // A large group can be split over several pages; merge its member chunks. + const groups = new Map(); + // member (user or group) ID -> IDs of groups it directly belongs to + const parentGroups = new Map>(); + for (const g of groupsRes.items) { + if (g["@removed"]) continue; + const existing = groups.get(g.id); + groups.set(g.id, { + id: g.id, + displayName: g.displayName ?? existing?.displayName ?? null + }); + for (const member of g["members@delta"] ?? []) { + if (member["@removed"]) continue; + if (!isUserRef(member) && !isGroupRef(member)) continue; + const parents = parentGroups.get(member.id) ?? new Set(); + parents.add(g.id); + parentGroups.set(member.id, parents); + } + } + + const transitiveGroups = (memberId: string): DirectoryGroup[] => { + const seen = new Set(); + const queue = [...(parentGroups.get(memberId) ?? [])]; + while (queue.length) { + const groupId = queue.pop()!; + if (seen.has(groupId)) continue; + seen.add(groupId); + queue.push(...(parentGroups.get(groupId) ?? [])); + } + return [...seen] + .map((id) => groups.get(id)) + .filter((g): g is DirectoryGroup => !!g); + }; + + const users = usersRes.items + .filter((u) => !u["@removed"]) + .map((u) => toDirectoryUser(u, transitiveGroups(u.id))); + + return { + kind: "full", + users, + nextCursor: this.cursorFrom(usersRes.deltaLink, groupsRes.deltaLink) + }; + } + + /** Returns null when a full fetch is required instead. */ + private async fetchIncremental( + graph: GraphClient, + cursor: AzureCursor, + toDirectoryUser: ( + u: GraphUser, + groups: DirectoryGroup[] + ) => DirectoryUser + ): Promise { + const usersRes = await graph.getAll(cursor.usersDeltaLink); + const groupsRes = await graph.getAll( + cursor.groupsDeltaLink + ); + + const removedExternalIds = new Set(); + const affectedUserIds = new Set(); + + for (const u of usersRes.items) { + if (u["@removed"]) { + removedExternalIds.add(u.id); + } else { + affectedUserIds.add(u.id); + } + } + + // Groups whose every (transitive) user member must be re-read. + const expandGroupIds = new Set(); + for (const g of groupsRes.items) { + if (g["@removed"]) { + // The former members of a deleted group aren't listed anywhere. + return null; + } + if (g.displayName !== undefined) { + // Possibly renamed: name-based role mappings may change for all members. + expandGroupIds.add(g.id); + } + for (const member of g["members@delta"] ?? []) { + if (isUserRef(member)) { + affectedUserIds.add(member.id); + } else if (isGroupRef(member)) { + expandGroupIds.add(member.id); + } + } + } + + for (const groupId of expandGroupIds) { + const members = await graph.getAll<{ id: string }>( + `${GRAPH}/groups/${groupId}/transitiveMembers/microsoft.graph.user?$select=id&$top=999` + ); + for (const m of members.items) affectedUserIds.add(m.id); + if (affectedUserIds.size > MAX_INCREMENTAL_AFFECTED_USERS) { + return null; + } + } + if (affectedUserIds.size > MAX_INCREMENTAL_AFFECTED_USERS) { + return null; + } + + const users: DirectoryUser[] = []; + for (const userId of affectedUserIds) { + if (removedExternalIds.has(userId)) continue; + const u = await graph.get( + `${GRAPH}/users/${userId}?$select=${USER_SELECT}` + ); + if (!u) { + removedExternalIds.add(userId); + continue; + } + const memberOf = await graph.getAll( + `${GRAPH}/users/${userId}/transitiveMemberOf/microsoft.graph.group?$select=id,displayName&$top=999` + ); + users.push( + toDirectoryUser( + u, + memberOf.items.map((g) => ({ + id: g.id, + displayName: g.displayName ?? null + })) + ) + ); + } + + return { + kind: "incremental", + users, + removedExternalIds: [...removedExternalIds], + nextCursor: this.cursorFrom(usersRes.deltaLink, groupsRes.deltaLink) + }; + } + + private cursorFrom( + usersDeltaLink: string | null, + groupsDeltaLink: string | null + ): string | null { + if (!usersDeltaLink || !groupsDeltaLink) return null; + const cursor: AzureCursor = { usersDeltaLink, groupsDeltaLink }; + return JSON.stringify(cursor); + } +} diff --git a/server/private/lib/directorySync/azure.ts b/server/private/lib/directorySync/azure.ts new file mode 100644 index 0000000000..e86e173b4e --- /dev/null +++ b/server/private/lib/directorySync/azure.ts @@ -0,0 +1,435 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +import { db, idp, idpOidcConfig, idpOrg } from "@server/db"; +import { eq, and } from "drizzle-orm"; +import type { + DirectoryFetchContext, + DirectoryFetchResult, + DirectorySyncProvider, + DirectoryUser +} from "./types"; +import { decrypt } from "@server/lib/crypto"; +import config from "@server/lib/config"; +import cache from "@server/lib/cache"; +import z from "zod"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import { ensureValidIdp, type IdpDbResponse } from "./ensureValidIdp"; + +const userBodySchema = z.object({ + "@odata.nextLink": z.url().optional(), + value: z.array( + z.object({ + id: z.uuid(), + mail: z.email().nullish(), + displayName: z.string().nullish(), + accountEnabled: z.boolean().nullish(), + userPrincipalName: z.string().nullish(), + otherMails: z.array(z.email()).nullish() + }) + ) +}); + +// entries flagged as removed (e.g. soft-deleted groups or removed members), these +// can show up even in initial delta rounds (no stored `deltaLink`) +const removedSchema = z.object({ reason: z.string() }).optional(); + +const groupBodySchema = z.object({ + "@odata.nextLink": z.url().optional(), + "@odata.deltaLink": z.url().optional(), + value: z.array( + z.object({ + id: z.uuid(), + // large groups can be returned several times with their members split + // across the entries, the repeated entries may not have `displayName` + displayName: z.string().nullish(), + "@removed": removedSchema, + // omitted when the group has no members + "members@delta": z + .array( + z.object({ + // user, group, device, servicePrincipal, orgContact... + "@odata.type": z.string(), + id: z.uuid(), + "@removed": removedSchema + }) + ) + .default([]) + }) + ) +}); + +const tokenBodySchema = z.object({ + expires_in: z.int(), + access_token: z.string() +}); + +type MicrosoftGraphUser = z.output["value"][number]; +type MicrosoftGraphGroup = z.output["value"][number]; + +type GraphGroupNode = { + displayName: string | null; + /** Groups that have this group as a direct member. */ + parentGroupIds: Set; + /** Users that are direct members of this group. */ + userIds: Set; +}; + +export class AzureDirectorySyncProvider implements DirectorySyncProvider { + #TOKEN_CACHE_KEY_PREFIX = "azureDirectorySyncToken"; + #microsoftGraphBaseUrl = "https://graph.microsoft.com/v1.0"; + + async fetch(ctx: DirectoryFetchContext): Promise { + const directoryUsers: Record = {}; + // 0. fetch idp and check that it is valid + const idpRes = await ensureValidIdp(ctx.idpId, "azure"); + + // 1. get access token (and cache) + const token = await this.#getToken(idpRes); + + // 2. get users & groups + const [userResponse, groupResponse] = await Promise.all([ + this.#fetchAllUsers(token, idpRes.idp.idpId), + this.#fetchAllGroups(token, idpRes.idp.idpId) + ]); + + for (const user of userResponse.users) { + let email = user.mail ?? user.otherMails?.[0]; + + if (!email && user.userPrincipalName) { + // uPn is formatted with `#EXT#` at the end, we remove that + const extSuffixRegex = /(\#EXT\#.*$)/; + const uPnWithoutEXT = user.userPrincipalName.replaceAll( + extSuffixRegex, + "" + ); + + // split uPn with `_` (Entra `_` replaces `@`) + let splitted = uPnWithoutEXT.split("_"); + if (splitted.length === 2) { + email = splitted.join("@"); + } + } + + if (!email) { + logger.warn( + `Directory sync: skipping Entra user ${user.userPrincipalName} (id: ${user.id}) for IdP ${idpRes.idp.idpId}: no email found in \`mail\`, \`otherMails\` or \`userPrincipalName\`` + ); + } else { + directoryUsers[user.id] = { + email, + externalId: user.id, + enabled: user.accountEnabled ?? true, + name: user.displayName ?? null, + username: email, + groups: [] + }; + } + } + + // 3. build the group tree, merging the repeated entries of large groups + const groupNodes = new Map(); + const getGroupNode = (groupId: string) => { + let node = groupNodes.get(groupId); + if (!node) { + node = { + displayName: null, + parentGroupIds: new Set(), + userIds: new Set() + }; + groupNodes.set(groupId, node); + } + return node; + }; + + for (const group of groupResponse.groups) { + if (group["@removed"]) { + continue; + } + + const node = getGroupNode(group.id); + if (group.displayName) { + node.displayName = group.displayName; + } + + for (const member of group["members@delta"]) { + if (member["@removed"]) { + continue; + } + + if (member["@odata.type"] === "#microsoft.graph.user") { + node.userIds.add(member.id); + } else if (member["@odata.type"] === "#microsoft.graph.group") { + getGroupNode(member.id).parentGroupIds.add(group.id); + } + } + } + + // 4. expand nested memberships: a user is a member of its direct groups + // and of every group above them + const ancestorsCache = new Map>(); + const getGroupWithAncestors = ( + groupId: string, + visiting = new Set() + ): Set => { + const cached = ancestorsCache.get(groupId); + if (cached) { + return cached; + } + + const result = new Set([groupId]); + visiting.add(groupId); + for (const parentId of groupNodes.get(groupId)?.parentGroupIds ?? + []) { + // guard against cycles + if (visiting.has(parentId)) { + continue; + } + for (const ancestorId of getGroupWithAncestors( + parentId, + visiting + )) { + result.add(ancestorId); + } + } + visiting.delete(groupId); + + ancestorsCache.set(groupId, result); + return result; + }; + + const userGroupIds = new Map>(); + for (const [groupId, node] of groupNodes) { + for (const userId of node.userIds) { + if (!directoryUsers[userId]) { + continue; + } + + let groupIds = userGroupIds.get(userId); + if (!groupIds) { + groupIds = new Set(); + userGroupIds.set(userId, groupIds); + } + for (const id of getGroupWithAncestors(groupId)) { + groupIds.add(id); + } + } + } + + for (const [userId, groupIds] of userGroupIds) { + directoryUsers[userId].groups = [...groupIds].map((id) => ({ + id, + displayName: groupNodes.get(id)?.displayName ?? null + })); + } + + return { + users: Object.values(directoryUsers) + // nextCursor: null + }; + } + + async #getToken(idpRes: IdpDbResponse): Promise { + const cacheKey = `${this.#TOKEN_CACHE_KEY_PREFIX}:${idpRes.idp.idpId}`; + const cached = await cache.get(cacheKey); + + if (cached) { + logger.info("Returning cached access token"); + return cached; + } + + logger.info("Fetching fresh access token"); + const key = config.getRawConfig().server.secret!; + + let clientId = ""; + let clientSecret = ""; + + if (idpRes.idp.type === "oidc") { + clientSecret = decrypt(idpRes.idpOidcConfig.clientSecret, key); + clientId = decrypt(idpRes.idpOidcConfig.clientId, key); + } + + const res = await fetch(idpRes.idpOidcConfig.tokenUrl, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded" + }, + body: new URLSearchParams({ + grant_type: "client_credentials", + client_id: clientId, + client_secret: clientSecret, + scope: "https://graph.microsoft.com/.default" + }) + }); + + const body: any = await res.json().catch(() => ({})); + if (!res.ok) { + const reason = + body.error_description ?? + body.error ?? + res.statusText ?? + "no error details in response"; + throw new Error( + `Directory sync: Entra token request for IdP ${idpRes.idp.idpId} failed (HTTP ${res.status}): ${reason}` + ); + } + + try { + const tokenResponse = tokenBodySchema.parse(body); + await cache.set( + cacheKey, + tokenResponse.access_token, + tokenResponse.expires_in - 30 // keep the response 30 seconds less than the token to make sure to always have a fresh token + ); + + logger.info( + `Got new access token (expires in: ${tokenResponse.expires_in}s)` + ); + + return tokenResponse.access_token; + } catch (e) { + throw new Error( + `Directory sync: unexpected Entra token response for IdP ${idpRes.idp.idpId}: ${fromError(e).toString()}`, + { cause: e } + ); + } + } + + async #fetchAllUsers(token: string, idpId: number) { + // we need to follow `@odata.nextLink` as-is until it is absent + // (microsoft graph returns a max of 999 users per page) + const sp = new URLSearchParams({ + $select: + "id,displayName,mail,userPrincipalName,accountEnabled,otherMails", + $top: "999" + }); + let nextUrl: string | undefined = + `${this.#microsoftGraphBaseUrl}/users?${sp.toString()}`; + const allUsers: MicrosoftGraphUser[] = []; + + while (nextUrl) { + logger.info(`Fetching URL: ${nextUrl}`); + const res = await fetch(nextUrl, { + headers: { + Authorization: `Bearer ${token}` + } + }); + + const body: any = await res.json().catch(() => ({})); + if (!res.ok) { + const reason = + body.error?.message ?? + body.error?.code ?? + res.statusText ?? + "no error details in response"; + throw new Error( + `Directory sync: Entra users request for IdP ${idpId} failed (HTTP ${res.status}): ${reason}` + ); + } + + try { + const response = userBodySchema.parse(body); + + allUsers.push(...response.value); + nextUrl = response["@odata.nextLink"]; + } catch (e) { + console.dir( + { + body + }, + { depth: null } + ); + throw new Error( + `Directory sync: unexpected Entra users response for IdP ${idpId}: ${fromError(e).toString()}`, + { cause: e } + ); + } + } + + logger.info(`Fetched ${allUsers.length} users from Microsoft Graph`); + return { + users: allUsers + }; + } + + async #fetchAllGroups(token: string, idpId: number) { + // we need to follow `@odata.nextLink` as-is until it is absent, + // the last page returns `@odata.deltaLink` instead + // (microsoft graph returns a max of 100 groups per page) + const MAX_PAGES = 3; + let pagesFound = 0; + const sp = new URLSearchParams({ + $select: "id,displayName,members" + }); + let nextUrl: string | undefined = + `${this.#microsoftGraphBaseUrl}/groups/delta?${sp.toString()}`; + let deltaLink: string | null = null; + const allGroups: MicrosoftGraphGroup[] = []; + + while (nextUrl) { + logger.info(`Fetching URL: ${nextUrl}`); + + const res = await fetch(nextUrl, { + headers: { + Authorization: `Bearer ${token}` + } + }); + + const body: any = await res.json().catch(() => ({})); + if (!res.ok) { + const reason = + body.error?.message ?? + body.error?.code ?? + res.statusText ?? + "no error details in response"; + throw new Error( + `Directory sync: Entra groups request for IdP ${idpId} failed (HTTP ${res.status}): ${reason}` + ); + } + + pagesFound += 1; + try { + const response = groupBodySchema.parse(body); + + allGroups.push(...response.value); + nextUrl = response["@odata.nextLink"]; + + // Sometimes, microsoft graph will send empty pages with a `@odata.nextLink` even + // though they are the last page, in that case, we stop the loop because this shouldn't happen + // DO NOT ASK ME WHY THIS IS THE CASE, IDK ¯\_(ツ)_/¯ + if (response.value.length === 0) { + nextUrl = undefined; + } + deltaLink = response["@odata.deltaLink"] ?? null; + } catch (e) { + console.dir( + { + body + }, + { depth: null } + ); + throw new Error( + `Directory sync: unexpected Entra groups response for IdP ${idpId}: ${fromError(e).toString()}`, + { cause: e } + ); + } + } + + logger.info(`Fetched ${allGroups.length} groups from Microsoft Graph`); + return { + groups: allGroups, + deltaLink + }; + } +} diff --git a/server/private/lib/directorySync/ensureValidIdp.ts b/server/private/lib/directorySync/ensureValidIdp.ts new file mode 100644 index 0000000000..40aaeead8d --- /dev/null +++ b/server/private/lib/directorySync/ensureValidIdp.ts @@ -0,0 +1,42 @@ +import { db, idp, idpOidcConfig, idpOrg, type IdpOidcConfig } from "@server/db"; +import { eq, and } from "drizzle-orm"; + +export async function ensureValidIdp( + idpId: number, + variant: IdpOidcConfig["variant"] +) { + const [idpRes] = await db + .select() + .from(idp) + .where(eq(idp.idpId, idpId)) + .innerJoin(idpOidcConfig, eq(idpOidcConfig.idpId, idp.idpId)) + .innerJoin(idpOrg, and(eq(idpOrg.idpId, idp.idpId))) + .limit(1); + + if (!idpRes) { + throw new Error( + `Directory sync: IdP ${idpId} not found, or it has no OIDC config or org mapping` + ); + } + + if (idpRes.idpOidcConfig.variant !== variant) { + throw new Error( + `Directory sync: IdP ${idpId} has variant "${idpRes.idpOidcConfig.variant}", expected "${variant}"` + ); + } + + if (!idpRes.idp.directorySyncEnabled) { + throw new Error( + `Directory sync: directory sync is not enabled on IdP ${idpId}` + ); + } + + if (idpRes.idp.autoProvision) { + throw new Error( + `Directory sync: IdP ${idpId} has both directory sync and auto-provisioning enabled; only one can be active` + ); + } + return idpRes; +} + +export type IdpDbResponse = Awaited>; diff --git a/server/private/lib/directorySync/google.ts b/server/private/lib/directorySync/google.ts new file mode 100644 index 0000000000..a6de54ac96 --- /dev/null +++ b/server/private/lib/directorySync/google.ts @@ -0,0 +1,16 @@ +import type { + DirectoryFetchContext, + DirectoryFetchResult, + DirectorySyncProvider, + DirectoryUser +} from "./types"; +import { ensureValidIdp } from "./ensureValidIdp"; + +export class AzureDirectorySyncProvider implements DirectorySyncProvider { + async fetch(ctx: DirectoryFetchContext): Promise { + const directoryUsers: Record = {}; + // 0. fetch idp and check that it is valid + const idpRes = await ensureValidIdp(ctx.idpId, "google"); + throw new Error("Method not implemented."); + } +} diff --git a/server/private/lib/directorySync/run.ts b/server/private/lib/directorySync/run.ts new file mode 100644 index 0000000000..697bffaa59 --- /dev/null +++ b/server/private/lib/directorySync/run.ts @@ -0,0 +1,52 @@ +// // Usage: npx tsx server/private/lib/directorySync/run.ts +// // Runs one directory sync for the IdP using its saved directory role mapping. +// import { AzureDirectorySyncProvider } from "./azure"; +// must be imported first, before any module that calls `z.openapi()` +import "@server/extendZod"; +import { runIdpDirectorySync } from "./syncWorkflow"; + +// pass `--dry-run` to only compute the sync plans, without applying them +const dryRun = process.argv.includes("--dry-run"); + +await runIdpDirectorySync(dryRun).then((changes) => { + console.dir( + { + changes + }, + { depth: null } + ); +}); +process.exit(0); + +// const idpId = Number(process.argv[2]); +// if (!idpId) { +// console.error("Usage: run.ts "); +// process.exit(1); +// } + +// const provider = new AzureDirectorySyncProvider(); + +// provider +// .fetch({ +// idpId: idpId, +// identifierPath: "email", +// cursor: null +// }) +// .then((report) => { +// console.dir({ report }, { depth: null }); +// process.exit(0); +// }) +// .catch((err) => { +// console.error(err); +// process.exit(1); +// }); + +// runDirectorySync(idpId) +// .then((report) => { +// console.log(JSON.stringify(report, null, 2)); +// process.exit(0); +// }) +// .catch((err) => { +// console.error(err); +// process.exit(1); +// }); diff --git a/server/private/lib/directorySync/syncWorkflow.ts b/server/private/lib/directorySync/syncWorkflow.ts new file mode 100644 index 0000000000..de2ec89759 --- /dev/null +++ b/server/private/lib/directorySync/syncWorkflow.ts @@ -0,0 +1,619 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +import { + db, + idp, + idpOidcConfig, + idpOrg, + orgs, + roles, + userOrgRoles, + userOrgs, + users, + type Role +} from "@server/db"; + +import { durationToMs } from "@app/lib/durationToMs"; +import { + parseDirectorySyncRoleMapping, + type DirectorySyncRoleMappingConfig +} from "@app/lib/idpDirectorySyncRoleMapping"; +import { generateId } from "@server/auth/sessions/app"; +import { calculateUserClientsForOrgs } from "@server/lib/calculateUserClientsForOrgs"; +import { assignUserToOrg, removeUserFromOrg } from "@server/lib/userOrg"; +import logger from "@server/logger"; +import { UserType } from "@server/types/UserTypes"; +import { and, eq, inArray, sql, type InferInsertModel } from "drizzle-orm"; +import { AzureDirectorySyncProvider } from "./azure"; +import type { + DirectoryFetchContext, + DirectorySyncProvider, + DirectoryUser +} from "./types"; + +type InsertUser = InferInsertModel; +type UpdateUser = Pick< + InsertUser, + "username" | "email" | "name" | "userId" | "externalUserId" +>; + +type SyncedRole = { id: number; name: string }; + +/** What happens to one user: their Pangolin profile and their membership in `orgId` are decided separately. */ +export type UserSyncEntry = { + /** `null` when the user is in Pangolin but gone from the directory. */ + directoryUser: DirectoryUser | null; + user: + | { operation: "create"; data: InsertUser } + | { operation: "update"; data: UpdateUser } + | { operation: "unchanged"; userId: string } + /** New directory user not provisioned in Pangolin. */ + | { operation: "skip" }; + + membership: + | { operation: "add" | "updateRoles"; roles: SyncedRole[] } + /** Removed from the org, the user itself is not deleted. */ + | { operation: "remove" } + | { + operation: "unchanged"; + reason: "noMatchingRoles" | "manualMembership" | "owner"; + }; +}; + +// the planned changes for one idpOrg, computed first and +// applied in a single transaction at the end +export type IdpOrgSyncPlan = { + idpId: number; + orgId: string; + entries: UserSyncEntry[]; +}; + +// Track if the offline checker interval is running +let directorySyncInterval: NodeJS.Timeout | null = null; +const DIRECTORY_SYNC_RUN_INTERVAL = durationToMs(5, "minutes"); + +export async function runIdpDirectorySync(dryRun: boolean = false) { + // to sync idps, we need + const idpsToSync = await db + .select() + .from(idp) + .innerJoin(idpOidcConfig, eq(idpOidcConfig.idpId, idp.idpId)) + .innerJoin(idpOrg, and(eq(idpOrg.idpId, idp.idpId))) + .where( + and( + inArray(idpOidcConfig.variant, ["azure", "google"]), + eq(idp.directorySyncEnabled, true), + eq(idp.autoProvision, false) + ) + ); + + const azureSyncProvider = new AzureDirectorySyncProvider(); + const fetchPromises = idpsToSync + .map(({ idp, idpOidcConfig, idpOrg }) => { + if (idpOidcConfig.variant === "azure") { + const ctx = { + idpId: idp.idpId, + orgId: idpOrg.orgId, + identifierPath: idpOidcConfig.identifierPath, + roleMapping: parseDirectorySyncRoleMapping( + idpOrg.directoryRoleMapping + ) + } satisfies DirectoryFetchContext; + return syncIdp(azureSyncProvider, ctx, dryRun); + } + return null; + }) + .filter((p) => p != null); + + return await Promise.allSettled(fetchPromises); +} + +async function syncIdp( + provider: DirectorySyncProvider, + ctx: DirectoryFetchContext, + dryRun: boolean +) { + // Note + // username is unique only within IDPs or within internal ? + // we can have two users with the same username, one in an IDP and another `internal` + // Which means: a user can have two accounts w/ the same username (email in azure for ex) + // and each account can have different roles within the same Org + + // ... + // Later: we might need to change the `identifierPath` for azure/google idps to `sub` (for ex) + // which is an immutable external ID that survives username/email changes + // but first we need to check if that ID is the same returned by Entra/google workspace + // if it's not the same ID, we might kepp `idpSyncUser` otherwise, we may need to see how + // we can maybe get rid of that table (or just not need it anymore) + + // ... + // For global idps, we will rely on org policies + // we will not store a default mapping for directory sync + // if the admin wants to add the synched IDP to an org, they can create an org policy + // & set the role mapping to `automatic` (for ex) + // for global idps, if there is not an org policy for a specific org, the user is not added to that org + + // 1. Fetch + const result = await provider.fetch(ctx); + + /** + * user => + * user (has roles within) - org + * + * algorithm: + + fetch users for org: + - if user is new to pangolin and idp: + - if fixed roles: + -> create new user in DB + - if group mapping: + - check that the user groups match at least one of the mapped groups: + -> create new user in DB + - if automatic groups: + - if create: + -> create new user in DB + - if not create: + - check that group names matches: + -> create new user in DB + + -> Add user to Org with the correct roles + + - if user is not new to pangolin and idp: + - if fixed roles or automatic groups with create : + - if user doesn't belongs to org: + -> add user to org with the correct roles + - else: + -> update user roles in org + - if group mapping matches any role: + -> update user roles in org + - if automatic groups without create still matches groups: + -> update user roles in org + - else: + -> remove user from org + + - if user exists in pangolin & IDP but not in remote users: + -> remove user from org + +for each remote user: + roleIds = resolveRoles(user, ctx.roleMapping) // fixed / group mapping / automatic + // [] = user doesn't qualify for this org + if user is disabled: roleIds = [] + + pangolinUser = match by externalUserId, then username + if no pangolinUser: + if roleIds empty: skip + create user + else: + update profile (name, email, username) // user-level, always safe + + membership = userOrg for (pangolinUser, orgId) + if no membership: + if roleIds non-empty: add to org with roleIds, flag = true + else if membership.flag is true: + if roleIds non-empty: set roles to roleIds + else: remove from org + else (manual membership): + skip // admin-owned, don't touch + +for each membership with flag = true whose user is not in the remote set: + remove from org // user row stays (may be in other orgs) + + + */ + + const existingUsers = await db + .select() + .from(users) + .where(eq(users.idpId, ctx.idpId)); + + const existingUserOrgs = await db + .select() + .from(users) + .innerJoin(userOrgs, eq(userOrgs.userId, users.userId)) + .where(and(eq(users.idpId, ctx.idpId), eq(userOrgs.orgId, ctx.orgId))); + + const existingPangolinUsers = new Map< + string, + (typeof existingUsers)[number] + >(); + for (const u of existingUsers) { + // rely on the external ID in the tenant if it exists, else use the username + if (u.externalUserId) { + existingPangolinUsers.set(u.externalUserId, u); + } else { + existingPangolinUsers.set(u.username, u); + } + } + + const existingPangolinUserOrgs = new Map< + string, + (typeof existingUserOrgs)[number] + >(); + for (const u of existingUserOrgs) { + existingPangolinUserOrgs.set(u.user.userId, u); + } + + const syncPlan: IdpOrgSyncPlan = { + idpId: ctx.idpId, + orgId: ctx.orgId, + entries: [] + }; + + for (const user of result.users) { + const roles = await resolveUserRoles( + user, + ctx.roleMapping, + ctx.orgId, + dryRun + ); + + const pangolinUser = + existingPangolinUsers.get(user.externalId) ?? + existingPangolinUsers.get(user.username); + + // this is a new user + if (!pangolinUser) { + // the user doesn't match any group/role on the org + if (roles.length === 0) { + syncPlan.entries.push({ + directoryUser: user, + user: { operation: "skip" }, + membership: { + operation: "unchanged", + reason: "noMatchingRoles" + } + }); + continue; + } + + syncPlan.entries.push({ + directoryUser: user, + user: { + operation: "create", + data: { + userId: generateId(15), + username: user.username, + externalUserId: user.externalId, + email: user.email, + name: user.name, + type: UserType.OIDC, + idpId: ctx.idpId, + emailVerified: true, // OIDC users are always verified + dateCreated: new Date().toISOString() + } + }, + membership: { operation: "add", roles } + }); + } else { + const membership = existingPangolinUserOrgs.get( + pangolinUser.userId + ); + + let membershipOperation: UserSyncEntry["membership"]; + + if (!membership) { + membershipOperation = + roles.length === 0 + ? // skip users which don't belong to the org + { operation: "unchanged", reason: "noMatchingRoles" } + : { operation: "add", roles }; + } else if ( + membership.userOrgs.autoProvisioned || + membership.userOrgs.directorySynched + ) { + if (membership.userOrgs.isOwner) { + membershipOperation = { + operation: "unchanged", + reason: "owner" + }; + } else if (roles.length === 0) { + membershipOperation = { operation: "remove" }; + } else { + membershipOperation = { operation: "updateRoles", roles }; + } + } else { + membershipOperation = { + operation: "unchanged", + reason: "manualMembership" + }; + } + + syncPlan.entries.push({ + directoryUser: user, + user: { + operation: "update", + data: { + userId: pangolinUser.userId, + username: user.username, + email: user.email, + name: user.name, + externalUserId: user.externalId + } + }, + membership: membershipOperation + }); + } + } + + // users still in the org through sync/provisioning but gone from the directory + const directoryUserIds = new Set( + syncPlan.entries.flatMap((e) => + e.user.operation === "update" ? [e.user.data.userId] : [] + ) + ); + for (const { user, userOrgs: membership } of existingUserOrgs) { + if (directoryUserIds.has(user.userId)) continue; + if ( + membership.isOwner || + (!membership.autoProvisioned && !membership.directorySynched) + ) + continue; + + syncPlan.entries.push({ + directoryUser: null, + user: { operation: "unchanged", userId: user.userId }, + membership: { operation: "remove" } + }); + } + + const [org] = await db.select().from(orgs).where(eq(orgs.orgId, ctx.orgId)); + if (!org) { + throw new Error(`Org ${ctx.orgId} not found for directory sync`); + } + + if (dryRun) { + return syncPlan; + } + + // the upsert can return an existing row's userId instead of the generated one + // if a user is synched in multiple orgs, the generated `userId` will be + // different when creating the same user, but the `externalId` will be the same + const userIdsByExternalId = new Map(); + + await db.transaction(async (trx) => { + // create new users + // upsert so the same directory user synced into several orgs isn't created twice + const usersToUpsert = syncPlan.entries.flatMap((e) => + e.user.operation === "create" ? [e.user.data] : [] + ); + if (usersToUpsert.length > 0) { + const upsertedUsers = await trx + .insert(users) + .values(usersToUpsert) + .onConflictDoUpdate({ + target: [users.idpId, users.externalUserId], + set: { + username: sql`excluded.username`, + email: sql`excluded.email`, + name: sql`excluded.name` + } + }) + .returning({ + userId: users.userId, + externalUserId: users.externalUserId + }); + + for (const u of upsertedUsers) { + userIdsByExternalId.set(u.externalUserId!, u.userId); + } + } + + // update existing users by userId, this also backfills `externalUserId` + // for users previously matched by username + const usersToUpdate = syncPlan.entries.flatMap((e) => + e.user.operation === "update" ? [e.user.data] : [] + ); + await Promise.all( + usersToUpdate.map((u) => + trx + .update(users) + .set(u) + .where( + and( + eq(users.userId, u.userId), + eq(users.idpId, ctx.idpId) + ) + ) + ) + ); + + // apply memberships sequentially, `assignUserToOrg` and `removeUserFromOrg` + // read the user's other orgs for billing + for (const entry of syncPlan.entries) { + const userId = getEntryUserId(entry, userIdsByExternalId); + if (!userId) continue; + + switch (entry.membership.operation) { + case "add": + await assignUserToOrg( + org, + { userId, orgId: ctx.orgId, directorySynched: true }, + entry.membership.roles.map((r) => r.id), + trx + ); + break; + case "updateRoles": + // remove previous roles + await trx + .delete(userOrgRoles) + .where( + and( + eq(userOrgRoles.userId, userId), + eq(userOrgRoles.orgId, ctx.orgId) + ) + ); + // add new roles + await trx.insert(userOrgRoles).values( + entry.membership.roles.map((r) => ({ + userId, + orgId: ctx.orgId, + roleId: r.id + })) + ); + break; + case "remove": + await removeUserFromOrg(org, userId, trx); + break; + } + } + }); + + for (const entry of syncPlan.entries) { + if (entry.membership.operation === "unchanged") continue; + const userId = getEntryUserId(entry, userIdsByExternalId); + if (!userId) continue; + + calculateUserClientsForOrgs(userId).catch((error) => { + logger.error( + "Error calculating user clients after directory sync", + { error } + ); + }); + } + + return syncPlan; +} + +function getEntryUserId( + entry: UserSyncEntry, + userIdsByExternalId: Map +) { + switch (entry.user.operation) { + case "create": + return userIdsByExternalId.get(entry.user.data.externalUserId!); + case "update": + return entry.user.data.userId; + case "unchanged": + return entry.user.userId; + case "skip": + return undefined; + } +} + +async function resolveUserRoles( + user: DirectoryUser, + roleMapping: DirectorySyncRoleMappingConfig, + orgId: string, + dryRun: boolean +): Promise { + if (!user.enabled) return []; + + let matchedRoles: Role[] = []; + + switch (roleMapping.mode) { + case "fixedRoles": { + if (roleMapping.fixedRoleNames.length > 0) { + matchedRoles = await db + .select() + .from(roles) + .where( + and( + eq(roles.orgId, orgId), + inArray(roles.name, roleMapping.fixedRoleNames) + ) + ); + } + break; + } + case "groupMapping": { + const roleNames: string[] = []; + const groupMapping = new Map( + roleMapping.groupRules.map((rule) => [ + rule.group, + rule.roleNames + ]) + ); + + for (const group of user.groups) { + let roles: string[] = []; + if (group.displayName) { + roles = + groupMapping.get(group.displayName) ?? + groupMapping.get(group.id) ?? + []; + } else { + roles = + groupMapping.get(group.displayName ?? group.id) ?? []; + } + roleNames.push(...roles); + } + + if (roleNames.length > 0) { + matchedRoles = await db + .select() + .from(roles) + .where( + and( + eq(roles.orgId, orgId), + inArray(roles.name, roleNames) + ) + ); + } + break; + } + case "automatic": + const roleNames = user.groups + .map((g) => g.displayName) + .filter((name) => name !== undefined && name !== null); + + if (roleNames.length > 0) { + matchedRoles = await db + .select() + .from(roles) + .where( + and( + eq(roles.orgId, orgId), + inArray(roles.name, roleNames) + ) + ); + } + + if (roleMapping.createMissingRoles) { + const missingRoles = new Set(roleNames).difference( + new Set(matchedRoles.map((r) => r.name)) + ); + + if (missingRoles.size === 0) { + break; + } + + if (dryRun) { + // the roles would be created when applying the plan, they don't have an id yet + return [ + ...matchedRoles.map((role) => ({ + id: role.roleId, + name: role.name + })), + ...[...missingRoles].map((name) => ({ id: -1, name })) + ]; + } + + const result = await db + .insert(roles) + .values( + [...missingRoles].map((role) => ({ + name: role, + orgId: orgId + })) + ) + .returning(); + + matchedRoles.push(...result); + } + + break; + } + + return matchedRoles.map((role) => ({ id: role.roleId, name: role.name })); +} diff --git a/server/private/lib/directorySync/types.ts b/server/private/lib/directorySync/types.ts new file mode 100644 index 0000000000..30b9b0b0de --- /dev/null +++ b/server/private/lib/directorySync/types.ts @@ -0,0 +1,58 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +import type { DirectorySyncRoleMappingConfig } from "@app/lib/idpDirectorySyncRoleMapping"; + +// Provider-neutral shapes used by the directory sync engine (runSync.ts). +// Each provider (Azure, Google, ...) turns its API into these. + +export type DirectoryGroup = { + /** Stable provider group ID. */ + id: string; + displayName: string | null; +}; + +export type DirectoryUser = { + /** Stable provider user ID (Entra object ID, Google directory ID). */ + externalId: string; + /** Value that login computes as `users.username` (already resolved from `identifierPath`). */ + username: string; + email: string | null; + name: string | null; + /** False when the account is disabled or suspended in the directory. */ + enabled: boolean; + /** Every group the user belongs to, nested memberships included. */ + groups: DirectoryGroup[]; +}; + +export type DirectoryFetchResult = { + users: DirectoryUser[]; + // nextCursor: string | null; +}; + +export type DirectoryFetchContext = { + idpId: number; + orgId: string; + roleMapping: DirectorySyncRoleMappingConfig; + /** `idpOidcConfig.identifierPath`, used to compute `DirectoryUser.username`. */ + identifierPath: string; + /** Opaque provider state from the previous successful run. Null = full fetch. */ + // cursor: string | null; +}; + +export interface DirectorySyncProvider { + fetch(ctx: DirectoryFetchContext): Promise; +} + +/** Thrown by a provider when the stored cursor can't be used anymore; the engine retries with a full fetch. */ +export class DirectoryCursorExpiredError extends Error {} diff --git a/server/private/routers/orgIdp/createOrgOidcIdp.ts b/server/private/routers/orgIdp/createOrgOidcIdp.ts index 8b214b8959..0ad2093350 100644 --- a/server/private/routers/orgIdp/createOrgOidcIdp.ts +++ b/server/private/routers/orgIdp/createOrgOidcIdp.ts @@ -42,9 +42,12 @@ const bodySchema = z.strictObject({ namePath: z.string().optional(), scopes: z.string().nonempty(), autoProvision: z.boolean().optional(), + directorySyncEnabled: z.boolean().optional(), variant: z.enum(["oidc", "google", "azure"]).optional().default("oidc"), roleMapping: z.string().optional(), orgMapping: z.string().nullish(), + directoryRoleMapping: z.string().nullish(), + directorySyncDeletions: z.boolean().optional(), tags: z.string().optional() }); @@ -122,6 +125,9 @@ export async function createOrgOidcIdp( variant, roleMapping, orgMapping: orgMappingBody, + directorySyncEnabled, + directoryRoleMapping, + directorySyncDeletions, tags } = parsedBody.data; @@ -138,6 +144,15 @@ export async function createOrgOidcIdp( } } + if (autoProvision && directorySyncEnabled) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Auto provisioning and directory sync cannot both be enabled" + ) + ); + } + const key = config.getRawConfig().server.secret!; const encryptedSecret = encrypt(clientSecret, key); @@ -150,6 +165,10 @@ export async function createOrgOidcIdp( .values({ name, autoProvision, + // directory sync is only supported for google/azure idps + directorySyncEnabled: + (variant === "google" || variant === "azure") && + !!directorySyncEnabled, type: "oidc", tags }) @@ -174,7 +193,9 @@ export async function createOrgOidcIdp( idpId: idpRes.idpId, orgId: orgId, roleMapping: roleMapping || null, - orgMapping: orgMappingBody + orgMapping: orgMappingBody, + directoryRoleMapping: directoryRoleMapping || null, + directorySyncDeletions: directorySyncDeletions ?? true }); }); diff --git a/server/private/routers/orgIdp/updateOrgOidcIdp.ts b/server/private/routers/orgIdp/updateOrgOidcIdp.ts index 9d4698e3c9..13ea23311f 100644 --- a/server/private/routers/orgIdp/updateOrgOidcIdp.ts +++ b/server/private/routers/orgIdp/updateOrgOidcIdp.ts @@ -47,8 +47,11 @@ const bodySchema = z.strictObject({ namePath: z.string().optional(), scopes: z.string().optional(), autoProvision: z.boolean().optional(), + directorySyncEnabled: z.boolean().optional(), roleMapping: z.string().optional(), orgMapping: z.string().nullish(), + directoryRoleMapping: z.string().nullish(), + directorySyncDeletions: z.boolean().optional(), tags: z.string().optional() }); @@ -59,7 +62,6 @@ const UpdateOrgIdpResponseDataSchema = z.object({ idpId: z.number() }); - registry.registerPath({ method: "post", path: "/org/{orgId}/idp/{idpId}/oidc", @@ -80,7 +82,9 @@ registry.registerPath({ description: "Successful response", content: { "application/json": { - schema: createApiResponseSchema(UpdateOrgIdpResponseDataSchema) + schema: createApiResponseSchema( + UpdateOrgIdpResponseDataSchema + ) } } } @@ -126,6 +130,9 @@ export async function updateOrgOidcIdp( name, roleMapping, orgMapping, + directorySyncEnabled, + directoryRoleMapping, + directorySyncDeletions, tags } = parsedBody.data; @@ -181,10 +188,39 @@ export async function updateOrgOidcIdp( : undefined; const encryptedClientId = clientId ? encrypt(clientId, key) : undefined; + const [existingOidcConfig] = await db + .select({ variant: idpOidcConfig.variant }) + .from(idpOidcConfig) + .where(eq(idpOidcConfig.idpId, idpId)); + + // auto provisioning and directory sync are mutually exclusive + const effectiveAutoProvision = + autoProvision ?? existingIdp.autoProvision; + if ( + effectiveAutoProvision && + (directorySyncEnabled ?? existingIdp.directorySyncEnabled) + ) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Auto provisioning and directory sync cannot both be enabled" + ) + ); + } + + // directory sync is only supported for google/azure idps + const directorySyncSupported = + existingOidcConfig?.variant === "google" || + existingOidcConfig?.variant === "azure"; + const effectiveDirectorySyncEnabled = directorySyncSupported + ? directorySyncEnabled + : false; + await db.transaction(async (trx) => { const idpData = { name, autoProvision, + directorySyncEnabled: effectiveDirectorySyncEnabled, tags }; @@ -224,6 +260,8 @@ export async function updateOrgOidcIdp( const idpOrgPolicyPatch: { roleMapping?: string; orgMapping?: string | null; + directoryRoleMapping?: string | null; + directorySyncDeletions?: boolean; } = {}; if (roleMapping !== undefined) { idpOrgPolicyPatch.roleMapping = roleMapping; @@ -231,6 +269,13 @@ export async function updateOrgOidcIdp( if (orgMapping !== undefined) { idpOrgPolicyPatch.orgMapping = orgMapping; } + if (directoryRoleMapping !== undefined) { + idpOrgPolicyPatch.directoryRoleMapping = directoryRoleMapping; + } + if (directorySyncDeletions !== undefined) { + idpOrgPolicyPatch.directorySyncDeletions = + directorySyncDeletions; + } if (Object.keys(idpOrgPolicyPatch).length > 0) { await trx .update(idpOrg) diff --git a/server/routers/badger/exchangeSession.ts b/server/routers/badger/exchangeSession.ts index 31600dc3e1..c0c77ed8dd 100644 --- a/server/routers/badger/exchangeSession.ts +++ b/server/routers/badger/exchangeSession.ts @@ -4,7 +4,12 @@ import createHttpError from "http-errors"; import { z } from "zod"; import { fromError } from "zod-validation-error"; import logger from "@server/logger"; -import { resourceAccessToken, resources, sessions } from "@server/db"; +import { + resourceAccessToken, + resourceSessions, + resources, + sessions +} from "@server/db"; import { db } from "@server/db"; import { and, eq, inArray, or, sql } from "drizzle-orm"; import { @@ -128,7 +133,18 @@ export async function exchangeSession( ); } - await db.delete(sessions).where(eq(sessions.sessionId, requestToken)); + // Enforce single use: consume the request token. Checking the deleted + // row count makes concurrent exchanges of the same token race safely. + const consumed = await db + .delete(resourceSessions) + .where(eq(resourceSessions.sessionId, requestSession.sessionId)) + .returning(); + + if (consumed.length === 0) { + return next( + createHttpError(HttpCode.UNAUTHORIZED, "Invalid request token") + ); + } const token = generateSessionToken(); diff --git a/server/startSchedulers.ts b/server/startSchedulers.ts index 4a67495cc0..3f0dcfb1a2 100644 --- a/server/startSchedulers.ts +++ b/server/startSchedulers.ts @@ -12,7 +12,7 @@ export function startSchedulers() { // updates into periodic bulk writes, preventing connection pool exhaustion. startPingAccumulator(); - if (build != "saas") { + if (build !== "saas") { startOlmOfflineChecker(); // this is to handle the offline check for olms startNewtOfflineChecker(); // this is to handle the offline check for newts } diff --git a/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx b/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx index e7e52b0683..83bb963713 100644 --- a/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx +++ b/src/app/[orgId]/settings/(private)/idp/[idpId]/general/page.tsx @@ -1,7 +1,28 @@ "use client"; -import { zodResolver } from "@hookform/resolvers/zod"; -import { z } from "zod"; +import AutoProvisionConfigWidget from "@app/components/AutoProvisionConfigWidget"; +import CopyToClipboard from "@app/components/CopyToClipboard"; +import DirectorySyncConfigWidget from "@app/components/DirectorySyncConfigWidget"; +import IdpAutoProvisionUsersDescription from "@app/components/IdpAutoProvisionUsersDescription"; +import IdpIdentifierChangeDialog from "@app/components/IdpIdentifierChangeDialog"; +import IdpTypeBadge from "@app/components/IdpTypeBadge"; +import { + InfoSection, + InfoSectionContent, + InfoSections, + InfoSectionTitle +} from "@app/components/InfoSection"; +import { PaidFeaturesAlert } from "@app/components/PaidFeaturesAlert"; +import { + SettingsContainer, + SettingsSection, + SettingsSectionBody, + SettingsSectionDescription, + SettingsSectionForm, + SettingsSectionGrid, + SettingsSectionHeader, + SettingsSectionTitle +} from "@app/components/Settings"; import { Button } from "@app/components/ui/button"; import { Form, @@ -13,42 +34,9 @@ import { FormMessage } from "@app/components/ui/form"; import { Input } from "@app/components/ui/input"; -import { useForm } from "react-hook-form"; -import { toast } from "@app/hooks/useToast"; -import { useRouter, useParams, redirect } from "next/navigation"; -import { - SettingsContainer, - SettingsSection, - SettingsSectionHeader, - SettingsSectionTitle, - SettingsSectionDescription, - SettingsSectionBody, - SettingsSectionForm, - SettingsSectionFooter, - SettingsSectionGrid -} from "@app/components/Settings"; -import { formatAxiosError } from "@app/lib/api"; -import { createApiClient } from "@app/lib/api"; import { useEnvContext } from "@app/hooks/useEnvContext"; -import { useState, useEffect } from "react"; -import { Alert, AlertDescription, AlertTitle } from "@app/components/ui/alert"; -import { InfoIcon, ExternalLink } from "lucide-react"; -import { - InfoSection, - InfoSectionContent, - InfoSections, - InfoSectionTitle -} from "@app/components/InfoSection"; -import CopyToClipboard from "@app/components/CopyToClipboard"; -import IdpTypeBadge from "@app/components/IdpTypeBadge"; -import { useTranslations } from "next-intl"; -import { AxiosResponse } from "axios"; -import { ListRolesResponse } from "@server/routers/role"; -import AutoProvisionConfigWidget from "@app/components/AutoProvisionConfigWidget"; -import IdpAutoProvisionUsersDescription from "@app/components/IdpAutoProvisionUsersDescription"; -import IdpIdentifierChangeDialog from "@app/components/IdpIdentifierChangeDialog"; -import { PaidFeaturesAlert } from "@app/components/PaidFeaturesAlert"; -import { tierMatrix } from "@server/lib/billing/tierMatrix"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; import { compileRoleMappingExpression, createMappingBuilderRule, @@ -57,6 +45,21 @@ import { MappingBuilderRule, RoleMappingMode } from "@app/lib/idpRoleMapping"; +import { + createDefaultDirectorySyncRoleMappingConfig, + DirectorySyncRoleMappingConfig, + parseDirectorySyncRoleMapping, + serializeDirectorySyncRoleMapping +} from "@app/lib/idpDirectorySyncRoleMapping"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { tierMatrix } from "@server/lib/billing/tierMatrix"; +import { ListRolesResponse } from "@server/routers/role"; +import { AxiosResponse } from "axios"; +import { useTranslations } from "next-intl"; +import { useParams, useRouter } from "next/navigation"; +import { useEffect, useState } from "react"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; export default function GeneralPage() { const { env } = useEnvContext(); @@ -75,6 +78,10 @@ export default function GeneralPage() { MappingBuilderRule[] >([createMappingBuilderRule()]); const [rawRoleExpression, setRawRoleExpression] = useState(""); + const [directoryRoleMapping, setDirectoryRoleMapping] = + useState( + createDefaultDirectorySyncRoleMappingConfig + ); const [variant, setVariant] = useState<"oidc" | "google" | "azure">("oidc"); const [originalIdentifierPath, setOriginalIdentifierPath] = useState(""); const [identifierConfirmOpen, setIdentifierConfirmOpen] = useState(false); @@ -105,6 +112,8 @@ export default function GeneralPage() { namePath: z.string().nullable().optional(), scopes: z.string().min(1, { message: t("idpScopeRequired") }), autoProvision: z.boolean().default(false), + directorySyncEnabled: z.boolean().default(false), + directorySyncDeletions: z.boolean().default(true), orgMapping: z.string().optional() }); @@ -118,6 +127,8 @@ export default function GeneralPage() { roleMapping: z.string().nullable().optional(), roleId: z.number().nullable().optional(), autoProvision: z.boolean().default(false), + directorySyncEnabled: z.boolean().default(false), + directorySyncDeletions: z.boolean().default(true), orgMapping: z.string().optional() }); @@ -132,6 +143,8 @@ export default function GeneralPage() { roleMapping: z.string().nullable().optional(), roleId: z.number().nullable().optional(), autoProvision: z.boolean().default(false), + directorySyncEnabled: z.boolean().default(false), + directorySyncDeletions: z.boolean().default(true), orgMapping: z.string().optional() }); @@ -139,9 +152,7 @@ export default function GeneralPage() { type GoogleFormValues = z.infer; type AzureFormValues = z.infer; type GeneralFormValues = - | OidcFormValues - | GoogleFormValues - | AzureFormValues; + OidcFormValues | GoogleFormValues | AzureFormValues; // Get the appropriate schema based on variant const getFormSchema = () => { @@ -168,6 +179,8 @@ export default function GeneralPage() { namePath: "name", scopes: "openid profile email", autoProvision: true, + directorySyncEnabled: false, + directorySyncDeletions: true, roleMapping: null, roleId: null, tenantId: "", @@ -220,6 +233,9 @@ export default function GeneralPage() { clientId: data.idpOidcConfig.clientId, clientSecret: data.idpOidcConfig.clientSecret, autoProvision: data.idp.autoProvision, + directorySyncEnabled: data.idp.directorySyncEnabled, + directorySyncDeletions: + data.idpOrg?.directorySyncDeletions ?? true, roleMapping: roleMapping || null, roleId: null, orgMapping: data.idpOrg?.orgMapping ?? "" @@ -254,6 +270,12 @@ export default function GeneralPage() { setRawRoleExpression( detectedRoleMappingConfig.rawExpression ); + + setDirectoryRoleMapping( + parseDirectorySyncRoleMapping( + data.idpOrg.directoryRoleMapping + ) + ); } } catch (e) { toast({ @@ -345,8 +367,14 @@ export default function GeneralPage() { clientId: data.clientId, clientSecret: data.clientSecret, autoProvision: data.autoProvision, + directorySyncEnabled: + (variant === "google" || variant === "azure") && + data.directorySyncEnabled, roleMapping: roleMappingExpression, - orgMapping: orgMappingTrimmed === "" ? null : orgMappingTrimmed + orgMapping: orgMappingTrimmed === "" ? null : orgMappingTrimmed, + directoryRoleMapping: + serializeDirectorySyncRoleMapping(directoryRoleMapping), + directorySyncDeletions: data.directorySyncDeletions }; // Add variant-specific fields @@ -557,6 +585,12 @@ export default function GeneralPage() { autoProvision={form.watch("autoProvision")} onAutoProvisionChange={(checked) => { form.setValue("autoProvision", checked); + if (checked) { + form.setValue( + "directorySyncEnabled", + false + ); + } }} orgId={orgId as string} roleMappingMode={roleMappingMode} @@ -588,6 +622,63 @@ export default function GeneralPage() { + {/* Directory Sync Settings */} + {(variant === "google" || variant === "azure") && ( + + + + {t("idpDirectorySync")} + + + {t("idpDirectorySyncDescription")} + + + +
+ + { + form.setValue( + "directorySyncEnabled", + checked + ); + if (checked) { + form.setValue( + "autoProvision", + false + ); + } + }} + orgId={orgId as string} + roleMapping={directoryRoleMapping} + syncDeletions={form.watch( + "directorySyncDeletions" + )} + onSyncDeletionsChange={(checked) => + form.setValue( + "directorySyncDeletions", + checked + ) + } + onRoleMappingChange={ + setDirectoryRoleMapping + } + /> + + +
+ + )} + {/* Google Configuration */} {variant === "google" && ( diff --git a/src/app/[orgId]/settings/(private)/idp/create/page.tsx b/src/app/[orgId]/settings/(private)/idp/create/page.tsx index 33ef71eee0..e72551ae49 100644 --- a/src/app/[orgId]/settings/(private)/idp/create/page.tsx +++ b/src/app/[orgId]/settings/(private)/idp/create/page.tsx @@ -1,6 +1,7 @@ "use client"; import AutoProvisionConfigWidget from "@app/components/AutoProvisionConfigWidget"; +import DirectorySyncConfigWidget from "@app/components/DirectorySyncConfigWidget"; import IdpAutoProvisionUsersDescription from "@app/components/IdpAutoProvisionUsersDescription"; import { PaidFeaturesAlert } from "@app/components/PaidFeaturesAlert"; import { @@ -48,6 +49,11 @@ import { MappingBuilderRule, RoleMappingMode } from "@app/lib/idpRoleMapping"; +import { + createDefaultDirectorySyncRoleMappingConfig, + DirectorySyncRoleMappingConfig, + serializeDirectorySyncRoleMapping +} from "@app/lib/idpDirectorySyncRoleMapping"; export default function Page() { const { env } = useEnvContext(); @@ -64,6 +70,10 @@ export default function Page() { MappingBuilderRule[] >([createMappingBuilderRule()]); const [rawRoleExpression, setRawRoleExpression] = useState(""); + const [directoryRoleMapping, setDirectoryRoleMapping] = + useState( + createDefaultDirectorySyncRoleMappingConfig + ); const t = useTranslations(); const { isPaidUser } = usePaidStatus(); @@ -92,7 +102,9 @@ export default function Page() { autoProvision: z.boolean().default(false), roleMapping: z.string().nullable().optional(), roleId: z.number().nullable().optional(), - orgMapping: z.string().optional() + orgMapping: z.string().optional(), + directorySyncEnabled: z.boolean().default(false), + directorySyncDeletions: z.boolean().default(true) }); type CreateIdpFormValues = z.infer; @@ -114,7 +126,9 @@ export default function Page() { autoProvision: false, roleMapping: null, roleId: null, - orgMapping: "" + orgMapping: "", + directorySyncEnabled: false, + directorySyncDeletions: true } }); @@ -122,9 +136,9 @@ export default function Page() { useEffect(() => { async function fetchRoles() { const res = await api - .get< - AxiosResponse - >(`/org/${params.orgId}/roles`) + .get>( + `/org/${params.orgId}/roles` + ) .catch((e) => { console.error(e); toast({ @@ -189,6 +203,12 @@ export default function Page() { emailPath: data.emailPath, namePath: data.namePath, autoProvision: data.autoProvision, + directorySyncEnabled: + (data.type === "google" || data.type === "azure") && + data.directorySyncEnabled, + directoryRoleMapping: + serializeDirectorySyncRoleMapping(directoryRoleMapping), + directorySyncDeletions: data.directorySyncDeletions, roleMapping: roleMappingExpression, scopes: data.scopes, variant: data.type @@ -333,6 +353,12 @@ export default function Page() { "autoProvision", checked ); + if (checked) { + form.setValue( + "directorySyncEnabled", + false + ); + } }} orgId={params.orgId as string} roleMappingMode={roleMappingMode} @@ -370,6 +396,68 @@ export default function Page() { + {/* Directory Sync Settings */} + {(form.watch("type") === "google" || + form.watch("type") === "azure") && ( + + + + {t("idpDirectorySync")} + + + {t("idpDirectorySyncDescription")} + + + +
+ + { + form.setValue( + "directorySyncEnabled", + checked + ); + if (checked) { + form.setValue( + "autoProvision", + false + ); + } + }} + orgId={params.orgId as string} + roleMapping={directoryRoleMapping} + onRoleMappingChange={ + setDirectoryRoleMapping + } + syncDeletions={ + form.watch( + "directorySyncDeletions" + ) ?? true + } + onSyncDeletionsChange={(checked) => + form.setValue( + "directorySyncDeletions", + checked + ) + } + /> + + +
+ + )} + {form.watch("type") === "google" && ( diff --git a/src/app/admin/idp/page.tsx b/src/app/admin/idp/page.tsx index 01aac2a1e6..8fda09cd62 100644 --- a/src/app/admin/idp/page.tsx +++ b/src/app/admin/idp/page.tsx @@ -2,7 +2,7 @@ import { internal } from "@app/lib/api"; import { authCookieHeader } from "@app/lib/api/cookies"; import { AxiosResponse } from "axios"; import SettingsSectionTitle from "@app/components/SettingsSectionTitle"; -import IdpTable, { IdpRow } from "@app/components/AdminIdpTable"; +import AdminIdpTable, { IdpRow } from "@app/components/AdminIdpTable"; import { getTranslations } from "next-intl/server"; export default async function IdpPage() { @@ -25,7 +25,7 @@ export default async function IdpPage() { title={t("idpManage")} description={t("idpManageDescription")} /> - + ); } diff --git a/src/components/AdminIdpTable.tsx b/src/components/AdminIdpTable.tsx index 75a7c54543..8dcc28d156 100644 --- a/src/components/AdminIdpTable.tsx +++ b/src/components/AdminIdpTable.tsx @@ -1,26 +1,23 @@ "use client"; -import { ColumnDef } from "@tanstack/react-table"; -import { ExtendedColumnDef } from "@app/components/ui/data-table"; import { IdpDataTable } from "@app/components/AdminIdpDataTable"; -import { Button } from "@app/components/ui/button"; -import { ArrowRight, ArrowUpDown, MoreHorizontal } from "lucide-react"; -import { useState } from "react"; import ConfirmDeleteDialog from "@app/components/ConfirmDeleteDialog"; -import { toast } from "@app/hooks/useToast"; -import { formatAxiosError } from "@app/lib/api"; -import { createApiClient } from "@app/lib/api"; -import { useEnvContext } from "@app/hooks/useEnvContext"; -import { Badge } from "@app/components/ui/badge"; -import { useRouter } from "next/navigation"; +import { Button } from "@app/components/ui/button"; +import { ExtendedColumnDef } from "@app/components/ui/data-table"; import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from "@app/components/ui/dropdown-menu"; -import Link from "next/link"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { toast } from "@app/hooks/useToast"; +import { createApiClient, formatAxiosError } from "@app/lib/api"; +import { ArrowRight, ArrowUpDown, MoreHorizontal } from "lucide-react"; import { useTranslations } from "next-intl"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useState } from "react"; import IdpTypeBadge from "./IdpTypeBadge"; export type IdpRow = { @@ -35,7 +32,7 @@ type Props = { idps: IdpRow[]; }; -export default function IdpTable({ idps }: Props) { +export default function AdminIdpTable({ idps }: Props) { const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); const [selectedIdp, setSelectedIdp] = useState(null); const api = createApiClient(useEnvContext()); diff --git a/src/components/AutoProvisionConfigWidget.tsx b/src/components/AutoProvisionConfigWidget.tsx index 4cf9394445..156f74c95a 100644 --- a/src/components/AutoProvisionConfigWidget.tsx +++ b/src/components/AutoProvisionConfigWidget.tsx @@ -86,7 +86,7 @@ export default function AutoProvisionConfigWidget({ diff --git a/src/components/DirectorySyncConfigWidget.tsx b/src/components/DirectorySyncConfigWidget.tsx new file mode 100644 index 0000000000..9085b5d949 --- /dev/null +++ b/src/components/DirectorySyncConfigWidget.tsx @@ -0,0 +1,76 @@ +"use client"; + +import DirectorySyncRoleMappingFields from "@app/components/DirectorySyncRoleMappingFields"; +import { SwitchInput } from "@app/components/SwitchInput"; +import { DirectorySyncRoleMappingConfig } from "@app/lib/idpDirectorySyncRoleMapping"; +import { useTranslations } from "next-intl"; + +type DirectorySyncConfigWidgetProps = { + directorySyncEnabled: boolean; + onDirectorySyncEnabledChange: (checked: boolean) => void; + roleMapping: DirectorySyncRoleMappingConfig; + onRoleMappingChange: (roleMapping: DirectorySyncRoleMappingConfig) => void; + syncDeletions: boolean; + onSyncDeletionsChange: (checked: boolean) => void; + orgId: string; + showDirectorySyncSwitch?: boolean; + disabled?: boolean; + roleMappingFieldIdPrefix?: string; + directorySyncSwitchId?: string; + syncDeletionsSwitchId?: string; +}; + +export default function DirectorySyncConfigWidget({ + directorySyncEnabled, + onDirectorySyncEnabledChange, + roleMapping, + onRoleMappingChange, + syncDeletions, + onSyncDeletionsChange, + orgId, + showDirectorySyncSwitch = true, + disabled = false, + roleMappingFieldIdPrefix = "org-idp-directory-sync", + directorySyncSwitchId = "directory-sync-toggle", + syncDeletionsSwitchId = "directory-sync-deletions-toggle" +}: DirectorySyncConfigWidgetProps) { + const t = useTranslations(); + + const showRoleMapping = + showDirectorySyncSwitch === false || directorySyncEnabled; + + return ( +
+ {showDirectorySyncSwitch && ( +
+ +
+ )} + + {showRoleMapping && ( +
+ + +
+ )} +
+ ); +} diff --git a/src/components/DirectorySyncRoleMappingFields.tsx b/src/components/DirectorySyncRoleMappingFields.tsx new file mode 100644 index 0000000000..31b8b3d08d --- /dev/null +++ b/src/components/DirectorySyncRoleMappingFields.tsx @@ -0,0 +1,309 @@ +"use client"; + +import { HorizontalTabs } from "@app/components/HorizontalTabs"; +import { Alert, AlertDescription } from "@app/components/ui/alert"; +import { FormDescription, FormLabel } from "@app/components/ui/form"; +import { Button } from "@app/components/ui/button"; +import { CheckboxWithLabel } from "@app/components/ui/checkbox"; +import { Input } from "@app/components/ui/input"; +import { InfoIcon } from "lucide-react"; +import { useTranslations } from "next-intl"; +import { ReactNode, useEffect } from "react"; +import { + createDirectoryGroupRoleRule, + DirectoryGroupRoleRule, + DirectorySyncRoleMappingConfig, + DirectorySyncRoleMappingMode +} from "@app/lib/idpDirectorySyncRoleMapping"; +import { usePaidStatus } from "@app/hooks/usePaidStatus"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { tierMatrix } from "@server/lib/billing/tierMatrix"; +import { build } from "@server/build"; +import { RolesSelector, SelectedRole } from "./roles-selector"; + +export type DirectorySyncRoleMappingFieldsProps = { + value: DirectorySyncRoleMappingConfig; + onChange: (value: DirectorySyncRoleMappingConfig) => void; + orgId: string; + /** Unique prefix for field ids when multiple instances exist on one page. */ + fieldIdPrefix?: string; +}; + +export default function DirectorySyncRoleMappingFields({ + value, + onChange, + orgId, + fieldIdPrefix = "directory-sync-role-mapping" +}: DirectorySyncRoleMappingFieldsProps) { + const t = useTranslations(); + const { env } = useEnvContext(); + const { isPaidUser } = usePaidStatus(); + + const supportsMultipleRolesPerUser = isPaidUser(tierMatrix.fullRbac); + const showSingleRoleDisclaimer = + !env.flags.disableEnterpriseFeatures && + !isPaidUser(tierMatrix.fullRbac); + + const { mode, fixedRoleNames, groupRules, createMissingRoles } = value; + + useEffect(() => { + if (!supportsMultipleRolesPerUser && fixedRoleNames.length > 1) { + onChange({ ...value, fixedRoleNames: [fixedRoleNames[0]] }); + } + }, [supportsMultipleRolesPerUser, fixedRoleNames, value, onChange]); + + const selectRoleNames = ( + nextTags: SelectedRole[], + current: string[] + ): string[] => { + let names = [...new Set(nextTags.map((tag) => tag.text))]; + if (!supportsMultipleRolesPerUser) { + if (names.length === 0 && current.length > 0) { + return [current[current.length - 1]!]; + } + if (names.length > 1) { + names = [names[names.length - 1]!]; + } + } + return names; + }; + + const setGroupRules = (rules: DirectoryGroupRoleRule[]) => + onChange({ + ...value, + groupRules: rules.length ? rules : [createDirectoryGroupRoleRule()] + }); + + const modes: { value: DirectorySyncRoleMappingMode; label: string }[] = [ + { + value: "automatic", + label: t("directorySyncRoleMappingModeAutomatic") + }, + { + value: "groupMapping", + label: t("directorySyncRoleMappingModeGroupMapping") + }, + { + value: "fixedRoles", + label: t("directorySyncRoleMappingModeFixedRoles") + } + ]; + + const groupRulesGridClass = + "md:grid md:grid-cols-[minmax(0,1fr)_minmax(0,1.75fr)_6rem] md:gap-x-3"; + + const panels: Record = { + fixedRoles: ( +
+ ({ + id: name, + text: name + }))} + mapRolesByName + orgId={orgId} + onSelectRoles={(nextTags) => + onChange({ + ...value, + fixedRoleNames: selectRoleNames( + nextTags, + fixedRoleNames + ) + }) + } + /> + + {t("directorySyncRoleMappingFixedRolesDescription")} + +
+ ), + groupMapping: ( +
+
+
+ + {t("directorySyncRoleMappingGroup")} + + + {t("roleMappingAssignRoles")} + + +
+ + {groupRules.map((rule, index) => ( +
+
+ + {t("directorySyncRoleMappingGroup")} + + + setGroupRules( + groupRules.map((row, i) => + i === index + ? { + ...row, + group: e.target.value + } + : row + ) + ) + } + placeholder={t( + "directorySyncRoleMappingGroupPlaceholder" + )} + /> +
+
+ + {t("roleMappingAssignRoles")} + + ({ id: name, text: name }) + )} + buttonText={t("roleMappingAssignRoles")} + mapRolesByName + orgId={orgId} + onSelectRoles={(nextTags) => + setGroupRules( + groupRules.map((row, i) => + i === index + ? { + ...row, + roleNames: + selectRoleNames( + nextTags, + row.roleNames + ) + } + : row + ) + ) + } + /> +
+
+ +
+
+ ))} +
+ + + + {t("directorySyncRoleMappingGroupMappingDescription")} + +
+ ), + automatic: ( +
+ + {t("directorySyncRoleMappingAutomaticDescription")} + + + onChange({ + ...value, + createMissingRoles: checked === true + }) + } + /> + + {t("directorySyncRoleMappingCreateMissingRolesDescription")} + +
+ ) + }; + + const skipNotices: Partial> = { + groupMapping: t("directorySyncRoleMappingGroupMappingSkipNotice"), + automatic: createMissingRoles + ? t("directorySyncRoleMappingAutomaticSkipNotice") + : t("directorySyncRoleMappingAutomaticNoCreateSkipNotice") + }; + + const activeTab = Math.max( + modes.findIndex((option) => option.value === mode), + 0 + ); + + return ( +
+
+ {showSingleRoleDisclaimer && ( + + {build === "saas" + ? t("singleRolePerUserPlanNotice") + : t("singleRolePerUserEditionNotice")} + + )} +
+ + ({ + title: option.label, + href: "#" + }))} + activeTab={activeTab} + onTabChange={(index) => + onChange({ ...value, mode: modes[index]!.value }) + } + > + {modes.map((option) => ( +
+ + {t("directorySyncRoleMappingDescription")} + + + {skipNotices[option.value] && ( + + + + {skipNotices[option.value]} + + + )} + {panels[option.value]} +
+ ))} + +
+ ); +} diff --git a/src/components/HorizontalTabs.tsx b/src/components/HorizontalTabs.tsx index 6c91dbfefd..725e2359b7 100644 --- a/src/components/HorizontalTabs.tsx +++ b/src/components/HorizontalTabs.tsx @@ -24,6 +24,10 @@ interface HorizontalTabsProps { disabled?: boolean; clientSide?: boolean; defaultTab?: number; + /** Client-side only: controlled active tab index. */ + activeTab?: number; + /** Client-side only: called with the clicked tab index. */ + onTabChange?: (index: number) => void; } export function HorizontalTabs({ @@ -31,13 +35,17 @@ export function HorizontalTabs({ items, disabled = false, clientSide = false, - defaultTab = 0 + defaultTab = 0, + activeTab, + onTabChange }: HorizontalTabsProps) { const pathname = usePathname(); const params = useParams(); const { licenseStatus, isUnlocked } = useLicenseStatusContext(); const t = useTranslations(); - const [activeClientTab, setActiveClientTab] = useState(defaultTab); + const [uncontrolledClientTab, setUncontrolledClientTab] = + useState(defaultTab); + const activeClientTab = activeTab ?? uncontrolledClientTab; function hydrateHref(href: string) { return href @@ -75,7 +83,8 @@ export function HorizontalTabs({ type="button" onClick={() => { if (!isDisabled) { - setActiveClientTab(index); + setUncontrolledClientTab(index); + onTabChange?.(index); } }} className={cn( diff --git a/src/lib/idpDirectorySyncRoleMapping.ts b/src/lib/idpDirectorySyncRoleMapping.ts new file mode 100644 index 0000000000..b93a2f20f7 --- /dev/null +++ b/src/lib/idpDirectorySyncRoleMapping.ts @@ -0,0 +1,147 @@ +export type DirectorySyncRoleMappingMode = + "fixedRoles" | "groupMapping" | "automatic"; + +export type DirectoryGroupRoleRule = { + /** Stable React list key; not persisted. */ + id?: string; + /** Directory group name or ID to match. */ + group: string; + roleNames: string[]; +}; + +export type DirectorySyncRoleMappingConfig = { + mode: DirectorySyncRoleMappingMode; + /** `fixedRoles`: roles given to every synced user. */ + fixedRoleNames: string[]; + /** `groupMapping`: explicit directory group → roles rules. */ + groupRules: DirectoryGroupRoleRule[]; + /** `automatic`: create a role named after a group when none exists. */ + createMissingRoles: boolean; +}; + +const DIRECTORY_SYNC_ROLE_MAPPING_VERSION = 1; + +const MODES: DirectorySyncRoleMappingMode[] = [ + "fixedRoles", + "groupMapping", + "automatic" +]; + +function newGroupRoleRuleId(): string { + if (typeof crypto !== "undefined" && "randomUUID" in crypto) { + return crypto.randomUUID(); + } + return `group-rule-${Date.now()}-${Math.random().toString(36).slice(2, 11)}`; +} + +export function createDirectoryGroupRoleRule(): DirectoryGroupRoleRule { + return { + id: newGroupRoleRuleId(), + group: "", + roleNames: [] + }; +} + +export function createDefaultDirectorySyncRoleMappingConfig(): DirectorySyncRoleMappingConfig { + return { + mode: "automatic", + fixedRoleNames: [], + groupRules: [createDirectoryGroupRoleRule()], + createMissingRoles: false + }; +} + +function toStringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((v): v is string => typeof v === "string") + : []; +} + +/** Parse the stored `idpOrg.directoryRoleMapping` value. Invalid values fall back to defaults. */ +export function parseDirectorySyncRoleMapping( + stored: string | null | undefined +): DirectorySyncRoleMappingConfig { + const defaults = createDefaultDirectorySyncRoleMappingConfig(); + if (!stored?.trim()) { + return defaults; + } + + let parsed: unknown; + try { + parsed = JSON.parse(stored); + } catch { + return defaults; + } + if (!parsed || typeof parsed !== "object") { + return defaults; + } + + const raw = parsed as Record; + const mode = MODES.includes(raw.mode as DirectorySyncRoleMappingMode) + ? (raw.mode as DirectorySyncRoleMappingMode) + : defaults.mode; + + const groupRules = Array.isArray(raw.groupRules) + ? raw.groupRules + .filter( + (rule): rule is Record => + !!rule && typeof rule === "object" + ) + .map((rule) => ({ + id: newGroupRoleRuleId(), + group: typeof rule.group === "string" ? rule.group : "", + roleNames: toStringArray(rule.roleNames) + })) + : []; + + return { + mode, + fixedRoleNames: toStringArray(raw.fixedRoleNames), + groupRules: groupRules.length ? groupRules : defaults.groupRules, + createMissingRoles: raw.createMissingRoles === true + }; +} + +/** + * Serialize for `idpOrg.directoryRoleMapping`. Only the active mode's fields are kept. + * Returns null when the active mode has nothing configured. + */ +export function serializeDirectorySyncRoleMapping( + config: DirectorySyncRoleMappingConfig +): string | null { + const base = { + version: DIRECTORY_SYNC_ROLE_MAPPING_VERSION, + mode: config.mode + }; + + switch (config.mode) { + case "fixedRoles": { + const fixedRoleNames = config.fixedRoleNames + .map((name) => name.trim()) + .filter(Boolean); + if (!fixedRoleNames.length) { + return null; + } + return JSON.stringify({ ...base, fixedRoleNames }); + } + case "groupMapping": { + const groupRules = config.groupRules + .map((rule) => ({ + group: rule.group.trim(), + roleNames: rule.roleNames + .map((name) => name.trim()) + .filter(Boolean) + })) + .filter((rule) => rule.group && rule.roleNames.length); + if (!groupRules.length) { + return null; + } + return JSON.stringify({ ...base, groupRules }); + } + case "automatic": + return JSON.stringify({ + ...base, + createMissingRoles: config.createMissingRoles + }); + } +}