feat: Identity Provider Directory Sync

messages/en-US.json

@@ -591,7 +591,7 @@
     "licenseReckeckAll": "Recheck All Keys",
     "licenseSiteUsage": "Sites Usage",
     "licenseSiteUsageDecsription": "View the number of sites using this license.",
-    "licenseNoSiteLimit": "There is no limit on the number of sites using an unlicensed host.",
+    "licenseNoSiteLimit": "There is no limit on the number of sites for this host.",
     "licensePurchase": "Purchase License",
     "licensePurchaseSites": "Purchase Additional Sites",
     "licenseSitesUsedMax": "{usedSites} of {maxSites} sites used",
@@ -600,7 +600,7 @@
     "licenseUsageDescription": "View the number of users and sites licensed for this host",
     "licenseUsageSites": "Sites",
     "licenseUsageUsers": "Users",
-    "licenseNoUserLimit": "There is no limit on the number of users using an unlicensed hot",
+    "licenseNoUserLimit": "There is no limit on the number of users for this host.",
     "licenseUsersUsedMax": "{usedUsers} of {maxUsers} users used",
     "licenseUsersUsed": "{count, plural, =0 {# users} one {# user} other {# users}}",
     "licenseUnlimited": "Unlimited",
@@ -1178,6 +1178,25 @@
     "idpAutoProvisionUsers": "Auto Provision Users",
     "idpAutoProvisionUsersDescription": "When enabled, users will be automatically created in the system upon first login with the ability to map users to roles and organizations.",
     "idpAutoProvisionConfigureAfterCreate": "You can configure auto provision settings once the identity provider is created.",
+    "idpDirectorySync": "Directory Sync",
+    "idpDirectorySyncDescription": "Periodically sync users from the identity provider directory. Cannot be enabled together with auto provisioning.",
+    "idpDirectorySyncDeletions": "Sync User Deletions",
+    "idpDirectorySyncDeletionsDescription": "When enabled, users who are deleted from the directory or no longer match the role mapping are removed from this organization.",
+    "directorySyncRoleMappingDescription": "Determine how roles are assigned to users synced from the directory.",
+    "directorySyncRoleMappingModeFixedRoles": "Fixed Roles",
+    "directorySyncRoleMappingModeGroupMapping": "Group Mapping",
+    "directorySyncRoleMappingModeAutomatic": "Automatic",
+    "directorySyncRoleMappingFixedRolesDescription": "Assign the same role set to every synced user.",
+    "directorySyncRoleMappingGroup": "Directory Group",
+    "directorySyncRoleMappingGroupPlaceholder": "Group name or ID",
+    "directorySyncRoleMappingAddGroupRule": "Add Group Mapping",
+    "directorySyncRoleMappingGroupMappingDescription": "Members of a directory group receive the roles mapped to that group.",
+    "directorySyncRoleMappingAutomaticDescription": "Each directory group a user belongs to is mapped to the role with the same name.",
+    "directorySyncRoleMappingCreateMissingRoles": "Create missing roles",
+    "directorySyncRoleMappingCreateMissingRolesDescription": "When enabled, a role is created for any directory group that doesn't have a matching role yet. Otherwise those groups are skipped.",
+    "directorySyncRoleMappingGroupMappingSkipNotice": "Only users who belong to at least one of the groups mapped below are synced. Users without a mapped group are not synced.",
+    "directorySyncRoleMappingAutomaticSkipNotice": "Users who don't belong to any directory group are not synced.",
+    "directorySyncRoleMappingAutomaticNoCreateSkipNotice": "Users who don't belong to any directory group, or whose groups have no role with the same name, are not synced.",
     "licenseBadge": "EE",
     "idpType": "Provider Type",
     "idpTypeDescription": "Select the type of identity provider you want to configure",

server/db/pg/schema/schema.ts

@@ -1209,19 +1209,25 @@ export const supporterKey = pgTable("supporterKey", {
 export const idp = pgTable("idp", {
     idpId: serial("idpId").primaryKey(),
     name: varchar("name").notNull(),
-    type: varchar("type").notNull(),
+    type: varchar("type").notNull().$type<"oidc" | "google" | "azure">(),
     defaultRoleMapping: varchar("defaultRoleMapping"),
     defaultOrgMapping: varchar("defaultOrgMapping"),
     autoProvision: boolean("autoProvision").notNull().default(false),
-    tags: text("tags")
+    tags: text("tags"),
+    directorySyncEnabled: boolean("directorySyncEnabled")
+        .notNull()
+        .default(false)
 });
 
 export const idpOidcConfig = pgTable("idpOidcConfig", {
     idpOauthConfigId: serial("idpOauthConfigId").primaryKey(),
     idpId: integer("idpId")
         .notNull()
         .references(() => idp.idpId, { onDelete: "cascade" }),
-    variant: varchar("variant").notNull().default("oidc"),
+    variant: varchar("variant")
+        .notNull()
+        .$type<"oidc" | "google" | "azure">()
+        .default("oidc"),
     clientId: varchar("clientId").notNull(),
     clientSecret: varchar("clientSecret").notNull(),
     authUrl: varchar("authUrl").notNull(),
@@ -1326,9 +1332,42 @@ export const idpOrg = pgTable("idpOrg", {
         .notNull()
         .references(() => orgs.orgId, { onDelete: "cascade" }),
     roleMapping: varchar("roleMapping"),
-    orgMapping: varchar("orgMapping")
+    orgMapping: varchar("orgMapping"),
+    directoryRoleMapping: varchar("directoryRoleMapping"),
+    directorySyncDeletions: boolean("directorySyncDeletions")
+        .notNull()
+        .default(true)
+});
+
+// Runtime state of the directory sync job, one row per IdP. Deleting the row
+// forces a full sync on the next run.
+export const idpSyncState = pgTable("idpSyncState", {
+    idpId: integer("idpId")
+        .primaryKey()
+        .references(() => idp.idpId, { onDelete: "cascade" }),
+    // Provider-defined opaque JSON (e.g. Azure delta links). Null = full sync.
+    cursor: text("cursor"),
+    lastSyncStartedAt: bigint("lastSyncStartedAt", { mode: "number" }),
+    lastSyncSucceededAt: bigint("lastSyncSucceededAt", { mode: "number" }),
+    lastError: text("lastError"),
+    consecutiveFailures: integer("consecutiveFailures").notNull().default(0)
 });
 
+// Links a directory user (stable provider ID) to the Pangolin user sync manages.
+export const idpSyncUser = pgTable(
+    "idpSyncUser",
+    {
+        idpId: integer("idpId")
+            .notNull()
+            .references(() => idp.idpId, { onDelete: "cascade" }),
+        externalId: varchar("externalId").notNull(),
+        userId: varchar("userId")
+            .notNull()
+            .references(() => users.userId, { onDelete: "cascade" })
+    },
+    (t) => [primaryKey({ columns: [t.idpId, t.externalId] })]
+);
+
 export const clients = pgTable(
     "clients",
     {

server/db/sqlite/schema/schema.ts

@@ -1512,23 +1512,31 @@ export const supporterKey = sqliteTable("supporterKey", {
 export const idp = sqliteTable("idp", {
     idpId: integer("idpId").primaryKey({ autoIncrement: true }),
     name: text("name").notNull(),
-    type: text("type").notNull(),
+    type: text("type").notNull().$type<"oidc" | "google" | "azure">(),
     defaultRoleMapping: text("defaultRoleMapping"),
     defaultOrgMapping: text("defaultOrgMapping"),
     autoProvision: integer("autoProvision", {
         mode: "boolean"
     })
         .notNull()
         .default(false),
-    tags: text("tags")
+    tags: text("tags"),
+    directorySyncEnabled: integer("directorySyncEnabled", {
+        mode: "boolean"
+    })
+        .notNull()
+        .default(false)
 });
 
 // Identity Provider OAuth Configuration
 export const idpOidcConfig = sqliteTable("idpOidcConfig", {
     idpOauthConfigId: integer("idpOauthConfigId").primaryKey({
         autoIncrement: true
     }),
-    variant: text("variant").notNull().default("oidc"),
+    variant: text("variant")
+        .notNull()
+        .$type<"oidc" | "google" | "azure">()
+        .default("oidc"),
     idpId: integer("idpId")
         .notNull()
         .references(() => idp.idpId, { onDelete: "cascade" }),
@@ -1638,9 +1646,44 @@ export const idpOrg = sqliteTable("idpOrg", {
         .notNull()
         .references(() => orgs.orgId, { onDelete: "cascade" }),
     roleMapping: text("roleMapping"),
-    orgMapping: text("orgMapping")
+    orgMapping: text("orgMapping"),
+    directoryRoleMapping: text("directoryRoleMapping"),
+    directorySyncDeletions: integer("directorySyncDeletions", {
+        mode: "boolean"
+    })
+        .notNull()
+        .default(true)
 });
 
+// Runtime state of the directory sync job, one row per IdP. Deleting the row
+// forces a full sync on the next run.
+export const idpSyncState = sqliteTable("idpSyncState", {
+    idpId: integer("idpId")
+        .primaryKey()
+        .references(() => idp.idpId, { onDelete: "cascade" }),
+    // Provider-defined opaque JSON (e.g. Azure delta links). Null = full sync.
+    cursor: text("cursor"),
+    lastSyncStartedAt: integer("lastSyncStartedAt"),
+    lastSyncSucceededAt: integer("lastSyncSucceededAt"),
+    lastError: text("lastError"),
+    consecutiveFailures: integer("consecutiveFailures").notNull().default(0)
+});
+
+// Links a directory user (stable provider ID) to the Pangolin user sync manages.
+export const idpSyncUser = sqliteTable(
+    "idpSyncUser",
+    {
+        idpId: integer("idpId")
+            .notNull()
+            .references(() => idp.idpId, { onDelete: "cascade" }),
+        externalId: text("externalId").notNull(),
+        userId: text("userId")
+            .notNull()
+            .references(() => users.userId, { onDelete: "cascade" })
+    },
+    (t) => [primaryKey({ columns: [t.idpId, t.externalId] })]
+);
+
 // Blueprint runs
 export const blueprints = sqliteTable("blueprints", {
     blueprintId: integer("blueprintId").primaryKey({

server/lib/traefik/TraefikConfigManager.ts

@@ -297,7 +297,7 @@ export class TraefikConfigManager {
                     const secondsUntilExpiry =
                         localState.expiresAt - nowInSeconds;
                     const daysUntilExpiry = secondsUntilExpiry / (60 * 60 * 24);
-                    if (daysUntilExpiry < 45) {
+                    if (daysUntilExpiry < 15) {
                         logger.info(
                             `Fetching certificates due to upcoming expiry for ${domain} (${Math.round(daysUntilExpiry)} days remaining)`
                         );
@@ -314,7 +314,7 @@ export class TraefikConfigManager {
                     const nowInSeconds = Math.floor(Date.now() / 1000);
                     const secondsUntilExpiry = state.expiresAt - nowInSeconds;
                     const daysUntilExpiry = secondsUntilExpiry / (60 * 60 * 24);
-                    if (daysUntilExpiry < 45) {
+                    if (daysUntilExpiry < 15) {
                         logger.info(
                             `Fetching certificates due to upcoming expiry for wildcard cert ${certDomain} (${Math.round(daysUntilExpiry)} days remaining)`
                         );

server/lib/traefik/loadBalancer.test.ts

@@ -0,0 +1,81 @@
+import "@server/extendZod";
+import { assertEqualsObj } from "@test/assert";
+import {
+    buildHttpLoadBalancerServers,
+    buildTcpUdpLoadBalancerServers
+} from "./loadBalancer";
+import type { TargetWithSite } from "./types";
+
+function makeTarget(ip: string, siteType: string): TargetWithSite {
+    return {
+        targetId: 1,
+        resourceId: 1,
+        providerId: null,
+        siteId: 1,
+        ip,
+        port: 8080,
+        internalPort: 9000,
+        method: "http",
+        enabled: true,
+        health: "healthy",
+        path: null,
+        pathMatchType: null,
+        rewritePath: null,
+        rewritePathType: null,
+        priority: 100,
+        mode: "http",
+        authToken: null,
+        site: {
+            siteId: 1,
+            type: siteType,
+            subnet: null,
+            exitNodeId: null,
+            online: true
+        }
+    };
+}
+
+function runTests() {
+    const ipCases = [
+        { ip: "2001:db8::1", host: "[2001:db8::1]" },
+        { ip: "192.0.2.1", host: "192.0.2.1" }
+    ];
+    const cases = [
+        ...ipCases,
+        { ip: "backend.example.com", host: "backend.example.com" }
+    ];
+
+    for (const { ip, host } of cases) {
+        const target = makeTarget(ip, "local");
+        assertEqualsObj(
+            buildHttpLoadBalancerServers([target]),
+            [{ url: `http://${host}:8080` }],
+            `Local HTTP target ${ip} should format its endpoint`
+        );
+        assertEqualsObj(
+            buildTcpUdpLoadBalancerServers([target]),
+            [{ address: `${host}:8080` }],
+            `Local TCP/UDP target ${ip} should format its endpoint`
+        );
+    }
+
+    for (const { ip, host } of ipCases) {
+        const target = makeTarget("backend.example.com", "newt");
+        target.site.subnet = `${ip}/${ip.includes(":") ? 128 : 32}`;
+
+        assertEqualsObj(
+            buildHttpLoadBalancerServers([target]),
+            [{ url: `http://${host}:9000` }],
+            "Newt HTTP target should use the subnet endpoint and internal port"
+        );
+        assertEqualsObj(
+            buildTcpUdpLoadBalancerServers([target]),
+            [{ address: `${host}:9000` }],
+            "Newt TCP/UDP target should use the subnet endpoint and internal port"
+        );
+    }
+
+    console.log("All load balancer endpoint tests passed!");
+}
+
+runTests();

server/lib/traefik/loadBalancer.ts

@@ -1,3 +1,4 @@
+import { formatEndpoint } from "@server/lib/ip";
 import { TargetWithSite } from "./types";
 
 /**
@@ -48,12 +49,12 @@ export function buildHttpLoadBalancerServers(targets: TargetWithSite[]) {
                 target.site.type === "wireguard"
             ) {
                 return {
-                    url: `${target.method}://${target.ip}:${target.port}`
+                    url: `${target.method}://${formatEndpoint(target.ip, target.port)}`
                 };
             } else if (target.site.type === "newt") {
                 const ip = target.site.subnet!.split("/")[0];
                 return {
-                    url: `${target.method}://${ip}:${target.internalPort}`
+                    url: `${target.method}://${formatEndpoint(ip, target.internalPort!)}`
                 };
             }
         })
@@ -111,12 +112,12 @@ export function buildTcpUdpLoadBalancerServers(targets: TargetWithSite[]) {
                 target.site.type === "wireguard"
             ) {
                 return {
-                    address: `${target.ip}:${target.port}`
+                    address: formatEndpoint(target.ip, target.port)
                 };
             } else if (target.site.type === "newt") {
                 const ip = target.site.subnet!.split("/")[0];
                 return {
-                    address: `${ip}:${target.internalPort}`
+                    address: formatEndpoint(ip, target.internalPort!)
                 };
             }
         });