diff --git a/.github/workflows/frontend-ci.yml b/.github/workflows/frontend-ci.yml index 8d3e2439a..04e150847 100644 --- a/.github/workflows/frontend-ci.yml +++ b/.github/workflows/frontend-ci.yml @@ -360,9 +360,15 @@ jobs: timeout-minutes: 40 env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/playwright_shared # pragma: allowlist secret + # Admin API-key store requires NOSUPERUSER/NOBYPASSRLS regengine_sysadmin + # (provisioned by scripts/setup_e2e_postgres.py). Fake CI-only password. + ADMIN_DATABASE_URL: postgresql://regengine_sysadmin:ci-e2e-only-sysadmin-not-a-real-secret@localhost:5432/playwright_shared # pragma: allowlist secret — CI-only fake password for E2E NOSUPERUSER role PGOPTIONS: "-c timezone=UTC -c search_path=public,fsma,extensions" REGENGINE_ENV: test REGENGINE_E2E_MODE: "true" + # Raise product-mutation slowapi budget for authenticated-core-journeys + # (55 creates). Production default stays 30/minute when this is unset. + REGENGINE_E2E_RELAX_PRODUCT_MUTATION_LIMIT: "true" REGENGINE_E2E_EMAIL_OUTBOX: /tmp/regengine-playwright-email-outbox.jsonl AUTH_SECRET_KEY: playwright-local-jwt-signing-key-2026 # pragma: allowlist secret JWT_SIGNING_KEY: playwright-local-jwt-signing-key-2026 # pragma: allowlist secret diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index b579e8804..f53d4f5cd 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -61,6 +61,19 @@ if (!externalBaseURL) { // per email domain), so the suites' combined traffic exceeds the 100/min // per-tenant production default. process.env.TENANT_RATE_LIMIT_RPM = process.env.TENANT_RATE_LIMIT_RPM || '100000'; + // authenticated-core-journeys creates 55 products; raise the product + // mutation budget only when this dedicated env var is set (server hook + // stays at 30/minute when unset). + process.env.REGENGINE_E2E_RELAX_PRODUCT_MUTATION_LIMIT = + process.env.REGENGINE_E2E_RELAX_PRODUCT_MUTATION_LIMIT || 'true'; + // Admin API-key store needs regengine_sysadmin (setup_e2e_postgres.py). + // CI sets ADMIN_DATABASE_URL; for local Playwright Postgres default it. + if (!process.env.ADMIN_DATABASE_URL && localSharedDatabaseUrl.includes('playwright_shared')) { + process.env.ADMIN_DATABASE_URL = + 'postgresql://regengine_sysadmin:ci-e2e-only-sysadmin-not-a-real-secret@127.0.0.1:5432/playwright_shared'; // pragma: allowlist secret — CI-only fake password for E2E NOSUPERUSER role + } else if (!process.env.ADMIN_DATABASE_URL) { + process.env.ADMIN_DATABASE_URL = localSharedDatabaseUrl; + } process.env.NEXT_PUBLIC_ENABLE_EXPERIMENTAL_FEATURES = process.env.NEXT_PUBLIC_ENABLE_EXPERIMENTAL_FEATURES || 'true'; } @@ -128,6 +141,12 @@ export default defineConfig({ REGENGINE_E2E_EMAIL_OUTBOX: process.env.REGENGINE_E2E_EMAIL_OUTBOX || localEmailOutboxPath, DATABASE_URL: process.env.DATABASE_URL || localSharedDatabaseUrl, + ADMIN_DATABASE_URL: + process.env.ADMIN_DATABASE_URL || + process.env.DATABASE_URL || + localSharedDatabaseUrl, + REGENGINE_E2E_RELAX_PRODUCT_MUTATION_LIMIT: + process.env.REGENGINE_E2E_RELAX_PRODUCT_MUTATION_LIMIT || 'true', ADMIN_FALLBACK_SQLITE: process.env.ADMIN_FALLBACK_SQLITE || localAdminDatabaseUrl, WEBAUTHN_EXPECTED_ORIGIN: diff --git a/frontend/tests/e2e/authenticated-core-journeys.spec.ts b/frontend/tests/e2e/authenticated-core-journeys.spec.ts index b73dd1711..2ad050911 100644 --- a/frontend/tests/e2e/authenticated-core-journeys.spec.ts +++ b/frontend/tests/e2e/authenticated-core-journeys.spec.ts @@ -70,6 +70,21 @@ function isLocalE2E(): boolean { return !process.env.PLAYWRIGHT_BASE_URL; } +/** + * Invite audit/cleanup helpers talk to node:sqlite against ADMIN_FALLBACK_SQLITE. + * When ADMIN_DATABASE_URL points at Postgres (CI Passwordless E2E / sysadmin + * API-key store), SessionLocal writes invites + audit_logs there instead, so + * the SQLite file has no invites table. Skip the SQLite assertions/cleanup in + * that mode; API + UI checks below still cover the journey. + */ +function adminUsesSqliteFallback(): boolean { + const adminUrl = (process.env.ADMIN_DATABASE_URL || '').trim().toLowerCase(); + if (adminUrl.startsWith('postgres')) return false; + if (adminUrl.startsWith('sqlite')) return true; + // Unset ADMIN_DATABASE_URL → local E2E uses ADMIN_FALLBACK_SQLITE. + return isLocalE2E(); +} + function parseInviteEmailOutboxLine(line: string): InviteEmailOutboxRecord | null { try { return JSON.parse(line) as InviteEmailOutboxRecord; @@ -99,6 +114,8 @@ function removeInviteEmailOutboxRecords(email: string): void { } function expectAdminAuditLog(tenantId: string, eventType: string, resourceId: string): void { + if (!adminUsesSqliteFallback()) return; + const database = new DatabaseSync(adminSqlitePath()); try { const row = database @@ -112,7 +129,7 @@ function expectAdminAuditLog(tenantId: string, eventType: string, resourceId: st function cleanupInviteFixture(tenantId: string | null, email: string, userId?: string, inviteId?: string): void { removeInviteEmailOutboxRecords(email); - if (!tenantId || !isLocalE2E()) return; + if (!tenantId || !isLocalE2E() || !adminUsesSqliteFallback()) return; const database = new DatabaseSync(adminSqlitePath()); try { diff --git a/scripts/seed_passwordless_e2e_user.py b/scripts/seed_passwordless_e2e_user.py index 62d351037..7d1c85101 100644 --- a/scripts/seed_passwordless_e2e_user.py +++ b/scripts/seed_passwordless_e2e_user.py @@ -94,6 +94,16 @@ async def _seed_api_key(tenant_id: str, email: str) -> None: def main() -> int: _require_e2e_mode() + # Admin SessionLocal reads ADMIN_DATABASE_URL. CI points that at the + # NOSUPERUSER regengine_sysadmin role for the running app (api-key RLS). + # Fixture seeding must insert orgs/users as the postgres superuser on + # DATABASE_URL so RLS does not block bootstrap rows. Mutating env here + # only affects this seed process (Playwright execFileSync), not the app. + database_url = os.environ.get("DATABASE_URL", "").strip() + if not database_url: + raise SystemExit("DATABASE_URL is required to seed passwordless E2E users") + os.environ["ADMIN_DATABASE_URL"] = database_url + from services.admin.app.auth_crypto import AuthCryptoService from services.admin.app.database import SessionLocal, init_db from services.admin.app.sqlalchemy_models import ( diff --git a/scripts/setup_e2e_postgres.py b/scripts/setup_e2e_postgres.py index bfdd987d8..bf2011169 100644 --- a/scripts/setup_e2e_postgres.py +++ b/scripts/setup_e2e_postgres.py @@ -6,12 +6,42 @@ import sys from pathlib import Path -from sqlalchemy import create_engine -from sqlalchemy.engine import make_url +from sqlalchemy import create_engine, text +from sqlalchemy.engine import Engine, make_url ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) +# CI-only fake password for the NOSUPERUSER/NOBYPASSRLS regengine_sysadmin +# LOGIN used by ADMIN_DATABASE_URL in Passwordless Auth E2E. Never a real +# production secret — gitleaks allowlisted with reason. +_CI_E2E_SYSADMIN_PASSWORD = "ci-e2e-only-sysadmin-not-a-real-secret" # pragma: allowlist secret — CI-only fake password for Playwright E2E regengine_sysadmin LOGIN; never used in production + + +def _enable_e2e_sysadmin_login(engine: Engine) -> None: + """Flip the provisioned NOLOGIN sysadmin role to a CI-only LOGIN. + + ``_provision_cluster`` creates ``regengine_sysadmin`` as NOLOGIN so + migrations can GRANT to it. The admin API-key store then requires an + actual connection as that NOSUPERUSER/NOBYPASSRLS role — enable LOGIN + with an obviously fake CI password and grant CONNECT on the E2E DB. + """ + # Build the ALTER from the CI-only constant. Single-quote doubling is the + # only escaping needed for a SQL string literal; parameterized ALTER ROLE + # PASSWORD is not portable across drivers. + pw_literal = _CI_E2E_SYSADMIN_PASSWORD.replace("'", "''") + with engine.begin() as conn: + conn.execute( + text( + "ALTER ROLE regengine_sysadmin WITH LOGIN " + f"PASSWORD '{pw_literal}' " + "NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION" + ) + ) + conn.execute( + text("GRANT CONNECT ON DATABASE playwright_shared TO regengine_sysadmin") + ) + def main(): url = os.environ.get("DATABASE_URL", "") @@ -37,6 +67,13 @@ def main(): env=os.environ | {"MIGRATIONS_DATABASE_URL": url}, check=True, ) + # After migrations GRANT to the role, enable the CI-only LOGIN that + # ADMIN_DATABASE_URL points at in frontend-ci.yml / start-e2e-backend.sh. + engine = create_engine(target.set(drivername="postgresql+psycopg")) + try: + _enable_e2e_sysadmin_login(engine) + finally: + engine.dispose() if __name__ == "__main__": diff --git a/scripts/start-e2e-backend.sh b/scripts/start-e2e-backend.sh index e43d82a0e..e4cc8366c 100644 --- a/scripts/start-e2e-backend.sh +++ b/scripts/start-e2e-backend.sh @@ -44,6 +44,20 @@ export REGENGINE_ENV="${REGENGINE_ENV:-development}" export AUTH_SECRET_KEY="${AUTH_SECRET_KEY:-playwright-local-jwt-signing-key-2026}" export JWT_SIGNING_KEY="${JWT_SIGNING_KEY:-$AUTH_SECRET_KEY}" export DATABASE_URL="${DATABASE_URL:-sqlite:///$shared_db_path}" +# Prefer an explicit ADMIN_DATABASE_URL. For the Playwright Postgres cluster, +# default to the CI-only regengine_sysadmin LOGIN created by +# setup_e2e_postgres.py (NOSUPERUSER/NOBYPASSRLS). Sqlite local falls back to +# DATABASE_URL. Fake CI-only password — never a production secret. +_default_admin_database_url="$DATABASE_URL" +case "$DATABASE_URL" in + postgresql://*playwright_shared*|postgres://*playwright_shared*) + _default_admin_database_url="postgresql://regengine_sysadmin:ci-e2e-only-sysadmin-not-a-real-secret@localhost:5432/playwright_shared" # pragma: allowlist secret — CI-only fake password for E2E NOSUPERUSER role + ;; +esac +export ADMIN_DATABASE_URL="${ADMIN_DATABASE_URL:-$_default_admin_database_url}" +# E2E-only: raise product-mutation rate limit. Hook is inert unless this env +# var is set; production start paths do not source this script. +export REGENGINE_E2E_RELAX_PRODUCT_MUTATION_LIMIT="${REGENGINE_E2E_RELAX_PRODUCT_MUTATION_LIMIT:-true}" export ADMIN_FALLBACK_SQLITE="${ADMIN_FALLBACK_SQLITE:-sqlite:///$admin_db_path}" export DISABLE_TASK_WORKER="${DISABLE_TASK_WORKER:-true}" export ALLOW_INMEMORY_SESSION_STORE="${ALLOW_INMEMORY_SESSION_STORE:-true}" diff --git a/services/admin/app/invite_routes.py b/services/admin/app/invite_routes.py index 1106b0a38..64932d16e 100644 --- a/services/admin/app/invite_routes.py +++ b/services/admin/app/invite_routes.py @@ -518,6 +518,12 @@ async def accept_invite( invite_tenant_id = _resolve_invite_tenant(db, token_hash) if invite_tenant_id is not None: TenantContext.set_tenant_context(db, invite_tenant_id) + # public.users / organization_memberships FORCE RLS policies for + # regengine_sysadmin require regengine.is_sysadmin (or a matching + # regengine.user_id) on WITH CHECK. Invite acceptance provisions a + # brand-new user row, so arm the admin GUC on this already-sysadmin + # connection. set_admin_context(true) is a no-op under SQLite. + TenantContext.set_admin_context(db, True) elif _uses_postgres(db): # Fail closed: unknown, revoked, accepted or expired token. Proceeding # would read invites with no tenant context and 500 instead of diff --git a/services/admin/app/user_routes.py b/services/admin/app/user_routes.py index 979fd520a..95b7dfc55 100644 --- a/services/admin/app/user_routes.py +++ b/services/admin/app/user_routes.py @@ -114,6 +114,15 @@ async def list_users( if not tenant_id: raise HTTPException(status_code=400, detail="Tenant context required") + # public.users FORCE RLS (users_self_or_admin) only exposes the caller's own + # row unless regengine.is_sysadmin is armed. memberships already permit + # tenant-scoped peer SELECT, so without this arm the UserModel join silently + # drops every co-member and the directory returns only the caller. Permission + # Checker("users.read") already authorized this request; keep the query + # scoped by MembershipModel.tenant_id so the arm does not widen tenants. + # set_admin_context(true) is a no-op under SQLite. + TenantContext.set_admin_context(db, True) + # Count total count_stmt = ( select(func.count()) diff --git a/services/admin/tests/test_user_routes_1342.py b/services/admin/tests/test_user_routes_1342.py index 067b1c9ab..3f5979c8d 100644 --- a/services/admin/tests/test_user_routes_1342.py +++ b/services/admin/tests/test_user_routes_1342.py @@ -99,10 +99,13 @@ def test_returns_user_list_with_active_status(self, monkeypatch): db = MagicMock() call_count = [0] - def _execute_side_effect(stmt): + def _execute_side_effect(*args, **kwargs): call_count[0] += 1 result = MagicMock() if call_count[0] == 1: + # set_admin_context arm (users FORCE RLS peer visibility) + return result + if call_count[0] == 2: # Count query result.scalar.return_value = 1 else: @@ -137,10 +140,12 @@ def test_inactive_membership_yields_inactive_status(self, monkeypatch): db = MagicMock() call_count = [0] - def _execute_side_effect(stmt): + def _execute_side_effect(*args, **kwargs): call_count[0] += 1 result = MagicMock() if call_count[0] == 1: + return result + if call_count[0] == 2: result.scalar.return_value = 1 else: result.all.return_value = [(user, membership, role)] @@ -165,10 +170,12 @@ def test_empty_result_returns_zero_total(self, monkeypatch): db = MagicMock() call_count = [0] - def _execute_side_effect(stmt): + def _execute_side_effect(*args, **kwargs): call_count[0] += 1 result = MagicMock() if call_count[0] == 1: + return result + if call_count[0] == 2: result.scalar.return_value = 0 else: result.all.return_value = [] @@ -204,7 +211,7 @@ def test_returns_role_list(self, monkeypatch): db = MagicMock() call_count = [0] - def _execute_side_effect(stmt): + def _execute_side_effect(*args, **kwargs): call_count[0] += 1 result = MagicMock() if call_count[0] == 1: @@ -237,7 +244,7 @@ def test_system_role_has_is_system_true(self, monkeypatch): db = MagicMock() call_count = [0] - def _execute_side_effect(stmt): + def _execute_side_effect(*args, **kwargs): call_count[0] += 1 result = MagicMock() if call_count[0] == 1: @@ -265,7 +272,7 @@ def test_empty_roles_returns_zero_total(self, monkeypatch): db = MagicMock() call_count = [0] - def _execute_side_effect(stmt): + def _execute_side_effect(*args, **kwargs): call_count[0] += 1 result = MagicMock() if call_count[0] == 1: diff --git a/services/ingestion/app/product_catalog.py b/services/ingestion/app/product_catalog.py index af3b607b8..9de10b562 100644 --- a/services/ingestion/app/product_catalog.py +++ b/services/ingestion/app/product_catalog.py @@ -33,9 +33,37 @@ # human actions (operators curating the FTL list), so a modest per-client # budget is ample while still blocking scripted abuse. GET routes are left # unthrottled here. -_ADD_PRODUCT_RATE_LIMIT = "30/minute" -_UPDATE_PRODUCT_RATE_LIMIT = "30/minute" -_DELETE_PRODUCT_RATE_LIMIT = "30/minute" +# +# The production default is always ``30/minute``. An elevated budget is used +# only when ``REGENGINE_E2E_RELAX_PRODUCT_MUTATION_LIMIT`` is explicitly set +# (the Passwordless Auth E2E job seeds 55 products in one minute). Never arm +# this from ``REGENGINE_E2E_MODE`` alone — production defaults must not change +# unless that dedicated env var is set. +_DEFAULT_PRODUCT_MUTATION_RATE_LIMIT = "30/minute" +_E2E_RELAXED_PRODUCT_MUTATION_RATE_LIMIT = "10000/minute" +_E2E_RELAX_PRODUCT_MUTATION_LIMIT_ENV = "REGENGINE_E2E_RELAX_PRODUCT_MUTATION_LIMIT" +_E2E_RELAX_TRUE_VALUES = frozenset({"1", "true", "yes", "on"}) + + +def resolve_product_mutation_rate_limit( + environ: dict[str, str] | None = None, +) -> str: + """Return the product-mutation rate-limit string. + + Unset / empty env → production default ``30/minute``. Truthy + ``REGENGINE_E2E_RELAX_PRODUCT_MUTATION_LIMIT`` → elevated E2E budget. + """ + env = os.environ if environ is None else environ + raw = str(env.get(_E2E_RELAX_PRODUCT_MUTATION_LIMIT_ENV, "")).strip().lower() + if raw in _E2E_RELAX_TRUE_VALUES: + return _E2E_RELAXED_PRODUCT_MUTATION_RATE_LIMIT + return _DEFAULT_PRODUCT_MUTATION_RATE_LIMIT + + +# slowapi accepts a callable that returns the limit string per request. +_ADD_PRODUCT_RATE_LIMIT = resolve_product_mutation_rate_limit +_UPDATE_PRODUCT_RATE_LIMIT = resolve_product_mutation_rate_limit +_DELETE_PRODUCT_RATE_LIMIT = resolve_product_mutation_rate_limit # Bounded list-item type reused by request models. _CatalogListItem = Annotated[str, StringConstraints(max_length=200)] diff --git a/services/ingestion/tests/test_product_catalog_hardening.py b/services/ingestion/tests/test_product_catalog_hardening.py index 62e3839ee..f94275591 100644 --- a/services/ingestion/tests/test_product_catalog_hardening.py +++ b/services/ingestion/tests/test_product_catalog_hardening.py @@ -88,6 +88,24 @@ def _valid_body() -> dict: # --------------------------------------------------------------------------- +def test_product_mutation_rate_limit_default_when_relax_env_unset( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Production default stays 30/minute when the E2E-only relax env is unset.""" + monkeypatch.delenv("REGENGINE_E2E_RELAX_PRODUCT_MUTATION_LIMIT", raising=False) + assert pc.resolve_product_mutation_rate_limit({}) == "30/minute" + assert pc.resolve_product_mutation_rate_limit() == "30/minute" + + +def test_product_mutation_rate_limit_relaxed_when_env_set() -> None: + assert ( + pc.resolve_product_mutation_rate_limit( + {"REGENGINE_E2E_RELAX_PRODUCT_MUTATION_LIMIT": "true"} + ) + == "10000/minute" + ) + + def test_add_product_rate_limited() -> None: client = TestClient(_build_app()) saw_429 = False