diff --git a/.github/prompts/pr-review.md b/.github/prompts/pr-review.md index 96172a9b..0e02fecc 100644 --- a/.github/prompts/pr-review.md +++ b/.github/prompts/pr-review.md @@ -31,7 +31,7 @@ Review against each of these, in this order: - SQL injection in any ClickHouse-bound path (`BindParams`, query builders, dynamic table names) - Broken authentication / authorization (JWT claim handling, role extraction, policy templating) - Sensitive data exposure (secrets in logs, error messages leaking internal state) - - Broken access control (policy bypass, raw-SQL outside the admin role — `policy.admin_role` — on `/v1/admin/query`) + - Broken access control (policy bypass, raw-SQL outside the admin role — `policy.admin_role` — on `/v1/ops/query`) - Security misconfiguration (CORS, TLS, default credentials, permissive defaults) - Insufficient logging / monitoring - SSRF, XXE, deserialization flaws if touched diff --git a/AGENTS.md b/AGENTS.md index c569499a..00076ca1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,7 +35,7 @@ Fourteen internal packages under `internal/` (plus `internal/testutil/` for shar - **`config/`** — YAML + env var config loading (cleanenv) - **`dedupe/`** — `Deduplicator` interface → `Embedded` (Pebble) — optional, controlled by `dedupe.enabled` - **`discovery/`** — `SchemaRegistry` that introspects ClickHouse `system.columns` + `Validate()` for ingest payloads + `CanonicalizeTimestamps()` rewriting top-level `DateTime`/`DateTime64` column values to the canonical RFC 3339 UTC wire form pre-publish (Key Design Decision #19) -- **`ingest/`** — Ingest worker pipeline (`worker.go`: JetStream input → per-table batch INSERT with DLQ output). The pipeline is **insert-only**. The wire format `EventMessage` (`types.go`) carries `{table_name, scope, received_timestamp, data}` and nothing else; the worker accepts whatever table name the envelope carries (table existence was already checked by the HTTP ingest handler, which `404`s an unknown table before publish; the worker doesn't re-validate), then bulk-INSERTs. In the embedded-NATS deployment (the default), the server runs with `DontListen: true` (`internal/mq/embedded.go`), so the only Publishers reachable on the `ingest.>` subjects are in-process Go code — today, only the HTTP `/v1/ingest?table={table}` handler. Non-insert mutations (`DELETE`/`UPDATE`/`TRUNCATE`/…) must go through `POST /v1/admin/query` under the admin role (the same `RequireAdmin` gate as the rest of `/v1/admin/*`), so non-admin callers never reach the proxy. A request with no token (or an invalid one) resolves to the `default_role`, which in a production config is not the admin role (setting them equal is a loudly-warned dev-only setting), so it can't reach this endpoint. Plus `Sweeper` (Active Sweeper for NATS message lifecycle) + `EventMessage`/`BufferConsumerName` types (`types.go`) +- **`ingest/`** — Ingest worker pipeline (`worker.go`: JetStream input → per-table batch INSERT with DLQ output). The pipeline is **insert-only**. The wire format `EventMessage` (`types.go`) carries `{table_name, scope, received_timestamp, data}` and nothing else; the worker accepts whatever table name the envelope carries (table existence was already checked by the HTTP ingest handler, which `404`s an unknown table before publish; the worker doesn't re-validate), then bulk-INSERTs. In the embedded-NATS deployment (the default), the server runs with `DontListen: true` (`internal/mq/embedded.go`), so the only Publishers reachable on the `ingest.>` subjects are in-process Go code — today, only the HTTP `/v1/ingest?table={table}` handler. Non-insert mutations (`DELETE`/`UPDATE`/`TRUNCATE`/…) must go through `POST /v1/ops/query` under the admin role (the same `RequireAdmin` gate as the rest of `/v1/ops/*`), so non-admin callers never reach the proxy. A request with no token (or an invalid one) resolves to the `default_role`, which in a production config is not the admin role (setting them equal is a loudly-warned dev-only setting), so it can't reach this endpoint. Plus `Sweeper` (Active Sweeper for NATS message lifecycle) + `EventMessage`/`BufferConsumerName` types (`types.go`) - **`mq/`** — `Publisher`/`Subscriber` interfaces → `EmbeddedNATS` + `RemoteNATS` - **`observability/`** — OpenTelemetry pipeline: `InitProvider` wires trace/metric/log providers via OTLP gRPC (each signal independently gated). A top-level `Prometheus` config block drives an optional `/metrics` scrape endpoint that runs independently of OTLP push — standalone (Alloy/Mimir scrape, no collector), alongside OTLP, or off. `NewLogger` produces a slog handler that fans out to stdout AND OTLP (stdout always 100%, OTLP sample-rate-aware). `TraceHandler` injects trace_id/span_id from active spans. `tracer.go` provides W3C trace context propagation over NATS headers. - **`pipes/`** — Named query pipes: `NamedQuery` type + NATS KV store (`WAVEHOUSE_PIPES`) + `.sql` file bootstrap @@ -57,7 +57,7 @@ The invariant index — what must stay true. Full narrative and rationale live i 8. **Optional dedup** — opt-in via `dedupe.enabled`; `dedupe.id_field` selects the JSON key. 9. **Singleflight** — `TieredCache` coalesces concurrent misses (`x/sync/singleflight`) to prevent cache stampede. 10. **Active Sweeper** — purges NATS messages that are both ACKed (written to CH) and older than the gap window; SSE gap-fill uses `DeliverByStartTime`, no in-process ring buffer. -11. **Hasura-style access control: fail-closed (security)** — `policy.IsAdmin` (role == `admin_role`, **exact case-sensitive**, default `"admin"`) is the single admin check, shared by `Evaluate`/`ResolveRole`/`Validate`/the `/v1/admin` gate/`RoleAllowed`. Empty/absent role matches nothing (no `"*"` wildcard); `Validate` rejects empty role keys; a `nil` policy (deleted) denies **everyone incl. admin** via a role — a total lockout for token-based callers, so bootstrap from the policy file, never an implicit admin grant (**exception:** the operator key's `auth.IsOperator` bit passes the `/v1/admin` gate even under a `nil` policy — a deliberate break-glass restore over HTTP, see #7). `default_role` is the one sanctioned roleless exception (`ResolveRole` maps empty → it pre-eval); `default_role == admin_role` is permitted but dev-only and loudly warned (`policy.DefaultRoleGrantsAdmin`). Preserve when touching `internal/policy` (policy twin of #13; see #159). Detail: architecture.md § `policy/`. +11. **Hasura-style access control: fail-closed (security)** — `policy.IsAdmin` (role == `admin_role`, **exact case-sensitive**, default `"admin"`) is the single admin check, shared by `Evaluate`/`ResolveRole`/`Validate`/the `/v1/ops` gate/`RoleAllowed`. Empty/absent role matches nothing (no `"*"` wildcard); `Validate` rejects empty role keys; a `nil` policy (deleted) denies **everyone incl. admin** via a role — a total lockout for token-based callers, so bootstrap from the policy file, never an implicit admin grant (**exception:** the operator key's `auth.IsOperator` bit passes the `/v1/ops` gate even under a `nil` policy — a deliberate break-glass restore over HTTP, see #7). `default_role` is the one sanctioned roleless exception (`ResolveRole` maps empty → it pre-eval); `default_role == admin_role` is permitted but dev-only and loudly warned (`policy.DefaultRoleGrantsAdmin`). Preserve when touching `internal/policy` (policy twin of #13; see #159). Detail: architecture.md § `policy/`. 12. **Structured queries: column authz fail-closed (security)** — `POST /v1/query?table={table}`: typed AST validated against schema, permission-enforced, timestamp-bucketed for cache, `DefaultMaxRows` (10,000) cap. Every column reference — projection, aggregation args, `filters`, `group_by`, `order_by`, `time_range` — is authorized inside `query.Build` (the single chokepoint that enumerates them all), so no clause can skip the role's `allow_columns`/`deny_columns` check (#223). A `select_all` read by a *column-restricted* role expands to its allowed columns via `policy.AllowedProjection`, never a bare `SELECT *`; *unrestricted*/admin roles keep `SELECT *` (`policy.RestrictsColumns` decides). Omitting `columns` selects nothing (`ErrEmptyProjection` → `200 []`); `["*"]` is the literal column `*` (schema-gated, not a wildcard); a table-granted role with no readable columns fails closed (`ErrNoReadableColumns` → `403`). Structured and live-stream (`stream.filterColumns`) reads share the one per-column decision `policy.IsColumnAllowed`, so column visibility can't drift. Row visibility has the same one-source guarantee (#319): `Evaluate` resolves a role's row-`filter` once (`resolvePredicates`), and both surfaces consume that single resolution — the query path renders it to SQL (`predicatesToSQL`), the stream evaluates it in memory per subscriber (`ResolvedPermissions.RowVisible`, whose type-aware comparison fails closed on anything it can't prove about the ingested payload — `policy.ColumnSpec`, with `DateTime`/`DateTime64` operands compared as instants through the ingest grammar (`discovery.Column.TimeParser`) and claim constants rendered canonically and digit-exact by the one shared rule `policy.CanonicalScalar` (#457 — which also refuses a float64 at/past 2^53 rather than match a neighboring ID, and whose ok=false — an absent claim, a structured value, no canonical form — makes the predicate match no rows on BOTH surfaces: `1 = 0` in SQL, every row withheld in memory); numeric comparison runs in the column's STORAGE domain (`policy.NumericSpec`, classified by `discovery.NumericStorageOf` — Float width rounding, Decimal scale truncation, integer exactness, both operands narrowed as ClickHouse narrows stored value and bound constant, out-of-range operands refused rather than modeled; the `tests/integration` differential oracle holds in-range verdicts equal to a live ClickHouse's and the never-admit-where-SQL-hides direction for the refused out-of-range ones); an event whose insert later fails into the DLQ is the one residual payload-vs-stored asymmetry, documented in the access-control enforcement caution) — so row visibility can't drift either. Preserve when touching `internal/query` or the structured-query handler. Detail: architecture.md § `query/`. 13. **Named query pipes: fail-closed (security)** — pre-defined SQL templates (Tinybird-style) with param binding + caching; `GET/POST /v1/pipes/{name}` sit outside `RequireAdmin`, so per-pipe `allowed_roles` is the *only* execute-path gate, via `policy.RoleAllowed`: exact allowlist membership (no `"*"`), admin always passes, empty/absent role and empty-string entries authorize nobody, and no `allowed_roles` → admin-only. Preserve and exercise via `testutil.RunRoleMatrix` / `StandardRoleMatrix` (see #159). Detail: architecture.md § `pipes/`. 14. **TypeScript SDK** — `@wavehouse/sdk`: zero-dep client, typed query builder, real-time SSE, live queries (incrementable/decomposable/poll aggregation), codegen CLI. The canonical client (see §SDK Sync). diff --git a/CHANGELOG.md b/CHANGELOG.md index f90e72b1..b36478e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **HTTP customization for the SDK — `options.headers`, `options.fetchOptions`, and `options.fetch`** (`clients/ts/src/types.ts`, `clients/ts/src/client.ts`, `clients/ts/src/http.ts`, `clients/ts/src/index.ts`, `clients/ts/src/client.test.ts`, `docs/src/content/docs/sdk/index.mdx`): closes #269. `ClientOptions` exposed only `maxRetries`, so a WaveHouse behind a header-gated proxy (Cloudflare Access, an mTLS sidecar, an auth gateway) or a cookie-authenticated origin was unreachable from the SDK — a defense-in-depth gate forced consumers off the client entirely. Three knobs land together, shaped after the conventions in Supabase's, OpenAI's, and Anthropic's clients rather than invented here. **`headers`** adds static headers to every REST request; names match case-insensitively as HTTP requires, and they apply *underneath* the SDK's own — `auth` keeps `Authorization`, and a request's `Content-Type`/`Accept` can't be displaced by a global one, since a header joined rather than replaced is how you ship `Content-Type: application/json, image/png`. **`fetchOptions`** merges extra `RequestInit` fields (`credentials: "include"` for the cookie case, `mode`, `cache`, or a runtime extension like Next.js's `next: { tags }`); the fields the SDK controls — `method`, `headers`, `body`, `signal` — always win, so it cannot corrupt the request. **`fetch`** replaces the HTTP implementation outright. All three are REST-only: `.stream()` and `.liveQuery()`'s live connection go through `EventSource`, which accepts neither headers nor a `fetch`, so a header-gated deployment can query but not stream until #203 changes that transport — stated plainly in the docs rather than left implicit, since the equivalent gap in Supabase's realtime client was found the hard way by a user whose RLS policies silently stopped matching. `.liveQuery()`'s initial backfill is an ordinary REST call and *is* covered. Per-call overrides and dynamic header callbacks are deliberately deferred to #459; the per-call slot (`.fetch(opts)`) already exists, so adding them later is additive. The exported `FetchLike` matches the standard `fetch` signature, written out rather than as `typeof fetch` because that resolves differently depending on whether the consumer's TypeScript `lib` includes DOM — the same fragility behind Supabase's long tail of `node-fetch` resolution issues. `options.fetch` accepts any `fetch`-compatible function (exported as the `FetchLike` type) and is used for every request, retries included, so middleware sees each attempt. The motivating case is a runtime bug consumers can't fix themselves: undici 8.8.0–8.9.0 stalls a request before it goes out when a keep-alive socket is reused while the event loop is idle ([nodejs/undici#5600](https://github.com/nodejs/undici/issues/5600), fixed in 8.10.0), and Node 26 bundles 8.9.0 — so calls that should take milliseconds don't, with no recourse inside the SDK. Severity varies with the runtime and the idle gap: the upstream report measured ~450–465 ms, and our own runs against an instant-answering server have ranged from ~100 ms to tens of seconds. Upgrading undici is the real fix, and `options.fetch` is how you get it without waiting for a new runtime: install undici yourself and route through it, passing its dispatcher **explicitly**. That last part is load-bearing and not obvious — undici keeps its connection pool on a shared `globalThis` symbol claimed by whichever copy loads first (Node claims it for the bundled copy on the first built-in `fetch` call, not at startup), so calling an installed 8.10.0's `fetch` without a `dispatcher` resolves whatever is on that symbol and can still stall through the bundled 8.9.0's pool. Measured with both copies loaded, 1.5 s idle gaps against a 10 ms server: `21, 1514, 1495, 583 ms` with an implied dispatcher versus `17, 14, 12, 13 ms` with an explicit `new Agent()`. (Tuning `keepAliveTimeout` is *not* a workaround — measured, it changes nothing, because the retirement timer is starved by the same idle event loop; `new Agent({ pipelining: 0 })` does work for anyone pinned to an affected version, at a connection per request.) The same hook covers the ordinary reasons an SDK grows one: proxies, client certificates, tracing or circuit-breaker middleware, and mocking HTTP in a consumer's own tests without monkey-patching a global. `fetch` stays optional all the way through to the internal `HttpContext` rather than being defaulted at construction, so the default path still calls the global directly — that keeps it late-bound (replacing `globalThis.fetch` after a client exists still works, which is what `vi.stubGlobal` does) and avoids invoking a detached `fetch` reference, which is not universally safe; both properties are pinned by tests. Only the SSE transport is exempt: the live connection behind `.stream()`/`.liveQuery()` goes through `EventSource`, which this does not replace — but `.liveQuery()`'s initial backfill is an ordinary request and does go through the supplied function. Implementations shipping their own request/response declarations (undici, `node-fetch`) need casts on the URL argument, the init, and the return value, since those types are separate from the ones behind the global `fetch`; the narrow documented runtime contract is what makes them safe — a string URL and plain `RequestInit` in, and only `.ok`/`.headers` plus `.text()` on success and `.status`/`.statusText`/`.json()` on a non-`ok` response read back. Abort handling is part of that contract: `ABORTED` requires a `DOMException` named `AbortError` (platform `fetch` and undici), and any other rejection is retried as `NETWORK_ERROR`. -- **Non-JWT operator key for full-access admin + break-glass recovery** (`internal/auth/auth.go`, `internal/auth/context.go`, `internal/auth/auth_test.go`, `internal/api/router.go`, `internal/api/router_test.go`, `internal/config/config.go`, `internal/config/config_test.go`, `cmd/wavehouse/main.go`, `config.yaml`, `deployments/compose/standalone.yaml`, `tests/e2e/fixtures/config.yaml`, `docs/src/content/docs/configuration.mdx`, `docs/src/content/docs/access-control.mdx`, `docs/src/content/docs/api.md`, `docs/src/content/docs/deployment.md`, `docs/src/content/docs/development.md`, `docs/src/content/docs/architecture.md`, `docs/src/content/docs/reverse-proxy.mdx`, `AGENTS.md`, `SECURITY.md`): closes #240; partially advances the auth-hardening epic [#228](https://github.com/Wave-RF/WaveHouse/issues/228) and the management/data-plane split [#359](https://github.com/Wave-RF/WaveHouse/issues/359). Adds an optional `auth.operator_key` (`WH_AUTH_OPERATOR_KEY`): a request presenting it — via an `Authorization: Operator ` header (forwarded verbatim by proxies, no collision with Bearer JWTs) or the `X-Operator-Key` alias — is authorized as a full-access platform operator — the whole data plane *and* the `/v1/admin/*` management surface — without minting a JWT and independently of the token verifier, giving the person running the deployment a role-free credential for bootstrap and break-glass. The middleware checks it before the Bearer token with a constant-time comparison (`crypto/subtle`) and stamps two things into the request context: the live `admin_role` (so the policy evaluator's admin bypass grants unrestricted data-plane access while a policy exists) and a platform-operator bit that `RequireAdmin` honors **even when the policy is `nil`/deleted** — the one HTTP path that can restore a wiped policy, which previously required SSH access and a reboot. Empty (the default) disables it, so existing deployments are unchanged; treat it as an admin secret (load from a secret store, serve only over TLS). A successful operator authentication is audit-logged at `INFO`; a request presenting a *non-matching* operator key is logged at `WARN` and counted by a new `wavehouse_auth_operator_key_failures_total` counter (a probing/brute-force signal on the most privileged credential in the system) before falling through to the normal token/default path — the middleware still never rejects. The `Authorization` auth-scheme is matched case-insensitively (RFC 7235) via a shared `authScheme` helper, which also makes the existing `Bearer` JWT scheme case-insensitive (previously it required the canonical `Bearer` casing). Explicitly out of scope, tracked in #359: scoping the operator credential to the management surface only, and capability-scoped admin permissions. +- **Non-JWT operator key for full-access admin + break-glass recovery** (`internal/auth/auth.go`, `internal/auth/context.go`, `internal/auth/auth_test.go`, `internal/api/router.go`, `internal/api/router_test.go`, `internal/config/config.go`, `internal/config/config_test.go`, `cmd/wavehouse/main.go`, `config.yaml`, `deployments/compose/standalone.yaml`, `tests/e2e/fixtures/config.yaml`, `docs/src/content/docs/configuration.mdx`, `docs/src/content/docs/access-control.mdx`, `docs/src/content/docs/api.md`, `docs/src/content/docs/deployment.md`, `docs/src/content/docs/development.md`, `docs/src/content/docs/architecture.md`, `docs/src/content/docs/reverse-proxy.mdx`, `AGENTS.md`, `SECURITY.md`): closes #240; partially advances the auth-hardening epic [#228](https://github.com/Wave-RF/WaveHouse/issues/228) and the management/data-plane split [#359](https://github.com/Wave-RF/WaveHouse/issues/359). Adds an optional `auth.operator_key` (`WH_AUTH_OPERATOR_KEY`): a request presenting it — via an `Authorization: Operator ` header (forwarded verbatim by proxies, no collision with Bearer JWTs) or the `X-Operator-Key` alias — is authorized as a full-access platform operator — the whole data plane *and* the `/v1/ops/*` management surface — without minting a JWT and independently of the token verifier, giving the person running the deployment a role-free credential for bootstrap and break-glass. The middleware checks it before the Bearer token with a constant-time comparison (`crypto/subtle`) and stamps two things into the request context: the live `admin_role` (so the policy evaluator's admin bypass grants unrestricted data-plane access while a policy exists) and a platform-operator bit that `RequireAdmin` honors **even when the policy is `nil`/deleted** — the one HTTP path that can restore a wiped policy, which previously required SSH access and a reboot. Empty (the default) disables it, so existing deployments are unchanged; treat it as an admin secret (load from a secret store, serve only over TLS). A successful operator authentication is audit-logged at `INFO`; a request presenting a *non-matching* operator key is logged at `WARN` and counted by a new `wavehouse_auth_operator_key_failures_total` counter (a probing/brute-force signal on the most privileged credential in the system) before falling through to the normal token/default path — the middleware still never rejects. The `Authorization` auth-scheme is matched case-insensitively (RFC 7235) via a shared `authScheme` helper, which also makes the existing `Bearer` JWT scheme case-insensitive (previously it required the canonical `Bearer` casing). Explicitly out of scope, tracked in #359: scoping the operator credential to the management surface only, and capability-scoped admin permissions. - **Missing-dedupe-id observability + optional strict mode** (`internal/api/ingest.go`, `internal/api/ingest_test.go`, `internal/config/config.go`, `cmd/wavehouse/main.go`, `config.yaml`, `deployments/compose/standalone.yaml`, `docs/src/content/docs/configuration.mdx`, `docs/src/content/docs/api.md`, `docs/src/content/docs/architecture.md`): closes #219. With dedupe enabled, a row missing the configured `id_field` can't be deduped — previously it was published with idempotency silently disabled and *no* log or metric, so a producer bug that dropped the id turned off the guarantee for those rows unnoticed. Now every such row is logged at `WARN` and counted by a new `wavehouse_ingest_dedupe_missing_id_total` counter (labeled by `table`), making the loss observable server-side. A new opt-in `dedupe.require_id` (`WH_DEDUPE_REQUIRE_ID`, default `false`) turns that signal into enforcement: a row missing the id is rejected (`400` for a single insert; a per-record failure in a batch) instead of published — a tripwire for producers that must guarantee the id (complements the client-side [#202](https://github.com/Wave-RF/WaveHouse/issues/202)). Default behavior is unchanged. - **"Durability & Storage" operations guide** (`docs/src/content/docs/durability.md` (new), `docs/src/config/sidebar.ts`, `docs/src/content/docs/reverse-proxy.mdx`, `docs/src/content/docs/configuration.mdx`, `docs/src/content/docs/deployment.md`): documents #84. A new Operations page making the embedded-JetStream durability contract explicit before the docs site publishes: a `200` from `POST /v1/ingest` means the event has been `fsync`'d to disk on the node (the server runs with `SyncAlways: true` in `internal/mq/embedded.go`), which makes the storage substrate's `fsync` tail the ingest latency floor. Covers the contract (and how it differs from JetStream's default page-cache-then-periodic-sync mode), why a slow `fsync` tail manifests as `create stream: ... context deadline exceeded` and `503` backpressure, a where-it's-cheap-vs-expensive substrate table (managed cloud block storage and PLP NVMe vs. ZFS-without-SLOG / qcow2-on-`ext4` / spinning disks), an `fio` recipe + verdict bands to measure your own storage (with the macOS `F_FULLFSYNC` honesty caveat), and the symptom checklist. Forward-references the configurable group-commit interval (`mq.sync_interval`, [#139](https://github.com/Wave-RF/WaveHouse/issues/139)) and the planned `wavehouse storage-check` preflight ([#84](https://github.com/Wave-RF/WaveHouse/issues/84)) without claiming either exists yet. Cross-linked from Configuration (Message Queue), Deployment (Persistent Storage), and the Ingest Pipeline's worker-side ack section; no code changes. - **"Behind a reverse proxy" deployment guide** (`docs/src/content/docs/reverse-proxy.mdx` (new), `docs/src/config/sidebar.ts`, `docs/src/content/docs/deployment.md`, `docs/src/content/docs/configuration.mdx`, `docs/src/content/docs/api.md`, `internal/api/stream.go`, `internal/api/stream_test.go`): closes #241. A new Operations page for the common "WaveHouse behind nginx / Caddy / Cloudflare Tunnel" setup, since several behaviors only matter behind a proxy and weren't documented together. Covers: TLS termination (WaveHouse serves plain HTTP and manages no certs); the request-body size limits and the division of responsibility (WaveHouse ships fixed in-code memory-safety backstops — 1 MiB control / 16 MiB ingest — while the proxy is the tunable *outer* limit, so a missing/loose proxy limit can't OOM the server); Server-Sent Events buffering + idle-timeout tuning (WaveHouse sends a `: connected` comment on open plus a periodic `:` keepalive comment so quiet streams survive proxy idle timeouts, [#226](https://github.com/Wave-RF/WaveHouse/issues/226)); the `?token=` / `since` / `Last-Event-ID` forwarding streams need; `X-Forwarded-For` trust (don't expose `:8080` directly — it's honored, so a direct client could spoof it); and which health paths to expose (`/livez`/`/readyz` internal-optional, `/v1/health` must stay public). Ships full example nginx, Caddy, and Cloudflare-Tunnel configs, and is cross-linked from Deployment, Configuration, and the API reference. One small code change lands with it: the SSE endpoint (`GET /v1/stream`) now sets `X-Accel-Buffering: no` so nginx-class proxies stream events without buffering out of the box (nginx strips the header before the client sees it; Caddy/Cloudflare ignore it). The health-probe guidance is also upgraded from "optional" to a recommendation — keep the bare `/livez`/`/readyz`/`/healthz` paths internal (a public `/readyz` turns each hit into a ClickHouse `Ping`) and expose only `/v1/health` publicly. @@ -25,6 +25,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **BREAKING (API): every admin-gated endpoint now lives under one `/v1/ops/*` prefix** (`internal/api/router.go`, `internal/api/errors.go`, `clients/ts/src/{sql,policy,pipes,schema,dlq,table,cli/codegen}.ts`, docs throughout, plus tests in `internal/api`, `tests/integration`, `tests/e2e/sdk`, and the SDK): the admin surface was split across two shapes — the `/v1/admin/*` subtree (raw SQL, policy CRUD, pipes CRUD) plus three individually-gated top-level routes (`GET /v1/schema`, `POST /v1/schema/refresh`, `GET /v1/dlq/stats`) — so the path alone couldn't tell you what the `RequireAdmin` gate covered. All of them merge into a single `/v1/ops` subtree behind one tree-level gate: `/v1/admin/{query,policy,pipes…}` → `/v1/ops/{query,policy,pipes…}`, `/v1/schema[/refresh]` → `/v1/ops/schema[/refresh]`, `/v1/dlq/stats` → `/v1/ops/dlq/stats`. The gate itself is unchanged (`policy.AdminRole`, operator-key break-glass included). No aliases are kept for the old paths (pre-1.0; the SDK's path constants are updated in the same change, so its method surface — `wh.sql`, `wh.policy`, `wh.pipes`, `wh.schema`, `wh.dlq` — is unaffected). If you fenced `/v1/admin/` at your reverse proxy or ingress (deny rule, IP allowlist, internal-only listener), that rule silently stops matching after this rename — move it to `/v1/ops/` (see `docs/src/content/docs/reverse-proxy.mdx` §"Fencing the admin surface"). One observable log change: an authorization denial on the former top-level schema route now records the tree pattern `route:"/v1/ops/*"` (as `/v1/admin/*` denials always did) instead of the full route template — the `gate:"admin"` attribute already identifies the check. The docs pass riding along also closed accuracy gaps surfaced in review: `reverse-proxy.mdx`'s body-cap table is relabeled by body shape instead of a control/data-plane split that clashed with the ops naming, the `RequireAdmin` denial contract now reads 401-for-a-present-but-invalid-token vs 403-for-a-non-admin-role everywhere it's described (`api.md`, `architecture.md`, `sdk/queries.md`), and the `gh attestation verify` examples in `deployment.md` and `SECURITY.md` pin `--signer-workflow` to the workflow that publishes the artifact, since `--repo` alone accepts an attestation from any workflow in the repo. + - **BREAKING (SDK): `PipeRef.fetch` no longer accepts a `limit` it silently ignored** (`clients/ts/src/pipes.ts`, `clients/ts/src/client.test.ts`, `docs/src/content/docs/sdk/pipes.md`, `docs/src/content/docs/sdk/reference.md`): closes #464, raised by CodeRabbit on #456. It took the same per-call options type as the query builder — which carries `limit` — but forwarded only `signal`, so `wh.pipe('top_pages').fetch({ limit: 10 })` type-checked, ran, and quietly returned whatever the pipe's SQL returned. `QueryBuilder.fetch` and `TableRef.fetch` both honour `limit`, so the inconsistency sat inside one shared type. There is nothing to forward: the endpoint binds the request body as the pipe's *parameters* (`internal/api/pipes.go` → `pipes.BindParams`), and a key the SQL doesn't declare is ignored, so a client-side row cap is not something the pipes surface offers. The parameter is now a dedicated `PipeRequestOptions` (exported) declaring `signal?: AbortSignal` and `limit?: never`, making the dead option a compile error rather than a silent no-op. `never` rather than simply omitting `limit`, because omitting it only rejects fresh object literals — TypeScript's excess-property check doesn't apply to a *variable*, so a shared `const opts: RequestOptions` carrying a limit would still have passed and still been dropped, which is the defect rather than a narrower version of it. Both cases are pinned by `@ts-expect-error` tests. **Note the collateral effect**, which is the half most consumers will actually meet: a value *declared* `RequestOptions` no longer assigns to a pipe `.fetch()` at all, even when it carries no limit at runtime, because the declared type permits one and assignability is decided on the type. Type a shared options object as `PipeRequestOptions` — the table and query-builder `.fetch()` accept it too, so it works everywhere — or inline `{ signal }` at the pipe call. Structural wrappers are unaffected: method parameters compare bivariantly, so an `interface Fetchable { fetch(opts?: RequestOptions): … }` is still satisfied by `PipeRef`. **Migration:** declare a `{{limit}}` parameter in the pipe's SQL and pass it as a pipe parameter — `wh.pipe(name, { limit })` — which is what the docs already showed. Pre-existing rather than introduced by #456, folded in there because that PR renames the type in question. - **BREAKING (SDK): `FetchOptions` is renamed `RequestOptions`** (`clients/ts/src/types.ts`, `clients/ts/src/index.ts`, `clients/ts/src/query-builder.ts`, `clients/ts/src/table.ts`, `clients/ts/src/pipes.ts`): the per-call options type accepted by `.fetch()`. The old name collided conceptually with the new `options.fetchOptions` — which, following OpenAI, Anthropic, and the wider ecosystem, means "extra `RequestInit` fields", not "options for our `.fetch()` method". Shipping both would have left `FetchOptions` and `fetchOptions` in the same SDK one capital letter apart, meaning unrelated things. `RequestOptions` is what Anthropic's SDK calls the identical concept. No deprecated alias: the type is unreferenced by anything consuming the pre-1.0 package, and keeping it would preserve exactly the ambiguity the rename removes. Renaming the import is the whole migration for this entry — note the separate `PipeRef.fetch` narrowing above, which is a behavioural break in the same file. The module-private `RequestOptions` in `http.ts` — the internal request descriptor — becomes `RequestSpec` to free the name. @@ -46,7 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security - **Live SSE streams now apply a role's row-`filter` per subscriber, closing a query/stream row-level-security drift** (`internal/stream/hub.go`, `internal/stream/subscriber.go`, `internal/stream/metrics.go`, `internal/policy/policy.go`, `internal/policy/{rowfilter,canonical,numeric}.go` (new — predicate evaluation, operand rendering, and storage-domain comparison as three focused files), `internal/discovery/validation.go`, `internal/discovery/timestamp.go`, `internal/api/stream.go`, `cmd/wavehouse/main.go`, `docs/src/content/docs/access-control.mdx`, `docs/src/content/docs/architecture.md`, `docs/src/content/docs/api.md`, `docs/src/content/docs/sdk/streaming.md`, `AGENTS.md`, `SECURITY.md`, `internal/stream/doc.go`, plus tests in `internal/policy/rowfilter_test.go` (new), `tests/integration/rowfilter_narrowing_test.go` (new), `internal/policy/policy_test.go`, `internal/discovery/validation_test.go`, `internal/discovery/timestamp_test.go`, `internal/stream/hub_test.go`, `tests/e2e/sdk/streaming.test.ts`): closes #319. The SSE delivery path stripped denied columns but never applied a role's row-level `filter` predicate, so a subscriber received rows the structured-query path would have filtered out for that same role — a data-exposure on the streaming surface for any table that combines a row-policy with a shared or role-scoped stream (harmless on the public Stats table today, which carries no restrictive row-policy, but real for any private/PII table fronted by a stream). The row-filter is now resolved once into predicates that feed **both** read surfaces — the query path renders them to SQL, the stream evaluates them in memory (`ResolvedPermissions.RowVisible`, evaluated per subscriber against that subscriber's claims via the same `Evaluate` call the query path uses) — so the two can't drift (the row-level analogue of the shared `IsColumnAllowed` decision from #223). Because a row-filter resolves against each subscriber's JWT claims, the #294/#353 once-per-role projection is now claims-aware: a role **without** a filter keeps the pure once-per-role fast path unchanged (zero regression on the public stream), while a role **with** a filter keeps the shared column projection but delivers it only to the subscribers whose claims admit each row (evaluated against the full event, so a filter may key on a column the role can't select). The in-memory comparison is type-aware and **fails closed on anything it can't prove** (`policy.ColumnKind`, seeded from the schema registry): numeric columns (`Int*`/`UInt*`/`Float*`/`Decimal*`) compare in the column's **storage domain**, built on the same canonical machinery #457 landed for claims rather than a parallel stack: BOTH operands render to exact canonical decimal form (`CanonicalScalar`/`CanonicalNumericLiteral` — digit-bounded, so an over-long client-controlled operand is refused before any superlinear work; the hub decodes payloads with `UseNumber`, so 64-bit IDs keep exact digits whether string-encoded — the JS-precision-loss escape hatch — or bare, and big integers stay byte-faithful on the SSE wire) and a digit-string comparator (`compareCanonicalDecimals`) orders them — the former float64/`math/big` comparison stack is deleted. The operands are then narrowed the way ClickHouse narrows the stored value at insert AND the bound constant at compare (`policy.NumericSpec`, classified per column by `discovery.NumericStorageOf`): `Float32`/`Float64` round to the column's width, `Decimal` truncates at its scale, and integer columns are exact at any width, refusing fractional operands and non-plain constant spellings (`'1e3'` errors ClickHouse's integer cast per query, so the stream withholds to match; Float and Decimal casts accept every JSON-number spelling — verified — so those compare by value). Both operands are also **range-gated** per column (`Int*`/`UInt*` width bounds, the Decimal precision budget): ClickHouse's reading of an out-of-range constant was measured to vary on a single release between a query error (a negative bound on an unsigned column — the role reads no rows), a mathematical promotion (`'256'` against a `UInt8`), and a width-boundary wrap onto a *different* value than written (`'9223372036854775808'` against an `Int64` compares as −2^63, where exact-precision comparison would have admitted the −2^63 rows SQL hides under `_neq`) — so the stream refuses all of them rather than model any one behavior, an out-of-range payload was never storable regardless, and the differential oracle asserts strict parity for the error class plus the never-admit-where-SQL-hides direction for the promotion/wrap class. This closes the review-raised ordering fail-open where a pre-narrowing payload (`16777217` into a `Float32`) satisfied `_gt: "16777216"` on the stream while the stored row (`16777216`) failed it in SQL, and `tests/integration/rowfilter_narrowing_test.go` holds every stream verdict equal to a live ClickHouse's across column shapes × payloads × constants × operators — ClickHouse itself as the oracle, the same differential rigor as #402's timestamp grammar. An operand outside the JSON number grammar (`NaN`, any `Inf` spelling) or beyond the float domain's range withholds the row — `String` columns compare bytewise (exactly ClickHouse's String semantics, equality *and* ordering) — `DateTime`/`DateTime64` columns compare as **instants**, both operands read through the very grammar ingest canonicalization (#402) uses (`discovery.Column.TimeParser`: same spellings and zone rule, truncated to the column's precision, out-of-range instants that insert-time saturation would move refused), so a zone-less filter constant — the spelling query-path SQL wants — matches the canonical RFC 3339 payload the stream carries, time-window `_gt`/`_lt` policies work, and either operand the grammar can't read withholds the row — and every other type (`Enum`, `UUID`, `Date`/`Date32`, `Bool`, IPs, `FixedString`, …) — plus every column when no schema is available (an unknown table, or the boot-time discovery-failure window where the server serves while retrying) — trusts byte-equality only: `_eq`/`_in` admit exactly the event's own text rendering, while `_neq`/`_gt`/`_lt` withhold the row, because a byte difference can be pure representation and admitting on it would deliver rows the query path excludes (an uppercase UUID under `_neq`; `"9" > "100"` as text under `_gt`). The filter *constants* are claims-derived, so the same exactness holds on that side too: the auth middleware parses JWTs with `WithJSONNumber` and every bound constant routes through the shared `policy.CanonicalScalar` rule (both landed with the claim-template entry below — before them, a Snowflake-scale `tenant` claim rendered as `"1e+16"` and the stream *delivered the float64-equal neighbor tenant's rows* while withholding the subscriber's own), and this PR extends `CanonicalScalar` in depth for claims maps that never passed through that parser (a plain `json.Unmarshal` decodes numbers as float64): a float64 at or past 2^53 is refused outright — the predicate then matches **no** rows on either surface (`1 = 0` in SQL, unconditionally false in memory), the same two-surface verdict an unresolvable claim gets — and smaller floats render positionally (`"1000000"`, never the `"1e+06"` spelling ClickHouse integer columns reject with a type error). Ambiguity therefore always costs availability (a row withheld), never confidentiality — a guarantee about the ingested payload the stream evaluates, whose one residual payload-vs-stored asymmetry — an event whose insert later fails outright (out-of-range value, batch error → DLQ) after it was already streamed — is documented in the enforcement caution — and each withhold is observable via the new `wavehouse_sse_rows_withheld_total` counter (labeled by table and role, on the live and replay paths alike), separating "no matching rows" from "a fail-closed filter is withholding everything". Subscriber claims are fixed at construction (a `stream.NewSubscriber` argument — no setter), so the fan-out's unsynchronized claims read is race-free structurally, with a dedicated `-race` test driving concurrent row-filtered broadcasts. Replay (gap-fill) applies the same per-connection row check via `Hub.ReplayProjector`, which now holds one policy snapshot for the whole gap-fill (one store read per replay instead of one per replayed event; a policy reload landing mid-replay applies from the first live event) and caches the per-table column-kind lookup across the replay loop. **Perf note:** on a topic whose role carries a row-filter, part of the #294/#353 once-per-role fan-out gain is traded back for correctness — `policy.Evaluate` runs per subscriber per event there (the column projection stays shared; roles without a filter are unaffected); #435 tracks memoizing it, and `BenchmarkBroadcast_RowFilteredFanout` exists to measure it. The resource limits (`max_rows`, `max_execution_time`, …) remain a query-path property and are still **not** applied to the stream (a separate, documented boundary). Supersedes the "stream path applies no row-level filter" invariant originally noted in the #294/#353 Changed entry (under **Changed** in this same block, which now points back here). -- **Row-filter claim templates now fail closed on every operator when the token doesn't carry the claim** (`internal/policy/policy.go`, `internal/policy/policy_test.go`, `internal/query/builder.go`, `internal/query/builder_test.go`, `internal/api/structured_query.go`, `internal/api/structured_query_test.go`, `internal/api/ingest.go`, `internal/api/ingest_test.go`, `internal/auth/auth.go`, `internal/auth/auth_test.go`, `internal/stream/hub_test.go`, `docs/src/content/docs/{access-control.mdx,configuration.mdx,deployment.md,pipes.mdx,architecture.md,api.md}`): closes [#385](https://github.com/Wave-RF/WaveHouse/issues/385) — the remaining fail-open half of the class [#224](https://github.com/Wave-RF/WaveHouse/issues/224) opened — and [#322](https://github.com/Wave-RF/WaveHouse/issues/322). An unresolvable `{{ jwt.* }}` template in a `filter` rendered as the empty string and still bound a real predicate for `_eq`/`_neq`/`_gt`/`_lt` — so a validly-signed token missing the claim (mixed IdP audiences, service tokens) got `WHERE tenant_id = ''` (leaking every empty-valued row), and `_neq`/`_gt` on a string column (`col != ''` / `col > ''`) matched essentially all rows, erasing the restriction entirely. Only a bare-claim `_in` failed closed. Now any filter template containing an unresolvable claim path emits the same constant-false predicate `_in` already used (`1 = 0`): on the structured-query path (`POST /v1/query`) the role sees no rows, matching what `access-control.mdx` promised all along (the SSE stream applies the same fail-closed rule per subscriber — see the row-`filter` entry above ([#381](https://github.com/Wave-RF/WaveHouse/pull/381)) — and named pipes authorize by role, not row filter). A template-free literal value — including an explicit `""` — still binds exactly as written, and an `_in` template with surrounding text joins the fail-closed path too (previously it bound the partial literal). A claim that resolves to a JSON **object or array** — usually a dropped path segment, `{{ jwt.app_metadata }}` where `{{ jwt.app_metadata.tenant_id }}` was meant — now also fails closed instead of binding its `map[…]`/`[…]` stringification (the bare-claim `_in` array keeps its multi-tenant semantics, and its **elements** now follow the same rule: one object, null, or nested-array element fails the whole set closed rather than binding a `map[…]`/`` rendering no row legitimately carries), and the JWT parser now decodes numeric claims as `json.Number` (`jwt.WithJSONNumber`), so a numeric id above 2^53 binds digit-exact instead of float64-rounding to a neighboring value. Every bound value flows through one rule, `policy.CanonicalScalar`, which also renders numeric claims in **canonical decimal form** rather than the token's spelling — `1.0`/`1e3` bind as `1`/`1000`, because the literal spelling raises a per-query `TYPE_MISMATCH` against a numeric ClickHouse column. The canonical form is **exact** at every width and precision — integers via big-int, fractions and exponents by digit-string arithmetic, never a float64 round-trip, so `0.1000000000000000000001` keeps every digit and `1e-400` fails closed instead of collapsing to `0` — while a magnitude only JSON can hold (`1e400`, `1e-400`) or a literal (or exact form) past 100 digits fails closed (the exact paths are length-bounded up front — big-int cost is superlinear in digit count and the ingest check path hands it client-controlled literals, so an unbounded literal was a single-request CPU sink) — and the insert-`check` comparison canonicalizes its payload side through the same function, so a numeric insert value matches a numeric claim by value, not by spelling. One deliberate JWT-validation shift rides along with `json.Number` decoding: a literal `exp: 0`, which float64 decoding special-cased as never-expiring, now reads as the epoch, so such tokens are rejected as expired and fall back to the roleless `default_role` like any invalid token. Breaking only for deployments that relied on the fail-open: a role whose token lacks a templated claim now reads nothing instead of *more* than intended. Insert-`check` `_eq` semantics are unchanged — the template still renders (unresolvable placeholder → empty string, surrounding literal text kept) and that rendered value is auto-injected (the required-value question is [#463](https://github.com/Wave-RF/WaveHouse/issues/463)) — but a `check: _in` template with surrounding text and an unresolvable claim now resolves to the empty set (every insert to that column rejected) instead of requiring membership in the partial literal, since the `_in` resolver is shared with the filter path. This PR also closes two adjacent fail-open paths in the same class surfaced in review: a claim template whose path is outside the `{{ jwt. }}` grammar (a hyphen, or a namespaced OIDC URL) is now **rejected when the policy is written** rather than bound as literal `{{…}}` text — a read leak for `_neq`/`_lt` and silent write corruption for `check`. That boundary is asymmetric for a running deployment: a bootstrap policy file carrying such a template makes the server **refuse to start** when the store is seeded from it (a populated KV store skips the file), an admin `PUT` on `/v1/admin/policy` (or a `POST` to its `validate` sibling) returns `400`, and a policy already stored in KV is *not* re-validated when a node loads it ([#461](https://github.com/Wave-RF/WaveHouse/issues/461)) — re-`PUT` it once after upgrading. And the row-filter predicate plus the role's `max_rows` cap are now emitted by `Build` itself, as part of the WHERE/LIMIT assembly it already does (the #322 half of this PR): splicing them into rendered SQL afterward let a crafted aggregation alias or `ORDER BY` alias-reference swallow the `WHERE` splice and delete the row filter — valid SQL returning the whole table, reachable precisely when a filter failed closed — and `ApplyMaxRows`'s uppercase-then-index offset drifted on length-changing runes (a column named `ıı`), silently dropping the cap. `InjectPermissionFilters`, `ApplyMaxRows`, and `findInsertPoint` are deleted, and no interim clause-keyword alias guard ships (one existed only between commits of this PR): a keyword-bearing alias (`Total order by region`) stays a legal identifier, contained by backtick quoting. One more member of that case-folding family: the aggregation-function allowlist is now **ASCII-exact** — `strings.ToLower` folds `İ` (U+0130) to `i`, so an aggregation named `mİn` passed the allowlist and reached ClickHouse verbatim as an unknown function, a per-query `500` where the builder's `400` belongs. Canonicalization is symmetric end to end: the insert-`check` comparison runs its **required side** through `CanonicalScalar` as well as the payload side, and a `check` value with **no placeholder** — which carries no JSON type — additionally matches by its numeric reading at compare time (a static `_eq: "1.0"` accepts an inserted `1.0` and an inserted `"1.0"` alike; without that reading the canonical payload side rejects every numeric insert the check was written to allow) while still binding and auto-injecting exactly as written, so read filters never move (`_neq: "1.0"` on a `String` version column keeps excluding exactly `'1.0'`). The second reading is gated **by type** (`policy.LiteralValue`, which `Evaluate` reserves for placeholder-free values), so a claim-derived value keeps strict canonical equality — a string-typed claim of `"1e3"` never accepts an inserted `1000`. The 100-digit literal bound counts **digits, not bytes** — sign, decimal point, and exponent marker ride free, so `-1e99` and its 101-byte written-out form both bind (the exact-form gate leaves two characters of slack past the bound, so a borderline exponent spelling like `1e101` can resolve where its written-out digits could not — the literal-side gate is the stricter of the two). The exact-form bound is **stricter than the old float64 path for wide magnitudes**: values like `1e150` or `1e-150`, whose exact decimal expansions exceed 100 digits, previously resolved (rounded) and now fail closed — on a read filter the role sees no rows; on an insert `check` the claim routes to the #463 auto-inject path, where an integer or `Decimal` column coerces the stamped `''` to `0`. **Upgrade note (data migration):** a pre-upgrade build auto-injected numeric claims above 2^53 in float64-rounded form, so rows it stamped carry a neighboring value of the true id (Snowflake-scale ids, ~1.7e18, are wide enough); the exact filter binds the true value, so those rows don't become wrong after upgrading — they become silently **unreachable** through the writer's own row filter. Before relying on the new filter, reconcile such rows (e.g. `ALTER TABLE … UPDATE` the scoped column from the rounded value to the exact claim value) — rounded and exact ids differ only above 2^53. +- **Row-filter claim templates now fail closed on every operator when the token doesn't carry the claim** (`internal/policy/policy.go`, `internal/policy/policy_test.go`, `internal/query/builder.go`, `internal/query/builder_test.go`, `internal/api/structured_query.go`, `internal/api/structured_query_test.go`, `internal/api/ingest.go`, `internal/api/ingest_test.go`, `internal/auth/auth.go`, `internal/auth/auth_test.go`, `internal/stream/hub_test.go`, `docs/src/content/docs/{access-control.mdx,configuration.mdx,deployment.md,pipes.mdx,architecture.md,api.md}`): closes [#385](https://github.com/Wave-RF/WaveHouse/issues/385) — the remaining fail-open half of the class [#224](https://github.com/Wave-RF/WaveHouse/issues/224) opened — and [#322](https://github.com/Wave-RF/WaveHouse/issues/322). An unresolvable `{{ jwt.* }}` template in a `filter` rendered as the empty string and still bound a real predicate for `_eq`/`_neq`/`_gt`/`_lt` — so a validly-signed token missing the claim (mixed IdP audiences, service tokens) got `WHERE tenant_id = ''` (leaking every empty-valued row), and `_neq`/`_gt` on a string column (`col != ''` / `col > ''`) matched essentially all rows, erasing the restriction entirely. Only a bare-claim `_in` failed closed. Now any filter template containing an unresolvable claim path emits the same constant-false predicate `_in` already used (`1 = 0`): on the structured-query path (`POST /v1/query`) the role sees no rows, matching what `access-control.mdx` promised all along (the SSE stream applies the same fail-closed rule per subscriber — see the row-`filter` entry above ([#381](https://github.com/Wave-RF/WaveHouse/pull/381)) — and named pipes authorize by role, not row filter). A template-free literal value — including an explicit `""` — still binds exactly as written, and an `_in` template with surrounding text joins the fail-closed path too (previously it bound the partial literal). A claim that resolves to a JSON **object or array** — usually a dropped path segment, `{{ jwt.app_metadata }}` where `{{ jwt.app_metadata.tenant_id }}` was meant — now also fails closed instead of binding its `map[…]`/`[…]` stringification (the bare-claim `_in` array keeps its multi-tenant semantics, and its **elements** now follow the same rule: one object, null, or nested-array element fails the whole set closed rather than binding a `map[…]`/`` rendering no row legitimately carries), and the JWT parser now decodes numeric claims as `json.Number` (`jwt.WithJSONNumber`), so a numeric id above 2^53 binds digit-exact instead of float64-rounding to a neighboring value. Every bound value flows through one rule, `policy.CanonicalScalar`, which also renders numeric claims in **canonical decimal form** rather than the token's spelling — `1.0`/`1e3` bind as `1`/`1000`, because the literal spelling raises a per-query `TYPE_MISMATCH` against a numeric ClickHouse column. The canonical form is **exact** at every width and precision — integers via big-int, fractions and exponents by digit-string arithmetic, never a float64 round-trip, so `0.1000000000000000000001` keeps every digit and `1e-400` fails closed instead of collapsing to `0` — while a magnitude only JSON can hold (`1e400`, `1e-400`) or a literal (or exact form) past 100 digits fails closed (the exact paths are length-bounded up front — big-int cost is superlinear in digit count and the ingest check path hands it client-controlled literals, so an unbounded literal was a single-request CPU sink) — and the insert-`check` comparison canonicalizes its payload side through the same function, so a numeric insert value matches a numeric claim by value, not by spelling. One deliberate JWT-validation shift rides along with `json.Number` decoding: a literal `exp: 0`, which float64 decoding special-cased as never-expiring, now reads as the epoch, so such tokens are rejected as expired and fall back to the roleless `default_role` like any invalid token. Breaking only for deployments that relied on the fail-open: a role whose token lacks a templated claim now reads nothing instead of *more* than intended. Insert-`check` `_eq` semantics are unchanged — the template still renders (unresolvable placeholder → empty string, surrounding literal text kept) and that rendered value is auto-injected (the required-value question is [#463](https://github.com/Wave-RF/WaveHouse/issues/463)) — but a `check: _in` template with surrounding text and an unresolvable claim now resolves to the empty set (every insert to that column rejected) instead of requiring membership in the partial literal, since the `_in` resolver is shared with the filter path. This PR also closes two adjacent fail-open paths in the same class surfaced in review: a claim template whose path is outside the `{{ jwt. }}` grammar (a hyphen, or a namespaced OIDC URL) is now **rejected when the policy is written** rather than bound as literal `{{…}}` text — a read leak for `_neq`/`_lt` and silent write corruption for `check`. That boundary is asymmetric for a running deployment: a bootstrap policy file carrying such a template makes the server **refuse to start** when the store is seeded from it (a populated KV store skips the file), an admin `PUT` on `/v1/ops/policy` (or a `POST` to its `validate` sibling) returns `400`, and a policy already stored in KV is *not* re-validated when a node loads it ([#461](https://github.com/Wave-RF/WaveHouse/issues/461)) — re-`PUT` it once after upgrading. And the row-filter predicate plus the role's `max_rows` cap are now emitted by `Build` itself, as part of the WHERE/LIMIT assembly it already does (the #322 half of this PR): splicing them into rendered SQL afterward let a crafted aggregation alias or `ORDER BY` alias-reference swallow the `WHERE` splice and delete the row filter — valid SQL returning the whole table, reachable precisely when a filter failed closed — and `ApplyMaxRows`'s uppercase-then-index offset drifted on length-changing runes (a column named `ıı`), silently dropping the cap. `InjectPermissionFilters`, `ApplyMaxRows`, and `findInsertPoint` are deleted, and no interim clause-keyword alias guard ships (one existed only between commits of this PR): a keyword-bearing alias (`Total order by region`) stays a legal identifier, contained by backtick quoting. One more member of that case-folding family: the aggregation-function allowlist is now **ASCII-exact** — `strings.ToLower` folds `İ` (U+0130) to `i`, so an aggregation named `mİn` passed the allowlist and reached ClickHouse verbatim as an unknown function, a per-query `500` where the builder's `400` belongs. Canonicalization is symmetric end to end: the insert-`check` comparison runs its **required side** through `CanonicalScalar` as well as the payload side, and a `check` value with **no placeholder** — which carries no JSON type — additionally matches by its numeric reading at compare time (a static `_eq: "1.0"` accepts an inserted `1.0` and an inserted `"1.0"` alike; without that reading the canonical payload side rejects every numeric insert the check was written to allow) while still binding and auto-injecting exactly as written, so read filters never move (`_neq: "1.0"` on a `String` version column keeps excluding exactly `'1.0'`). The second reading is gated **by type** (`policy.LiteralValue`, which `Evaluate` reserves for placeholder-free values), so a claim-derived value keeps strict canonical equality — a string-typed claim of `"1e3"` never accepts an inserted `1000`. The 100-digit literal bound counts **digits, not bytes** — sign, decimal point, and exponent marker ride free, so `-1e99` and its 101-byte written-out form both bind (the exact-form gate leaves two characters of slack past the bound, so a borderline exponent spelling like `1e101` can resolve where its written-out digits could not — the literal-side gate is the stricter of the two). The exact-form bound is **stricter than the old float64 path for wide magnitudes**: values like `1e150` or `1e-150`, whose exact decimal expansions exceed 100 digits, previously resolved (rounded) and now fail closed — on a read filter the role sees no rows; on an insert `check` the claim routes to the #463 auto-inject path, where an integer or `Decimal` column coerces the stamped `''` to `0`. **Upgrade note (data migration):** a pre-upgrade build auto-injected numeric claims above 2^53 in float64-rounded form, so rows it stamped carry a neighboring value of the true id (Snowflake-scale ids, ~1.7e18, are wide enough); the exact filter binds the true value, so those rows don't become wrong after upgrading — they become silently **unreachable** through the writer's own row filter. Before relying on the new filter, reconcile such rows (e.g. `ALTER TABLE … UPDATE` the scoped column from the rounded value to the exact claim value) — rounded and exact ids differ only above 2^53. - **A `?token=` query credential is now stripped from the request URL whichever credential wins, not only when it is the one used** (`internal/auth/auth.go`, `internal/auth/auth_test.go`, `docs/src/content/docs/api.md`, `docs/src/content/docs/reverse-proxy.mdx`, `docs/src/content/docs/sdk/index.mdx`): raised by CodeRabbit on [#448](https://github.com/Wave-RF/WaveHouse/pull/448). `bearerToken` returned from the `Authorization: Bearer` branch *before* the strip, so a request presenting both credentials — which any caller can do — left the unused JWT sitting in `r.URL` for the rest of the request's life. The operator-key path had the same shape one frame up, returning before `bearerToken` ran at all, so the strip is now resolved ahead of that branch too. Not an active leak today (WaveHouse's own request logging only ever records `r.URL.Path`, and the OTel HTTP instrumentation records no query attribute), so this is defense in depth rather than a fix for an observed exposure — but it closed an inconsistency in an invariant the code already asserted on the query-only path, where any later handler or future logging change would have turned it into one. The strip now runs once, before either credential path returns; header precedence is unchanged, unrelated query parameters survive, and both cases are pinned by tests. The docs correspondingly drop the "the header path leaves the query parameter untouched" caveat that described the old behavior. - **Policy `_in` is now enforced on both the row-`filter` and insert-`check` paths, closing a fail-open row-security gap** (`internal/policy/policy.go`, `internal/api/ingest.go`, `docs/src/content/docs/access-control.mdx`, plus tests in `internal/policy/policy_test.go`, `internal/api/ingest_test.go`): closes #224. The `Filter` schema accepted `_in` but the engine never read it: on the row-`filter`/SELECT path `resolveFilters` had no `_in` branch, so a row-security filter like `tenant_id: { _in: … }` produced **no `WHERE` predicate** and the role saw every row instead of its tenant subset (a fail-open, same family as #223); on the `check`/INSERT path only `_eq` was honored, silently dropping any other operator. `_in` now takes a single claim that resolves to a JSON **array** (the multi-tenant case — a token's `tenant_ids` list) and emits `col IN (?, …)` with one bound param per element; a scalar claim is a one-element set, and an empty/absent claim matches **no rows** (fail-closed) rather than widening to all of them. On the insert path an `_in` check requires the column be present and one of the set — there is no single value to auto-inject as `_eq` does, so an omitted column is rejected (`403 check failed`). The comparison operators are enforced on `filter` (`_eq`/`_neq`/`_gt`/`_lt`/`_in` all produce predicates now, so nothing is rejected there) and, on `check`, `_neq`/`_gt`/`_lt` become a loud config-load rejection (no insert-time semantics; `check` honors `_eq` + `_in`). The `_in` value stays a single templated string in the wire schema (Go `Filter.In`, SDK `PolicyFilter._in`), matching the established "set = array" shape of the caller-query `in` operator. - **`denied_aggregations` is now enforced case-insensitively against the caller-supplied function name, closing a policy bypass** (`internal/policy/policy.go`, plus tests in `internal/policy/policy_test.go`): closes #318. `IsAggregationAllowed` lower-cased the aggregation name only *after* the deny-list loop and the empty-allow-list early return, so the deny check compared a lower-cased deny entry against the raw caller input — a denied aggregation slipped past simply by changing case (`SUM` bypassed a `sum` deny entry, and with an empty allow list the call was then permitted). `isValidAggFn` already accepts any casing and the SQL builder emits the function name verbatim, so the denied aggregation actually executed. The case fold now happens once, before the deny check, so deny wins regardless of caller casing — matching the case-insensitive contract the access-control docs already specified. @@ -58,11 +60,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Column authorization now lives in the one place that already enumerates *every* column reference — the SQL builder (`query.Build`, made policy-aware) — so the projection, aggregation arguments, `filters`, `group_by`, `order_by`, and `time_range` are all checked against the role's allowlist in a single pass, and no current or future clause can silently skip the check. A full-row read is now requested explicitly with `"select_all": true`, which for a column-restricted role expands to exactly that role's allowed columns, so denied columns never reach ClickHouse; unrestricted roles and admin keep `SELECT *`; a role allowed the table but no columns fails closed with `403`. **Omitting `columns` now returns nothing** (an empty result) rather than expanding to a projection — so a hidden column can't leak by being left out at all. Because the generated SQL is now role-specific, the read cache key partitions by column visibility for free — one role can no longer be served another's cached columns. The structured-query and live-stream paths now share the same per-column decision (`policy.IsColumnAllowed`, plus new `AllowedProjection`/`RestrictsColumns` helpers), so the two read surfaces enforce identical column visibility and can't drift apart. New unit, handler, and end-to-end (SDK → WaveHouse → ClickHouse) tests cover each bypass shape as explicit red-team cases. One intended behavior change falls out of authorizing `order_by`/`group_by`/`filter`: a column-restricted role can no longer order, group, or filter by a column it cannot read — those clauses must reference only readable columns (unrestricted roles and admin are unaffected). The SDK's bare `.fetch()` maps to `select_all`, so it still returns a full row set (the same-release #270 fix already dropped its implicit `ORDER BY received_timestamp` default). - **Structured queries accept any identifier ClickHouse accepts (bring-your-own schemas), with every identifier backtick-quoted to stay injection-safe** (`internal/chsql/` (new), `internal/query/builder.go`, `internal/query/ast.go`, `internal/policy/policy.go`, `internal/api/structured_query.go`, `clients/ts/src/query-builder.ts`, `clients/ts/src/types.ts`, `docs/src/content/docs/access-control.md`, `docs/src/content/docs/api.md`, plus tests in `internal/query/{builder,ast}_test.go`, `internal/api/structured_query_test.go`, `internal/policy/policy_test.go`, `internal/chsql/chsql_test.go`, `tests/integration/identifier_roundtrip_test.go` (new), `tests/e2e/sdk/{query,cache,ingest}.test.ts`): table names, column names, and aggregation aliases may now contain dots, spaces, unicode, SQL keywords, even embedded backticks/backslashes — anything ClickHouse permits in a quoted identifier. The builder and the policy row-filter path route every identifier through one shared `chsql.QuoteIdent` that escapes exactly as ClickHouse's own `backQuote()` does (`\` → `\\`, `` ` `` → `` \` ``, verified against `SHOW CREATE TABLE` on a live server). This replaces the prior safe-identifier regex, which rejected legitimate columns in existing customer schemas. It also **subsumes the earlier aggregation-alias injection hardening**: an alias such as `n FROM secrets --` is now neutralized by quoting (it becomes an inert column label) rather than rejected, so callers keep arbitrary alias names while the reparent/statement-break/subquery payloads still cannot break out. Column and table references stay bounded by schema membership (the injection boundary for those) plus the per-column allowlist (#223); aliases carry no membership but are fully contained by quoting. The lone refusal is an identifier literally containing `?` — clickhouse-go's positional value-binder counts every `?` in the query text, even inside a quoted identifier, so such a name is rejected fail-closed rather than risk shifting value parameters (tracked in [#279](https://github.com/Wave-RF/WaveHouse/issues/279)). Two related API changes fall out of treating names literally: in a query, **`*` is now a literal column name** (quoted, gated by schema membership), not a wildcard — the all-columns read moved to an explicit **`select_all`** boolean (`columns` and `select_all` are mutually exclusive; omitting both selects nothing), and the SDK gains `.selectAll()` with a bare `.fetch()` mapping to it. `count(*)` still means count-all, and `allow_columns: ["*"]` is still the policy wildcard. The `columns` field also now accepts a single string as shorthand for a one-element list. Values are unchanged (positional `?`, driver-escaped). New real-ClickHouse round-trip tests cover weird column/table/alias names (including embedded backticks and backslashes), injection containment, and the `*`/`select_all` contract; server-side `{name:Identifier}` params were evaluated and rejected because clickhouse-go cannot mix them with positional value binding. - **Pipe parameters can no longer inject SQL through a non-scalar value** (`internal/pipes/pipes.go`, plus tests in `internal/pipes/pipes_test.go`, `internal/api/pipes_test.go`, `tests/e2e/sdk/admin.test.ts`, and docs `docs/src/content/docs/{pipes.mdx,api.md,architecture.md}`): closes #317. Pipes bind parameters by inlining escaped literals into the SQL template (no positional `?` binds), and `formatParamValue`'s fallback branch emitted whatever Go's `fmt.Sprintf("%v")` produced — so a JSON **array** or **object** supplied as a parameter value reached ClickHouse **raw and unescaped**. In a string-context placeholder an array element's inner `'` terminated the literal, giving a clean `UNION`/boolean injection that bypassed the per-column allowlist the structured-query path enforces. The execute route (`GET/POST /v1/pipes/{name}`) is authorized only by the pipe's `allowed_roles` (commonly `[public]`), so this was reachable unauthenticated. `formatParamValue` is now recursive and fails closed: every scalar leaf — including each element of an array — is escaped, a JSON array renders as a parenthesized `(v1, v2, …)` list (the same `IN (…)` shape the structured-query builder emits, so list parameters such as `WHERE id IN {{ids}}` now work safely), and a value with no scalar SQL form — a JSON object, or an empty array that would render as the invalid `IN ()` — is rejected with a `400` rather than emitted raw. A numeric-looking query-string value still renders bare (so `?limit=100` works as a number) via a strict numeric-literal check; any other string is single-quote-escaped, so it can never become invalid or injectable bare SQL. The declared parameter `type` stays advisory metadata (documenting intent for callers and the SDK), with binding keyed off the runtime value as before. New unit tests cover the array/object/empty/nested-list cases plus the original injection payloads as red-team regressions, new handler tests cover the array-binds and object-rejected paths, and an end-to-end SDK test executes an `IN`-list pipe against real ClickHouse with a quote-bearing element to exercise escaping. -- **The public read endpoints now bound the request body they decode, closing a single-request OOM vector** (`internal/api/query.go`, `internal/api/structured_query.go`, `internal/api/pipes.go`, `internal/api/policy.go`, `docs/src/content/docs/api.md`, plus tests in `internal/api/structured_query_test.go`, `internal/api/pipes_test.go`, and `internal/api/policy_test.go`): closes #315. `POST /v1/query` and `GET/POST /v1/pipes/{name}` called `json.NewDecoder(r.Body).Decode(...)` with no `http.MaxBytesReader`, so a large JSON array — e.g. a giant `in`-list `filters[].value` — decoded whole. JSON amplifies roughly an order of magnitude (~13×) bytes→live-heap into Go's in-memory representation (interface boxing, slice-growth doubling, GC headroom), so a single ~50 MiB request inflated to hundreds of MiB and a few concurrent ones OOM-killed the box — a *memory*, not throughput, vector that a per-IP rate cap doesn't mitigate. Both decoders now wrap the body in a new 1 MiB `maxControlBodyBytes` cap (a query/pipe-parameter body is bounded by nature, far under 1 MiB even with a large `in`-list) and return `413 request body exceeded 1048576 bytes` via the same `writeMaxBytesError` mapping the ingest/admin handlers already use — bringing the public read path to body-cap parity with the previously-hardened `/v1/ingest` and `/v1/admin/query` (16 MiB, the data-plane cap). The same cap is extended to the remaining body-decoding handlers in the package so none is left as an OOM gap — the admin pipe-definition `PUT /v1/admin/pipes/{name}` and the admin policy decoders `PUT /v1/admin/policy` + `POST /v1/admin/policy/validate` (a pipe or policy document is a bounded description). The pipe `Execute` parameter path keeps its lenient fallback for a malformed-but-within-cap body (parameters may come from the query string alone) and only hard-stops with `413` on an over-cap body. The cap is a fixed in-code backstop, not a tuning knob — operators set their own outer limit at the [reverse proxy](https://wavehouse.dev/reverse-proxy#request-body-size-limits); see the new deployment guide in this release. +- **The public read endpoints now bound the request body they decode, closing a single-request OOM vector** (`internal/api/query.go`, `internal/api/structured_query.go`, `internal/api/pipes.go`, `internal/api/policy.go`, `docs/src/content/docs/api.md`, plus tests in `internal/api/structured_query_test.go`, `internal/api/pipes_test.go`, and `internal/api/policy_test.go`): closes #315. `POST /v1/query` and `GET/POST /v1/pipes/{name}` called `json.NewDecoder(r.Body).Decode(...)` with no `http.MaxBytesReader`, so a large JSON array — e.g. a giant `in`-list `filters[].value` — decoded whole. JSON amplifies roughly an order of magnitude (~13×) bytes→live-heap into Go's in-memory representation (interface boxing, slice-growth doubling, GC headroom), so a single ~50 MiB request inflated to hundreds of MiB and a few concurrent ones OOM-killed the box — a *memory*, not throughput, vector that a per-IP rate cap doesn't mitigate. Both decoders now wrap the body in a new 1 MiB `maxControlBodyBytes` cap (a query/pipe-parameter body is bounded by nature, far under 1 MiB even with a large `in`-list) and return `413 request body exceeded 1048576 bytes` via the same `writeMaxBytesError` mapping the ingest/admin handlers already use — bringing the public read path to body-cap parity with the previously-hardened `/v1/ingest` and `/v1/ops/query` (16 MiB, the data-plane cap). The same cap is extended to the remaining body-decoding handlers in the package so none is left as an OOM gap — the admin pipe-definition `PUT /v1/ops/pipes/{name}` and the admin policy decoders `PUT /v1/ops/policy` + `POST /v1/ops/policy/validate` (a pipe or policy document is a bounded description). The pipe `Execute` parameter path keeps its lenient fallback for a malformed-but-within-cap body (parameters may come from the query string alone) and only hard-stops with `413` on an over-cap body. The cap is a fixed in-code backstop, not a tuning knob — operators set their own outer limit at the [reverse proxy](https://wavehouse.dev/reverse-proxy#request-body-size-limits); see the new deployment guide in this release. - **Dropped chi's deprecated `middleware.RealIP` from the router** (`internal/api/router.go`, `docs/src/content/docs/reverse-proxy.mdx`): closes #281. It rewrote `r.RemoteAddr` from spoofable forwarded headers (`X-Forwarded-For` / `X-Real-IP` / `True-Client-IP`) on every request, whether or not a trusted proxy set them — the IP-spoofing GHSAs chi deprecated it for (SA1019; GHSA-3fxj-6jh8-hvhx / GHSA-rjr7-jggh-pgcp / GHSA-9g5q-2w5x-hmxf), which also blocked the weekly go-deps dependabot group on the chi bump (#209). Nothing in WaveHouse read `r.RemoteAddr` (no per-IP logic — rate limiting is the reverse proxy's job), so removing it is behavior-preserving and drops the spoofing vector; `r.RemoteAddr` is now the honest immediate peer (the proxy, when one fronts WaveHouse). Capturing the real client IP **safely** — trusted-proxy-aware and surfaced in traces and logs, plus gating trace-context propagation on the same trust boundary — is tracked as a follow-up (#333). ### Fixed +- **`TestDispatchLoop_PerTableBatching_NoCrossTableContamination` flake on contended CI runners** (`internal/ingest/worker_test.go`): pre-existing (the test and the code it pins are untouched by the `/v1/ops` rename — it just happened to red the rename PR's CI). The test proves table B flushes on its own **size** trigger rather than being stranded behind the `maxWait` **timer** by table A's earlier events, and it discriminated the two by demanding the whole publish→consume→flush pipeline complete within `maxWait − 250ms` = 1.75s of wall clock — which a busy runner can miss for reasons unrelated to the regression. Now `maxWait` is 30s with the assertion window set to production's `defaultMaxWait` (5s), per review: a stranded batch can't flush before 30s, while the size-trigger flush must land inside the real 5s timer bound — so the test still proves the flush beats the timer production actually runs, with ~3x the old wall-clock headroom (the flush itself completes in well under a second) and 6x separation from the stranded case. Don't shrink `maxWait` back toward the window. + +- **`wavehouse-codegen` now parses `/v1/ops/schema` as the array the server returns** (`clients/ts/src/cli/codegen.ts`, `clients/ts/src/cli/codegen.test.ts` (new), `docs/src/content/docs/sdk/reference.md`): closes #388. The CLI typed the schema response as a name-keyed map and iterated `Object.keys`, but the endpoint returns a JSON array of tables, so against a live server the generated file keyed the `Database` interface by array index (`0: 0Row;`) and didn't compile. The CLI now consumes the array (rejecting any other response shape — malformed table or column members included — loudly rather than crashing mid-generation), sorts tables by name for stable output, and gained unit tests over the wire shape, the ClickHouse→TypeScript type mapper, and the generator; `main()` now runs only when the file is invoked as a script (bin/tsx — unchanged behavior there), so the pure pieces are importable by tests. `wh.schema.list()` was never affected — the client method already transformed the array. + - **SSE row `DateTime` columns now arrive in canonical RFC 3339 UTC, matching `/v1/query`, instead of the producer's spelling** (`internal/discovery/timestamp.go` (new), `internal/discovery/{discovery.go,validation.go}`, `internal/api/{ingest.go,clickhouse_exec.go}`, `internal/ingest/worker.go`, `docs/src/content/docs/{api.md,architecture.md,ingest-pipeline.md,sdk/streaming.md}`, plus tests in `internal/discovery/{timestamp_test.go (new),discovery_test.go}`, `internal/api/{ingest_test.go,boot_chain_test.go}`, `tests/e2e/sdk/streaming.test.ts`): closes #372. The stream fans out the pre-insert payload verbatim, so a row `DateTime` reached SSE subscribers in whatever spelling the producer sent — typically the zone-less ClickHouse-native form — while `/v1/query` rendered the stored value as RFC 3339 `Z`; JavaScript's `Date.parse` reads a zone-less string as **local** time, so an SDK `liveQuery` stitching backfill + live landed the two paths hours apart (sign-flipping with the viewer's UTC offset — broke ordering and the "N min ago" label in the Stats live feed). Ingest now **canonicalizes** every top-level `DateTime`/`DateTime64` column value in the accepted input forms to RFC 3339 UTC (fraction truncated to the column's precision, `time.RFC3339Nano`-trimmed — byte-matching `/v1/query`'s `time.Time` marshaling) after validation and before the NATS publish, so the one payload every consumer shares — SSE subscribers, the ClickHouse insert, the DLQ — carries a single unambiguous spelling for every value it rewrote (a fail-open pass-through keeps the producer's spelling on that shared path, while `/v1/query`, rendering from storage, stays canonical — so for a pass-through ClickHouse accepts, the streamed and queried renderings diverge). Inputs stay liberal but are mirrored per column kind, exactly as ClickHouse reads them (#402 review): RFC 3339 with any offset (`.`-fractions only — ClickHouse has no `,` separator), `YYYY-MM-DD[ T]HH:MM:SS[.fff]`, `YYYY-MM-DD`, a Unix-seconds string of 9–10 digits (a fraction after it is honored only for `DateTime64` columns, parsed as an exact decimal and truncated at nine digits — never a `float64` round-trip, which corrupts nanoseconds and can round across the second; other digit lengths are ClickHouse's own forms — calendar `YYYYMMDD`/`YYYYMMDDhhmmss` or its 13/16/19-digit ms/µs/ns epochs — and pass through), and **integer** JSON numbers read the way ClickHouse reads them: Unix seconds for `DateTime`, **ticks at the column's scale** for `DateTime64` (the ms epoch `1750478400500` into a `DateTime64(3)` is a valid 2025 instant; an epoch-*seconds* number there is a 1970 instant — rewriting either as seconds would change what ClickHouse stores; non-integer numbers pass through, ClickHouse rejects them). Values whose instant lies outside the column type's range also pass through — ClickHouse *saturates* out-of-range values spelling-dependently (local time-of-day is kept while the date clamps; a `DateTime64(9)` column even rejects the insert past the Int64-nanosecond ceiling, a bound WaveHouse's rewrite window conservatively applies at precision ≥ 7), so only the producer's own spelling may be the one that saturates. Go `LoadLocation`'s `''`/`'Local'` environment quirks are rejected as unresolvable zones. A zone-less string is interpreted in the column's declared time zone, else the server's (discovered once per schema refresh via `SELECT timezone()`) — the same rule ClickHouse itself applies, so canonicalization never changes which instant is stored (differentially fuzzed against a live ClickHouse — ~35k generated inputs × six column shapes, raw vs canonicalized inserts, zero divergences — with every divergence class found along the way pinned in `tests/integration/timestamp_canonicalization_test.go`). Each timestamp column's spec (precision + resolved zone) is precomputed at schema discovery and cached on the column, so the per-record ingest path parses no type strings, loads no zones, and takes no locks. Canonicalization is **fail-open** (#402 review): a value outside the accepted forms publishes verbatim — ClickHouse's more liberal `best_effort` parser stays the arbiter of insertability, and a value it too rejects surfaces via the DLQ as before — and an unresolvable zone (no tzdata embedded — named zones resolve from the runtime's zone database, which the bundled distroless images ship) warns and passes through, never a failed refresh or a silent UTC reinterpretation that would move instants (`Etc/UTC` is special-cased to UTC); ingest never rejects a record over its timestamp spelling — fail-closed enforcement of the canonical form is the stream row-filter's (#381). The worker's `INSERT … FORMAT JSONEachRow` pins `date_time_input_format=best_effort` — the server default since ClickHouse 26.5, and on older servers the `basic` default rejects the canonical form's zone suffix (verified live); pre-upgrade messages still in NATS — zone-less strings, Unix numbers — parse exactly as before under either setting (bare digit-strings outside the 9–10-digit Unix-seconds shape are the one divergence: `best_effort` reads them as calendar/epoch forms where `basic` read Unix seconds). Boundaries: `Date`/`Date32` columns pass through untouched (day precision, no zone ambiguity on this path); events published before the upgrade replay in their original spelling until the sweeper retires them; a column with an explicit non-UTC zone renders as `Z` on both paths (`/v1/query` likewise normalizes `DateTime` values to UTC), so the declared zone affects how zone-less inputs are read but never the output spelling — byte-identical regardless of zone or `Nullable` wrapping (`/v1/query` unwraps nullable timestamps to the same UTC form; a SQL `NULL` renders as JSON `null`). - **The E2E harness enforces its own poll budgets, and no longer inherits an idle pooled connection** (`tests/e2e/sdk/helpers.ts`, `tests/e2e/sdk/helpers.test.ts` (new), `tests/e2e/sdk/setup.ts`, `tests/e2e/sdk/vitest.config.ts`, `tests/e2e/sdk/{batching,cache,dlq,ingest,ndjson,query,stress}.test.ts`, `scripts/orchestrator/main.go`, `docs/src/content/docs/development.md`, `docs/src/content/docs/sdk/reference.md`): closes #440. Two defects, the first of which hid the second. `waitForCondition` checked the clock only on loop entry, so a single slow `fn()` overran the advertised budget without bound — a 10s budget was measured running 28s, past the caller's `testTimeout`, so vitest killed the test first and reported a timeout naming neither the condition nor how long the poll actually waited. It now races `fn()` against the deadline, aborts the in-flight call via an `AbortSignal` handed to `fn`, and reports poll shape on failure (`N poll(s), slowest Xms`) — which separates "the write never landed" (many fast polls) from "the polling itself was starved" (few slow ones). That reporting is what exposed the second defect: `chQuery` used the global `fetch`, which reuses pooled connections, and undici 8.8.0–8.9.0 stalls for seconds before writing a request onto a socket that has been idle a few seconds ([nodejs/undici#5600](https://github.com/nodejs/undici/issues/5600), a scheduling regression in `scheduleIdleSocketValidation()`, fixed in 8.10.0). This suite has multi-second idle gaps by construction — the 5s ingest linger sits between every write and the first poll of its visibility wait — so every visibility wait sat in the triggering window; local `make test-e2e` went from 2 pass/3 fail to 5 pass/0 fail, and every run got faster (115.7–124.5s vs 128.7–137.1s). Node 26 bundles undici 8.9.0 while CI runs Node 22 (undici 6.28.0) via `.nvmrc`, which is why this was invisible to CI and to developers on other Node lines. `chQuery` additionally takes a per-request ceiling (`E2E_CH_QUERY_TIMEOUT_MS`, default 10 000 ms) and honours the caller's signal, threaded through 19 call sites, so an abandoned poll tears its request down rather than running on unobserved. Also here: `batching`'s visibility wait had ~700ms of headroom over the 5s linger where every other wait allows 10s (widened — the `>= 4500ms` lower bound that carries the test's meaning is unchanged); the E2E setup banner prints the active node/undici version and warns when the local major differs from `.nvmrc`; the orchestrator kills any orphaned `wavehouse-cov` before starting, loudly, and aborts only if the kill itself fails (a killed run leaves one, and it corrupts the next run through the shared `tmp/data` and log file, presenting as a dozen unrelated tests failing to see their rows in a log that blames a container which no longer exists); a new `E2E_NO_COVERAGE=1` drops `--coverage` for local debugging, ignored under the gating targets so it cannot green a coverage gate by omission; and `vitest.config.ts` moves from `__dirname` to `import.meta.dirname`, silencing the Vite 8 `configLoader: 'native'` warning. @@ -81,7 +87,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Mermaid node labels no longer clip their last glyph, and the comparison diagram pair stacks vertically instead of shrinking side-by-side** (`docs/src/config/mermaid-theme.mjs`, `docs/package.json`, `pnpm-lock.yaml`, `docs/src/styles/global.css`, upstream [`astro-themed-mermaid`](https://github.com/Wave-RF/astro-themed-mermaid) → v0.3.0): the build-time Chromium measured each node's box at the default font-weight 400 while `global.css` displays labels at weight 500, so every box came out ~1px too narrow and the longest label clipped on the right (`Buffer Consumer` → `Buffer Consume` — present across all 116 nodes on the live preview). Fixed upstream via a new `measurementCss` option on `astro-themed-mermaid` that injects label metrics into the render page so Mermaid measures what the browser shows; `mermaid-theme.mjs` passes the node-label weight + letter-spacing (plus a 1.5px safety pad) and resets cluster titles to their default measurement (they render as `overflow:visible` pills sized separately by the plugin, so a weight bump there would only mis-center them). Selectors are bare (`.nodeLabel p`, not `svg[aria-roledescription…] .nodeLabel p`) because Mermaid measures the label before the flowchart `` wrapper exists. Separately, the `.diagram-pair` wrapper (the DIY-vs-WaveHouse comparison on `why-wavehouse`) dropped its ≥1500px flex-row variant: two detailed diagrams side-by-side each shrank to ~450px on a wide monitor, so they now always stack vertically at the full content width — matching the new AGENTS.md "Authoring Mermaid diagrams" house style (favour vertical/`TB` over `LR`). - **Embedded NATS no longer installs its own process-wide SIGINT handler** (`internal/mq/embedded.go`, `internal/testutil/natsjs.go`): closes #287. nats-server's `Start()` registers a signal handler unless `NoSigs` is set, and its SIGINT path is `Shutdown(); WaitForShutdown(); os.Exit(0)` — racing WaveHouse's own graceful shutdown on every Ctrl-C/SIGTERM-adjacent stop. Depending on who won: the NATS handler's `os.Exit(0)` skipped all of `run()`'s deferred cleanup (dedup/cache/ClickHouse close) while still exiting 0, or the two concurrent `Server.Shutdown()` calls panicked (`close of nil channel` in `shutdownEventing`, nats-server v2.14.0 events.go:1907) with exit status 2 — and a panic exit also skips Go coverage flushing, which surfaced as the e2e suite suddenly reporting 0.0% coverage in `make ci` (the orchestrator SIGINTs the instrumented binary precisely to flush counters). Now `NoSigs: true`: WaveHouse's shutdown path is the single lifecycle owner and already stops the embedded server via `EmbeddedNATS.Close()`; the twin options literal in `testutil.NewJetStream` (pipes/policy KV tests) gets the same flag so a Ctrl-C during a local test run can't `os.Exit(0)` the test binary mid-suite. Removing the early-exit also exposed how long a *clean* graceful exit actually takes when no OTel collector is listening — ~15s of serialized exporter dial-backoff (filed as #288) — so the e2e orchestrator's SIGINT-to-kill budget rises 10s → 30s (`scripts/orchestrator/main.go`; fast exits are unaffected, the wait selects on process exit). Both shutdown paths now also `WaitForShutdown()` after `Shutdown()` — owning the lifecycle means waiting it out, so the process can't exit (or `t.TempDir()` cleanup race) while JetStream is still tearing down. *Superseded in part — the 10s → 30s orchestrator-budget bump was reverted back to 10s once OTel provider shutdown became concurrent and context-bounded (`internal/observability/provider.go`), which caps a collectorless clean exit at ~3s instead of ~15s; the `NoSigs` lifecycle fix above is unaffected.* -- **Docs corrections surfaced by this branch's review pass** (`docs/src/content/docs/{sdk,getting-started,development,api,configuration,deployment}.md`, `docs/src/content/docs/index.mdx`): All ten `POST /v1/ingest?table=` curl examples quote their URLs — unquoted `?` aborts on zsh (macOS default) with `no matches found` before curl runs, and the very first runnable command on the landing page was one of them. The Getting Started quickstart warns that the first ingest after creating a table can `404 unknown table` for up to the 60-second schema-refresh interval (and that `POST /v1/schema/refresh` is admin-only — also tagged in configuration.md and deployment.md, since the quickstart's trial role can't call it). The e2e test-file lists in sdk.md and development.md gain the `ndjson` suite (#217). sdk.md stops presenting `wh.dlq.stream()` as working — there is no server-side DLQ stream yet (the SSE bridge only carries `ingest.>`), so the documented call connected and received nothing; it's now caveated against #197 in all three places it appears. The admin-only sweep reaches the rest of the reference: api.md's schema/DLQ sections get local admin callouts + 401/403 (and 500-on-refresh) error rows matching the gates in `internal/api/router.go`, `wh.dlq` joins `wh.pipes`/`wh.policy`/`wh.schema` in stating the role requirement, and the SDK README's codegen section warns that `/v1/schema` is admin-only (`--auth `); the `/v1/admin/query` curl example gains the `Authorization` header it always needed (it was the only admin example that 403'd as written), and the JWT-testing section explains where `change-me-in-production` comes from and that the compose stack must set `WH_AUTH_JWT_SECRET`. (A pre-existing secretless-deployment auth-forgery vulnerability surfaced during review — empty-string-HMAC admin tokens validate against a no-secret instance — is tracked in #291 for a security-reviewed code fix; this PR removes the false "secretless ⇒ no token validates / pure public deployment" framing that masks it from every doc it ships — `api.md`, `access-control.md`, and `configuration.md` — leaving the legitimate tokenless-quickstart wording in `getting-started.md` intact.) +- **Docs corrections surfaced by this branch's review pass** (`docs/src/content/docs/{sdk,getting-started,development,api,configuration,deployment}.md`, `docs/src/content/docs/index.mdx`): All ten `POST /v1/ingest?table=` curl examples quote their URLs — unquoted `?` aborts on zsh (macOS default) with `no matches found` before curl runs, and the very first runnable command on the landing page was one of them. The Getting Started quickstart warns that the first ingest after creating a table can `404 unknown table` for up to the 60-second schema-refresh interval (and that `POST /v1/ops/schema/refresh` is admin-only — also tagged in configuration.md and deployment.md, since the quickstart's trial role can't call it). The e2e test-file lists in sdk.md and development.md gain the `ndjson` suite (#217). sdk.md stops presenting `wh.dlq.stream()` as working — there is no server-side DLQ stream yet (the SSE bridge only carries `ingest.>`), so the documented call connected and received nothing; it's now caveated against #197 in all three places it appears. The admin-only sweep reaches the rest of the reference: api.md's schema/DLQ sections get local admin callouts + 401/403 (and 500-on-refresh) error rows matching the gates in `internal/api/router.go`, `wh.dlq` joins `wh.pipes`/`wh.policy`/`wh.schema` in stating the role requirement, and the SDK README's codegen section warns that `/v1/ops/schema` is admin-only (`--auth `); the `/v1/ops/query` curl example gains the `Authorization` header it always needed (it was the only admin example that 403'd as written), and the JWT-testing section explains where `change-me-in-production` comes from and that the compose stack must set `WH_AUTH_JWT_SECRET`. (A pre-existing secretless-deployment auth-forgery vulnerability surfaced during review — empty-string-HMAC admin tokens validate against a no-secret instance — is tracked in #291 for a security-reviewed code fix; this PR removes the false "secretless ⇒ no token validates / pure public deployment" framing that masks it from every doc it ships — `api.md`, `access-control.md`, and `configuration.md` — leaving the legitimate tokenless-quickstart wording in `getting-started.md` intact.) - **`time_range.since`/`until` accept day/week suffixes (`7d`, `2w`) and reject unparseable values with a 400 instead of an opaque ClickHouse error** (`internal/query/builder.go`, `internal/query/builder_test.go`, `docs/src/content/docs/api.md`, `docs/src/content/docs/sdk.md`): closes #285. Go's `time.ParseDuration` stops at hours, so a documented `"7d"` (sdk.md) failed to parse and fell through to ClickHouse as a raw `DateTime` literal → opaque `Cannot read DateTime` error (surfaced as a retryable 500, per #271). `resolveTimeValue` now pre-expands `Nd`→`N*24h` / `Nw`→`N*168h` (Prometheus/ClickHouse-style) before parsing, and any value that is neither a relative duration nor an RFC3339 timestamp now fails closed — the builder maps it to a 400 with a clear message rather than handing the string to ClickHouse. @@ -123,7 +129,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Browser-first SDK distribution: an IIFE global build, CDN entry points, a `wavehouse-codegen` bin, and a Node 18 floor** (`clients/ts/tsup.config.ts`, `clients/ts/package.json`, `clients/ts/README.md`, `docs/src/content/docs/sdk.md`, `docs/src/content/docs/development.md`, `pnpm-workspace.yaml`): `@wavehouse/sdk` already shipped browser-ready ESM/CJS (zero deps, native `fetch`/`EventSource`) but documented only the `npm install` + bundler path. The build now also emits a minified, self-contained **IIFE bundle** (`dist/index.global.js`) that defines a `WaveHouse` global, wired to new `unpkg`/`jsdelivr` package fields — so `` then `WaveHouse.createClient({ … })` works on a no-build, FTP-deployed page — and the SDK README + `sdk.md` gain a "No build step (CDN)" section covering both the ESM-CDN (`` envelopes is gone from the wire format** (`internal/ingest/bento.go`, `internal/ingest/bento_test.go`, `tests/integration/ingest_test.go`, `tests/integration/dlq_test.go`, `docs/src/content/docs/architecture.md`, `docs/src/content/docs/api.md`, `AGENTS.md`): the worker accepts `{table_name, scope, received_timestamp, data}` envelopes (the existing `EventMessage` shape) and bulk-INSERTs them. All non-insert mutations (`DELETE`/`UPDATE`/`TRUNCATE`/`DROP`/`ALTER`/`REPLACE`/…) must go through `POST /v1/admin/query` under the admin role (the raw-SQL endpoint moved under `/v1/admin/*` in a follow-up commit on this same Unreleased section — see the Security entry below; the `RawSQL: true` policy field is gone too). The rationale: our policy engine authorizes mutations by evaluating row/column rules against the *payload* of an operation, which works for inserts but not for predicate-driven mutations (we can't prove a `WHERE` clause is satisfiable only for rows the caller is allowed to touch, and `WHERE 1=1` would otherwise nuke a table). Rather than ship a partial enforcement story, the pipelined surface narrows to the shape policy can authorize today; `/v1/admin/query` becomes the single sanctioned admin-equivalent surface for everything else. The embedded NATS server runs with `DontListen: true` (`internal/mq/embedded.go`), so the only Publishers reachable on `ingest.>` are in-process Go code (today, only the HTTP `/v1/ingest/{table}` handler), and `EventMessage` carries no `action` field so the legitimate producer never sets one. The `delete-envelope` DLQ shape and the `Wave-DLQ-Type` NATS header it required are gone with this — `dlq.
` now carries only insert-failure payloads (the inner data object), so DLQ consumers no longer need to discriminate. The `jsInput.chConn` and `jsInput.js` fields are dropped along with the `chConn driver.Conn` parameter on `StartIngestWorker`. Re-introducing pipelined / structured mutations is deferred until the policy engine can authorize predicates — track separately if/when scheduled. +- **BREAKING: the ingest pipeline is now insert-only; the `action` field on `ingest.
` envelopes is gone from the wire format** (`internal/ingest/bento.go`, `internal/ingest/bento_test.go`, `tests/integration/ingest_test.go`, `tests/integration/dlq_test.go`, `docs/src/content/docs/architecture.md`, `docs/src/content/docs/api.md`, `AGENTS.md`): the worker accepts `{table_name, scope, received_timestamp, data}` envelopes (the existing `EventMessage` shape) and bulk-INSERTs them. All non-insert mutations (`DELETE`/`UPDATE`/`TRUNCATE`/`DROP`/`ALTER`/`REPLACE`/…) must go through `POST /v1/ops/query` under the admin role (the raw-SQL endpoint moved under `/v1/ops/*` in a follow-up commit on this same Unreleased section — see the Security entry below; the `RawSQL: true` policy field is gone too). The rationale: our policy engine authorizes mutations by evaluating row/column rules against the *payload* of an operation, which works for inserts but not for predicate-driven mutations (we can't prove a `WHERE` clause is satisfiable only for rows the caller is allowed to touch, and `WHERE 1=1` would otherwise nuke a table). Rather than ship a partial enforcement story, the pipelined surface narrows to the shape policy can authorize today; `/v1/ops/query` becomes the single sanctioned admin-equivalent surface for everything else. The embedded NATS server runs with `DontListen: true` (`internal/mq/embedded.go`), so the only Publishers reachable on `ingest.>` are in-process Go code (today, only the HTTP `/v1/ingest/{table}` handler), and `EventMessage` carries no `action` field so the legitimate producer never sets one. The `delete-envelope` DLQ shape and the `Wave-DLQ-Type` NATS header it required are gone with this — `dlq.
` now carries only insert-failure payloads (the inner data object), so DLQ consumers no longer need to discriminate. The `jsInput.chConn` and `jsInput.js` fields are dropped along with the `chConn driver.Conn` parameter on `StartIngestWorker`. Re-introducing pipelined / structured mutations is deferred until the policy engine can authorize predicates — track separately if/when scheduled. - **Hub wildcard subscriptions** (`internal/api/hub.go`, `internal/api/hub_test.go`): dropped the NATS-style `*` / `>` pattern matching from `Hub.Broadcast`, the wildcard pattern loop, the `sent` dedup map, the `matchTopic` helper, and the eight wildcard tests (plus `TestMatchTopic`). After the #89 MVP cuts every producer publishes a concrete `ingest.
` subject and the SDK only ever subscribes to one concrete subject, so the wildcard fan-out was unused machinery. Closes #100 (part of #87). Net −210 lines (mostly tests). - **`project-orchestrator.yml` workflow + its three composite-action artifacts** (`.github/workflows/project-orchestrator.yml`, `.github/actions/board-upsert-status/`, `.github/actions/set-linked-issues-status/`, `.github/scripts/board-fetch-item.sh`, `AGENTS.md`, `CHANGELOG.md`): −887 lines net. The orchestrator was the largest single source of cross-trigger complexity on this repo (3-4 workflow_run-chained runs per PR push, `statusCheckRollup` GraphQL perms quirks, integration-token `NONE` for private-org members) for behaviour that is mostly either provided natively by GitHub or a one-click manual operation on a 4-person team. Replaced by: reviewer-assign step in `housekeeping.yml` that fires once on `pull_request_target: opened` / `ready_for_review` (not per-synchronize, so it doesn't re-spam after `dismiss_stale_reviews_on_push`), plus GitHub's native Projects v2 workflows (`Auto-add to project`, `Item added`, `Pull request merged`) configured in the project UI. Trade-offs explicit in the PR body: drafts no longer auto-flip on bot-clean, `CHANGES_REQUESTED` doesn't auto-move the board card, linked-issue card mirroring is dropped. AGENTS.md §"Governance Files" + §"Task Board state machine" + §"Review tooling reference" all rewritten to match. `dependabot-automerge.yml` trimmed in parallel: no more board-upsert step (native handles placement), `PROJECT_BOARD_TOKEN` guard removed (no longer used in this workflow), reviewer list sourced from `board-config.env`'s `ADMINS` via `replace()`, major-bump comment uses the marker-comment upsert pattern from `housekeeping.yml`. - **`STATUS_*` and old `ADMINS` consumers in `board-config.env`** — STATUS option IDs had only orchestrator-side consumers and are now unreferenced. `ADMINS` was restored to `board-config.env` after the initial orchestrator-removal commit dropped it (Gemini and Claude both flagged the resulting drift across three inlined copies); both `housekeeping.yml` and `dependabot-automerge.yml` now load `ADMINS` from `board-config.env`. `admin-approval.yml` keeps its own inline copy with the documented latency-avoidance reasoning. @@ -217,17 +223,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security -- **BREAKING: raw SQL moves from `POST /v1/query` to `POST /v1/admin/query` and is gated on the admin role** (`internal/api/router.go`, `internal/api/router_test.go`, `internal/api/query.go`, `internal/api/query_test.go`, `internal/policy/policy.go`, `internal/policy/policy_test.go`, `cmd/wavehouse/main.go`, `docs/src/content/docs/api.md`, `docs/src/content/docs/architecture.md`, `docs/src/content/docs/getting-started.md`, `docs/src/content/docs/development.md`, `docs/src/content/docs/why-wavehouse.md`, `clients/ts/src/sql.ts`, `clients/ts/src/types.ts`, `clients/ts/src/client.test.ts`, `clients/ts/src/namespaces.test.ts`, `tests/integration/query_test.go`, `tests/e2e/sdk/admin.test.ts`, `tests/e2e/sdk/setup.ts`, `tests/e2e/sdk/query.test.ts`, `internal/ingest/bento.go`, `README.md`, `SECURITY.md`, `AGENTS.md`, `.gemini/styleguide.md`, `.github/prompts/pr-review.md`). Previously, raw SQL was mounted at `/v1/query` under the `/v1` auth middleware with the authorization decision deferred to the handler — any caller with a `policy.RolePermissions.raw_sql: true` grant on any table could submit raw SQL and slip through as a non-admin. The route now lives under the `/v1/admin/*` group, so its URL telegraphs the access level, and it shares the surrounding `RequireAdmin` gate with the rest of that tree (policy CRUD, pipes CRUD). Raw SQL has no per-statement scope check (we cannot authorize predicates without a full SQL parser), so the admin gate is the entire authorization story; a separate, tighter gate just for raw SQL would be redundant, since the whole `/v1/admin/*` tree already requires the admin role. The `policy.RolePermissions.raw_sql` (`RawSQL`) field is **removed** outright from `Policy`/`ResolvedPermissions` and from the `Evaluate` return value — operators with `raw_sql: true` in `policy.yaml` will see a YAML-load warning for the unknown field but the field is otherwise ignored; the equivalent capability is now expressed only as "issue the JWT with the admin role." The in-handler `PolicyStore` plumbing and the `if h.PolicyStore != nil { … }` raw-SQL check inside `QueryHandler.Handle` are deleted along with the field. A new router-level test (`TestNewRouter_RawSQLAdminGate`) pins the contract: admin reaches the handler, while service, viewer, and no-role requests are all 403 (`service` is no longer privileged). The four obsolete handler-level policy tests (`TestQueryHandler_Policy*`, `TestQueryHandler_NoPolicyAllowsAll`) are removed. The shared `/v1/admin/*` gate is declared once as a `requireAdmin` (`RequireAdmin`) local at the top of the `/v1` route closure. The normal surfaces for non-admin callers — `POST /v1/ingest?table={table}`, `POST /v1/query?table={table}`, `GET/POST /v1/pipes/{name}` — are unchanged. -- **Per-pipe `allowed_roles` now fails closed on an empty role** (`internal/api/pipes.go`, `internal/api/pipes_test.go`, `docs/src/content/docs/api.md`): `PipesHandler.Execute` enforced a pipe's `allowed_roles` allowlist only when the request carried a non-empty role, so an empty or absent role skipped the check and the restricted pipe was served. Per-pipe `allowed_roles` is the only authorization gate on the execute path (`GET/POST /v1/pipes/{name}` sit outside the `/v1/admin/*` `RequireAdmin` gate), so any roleless request reached a restricted pipe unchecked — triggered whenever a request carried no token, or a token missing the configured `auth.role_claim` (either way the resolved role is empty). Removed the `if role != ""` guard so an empty role flows into the scan, matches nothing, and returns `403`; empty allowlist entries are skipped so a stray `""` can't authorize an empty role. The gap that hid this — every prior role test set a non-empty role — is closed by consolidating the four standalone role tests into a table-driven `TestPipesHandler_Execute_RoleAuthorization` matrix that pins the empty/absent-role rows, plus a focused `TestPipesHandler_Execute_RestrictedPipe_EmptyRoleDenied` regression. Closes #159. +- **BREAKING: raw SQL moves from `POST /v1/query` to `POST /v1/ops/query` and is gated on the admin role** (`internal/api/router.go`, `internal/api/router_test.go`, `internal/api/query.go`, `internal/api/query_test.go`, `internal/policy/policy.go`, `internal/policy/policy_test.go`, `cmd/wavehouse/main.go`, `docs/src/content/docs/api.md`, `docs/src/content/docs/architecture.md`, `docs/src/content/docs/getting-started.md`, `docs/src/content/docs/development.md`, `docs/src/content/docs/why-wavehouse.md`, `clients/ts/src/sql.ts`, `clients/ts/src/types.ts`, `clients/ts/src/client.test.ts`, `clients/ts/src/namespaces.test.ts`, `tests/integration/query_test.go`, `tests/e2e/sdk/admin.test.ts`, `tests/e2e/sdk/setup.ts`, `tests/e2e/sdk/query.test.ts`, `internal/ingest/bento.go`, `README.md`, `SECURITY.md`, `AGENTS.md`, `.gemini/styleguide.md`, `.github/prompts/pr-review.md`). Previously, raw SQL was mounted at `/v1/query` under the `/v1` auth middleware with the authorization decision deferred to the handler — any caller with a `policy.RolePermissions.raw_sql: true` grant on any table could submit raw SQL and slip through as a non-admin. The route now lives under the `/v1/ops/*` group, so its URL telegraphs the access level, and it shares the surrounding `RequireAdmin` gate with the rest of that tree (policy CRUD, pipes CRUD). Raw SQL has no per-statement scope check (we cannot authorize predicates without a full SQL parser), so the admin gate is the entire authorization story; a separate, tighter gate just for raw SQL would be redundant, since the whole `/v1/ops/*` tree already requires the admin role. The `policy.RolePermissions.raw_sql` (`RawSQL`) field is **removed** outright from `Policy`/`ResolvedPermissions` and from the `Evaluate` return value — operators with `raw_sql: true` in `policy.yaml` will see a YAML-load warning for the unknown field but the field is otherwise ignored; the equivalent capability is now expressed only as "issue the JWT with the admin role." The in-handler `PolicyStore` plumbing and the `if h.PolicyStore != nil { … }` raw-SQL check inside `QueryHandler.Handle` are deleted along with the field. A new router-level test (`TestNewRouter_RawSQLAdminGate`) pins the contract: admin reaches the handler, while service, viewer, and no-role requests are all 403 (`service` is no longer privileged). The four obsolete handler-level policy tests (`TestQueryHandler_Policy*`, `TestQueryHandler_NoPolicyAllowsAll`) are removed. The shared `/v1/ops/*` gate is declared once as a `requireAdmin` (`RequireAdmin`) local at the top of the `/v1` route closure. The normal surfaces for non-admin callers — `POST /v1/ingest?table={table}`, `POST /v1/query?table={table}`, `GET/POST /v1/pipes/{name}` — are unchanged. +- **Per-pipe `allowed_roles` now fails closed on an empty role** (`internal/api/pipes.go`, `internal/api/pipes_test.go`, `docs/src/content/docs/api.md`): `PipesHandler.Execute` enforced a pipe's `allowed_roles` allowlist only when the request carried a non-empty role, so an empty or absent role skipped the check and the restricted pipe was served. Per-pipe `allowed_roles` is the only authorization gate on the execute path (`GET/POST /v1/pipes/{name}` sit outside the `/v1/ops/*` `RequireAdmin` gate), so any roleless request reached a restricted pipe unchecked — triggered whenever a request carried no token, or a token missing the configured `auth.role_claim` (either way the resolved role is empty). Removed the `if role != ""` guard so an empty role flows into the scan, matches nothing, and returns `403`; empty allowlist entries are skipped so a stray `""` can't authorize an empty role. The gap that hid this — every prior role test set a non-empty role — is closed by consolidating the four standalone role tests into a table-driven `TestPipesHandler_Execute_RoleAuthorization` matrix that pins the empty/absent-role rows, plus a focused `TestPipesHandler_Execute_RestrictedPipe_EmptyRoleDenied` regression. Closes #159. - **Access-control policies reject empty role names and never match a roleless request to an empty role key** (`internal/policy/policy.go`, `internal/policy/policy_test.go`): the #159 step-3 audit (cross-check `internal/policy/` against empty roles) surfaced the policy-side twin of the empty-`allowed_roles`-entry footgun. `Evaluate` did a direct `rolePerms[role]` lookup, so a stray `""` role key in `policy.yaml` would have authorized any request whose resolved role is empty (no token, or a JWT missing `auth.role_claim`) on the policy-gated paths (`POST /v1/query?table={table}`, `POST /v1/ingest?table={table}`, the SSE stream). Two complementary guards: `Validate` now rejects empty/whitespace role keys at write/bootstrap time, and `Evaluate` skips the direct key lookup for an empty role so a roleless request can only be authorized by the configured `admin_role` (or a `default_role` that resolves to a real listed role) — fail-closed even if a malformed policy reaches the engine from KV. Regression tests pin both guards. Part of #159. - **`minimumReleaseAge: 10080` (7 days) on every pnpm workspace** (`clients/ts/pnpm-workspace.yaml`, `docs/pnpm-workspace.yaml`, `tests/e2e/sdk/pnpm-workspace.yaml`): pnpm 11 will refuse to install any package published in the last seven days, giving npm and security researchers time to flag a compromised release before it lands in our lockfile. Existing locked versions are grandfathered. For a one-off override on an urgent hotfix release, list the package under `minimumReleaseAgeExclude:` in the same file. Part of #160. - **Structured-query / pipes cache no longer collides on string params containing NUL bytes or on string-vs-int values with the same textual rendering** (`internal/api/cache_key.go`, `internal/api/cache_key_test.go`). The prior `queryCacheKey` framing wrote `"\x00%v"` between params, so `("SELECT 1", ["foo\x00bar"])` and `("SELECT 1", ["foo", "bar"])` produced the same SHA-256 digest — a cached row from one query could be served for the other (cross-query data exposure if either tuple could be reached by a caller of the structured-query or pipes endpoints with user-controllable string inputs). A separate construction reached the same boundary by ending the raw SQL with bytes that mimicked a param frame. Reframed each section with a 1-byte type marker plus 8-byte big-endian length, and JSON-marshalled each param as `{type, value}` so the type tag also separates `"42"` (string) from `42` (int). Table-driven `TestQueryCacheKey` covers the NUL-collision regression, the SQL-mimics-a-param-frame regression, and the type-distinct regression. Net effect on deploy: a one-time cold cache for `/v1/tables/{table}/query` and `/v1/pipes/{name}` consumers; raw-SQL has no cache so no impact there. Reported by CodeRabbit on PR #164 with empirical digest reproduction. ### Fixed -- **`policy.NewStore` fails loud on a broken bootstrap, and `policy.file_path` no longer has a default** (`internal/policy/store.go`, `internal/policy/store_test.go`, `internal/config/config.go`, `internal/config/config_test.go`, `config.yaml`, `docs/src/content/docs/configuration.md`, `docs/src/content/docs/deployment.md`, `tests/e2e/fixtures/policy.yaml` (new), `scripts/orchestrator/main.go`). The old `NewStore` swallowed every bootstrap failure — missing file, malformed YAML/JSON, `Validate` rejection, KV `Put` failure — as a Warn log, leaving the cache `nil`. With the RBAC work elsewhere in this section making `Evaluate(nil)`/`IsAdmin(nil)` deny everyone (including the admin role, no implicit grant), that masked path turned into a silent total lockout: the process bound `:8080` and `/health` returned 200, but every authenticated request 403'd. The masking test `TestStore_NewStore_BootstrapMissingFileIsNotFatal` codified the silent swallow as if it were intentional (`require.NoError` + `assert.Nil(store.Get())`), which is why the bug shipped — `internal/api/`, `tests/integration/`, and most other call-sites use `policy.NewMemoryStore` and bypass `NewStore` entirely, so only the e2e suite (the one place an actual `NewStore` boot has to succeed) could ever notice, and it didn't until the RBAC change closed the implicit-admin escape hatch. Three changes ship together: (1) `NewStore` now load-first from KV (KV is authoritative; the file is a one-shot seed for fresh deployments), distinguishes `jetstream.ErrKeyNotFound` from other KV errors via `errors.Is` (a transient KV read failure no longer downgrades to "treat as empty + bootstrap"), and propagates every bootstrap failure as a fatal error so a misconfigured deployment refuses to boot instead of silently denying every request — when `bootstrapPath == ""` and KV is empty the store still comes up but logs a loud operator-facing warning so a deliberate "I'll seed via API" choice is distinguishable from a silent lockout in stdout scraping. (2) **BREAKING:** `policy.file_path` has no default anymore — the prior `env-default:"policy.yaml"` quietly opted every deployment into a `policy.yaml` lookup in CWD that the operator never asked for, and combined with (1) that would refuse boot on any image that didn't ship that exact file. Set `WH_POLICY_FILE_PATH` / `policy.file_path` explicitly to opt into bootstrap; leave empty and seed via `PUT /v1/admin/policy`. The e2e orchestrator (`scripts/orchestrator/main.go`) already set the env var explicitly so it's unaffected; `cfg.Policy.FilePath` now defaults to `""` in `TestLoad_Defaults`. (3) New `tests/e2e/fixtures/policy.yaml` — a minimal seed (no `default_role`, viewer perms on `clicks`/`events`/`users` to mirror `tests/e2e/sdk/setup.ts`) — so the orchestrator-launched stack boots cleanly under the strict contract; `setup.ts` still PUTs the test policy on top of it before any test runs. Tests: the misleading `IsNotFatal` test is renamed to `BootstrapMissingFileIsFatal` and now asserts the error; new tests cover invalid YAML, invalid JSON, invalid policy (Validate failure on bootstrap), and `KVTakesPrecedenceOverFile` (the post-first-run restart case where the seed file may have been moved or deleted — KV still wins, missing file is no longer fatal); the `EmptyKV` test also asserts the operator-facing warning so a future silent-lockout regression is caught. `internal/policy` unit coverage rises to 93.6%. +- **`policy.NewStore` fails loud on a broken bootstrap, and `policy.file_path` no longer has a default** (`internal/policy/store.go`, `internal/policy/store_test.go`, `internal/config/config.go`, `internal/config/config_test.go`, `config.yaml`, `docs/src/content/docs/configuration.md`, `docs/src/content/docs/deployment.md`, `tests/e2e/fixtures/policy.yaml` (new), `scripts/orchestrator/main.go`). The old `NewStore` swallowed every bootstrap failure — missing file, malformed YAML/JSON, `Validate` rejection, KV `Put` failure — as a Warn log, leaving the cache `nil`. With the RBAC work elsewhere in this section making `Evaluate(nil)`/`IsAdmin(nil)` deny everyone (including the admin role, no implicit grant), that masked path turned into a silent total lockout: the process bound `:8080` and `/health` returned 200, but every authenticated request 403'd. The masking test `TestStore_NewStore_BootstrapMissingFileIsNotFatal` codified the silent swallow as if it were intentional (`require.NoError` + `assert.Nil(store.Get())`), which is why the bug shipped — `internal/api/`, `tests/integration/`, and most other call-sites use `policy.NewMemoryStore` and bypass `NewStore` entirely, so only the e2e suite (the one place an actual `NewStore` boot has to succeed) could ever notice, and it didn't until the RBAC change closed the implicit-admin escape hatch. Three changes ship together: (1) `NewStore` now load-first from KV (KV is authoritative; the file is a one-shot seed for fresh deployments), distinguishes `jetstream.ErrKeyNotFound` from other KV errors via `errors.Is` (a transient KV read failure no longer downgrades to "treat as empty + bootstrap"), and propagates every bootstrap failure as a fatal error so a misconfigured deployment refuses to boot instead of silently denying every request — when `bootstrapPath == ""` and KV is empty the store still comes up but logs a loud operator-facing warning so a deliberate "I'll seed via API" choice is distinguishable from a silent lockout in stdout scraping. (2) **BREAKING:** `policy.file_path` has no default anymore — the prior `env-default:"policy.yaml"` quietly opted every deployment into a `policy.yaml` lookup in CWD that the operator never asked for, and combined with (1) that would refuse boot on any image that didn't ship that exact file. Set `WH_POLICY_FILE_PATH` / `policy.file_path` explicitly to opt into bootstrap; leave empty and seed via `PUT /v1/ops/policy`. The e2e orchestrator (`scripts/orchestrator/main.go`) already set the env var explicitly so it's unaffected; `cfg.Policy.FilePath` now defaults to `""` in `TestLoad_Defaults`. (3) New `tests/e2e/fixtures/policy.yaml` — a minimal seed (no `default_role`, viewer perms on `clicks`/`events`/`users` to mirror `tests/e2e/sdk/setup.ts`) — so the orchestrator-launched stack boots cleanly under the strict contract; `setup.ts` still PUTs the test policy on top of it before any test runs. Tests: the misleading `IsNotFatal` test is renamed to `BootstrapMissingFileIsFatal` and now asserts the error; new tests cover invalid YAML, invalid JSON, invalid policy (Validate failure on bootstrap), and `KVTakesPrecedenceOverFile` (the post-first-run restart case where the seed file may have been moved or deleted — KV still wins, missing file is no longer fatal); the `EmptyKV` test also asserts the operator-facing warning so a future silent-lockout regression is caught. `internal/policy` unit coverage rises to 93.6%. - **`isMutation` no longer misclassifies CTE-prefixed reads as mutations when the SQL contains an identifier that collides with a mutation-verb name** (`internal/api/clickhouse_exec.go`, `internal/api/clickhouse_exec_test.go`). `containsMutationVerbAtTopLevel` previously checked every depth-0 identifier against `mutationVerbs`, so for `WITH x AS (SELECT 1) SELECT * FROM system.tables` the database name `system` matched the `SYSTEM` verb and routed the read through `Exec` — `executeCHQuery` returned an empty `[]` instead of the actual rows, silently. The same false-positive class trips for CTE aliases whose own name collides with a mutation verb (`WITH set AS (...)`, `WITH alter AS (...)`, etc.) and for table aliases (`SELECT * FROM cte AS set`). Two-part fix: (a) identifiers whose next non-whitespace token is `AS` or `(` are recognised as CTE definition names and skipped without checking the keyword sets, which handles the CTE-alias-collides-with-verb case; (b) among the remaining identifiers, the scanner stops on the FIRST that's a known statement keyword (mutation OR read-class — a new `nonMutationVerbs` set covers `SELECT`/`SHOW`/`DESCRIBE`/`DESC`/`EXPLAIN`/`EXISTS`/`CHECK`) and decides based on `mutationVerbs` membership. Affects callers of `executeCHQuery` (structured-query and pipes handlers). Regression tests added: `system.tables` / `system.columns` reads, CTE aliases named `set`/`alter`/`drop`, a multi-CTE case with two mutation-verb-name aliases, and a control case that reads `system.tables` inside a CTE body but actually issues an `INSERT` (must still be classified as mutation). Reported by CodeRabbit on PR #164. -- **The raw-SQL endpoint returns HTTP 200 with `[]` for mutation/DDL statements instead of HTTP 500** (`internal/api/query.go`, `internal/api/query_test.go`). **Note (largely superseded — see the HTTP-proxy entry above):** the in-handler verb classifier + cache-bypass approach described below was the state at this commit. The subsequent HTTP-proxy refactor moved the verb classifier to `internal/api/clickhouse_exec.go` (still used by the structured-query and pipes handlers) and the `/v1/admin/query` path now lets ClickHouse classify natively and forwards mutations' empty body as `[]`. The mutation-cache-bypass machinery is moot — there's no cache on that path at all anymore. The HTTP 200 + `[]` contract for successful no-result mutations is preserved, just delivered through the proxy. `executeQuery` previously routed every statement through `driver.Query`, which expects a result set. ClickHouse DDL/DML like `TRUNCATE`, `DROP`, `DELETE`, `ALTER` returns no result set, and the clickhouse-go driver surfaced that as an internal error that bubbled out as HTTP 500. The handler now classifies by the leading SQL verb (`INSERT`/`UPDATE`/`DELETE`/`TRUNCATE`/`DROP`/`ALTER`/`CREATE`/`RENAME`/`OPTIMIZE`/`REPLACE`/`GRANT`/`REVOKE`/`ATTACH`/`DETACH`/`KILL`/`SET`/`USE`/`SYSTEM` → `Exec`; everything else → `Query`) and marshals `[]` on `Exec` success — the same "no rows" shape an empty `SELECT` returns. Leading whitespace and `--` / `/* */` SQL comments are stripped before the verb is classified. Mutations also **bypass the TieredCache + singleflight path entirely** in `QueryHandler.Handle`: caching `[]` under the SQL cache key would let a second identical TRUNCATE/INSERT/etc. hit the cache and skip ClickHouse within `DefaultTTL` (silent data loss on the second request); singleflight collapsing concurrent identical mutations would drop one of two writes. `X-Cache: MISS` is set on every mutation response since the cache is never consulted. Closes #118. Now that the raw-SQL endpoint is the only sanctioned surface for non-insert mutations (see `Removed` above), this response-shape and cache-bypass fix is part of locking down that surface. (Originally landed at `/v1/query`; the endpoint moved to `/v1/admin/query` later in this same Unreleased section — see the Security entry below.) +- **The raw-SQL endpoint returns HTTP 200 with `[]` for mutation/DDL statements instead of HTTP 500** (`internal/api/query.go`, `internal/api/query_test.go`). **Note (largely superseded — see the HTTP-proxy entry above):** the in-handler verb classifier + cache-bypass approach described below was the state at this commit. The subsequent HTTP-proxy refactor moved the verb classifier to `internal/api/clickhouse_exec.go` (still used by the structured-query and pipes handlers) and the `/v1/ops/query` path now lets ClickHouse classify natively and forwards mutations' empty body as `[]`. The mutation-cache-bypass machinery is moot — there's no cache on that path at all anymore. The HTTP 200 + `[]` contract for successful no-result mutations is preserved, just delivered through the proxy. `executeQuery` previously routed every statement through `driver.Query`, which expects a result set. ClickHouse DDL/DML like `TRUNCATE`, `DROP`, `DELETE`, `ALTER` returns no result set, and the clickhouse-go driver surfaced that as an internal error that bubbled out as HTTP 500. The handler now classifies by the leading SQL verb (`INSERT`/`UPDATE`/`DELETE`/`TRUNCATE`/`DROP`/`ALTER`/`CREATE`/`RENAME`/`OPTIMIZE`/`REPLACE`/`GRANT`/`REVOKE`/`ATTACH`/`DETACH`/`KILL`/`SET`/`USE`/`SYSTEM` → `Exec`; everything else → `Query`) and marshals `[]` on `Exec` success — the same "no rows" shape an empty `SELECT` returns. Leading whitespace and `--` / `/* */` SQL comments are stripped before the verb is classified. Mutations also **bypass the TieredCache + singleflight path entirely** in `QueryHandler.Handle`: caching `[]` under the SQL cache key would let a second identical TRUNCATE/INSERT/etc. hit the cache and skip ClickHouse within `DefaultTTL` (silent data loss on the second request); singleflight collapsing concurrent identical mutations would drop one of two writes. `X-Cache: MISS` is set on every mutation response since the cache is never consulted. Closes #118. Now that the raw-SQL endpoint is the only sanctioned surface for non-insert mutations (see `Removed` above), this response-shape and cache-bypass fix is part of locking down that surface. (Originally landed at `/v1/query`; the endpoint moved to `/v1/ops/query` later in this same Unreleased section — see the Security entry below.) - **Ingest worker no longer infinite-retries permanent delete errors** (`internal/ingest/bento.go`, `internal/ingest/bento_test.go`, `docs/src/content/docs/architecture.md`, `AGENTS.md`): `jsInput.Read`'s `action: "delete"` block called `m.Nak()` on every `chConn.Exec` failure, which JetStream interprets as "redeliver immediately." A delete whose error was *deterministic* (syntax error, unknown table, malformed identifier) would loop forever — clogging the buffer consumer, burning CPU, and spamming logs with the same message. Phase 1 of issue #91: every delete-Exec error is now treated as permanent. The original NATS envelope is published to `dlq.
` (reusing the existing `bentoDLQDropped` counter when the DLQ publish itself fails) and the message is `DoubleAck`'d so it leaves the main queue. Issue #91 stays open after this lands as the Phase 2 tracker for transient-vs-permanent error classification (timeouts and network errors should still `Nak()` for retry); Phase 1 alone is the stopgap, Phase 2 makes the trade-off acceptable in production. *Superseded by the insert-only lock in `Removed` above — both the delete branch and the DLQ delete-envelope shape it required are gone.* - **CORS middleware: spec-compliant wildcard, no credentials, no header decoration on same-origin** (`internal/api/router.go`, `internal/api/router_test.go`, `AGENTS.md`, `config.yaml`, `docs/src/content/docs/configuration.md`, `docs/src/content/docs/deployment.md`): closes [#29](https://github.com/Wave-RF/WaveHouse/issues/29) and bookends [#30](https://github.com/Wave-RF/WaveHouse/issues/30). Three behavior changes, one rationale. (1) Dropped `Access-Control-Allow-Credentials: true` entirely — WaveHouse is a Bearer-token API (`Authorization: Bearer `), cookies are never used (verified: no `http.Cookie` / `SetCookie` anywhere in the Go tree, TS SDK sends no `credentials: 'include'`), so credentials mode is unnecessary AND the previous combination of `Allow-Credentials: true` with `Allow-Origin: *` violated the CORS spec — browsers reject that pairing, which silently broke any client that ever set `credentials: 'include'`. (2) Requests with no `Origin` header (same-origin browser navigation, server-to-server, curl) now skip the CORS decoration entirely instead of unconditionally stamping `Allow-Methods`/`Allow-Headers`/`Allow-Credentials` on every response. (3) A preflight from a disallowed origin still returns 204 but with no CORS headers, which the browser treats as preflight failure — same outcome as before but without leaking the methods/headers list to origins that aren't allowed. Allowlist mode sets `Vary: Origin` on both hits *and* rejects so shared caches can't memoize a headerless reject under the URL alone and replay it to a later allowed-origin request. Test coverage in `router_test.go` pins each branch: wildcard echoes `*`, allowlist hit echoes origin + `Vary: Origin`, allowlist miss gets no `Allow-Origin` but still gets `Vary: Origin` (both for regular and OPTIONS), no-Origin requests pass through clean, and a table-driven test asserts `Allow-Credentials` is never emitted across wildcard / empty-allowlist / allowlist-hit. Posture is documented as `AGENTS.md` §"Key Design Decisions" item 16 so future contributors don't reintroduce credentials or cookie auth without a design discussion. Config sample updated with the dev recipe (point at `http://localhost:3000` etc. instead of `*` once a frontend is built). - **OTel shutdown no longer hangs process exit when the collector is unreachable** (`cmd/wavehouse/main.go`): the `defer otelShutdown(context.Background())` was unbounded, and the OTel SDK's batch processors don't fully honor the shutdown context against an unreachable gRPC endpoint (the dial retries with backoff continue past the deadline). Bounded the shutdown to 5s. Discovered while writing `tests/integration/otel_test.go` `TestOTel_UnreachableEndpoint_DoesNotBlockStartupOrEmits`. *Superseded — the flat 5s bound in `cmd/wavehouse/main.go` is now 3s, and the bounding itself moved into `internal/observability/provider.go`, which shuts the trace/metric/log providers down concurrently and returns the instant the caller's context deadline passes (the experimental logs SDK's `BatchProcessor.Shutdown` ignores `ctx` and would otherwise block ~10s in gRPC backoff against an unreachable collector). Regression-guarded by `TestInitProvider_ShutdownParallelBounded`.* @@ -359,7 +365,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Auth `?token=` query parameter fallback**: JWT can now be passed via `?token=` query parameter for WebSocket and SSE connections where custom headers are not possible. The `Authorization` header takes precedence. Token is stripped from URL after extraction. - **SSE `id:` field and `Last-Event-ID` reconnection**: SSE events now include an `id:` field set to `received_timestamp`, enabling native `EventSource` automatic reconnection. The `Last-Event-ID` request header overrides `?since=` for seamless gap-fill on reconnect. - **WebSocket multiplexing**: WebSocket connections now support in-band JSON commands (`{"action":"subscribe","topic":"..."}` / `{"action":"unsubscribe","topic":"..."}`) for dynamic multi-topic subscriptions over a single connection. Outbound messages are wrapped in `{"topic":"...","data":{...}}` envelopes. Backward compatible with `?topic=` query parameter. -- **DLQ stats table filter**: `GET /v1/dlq/stats` now accepts optional `?table=` query parameter to filter stats to a specific table. +- **DLQ stats table filter**: `GET /v1/ops/dlq/stats` now accepts optional `?table=` query parameter to filter stats to a specific table. - **TypeScript SDK `SharedWSManager`**: Single multiplexed WebSocket per client with ref-counted subscriptions, auto-reconnect with exponential backoff, and client-side NATS-style wildcard dispatch. - **TypeScript SDK `LiveQuery`**: Stream-first backfill orchestrator — subscribes to the stream immediately, buffers events, fetches historical data, deduplicates by timestamp, then resumes live updates. Available via `queryBuilder.liveQuery(subscriber, opts?)`. - **TypeScript SDK `FilteredStreamController`**: Streams created from query builders with active filters/columns now apply client-side filtering (eq, neq, gt, gte, lt, lte, in, like, not_like) and column projection. @@ -367,7 +373,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **TypeScript SDK SSE connection counter**: Warns when opening more than 5 concurrent SSE connections (browser limit). - **TypeScript SDK default query limit**: `QueryBuilder.DEFAULT_LIMIT = 1000` applied when no explicit limit is set, preventing unbounded result sets. - **TypeScript SDK unit tests**: Comprehensive Vitest test suite for `@wavehouse/sdk` — `errors.test.ts`, `http.test.ts`, `query-builder.test.ts`, `table.test.ts`, `pipes.test.ts`, `namespaces.test.ts` (sql, schema, policy, DLQ, sys), `client.test.ts` (factory, namespace wiring, transport selection), `stream/controller.test.ts` (subscribe, unsubscribe, async iterator, ref counting). -- **TypeScript SDK codegen CLI**: `npm run codegen -- --url --out ` introspects `/v1/schema` and generates a TypeScript `Database` interface with ClickHouse-to-TypeScript type mapping (String, numeric, DateTime, Array, Map, Nullable, LowCardinality, etc.). +- **TypeScript SDK codegen CLI**: `npm run codegen -- --url --out ` introspects `/v1/ops/schema` and generates a TypeScript `Database` interface with ClickHouse-to-TypeScript type mapping (String, numeric, DateTime, Array, Map, Nullable, LowCardinality, etc.). - **TypeScript SDK playground**: Three runnable scripts (`playground:public`, `playground:auth`, `playground:admin`) demonstrating unauthenticated queries/SSE, JWT auth/WebSocket streaming, and admin workflows (schema, policy, pipes, DLQ, raw SQL). Includes Docker Compose file and setup/seed script. ### Removed @@ -448,15 +454,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Docs — deployment**: Fixed `cluster.yaml` → `clustered.yaml` Docker Compose file references. - **Docs — development**: Updated Makefile targets table (removed nonexistent `build-all`/`compose-cluster`, added all new targets), added missing linters to list, added missing packages to project structure, updated vulnerability scanning to use `make vulncheck`. - **`config.yaml`**: Fixed `policy.file_path` and `pipes.directory` defaults to match `config.go` struct tags. -- **DLQ stats per-subject counts**: `GET /v1/dlq/stats` now passes `WithSubjectFilter(">")` to NATS `Stream.Info()`, fixing empty per-table breakdown in the response. +- **DLQ stats per-subject counts**: `GET /v1/ops/dlq/stats` now passes `WithSubjectFilter(">")` to NATS `Stream.Info()`, fixing empty per-table breakdown in the response. - **Bento DLQ subject routing**: Bento ingest worker now sets `table_name` metadata on messages so the DLQ output routes to `dlq.
` instead of `dlq.unknown`. - **JWKS authentication**: New `auth.jwks_url` config for public key validation via JWKS endpoint. JWKS is tried first, falling back to HMAC secret. Powered by `keyfunc/v3`. - **Role-based access control**: JWT role extraction from configurable claim path (`auth.role_claim`). Built-in `admin`/`service` roles with full access; other roles governed by policies. - **Hasura-style access control policies**: Per-table, per-role column and row-level permissions with JWT claim templating (`{{ jwt.path }}`). Stored in NATS KV (`WAVEHOUSE_POLICY`) with file-based YAML/JSON bootstrap and cluster-wide sync via KV Watch. -- **Policy admin API**: `GET/PUT /v1/admin/policy` for CRUD, `POST /v1/admin/policy/validate` for dry-run validation. +- **Policy admin API**: `GET/PUT /v1/ops/policy` for CRUD, `POST /v1/ops/policy/validate` for dry-run validation. - **Structured query endpoint**: `POST /v1/tables/{table}/query` accepts a type-safe query AST (columns, aggregations, filters, group by, order by, limit, time range). Validated against schema, permissions enforced, converted to parameterized SQL. - **Timestamp bucketing**: Structured queries truncate time ranges to configurable buckets (`cache.timestamp_bucket_seconds`, default 60s) to improve cache hit rates. -- **Named query pipes**: Pre-defined SQL templates with parameter binding, role restrictions, and caching. `GET/POST /v1/pipes/{name}` for execution. Admin CRUD at `/v1/admin/pipes/*`. Bootstrap from `.sql` files via `pipes.directory`. +- **Named query pipes**: Pre-defined SQL templates with parameter binding, role restrictions, and caching. `GET/POST /v1/pipes/{name}` for execution. Admin CRUD at `/v1/ops/pipes/*`. Bootstrap from `.sql` files via `pipes.directory`. - **Ingest permission enforcement**: When policies are active, ingest checks insert permission, validates allowed columns, enforces check rules, and auto-injects claim-derived values. - **Stream permission filtering**: SSE and WebSocket streams filter events per role — denied columns are removed and unauthorized tables are skipped. - **Dev mode**: `auth.dev_mode` skips JWT validation and treats all requests as admin (development only). @@ -466,9 +472,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **TypeScript SDK** (`clients/ts/`): `@wavehouse/sdk` — zero-dependency client with type-safe query builder, real-time SSE streaming, live queries with smart aggregation updates (incrementable/decomposable/poll), and codegen CLI for generating typed interfaces from ClickHouse schemas. - **Schema discovery**: New `internal/discovery/` package introspects ClickHouse `system.columns` to build a live schema registry. Schemas are cached and auto-refreshed on a configurable interval (`schema.refresh_interval` / `WH_SCHEMA_REFRESH_INTERVAL`). - **Schema validation**: Ingest payloads are validated against discovered ClickHouse schemas — unknown fields, type mismatches, and non-nullable violations are rejected with descriptive 400 errors. -- **Schema API endpoints**: `GET /v1/schema` (list all tables), `GET /v1/schema/{table}` (single table), `POST /v1/schema/refresh` (force refresh). +- **Schema API endpoints**: `GET /v1/ops/schema` (list all tables), `GET /v1/ops/schema?table={table}` (single table), `POST /v1/ops/schema/refresh` (force refresh). - **Dead Letter Queue (DLQ)**: Failed batch inserts are published to a separate NATS stream (`WAVEHOUSE_DLQ`) instead of being silently lost. Controlled by `dlq.enabled` / `WH_DLQ_ENABLED`. -- **DLQ stats endpoint**: `GET /v1/dlq/stats` returns pending message count and consumer info. +- **DLQ stats endpoint**: `GET /v1/ops/dlq/stats` returns pending message count and consumer info. - **Optional authentication**: JWT auth is now opt-in via `auth.enabled` / `WH_AUTH_ENABLED` (defaults to `false`). When disabled, all `/v1/*` routes are open. - **Optional deduplication**: Dedup is now opt-in via `dedupe.enabled` / `WH_DEDUPE_ENABLED` (defaults to `false`). When enabled, specify the dedup key field with `dedupe.id_field` / `WH_DEDUPE_ID_FIELD`. - **Table-based ingest routing**: Ingest endpoint is now `POST /v1/ingest/{table}` — the table name comes from the URL path. diff --git a/SECURITY.md b/SECURITY.md index 66c10bda..96d7c5a1 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -27,8 +27,8 @@ WaveHouse handles data and enforces strict isolation: - **JWT validation**: The JWT middleware always runs (there is no on/off switch). Signing supports either an HMAC shared secret or a remote JWKS endpoint (`auth.jwks_url`). Accepted signing algorithms are restricted to the configured verifier's family — HMAC accepts only `HS256/384/512`, JWKS only the asymmetric set (`RS*`/`ES*`/`PS*`/`EdDSA`) — and the token's `alg` header is validated before any key material is used, so `alg: none` and algorithm-confusion attacks (re-signing with `HS256` against a JWKS deployment's public key) are rejected. A request with no token, or an invalid/expired one, falls back to the policy `default_role`; elevated access requires a valid token, and a denied request that carried a bad token fails loud (`401`) rather than as a bare `403`. - **Role-based access control**: Roles are extracted from a configurable JWT claim path. Non-admin roles have per-table, per-column policies enforced on ingest, query, and the live SSE stream; row-level rules split by path — insert `check` constraints are enforced (and auto-injected) on ingest, while select `filter` predicates apply to structured queries and the live SSE stream (the stream's in-memory row-filter comparison has a documented fail-closed boundary — see the [access-control docs](https://wavehouse.dev/access-control#where-each-rule-is-enforced)); the admin role (`policy.admin_role`, `"admin"` by default, exact case-sensitive match) bypasses them. The configured non-JWT operator key (`auth.operator_key`) likewise bypasses per-role policy — a matching request is authorized as a full-access platform operator without a JWT; treat it as an admin secret. A request presenting a *non-matching* operator key is logged at `WARN` and counted by `wavehouse_auth_operator_key_failures_total`, so probing of that credential is observable and alertable. - **Input validation**: JSON payloads are validated against ClickHouse schemas before processing. -- **Query passthrough**: Raw SQL via `POST /v1/admin/query` is restricted to the admin role — the same `RequireAdmin` gate as the rest of `/v1/admin/*`. A request with no/invalid token resolves to the `default_role`, which in a production config is not the admin role (setting `default_role` equal to the admin role is a loudly-warned, dev-only escape hatch), so it cannot reach this endpoint — the one exception is a request presenting the configured `auth.operator_key`, which reaches the whole `/v1/admin/*` surface (including this endpoint) without a JWT and even under a deleted policy, so treat that key as an admin secret. Raw SQL has no per-statement scope check (a full SQL parser would be needed to authorize predicates), so the role gate is the entire authorization story. Non-admin callers use structured queries (`POST /v1/query?table={table}`, validated against schema with permission injection) or named pipes (`GET/POST /v1/pipes/{name}`); raw-SQL grants to non-admin roles via the policy engine are no longer supported (the `raw_sql` field on policies has been removed). -- **Supply chain**: Third-party GitHub Actions are pinned to full commit SHAs (enforced by the repository's Actions settings — `sha_pinning_required`). `govulncheck` runs on every push/PR. Dependabot opens weekly grouped PRs for Go modules, GitHub Actions, and the npm packages — one grouped PR covering the docs site, TS SDK, and E2E tests via the root pnpm workspace. Released artifacts ship signed [Sigstore](https://www.sigstore.dev/) build-provenance attestations — verify the container image with `gh attestation verify oci://ghcr.io/wave-rf/wavehouse: --repo Wave-RF/WaveHouse`, a downloaded release-binary archive with `gh attestation verify --repo Wave-RF/WaveHouse`, and the `@wavehouse/sdk` package via its npm provenance badge or `npm audit signatures`. (Provenance covers the published binaries and image, not `go install`, which compiles from source.) +- **Query passthrough**: Raw SQL via `POST /v1/ops/query` is restricted to the admin role — the same `RequireAdmin` gate as the rest of `/v1/ops/*`. A request with no/invalid token resolves to the `default_role`, which in a production config is not the admin role (setting `default_role` equal to the admin role is a loudly-warned, dev-only escape hatch), so it cannot reach this endpoint — the one exception is a request presenting the configured `auth.operator_key`, which reaches the whole `/v1/ops/*` surface (including this endpoint) without a JWT and even under a deleted policy, so treat that key as an admin secret. Raw SQL has no per-statement scope check (a full SQL parser would be needed to authorize predicates), so the role gate is the entire authorization story. Non-admin callers use structured queries (`POST /v1/query?table={table}`, validated against schema with permission injection) or named pipes (`GET/POST /v1/pipes/{name}`); raw-SQL grants to non-admin roles via the policy engine are no longer supported (the `raw_sql` field on policies has been removed). +- **Supply chain**: Third-party GitHub Actions are pinned to full commit SHAs (enforced by the repository's Actions settings — `sha_pinning_required`). `govulncheck` runs on every push/PR. Dependabot opens weekly grouped PRs for Go modules, GitHub Actions, and the npm packages — one grouped PR covering the docs site, TS SDK, and E2E tests via the root pnpm workspace. Released artifacts ship signed [Sigstore](https://www.sigstore.dev/) build-provenance attestations — verify the container image with `gh attestation verify oci://ghcr.io/wave-rf/wavehouse: --repo Wave-RF/WaveHouse --signer-workflow Wave-RF/WaveHouse/.github/workflows/release.yml` (the rolling `:dev` image is signed by `publish-dev.yml` — swap the `--signer-workflow`; `--repo` alone accepts an attestation from any workflow in the repo), a downloaded release-binary archive with `gh attestation verify --repo Wave-RF/WaveHouse --signer-workflow Wave-RF/WaveHouse/.github/workflows/release.yml`, and the `@wavehouse/sdk` package via its npm provenance badge or `npm audit signatures`. (Provenance covers the published binaries and image, not `go install`, which compiles from source.) ## Disclosure Policy diff --git a/clients/ts/README.md b/clients/ts/README.md index e4f9e794..7f2cdbab 100644 --- a/clients/ts/README.md +++ b/clients/ts/README.md @@ -150,7 +150,7 @@ Generate TypeScript types from your live WaveHouse schema. The package ships a ` npx wavehouse-codegen --url http://localhost:8080 --out ./src/db.d.ts ``` -This introspects `/v1/schema`, maps ClickHouse column types to TypeScript, and outputs a `Database` interface you can pass to `createClient()`. `/v1/schema` is **admin-only** — pass an admin-role token with `--auth ` (or `-a`) against any non-dev policy. +This introspects `/v1/ops/schema`, maps ClickHouse column types to TypeScript, and outputs a `Database` interface you can pass to `createClient()`. `/v1/ops/schema` is **admin-only** — pass an admin-role token with `--auth ` (or `-a`) against any non-dev policy. ## Development & Testing diff --git a/clients/ts/src/cli/codegen.test.ts b/clients/ts/src/cli/codegen.test.ts new file mode 100644 index 00000000..710beef5 --- /dev/null +++ b/clients/ts/src/cli/codegen.test.ts @@ -0,0 +1,131 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { chTypeToTS, fetchSchemas, generateTypes } from "./codegen.js"; + +// The server's /v1/ops/schema wire shape: a JSON ARRAY of tables (Go +// SchemaRegistry.List() → []TableSchema), not a name-keyed map. Regression +// fixture for the map-parse bug that generated non-compiling `0: 0Row` +// output against a live server (#388). Deliberately unsorted to pin the +// by-name output ordering. +const wireSchemas = [ + { + name: "user_events", + columns: [ + { name: "id", type: "UInt64", is_nullable: false, has_default: false }, + { name: "note", type: "Nullable(String)", is_nullable: true, has_default: false }, + ], + }, + { + name: "clicks", + columns: [ + { name: "page", type: "String", is_nullable: false, has_default: false }, + { name: "count", type: "UInt32", is_nullable: false, has_default: true }, + ], + }, +]; + +describe("fetchSchemas", () => { + let fetchSpy: ReturnType; + + beforeEach(() => { + fetchSpy = vi.fn(); + vi.stubGlobal("fetch", fetchSpy); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("parses the array the server returns", async () => { + fetchSpy.mockResolvedValue(new Response(JSON.stringify(wireSchemas), { status: 200 })); + + const schemas = await fetchSchemas("http://localhost:8080"); + + expect(schemas).toEqual(wireSchemas); + const [url] = fetchSpy.mock.calls[0]; + expect(String(url)).toContain("/v1/ops/schema"); + }); + + it("sends the bearer token", async () => { + fetchSpy.mockResolvedValue(new Response("[]", { status: 200 })); + + await fetchSchemas("http://localhost:8080", "tok"); + + const [, init] = fetchSpy.mock.calls[0]; + expect(init.headers.Authorization).toBe("Bearer tok"); + }); + + it("rejects a non-array body instead of generating garbage", async () => { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify({ clicks: wireSchemas[1] }), { status: 200 }), + ); + + await expect(fetchSchemas("http://localhost:8080")).rejects.toThrow(/JSON array/); + }); + + it.each([ + ["a null member", [null]], + ["a table missing its columns", [{ name: "clicks" }]], + ["a column missing its type", [{ name: "clicks", columns: [{ name: "page" }] }]], + ])("rejects an array with %s instead of crashing mid-generation", async (_desc, body) => { + fetchSpy.mockResolvedValue(new Response(JSON.stringify(body), { status: 200 })); + + await expect(fetchSchemas("http://localhost:8080")).rejects.toThrow(/JSON array/); + }); + + it("surfaces an HTTP error status", async () => { + fetchSpy.mockResolvedValue(new Response('{"error":"forbidden"}', { status: 403 })); + + await expect(fetchSchemas("http://localhost:8080")).rejects.toThrow(/403/); + }); +}); + +describe("generateTypes", () => { + it("keys the Database interface by table name, sorted, from the wire-shape array", () => { + const out = generateTypes(wireSchemas); + + expect(out).toContain("export interface Database {"); + expect(out).toContain(" clicks: ClicksRow;"); + expect(out).toContain(" user_events: UserEventsRow;"); + expect(out.indexOf("clicks: ClicksRow")).toBeLessThan( + out.indexOf("user_events: UserEventsRow"), + ); + // The old map-parse bug keyed rows by array index ("0: 0Row;"). + expect(out).not.toMatch(/^\s*\d+\??:/m); + + expect(out).toContain("export interface ClicksRow {"); + expect(out).toContain(" page: string;"); + expect(out).toContain(" count?: number;"); // has_default → optional + expect(out).toContain("export interface UserEventsRow {"); + expect(out).toContain(" id: number;"); + expect(out).toContain(" note: string | null;"); // Nullable → | null + }); + + it("orders tables by code unit, independent of the host locale", () => { + // "_" (0x5F) sorts before "b" (0x62) by code unit; many ICU locales + // collate punctuation-insensitively and would flip the pair. + const out = generateTypes([ + { name: "ab", columns: [] }, + { name: "a_b", columns: [] }, + ]); + + expect(out.indexOf(" a_b: ABRow;")).toBeGreaterThan(-1); + expect(out.indexOf(" a_b: ABRow;")).toBeLessThan(out.indexOf(" ab: AbRow;")); + }); +}); + +describe("chTypeToTS", () => { + it.each([ + ["String", "string"], + ["Nullable(String)", "string | null"], + ["LowCardinality(String)", "string"], + ["UInt64", "number"], + ["Bool", "boolean"], + ["DateTime64(3, 'UTC')", "string"], + ["Array(UInt8)", "number[]"], + ["Map(String, UInt64)", "Record"], + ["Tuple(String, UInt8)", "unknown[]"], + ["AggregateFunction(sum, UInt64)", "unknown"], + ])("%s → %s", (ch, ts) => { + expect(chTypeToTS(ch)).toBe(ts); + }); +}); diff --git a/clients/ts/src/cli/codegen.ts b/clients/ts/src/cli/codegen.ts index 12f16a2c..869364ac 100644 --- a/clients/ts/src/cli/codegen.ts +++ b/clients/ts/src/cli/codegen.ts @@ -3,7 +3,7 @@ /** * WaveHouse Codegen CLI * - * Introspects a WaveHouse server's /v1/schema endpoint and generates + * Introspects a WaveHouse server's /v1/ops/schema endpoint and generates * a TypeScript Database interface for use with createClient(). * * Usage: @@ -11,6 +11,8 @@ * npm run codegen -- --url http://localhost:8080 --out ./db.d.ts */ +import { realpathSync } from "node:fs"; +import { pathToFileURL } from "node:url"; import { resolveURL } from "../url.js"; // ── Arg parsing (zero deps) ──────────────────────────────────────────────── @@ -159,21 +161,44 @@ interface TableSchema { columns: Column[]; } -type Schemas = Record; +// Runtime guards mirroring the interfaces above, limited to the fields whose +// absence would crash generation (`is_nullable` is unused and `has_default` is +// truthiness-safe, so a server that omits a boolean still generates fine) — a +// malformed member should fail fetchSchemas' loud shape error, not surface as +// a TypeError mid-generation. +function isColumn(value: unknown): value is Column { + if (typeof value !== "object" || value === null) return false; + const col = value as Record; + return typeof col.name === "string" && typeof col.type === "string"; +} + +function isTableSchema(value: unknown): value is TableSchema { + if (typeof value !== "object" || value === null) return false; + const table = value as Record; + return ( + typeof table.name === "string" && Array.isArray(table.columns) && table.columns.every(isColumn) + ); +} -async function fetchSchemas(url: string, auth?: string): Promise { +async function fetchSchemas(url: string, auth?: string): Promise { const headers: Record = {}; if (auth) headers.Authorization = `Bearer ${auth}`; - const res = await fetch(resolveURL(url, "/v1/schema").toString(), { headers }); + const res = await fetch(resolveURL(url, "/v1/ops/schema").toString(), { headers }); if (!res.ok) { const text = await res.text(); throw new Error(`Schema fetch failed (${res.status}): ${text}`); } - return (await res.json()) as Schemas; + // /v1/ops/schema returns a JSON array of tables — reject any other shape, + // malformed members included, loudly rather than crashing mid-generation. + const body = (await res.json()) as unknown; + if (!Array.isArray(body) || !body.every(isTableSchema)) { + throw new Error("Unexpected /v1/ops/schema response: expected a JSON array of table schemas"); + } + return body as TableSchema[]; } -function generateTypes(schemas: Schemas): string { +function generateTypes(schemas: TableSchema[]): string { const lines: string[] = [ "// Auto-generated by @wavehouse/sdk codegen", `// Generated at: ${new Date().toISOString()}`, @@ -182,22 +207,21 @@ function generateTypes(schemas: Schemas): string { "export interface Database {", ]; - const tableNames = Object.keys(schemas).sort(); - for (const tableName of tableNames) { - const _schema = schemas[tableName]; - const rowType = `${pascalCase(tableName)}Row`; - lines.push(` ${tableName}: ${rowType};`); + // Code-unit comparison, not localeCompare: db.d.ts is a committed artifact, + // so its ordering must not vary with the generating host's ICU locale. + const tables = [...schemas].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + for (const table of tables) { + lines.push(` ${table.name}: ${pascalCase(table.name)}Row;`); } lines.push("}"); lines.push(""); // Generate row interfaces - for (const tableName of tableNames) { - const schema = schemas[tableName]; - const rowType = `${pascalCase(tableName)}Row`; + for (const table of tables) { + const rowType = `${pascalCase(table.name)}Row`; lines.push(`export interface ${rowType} {`); - for (const col of schema.columns) { + for (const col of table.columns) { const tsType = chTypeToTS(col.type); const optional = col.has_default ? "?" : ""; lines.push(` ${col.name}${optional}: ${tsType};`); @@ -224,14 +248,13 @@ async function main() { console.log(`Fetching schema from ${args.url}...`); const schemas = await fetchSchemas(args.url, args.auth); - const tableCount = Object.keys(schemas).length; - if (tableCount === 0) { + if (schemas.length === 0) { console.warn("No tables found. Is WaveHouse running with tables in ClickHouse?"); process.exit(1); } - console.log(`Found ${tableCount} table(s): ${Object.keys(schemas).join(", ")}`); + console.log(`Found ${schemas.length} table(s): ${schemas.map((t) => t.name).join(", ")}`); const output = generateTypes(schemas); @@ -243,7 +266,25 @@ async function main() { console.log(`✓ Types written to ${outPath}`); } -main().catch((err) => { - console.error("Codegen failed:", err.message); - process.exit(1); -}); +// Exported for unit tests; the package entry point (src/index.ts) does not +// re-export the CLI. +export { chTypeToTS, fetchSchemas, generateTypes }; + +// Run only when invoked as a script (the `wavehouse-codegen` bin or tsx), not +// when imported by tests. realpath the argv side: npm bin shims are symlinks, +// while the ESM loader resolves import.meta.url to the real file. +function isDirectInvocation(): boolean { + if (!process.argv[1]) return false; + try { + return import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href; + } catch { + return false; + } +} + +if (isDirectInvocation()) { + main().catch((err) => { + console.error("Codegen failed:", err.message); + process.exit(1); + }); +} diff --git a/clients/ts/src/client.test.ts b/clients/ts/src/client.test.ts index 9fa69ab5..1fc9a22b 100644 --- a/clients/ts/src/client.test.ts +++ b/clients/ts/src/client.test.ts @@ -144,14 +144,14 @@ describe("WaveHouseClient.pipe()", () => { }); describe("WaveHouseClient.sql()", () => { - it("delegates to sql() and POSTs to /v1/admin/query", async () => { + it("delegates to sql() and POSTs to /v1/ops/query", async () => { fetchSpy.mockResolvedValue(new Response(JSON.stringify([{ count: 42 }]), { status: 200 })); const client = createClient({ baseURL: "http://localhost:8080" }); const result = await client.sql("SELECT count() FROM clicks"); expect(result.data).toEqual([{ count: 42 }]); - expect(fetchSpy.mock.calls[0][0]).toContain("/v1/admin/query"); + expect(fetchSpy.mock.calls[0][0]).toContain("/v1/ops/query"); }); it("throws a migration-clear error when called with a legacy params array", () => { diff --git a/clients/ts/src/client.ts b/clients/ts/src/client.ts index bb3987a9..3c26c63d 100644 --- a/clients/ts/src/client.ts +++ b/clients/ts/src/client.ts @@ -84,7 +84,7 @@ export class WaveHouseClient { // SQL errors. Throw a clear runtime error pointing at the migration. if (Array.isArray(opts)) { throw new Error( - "[WaveHouse SDK] client.sql(sql, params) was removed. The /v1/admin/query endpoint does not accept positional `?` params. Inline literals into the SQL, or use the structured query builder (wh.from(table)…) for safe binding from user input.", + "[WaveHouse SDK] client.sql(sql, params) was removed. The /v1/ops/query endpoint does not accept positional `?` params. Inline literals into the SQL, or use the structured query builder (wh.from(table)…) for safe binding from user input.", ); } return sql(this._ctx, query, opts); diff --git a/clients/ts/src/dlq.ts b/clients/ts/src/dlq.ts index 01349318..a783e8eb 100644 --- a/clients/ts/src/dlq.ts +++ b/clients/ts/src/dlq.ts @@ -19,7 +19,7 @@ export class DLQNamespace { async list(opts?: { signal?: AbortSignal }): Promise> { const { data, error } = await request(this._ctx, { method: "GET", - path: "/v1/dlq/stats", + path: "/v1/ops/dlq/stats", signal: opts?.signal, }); if (error) return err(error); @@ -30,7 +30,7 @@ export class DLQNamespace { async table(name: string, opts?: { signal?: AbortSignal }): Promise> { const { data, error } = await request(this._ctx, { method: "GET", - path: "/v1/dlq/stats", + path: "/v1/ops/dlq/stats", params: { table: name }, signal: opts?.signal, }); diff --git a/clients/ts/src/http.test.ts b/clients/ts/src/http.test.ts index 6b3e3f30..2d510dbb 100644 --- a/clients/ts/src/http.test.ts +++ b/clients/ts/src/http.test.ts @@ -76,7 +76,7 @@ describe("request", () => { await request(makeCtx({ auth: async () => "my-token" }), { method: "GET", - path: "/v1/schema", + path: "/v1/ops/schema", }); const [, init] = fetchSpy.mock.calls[0]; @@ -88,7 +88,7 @@ describe("request", () => { await request(makeCtx({ auth: async () => "" }), { method: "GET", - path: "/v1/schema", + path: "/v1/ops/schema", }); const [, init] = fetchSpy.mock.calls[0]; @@ -100,7 +100,7 @@ describe("request", () => { new Response(JSON.stringify({ error: "unknown table: foo" }), { status: 404 }), ); - const result = await request(makeCtx(), { method: "GET", path: "/v1/schema?table=foo" }); + const result = await request(makeCtx(), { method: "GET", path: "/v1/ops/schema?table=foo" }); expect(result.data).toBeNull(); expect(result.error?.status).toBe(404); @@ -172,7 +172,7 @@ describe("request", () => { await request(makeCtx(), { method: "GET", - path: "/v1/dlq/stats", + path: "/v1/ops/dlq/stats", params: { table: "clicks" }, }); @@ -197,12 +197,12 @@ describe("request", () => { await request(makeCtx({ baseURL: "https://app.example.com/api/warehouse/" }), { method: "GET", - path: "/v1/dlq/stats", + path: "/v1/ops/dlq/stats", params: { table: "clicks" }, }); const [url] = fetchSpy.mock.calls[0]; - expect(url).toBe("https://app.example.com/api/warehouse/v1/dlq/stats?table=clicks"); + expect(url).toBe("https://app.example.com/api/warehouse/v1/ops/dlq/stats?table=clicks"); }); it("handles empty response body", async () => { @@ -210,7 +210,7 @@ describe("request", () => { const result = await request(makeCtx(), { method: "POST", - path: "/v1/schema/refresh", + path: "/v1/ops/schema/refresh", }); expect(result.data).toBeUndefined(); diff --git a/clients/ts/src/namespaces.test.ts b/clients/ts/src/namespaces.test.ts index 3040806a..a25805e9 100644 --- a/clients/ts/src/namespaces.test.ts +++ b/clients/ts/src/namespaces.test.ts @@ -22,12 +22,12 @@ describe("sql", () => { afterEach(() => vi.restoreAllMocks()); - it("POSTs to /v1/admin/query with sql field", async () => { + it("POSTs to /v1/ops/query with sql field", async () => { const result = await sql(makeCtx(), "SELECT count() FROM clicks"); expect(result.data).toEqual([{ count: 10 }]); const [url, init] = fetchSpy.mock.calls[0]; - expect(url).toContain("/v1/admin/query"); + expect(url).toContain("/v1/ops/query"); expect(JSON.parse(init.body)).toEqual({ sql: "SELECT count() FROM clicks" }); }); @@ -68,7 +68,7 @@ describe("SchemaNamespace", () => { afterEach(() => vi.restoreAllMocks()); - it("list() GETs /v1/schema", async () => { + it("list() GETs /v1/ops/schema", async () => { const schemas = { clicks: { name: "clicks", columns: [] } }; fetchSpy.mockResolvedValue(new Response(JSON.stringify(schemas), { status: 200 })); @@ -76,10 +76,10 @@ describe("SchemaNamespace", () => { const result = await ns.list(); expect(result.data).toEqual(schemas); - expect(fetchSpy.mock.calls[0][0]).toContain("/v1/schema"); + expect(fetchSpy.mock.calls[0][0]).toContain("/v1/ops/schema"); }); - it("refresh() POSTs to /v1/schema/refresh", async () => { + it("refresh() POSTs to /v1/ops/schema/refresh", async () => { fetchSpy.mockResolvedValue(new Response(JSON.stringify({}), { status: 200 })); const ns = new SchemaNamespace(makeCtx()); @@ -87,7 +87,7 @@ describe("SchemaNamespace", () => { expect(result.error).toBeNull(); expect(fetchSpy.mock.calls[0][1].method).toBe("POST"); - expect(fetchSpy.mock.calls[0][0]).toContain("/v1/schema/refresh"); + expect(fetchSpy.mock.calls[0][0]).toContain("/v1/ops/schema/refresh"); }); }); @@ -99,7 +99,7 @@ describe("PolicyNamespace", () => { afterEach(() => vi.restoreAllMocks()); - it("get() GETs /v1/admin/policy", async () => { + it("get() GETs /v1/ops/policy", async () => { const policy = { tables: {} }; fetchSpy.mockResolvedValue(new Response(JSON.stringify(policy), { status: 200 })); @@ -107,10 +107,10 @@ describe("PolicyNamespace", () => { const result = await ns.get(); expect(result.data).toEqual(policy); - expect(fetchSpy.mock.calls[0][0]).toContain("/v1/admin/policy"); + expect(fetchSpy.mock.calls[0][0]).toContain("/v1/ops/policy"); }); - it("set() PUTs /v1/admin/policy", async () => { + it("set() PUTs /v1/ops/policy", async () => { fetchSpy.mockResolvedValue(new Response(JSON.stringify({ ok: true }), { status: 200 })); const ns = new PolicyNamespace(makeCtx()); @@ -120,14 +120,14 @@ describe("PolicyNamespace", () => { expect(fetchSpy.mock.calls[0][1].method).toBe("PUT"); }); - it("validate() POSTs /v1/admin/policy/validate", async () => { + it("validate() POSTs /v1/ops/policy/validate", async () => { fetchSpy.mockResolvedValue(new Response(JSON.stringify({ valid: true }), { status: 200 })); const ns = new PolicyNamespace(makeCtx()); const result = await ns.validate({ tables: {} }); expect(result.data).toEqual({ valid: true }); - expect(fetchSpy.mock.calls[0][0]).toContain("/v1/admin/policy/validate"); + expect(fetchSpy.mock.calls[0][0]).toContain("/v1/ops/policy/validate"); }); }); @@ -141,7 +141,7 @@ describe("DLQNamespace", () => { afterEach(() => vi.restoreAllMocks()); - it("list() GETs /v1/dlq/stats", async () => { + it("list() GETs /v1/ops/dlq/stats", async () => { const stats = { tables: { clicks: 5 }, total: 5 }; fetchSpy.mockResolvedValue(new Response(JSON.stringify(stats), { status: 200 })); diff --git a/clients/ts/src/pipes.test.ts b/clients/ts/src/pipes.test.ts index be186085..7786c632 100644 --- a/clients/ts/src/pipes.test.ts +++ b/clients/ts/src/pipes.test.ts @@ -79,7 +79,7 @@ describe("PipesNamespace", () => { vi.restoreAllMocks(); }); - it("list() GETs /v1/admin/pipes", async () => { + it("list() GETs /v1/ops/pipes", async () => { const pipes = [{ name: "p1", sql: "SELECT 1" }]; fetchSpy.mockResolvedValue(new Response(JSON.stringify(pipes), { status: 200 })); @@ -87,10 +87,10 @@ describe("PipesNamespace", () => { const result = await ns.list(); expect(result.data).toEqual(pipes); - expect(fetchSpy.mock.calls[0][0]).toContain("/v1/admin/pipes"); + expect(fetchSpy.mock.calls[0][0]).toContain("/v1/ops/pipes"); }); - it("get() GETs /v1/admin/pipes/{name}", async () => { + it("get() GETs /v1/ops/pipes/{name}", async () => { fetchSpy.mockResolvedValue( new Response(JSON.stringify({ name: "p1", sql: "SELECT 1" }), { status: 200 }), ); @@ -99,10 +99,10 @@ describe("PipesNamespace", () => { const result = await ns.get("p1"); expect(result.data?.name).toBe("p1"); - expect(fetchSpy.mock.calls[0][0]).toContain("/v1/admin/pipes/p1"); + expect(fetchSpy.mock.calls[0][0]).toContain("/v1/ops/pipes/p1"); }); - it("set() PUTs /v1/admin/pipes/{name}", async () => { + it("set() PUTs /v1/ops/pipes/{name}", async () => { fetchSpy.mockResolvedValue(new Response(JSON.stringify({ ok: true }), { status: 200 })); const ns = new PipesNamespace(makeCtx()); @@ -110,11 +110,11 @@ describe("PipesNamespace", () => { expect(result.error).toBeNull(); const [url, init] = fetchSpy.mock.calls[0]; - expect(url).toContain("/v1/admin/pipes/p1"); + expect(url).toContain("/v1/ops/pipes/p1"); expect(init.method).toBe("PUT"); }); - it("delete() DELETEs /v1/admin/pipes/{name}", async () => { + it("delete() DELETEs /v1/ops/pipes/{name}", async () => { fetchSpy.mockResolvedValue(new Response(JSON.stringify({ ok: true }), { status: 200 })); const ns = new PipesNamespace(makeCtx()); diff --git a/clients/ts/src/pipes.ts b/clients/ts/src/pipes.ts index e20c4ae4..27c3229c 100644 --- a/clients/ts/src/pipes.ts +++ b/clients/ts/src/pipes.ts @@ -68,7 +68,7 @@ export class PipesNamespace { async list(opts?: { signal?: AbortSignal }): Promise> { const { data, error } = await request(this._ctx, { method: "GET", - path: "/v1/admin/pipes", + path: "/v1/ops/pipes", signal: opts?.signal, }); if (error) return err(error); @@ -79,7 +79,7 @@ export class PipesNamespace { async get(name: string, opts?: { signal?: AbortSignal }): Promise> { const { data, error } = await request(this._ctx, { method: "GET", - path: `/v1/admin/pipes/${encodeURIComponent(name)}`, + path: `/v1/ops/pipes/${encodeURIComponent(name)}`, signal: opts?.signal, }); if (error) return err(error); @@ -94,7 +94,7 @@ export class PipesNamespace { ): Promise> { const { error } = await request<{ ok: boolean }>(this._ctx, { method: "PUT", - path: `/v1/admin/pipes/${encodeURIComponent(name)}`, + path: `/v1/ops/pipes/${encodeURIComponent(name)}`, body: def, signal: opts?.signal, }); @@ -106,7 +106,7 @@ export class PipesNamespace { async delete(name: string, opts?: { signal?: AbortSignal }): Promise> { const { error } = await request<{ ok: boolean }>(this._ctx, { method: "DELETE", - path: `/v1/admin/pipes/${encodeURIComponent(name)}`, + path: `/v1/ops/pipes/${encodeURIComponent(name)}`, signal: opts?.signal, }); if (error) return err(error); diff --git a/clients/ts/src/policy.ts b/clients/ts/src/policy.ts index b3c48cf4..8c2df43f 100644 --- a/clients/ts/src/policy.ts +++ b/clients/ts/src/policy.ts @@ -14,7 +14,7 @@ export class PolicyNamespace { async get(opts?: { signal?: AbortSignal }): Promise> { const { data, error } = await request(this._ctx, { method: "GET", - path: "/v1/admin/policy", + path: "/v1/ops/policy", signal: opts?.signal, }); if (error) return err(error); @@ -25,7 +25,7 @@ export class PolicyNamespace { async set(policy: Policy, opts?: { signal?: AbortSignal }): Promise> { const { error } = await request<{ ok: boolean }>(this._ctx, { method: "PUT", - path: "/v1/admin/policy", + path: "/v1/ops/policy", body: policy, signal: opts?.signal, }); @@ -40,7 +40,7 @@ export class PolicyNamespace { ): Promise> { const { data, error } = await request(this._ctx, { method: "POST", - path: "/v1/admin/policy/validate", + path: "/v1/ops/policy/validate", body: policy, signal: opts?.signal, }); diff --git a/clients/ts/src/schema.ts b/clients/ts/src/schema.ts index 03fa08d6..397d897b 100644 --- a/clients/ts/src/schema.ts +++ b/clients/ts/src/schema.ts @@ -15,7 +15,7 @@ export class SchemaNamespace { // The backend returns TableSchema[] — transform to Record. const { data, error } = await request(this._ctx, { method: "GET", - path: "/v1/schema", + path: "/v1/ops/schema", signal: opts?.signal, }); if (error) return err(error); @@ -38,7 +38,7 @@ export class SchemaNamespace { async refresh(opts?: { signal?: AbortSignal }): Promise> { const { error } = await request(this._ctx, { method: "POST", - path: "/v1/schema/refresh", + path: "/v1/ops/schema/refresh", signal: opts?.signal, }); if (error) return err(error); diff --git a/clients/ts/src/sql.ts b/clients/ts/src/sql.ts index bcc60253..4df26dae 100644 --- a/clients/ts/src/sql.ts +++ b/clients/ts/src/sql.ts @@ -5,8 +5,8 @@ import type { HttpContext, Result } from "./types.js"; /** * Execute a raw SQL query against ClickHouse. * - * Backed by `POST /v1/admin/query`, which requires the admin role — the same - * gate as the rest of `/v1/admin/*`. Callers must hold a JWT whose role is the + * Backed by `POST /v1/ops/query`, which requires the admin role — the same + * gate as the rest of `/v1/ops/*`. Callers must hold a JWT whose role is the * configured `admin_role` (`"admin"` by default); there is no separate * `service` role. The JWT middleware always runs, so a request with no token * or an invalid one is denied, never granted. Non-admin use cases should use @@ -23,7 +23,7 @@ import type { HttpContext, Result } from "./types.js"; * compatible with `sql()` — the proxy passes the upstream Content-Type * through, and the SDK's JSON decoder throws on a non-JSON body, which the * retry layer surfaces as a `NETWORK_ERROR` result (not a structured - * format-mismatch error). For CSV/TSV exports, hit `/v1/admin/query` + * format-mismatch error). For CSV/TSV exports, hit `/v1/ops/query` * directly with `fetch()` and read the body as text. * * **No parameter binding.** Positional `?` substitution is not supported. @@ -41,7 +41,7 @@ export async function sql>( ): Promise> { const { data, error } = await request(ctx, { method: "POST", - path: "/v1/admin/query", + path: "/v1/ops/query", body: { sql: query }, signal: opts?.signal, }); diff --git a/clients/ts/src/table.test.ts b/clients/ts/src/table.test.ts index becec5f8..7c317361 100644 --- a/clients/ts/src/table.test.ts +++ b/clients/ts/src/table.test.ts @@ -205,7 +205,7 @@ describe("TableRef", () => { // --- schema --- - it("schema() sends GET to /v1/schema?table={table}", async () => { + it("schema() sends GET to /v1/ops/schema?table={table}", async () => { const schema = { name: "clicks", columns: [{ name: "page", type: "String", is_nullable: false, has_default: false }], @@ -215,7 +215,7 @@ describe("TableRef", () => { const result = await table().schema(); expect(result.data).toEqual(schema); - expect(fetchSpy.mock.calls[0][0]).toContain("/v1/schema?table=clicks"); + expect(fetchSpy.mock.calls[0][0]).toContain("/v1/ops/schema?table=clicks"); }); // --- stream --- diff --git a/clients/ts/src/table.ts b/clients/ts/src/table.ts index 18c53146..f646858d 100644 --- a/clients/ts/src/table.ts +++ b/clients/ts/src/table.ts @@ -184,7 +184,7 @@ export class TableRef> { async schema(opts?: { signal?: AbortSignal }): Promise> { const { data, error } = await request(this._ctx, { method: "GET", - path: `/v1/schema?table=${encodeURIComponent(this._table)}`, + path: `/v1/ops/schema?table=${encodeURIComponent(this._table)}`, signal: opts?.signal, }); if (error) return err(error); diff --git a/clients/ts/src/url.test.ts b/clients/ts/src/url.test.ts index d9acd9f4..d68f6d88 100644 --- a/clients/ts/src/url.test.ts +++ b/clients/ts/src/url.test.ts @@ -22,14 +22,14 @@ describe("resolveURL", () => { }); it("preserves a multi-segment prefix", () => { - expect(resolveURL("https://example.com/a/b/c", "/v1/admin/pipes").toString()).toBe( - "https://example.com/a/b/c/v1/admin/pipes", + expect(resolveURL("https://example.com/a/b/c", "/v1/ops/pipes").toString()).toBe( + "https://example.com/a/b/c/v1/ops/pipes", ); }); it("accepts a path with or without a leading slash", () => { - expect(resolveURL("https://example.com/api", "v1/schema").toString()).toBe( - resolveURL("https://example.com/api", "/v1/schema").toString(), + expect(resolveURL("https://example.com/api", "v1/ops/schema").toString()).toBe( + resolveURL("https://example.com/api", "/v1/ops/schema").toString(), ); }); @@ -40,8 +40,8 @@ describe("resolveURL", () => { }); it("appends params after the prefix", () => { - const url = resolveURL("https://example.com/api", "/v1/dlq/stats", { table: "clicks" }); - expect(url.toString()).toBe("https://example.com/api/v1/dlq/stats?table=clicks"); + const url = resolveURL("https://example.com/api", "/v1/ops/dlq/stats", { table: "clicks" }); + expect(url.toString()).toBe("https://example.com/api/v1/ops/dlq/stats?table=clicks"); }); it("merges params with a query string already on the path", () => { @@ -52,7 +52,7 @@ describe("resolveURL", () => { }); it("percent-encodes param values", () => { - const url = resolveURL("https://example.com", "/v1/schema", { table: "a b&c" }); + const url = resolveURL("https://example.com", "/v1/ops/schema", { table: "a b&c" }); expect(url.searchParams.get("table")).toBe("a b&c"); expect(url.toString()).toContain("table=a+b%26c"); }); diff --git a/cmd/wavehouse/main.go b/cmd/wavehouse/main.go index 7ae3e21a..6616f7bf 100644 --- a/cmd/wavehouse/main.go +++ b/cmd/wavehouse/main.go @@ -346,7 +346,7 @@ func run() int { } // TODO: is this really the best/right way to do this? - // /v1/admin/query proxies straight to ClickHouse over HTTP — no native + // /v1/ops/query proxies straight to ClickHouse over HTTP — no native // driver involvement. Construct the base URL from the same fields the // ingest worker uses, defaulting the scheme to http if blank. queryHost, _, err := net.SplitHostPort(cfg.ClickHouse.Addr) diff --git a/deployments/compose/dev-policy.yaml b/deployments/compose/dev-policy.yaml index 1f57a6a6..dd107021 100644 --- a/deployments/compose/dev-policy.yaml +++ b/deployments/compose/dev-policy.yaml @@ -7,7 +7,7 @@ # so `git clone` → ingest → query → stream works out of the box. # # It is intentionally NOT admin and NOT a blanket grant: -# - no raw SQL (/v1/admin/query), no policy/pipe CRUD, no schema/DLQ admin — +# - no raw SQL (/v1/ops/query), no policy/pipe CRUD, no schema/DLQ admin — # those stay locked (they require the admin role); # - it only grants the tables named below, so if this file ever rides into a # real deployment it grants nothing there (those demo tables won't exist) @@ -19,7 +19,7 @@ # require auth). See docs/src/content/docs/access-control.mdx for the full model. # # Seeds NATS KV on FIRST boot only; after that KV is authoritative and runtime -# edits go through PUT /v1/admin/policy. Wired in via WH_POLICY_FILE_PATH. +# edits go through PUT /v1/ops/policy. Wired in via WH_POLICY_FILE_PATH. default_role: public tables: clicks: diff --git a/docs/src/content/docs/access-control.mdx b/docs/src/content/docs/access-control.mdx index c2838a01..2328e0be 100644 --- a/docs/src/content/docs/access-control.mdx +++ b/docs/src/content/docs/access-control.mdx @@ -54,7 +54,7 @@ For authorization, the behavior that matters is what happens with **no** role: t Setting `default_role` does nothing on its own; the role you name still needs entries under `tables` to grant any access. Think of it as "which role does an anonymous caller assume", not "what can anonymous callers do". :::caution[`default_role: admin` is a dev-only footgun] -Setting `default_role` equal to `admin_role` is permitted — it makes every unauthenticated request a full admin, including `/v1/admin/*`, which is handy for local development with no tokens. It is **never** for production. Every node that loads such a policy logs a loud `WARN` on startup and on every update. +Setting `default_role` equal to `admin_role` is permitted — it makes every unauthenticated request a full admin, including `/v1/ops/*`, which is handy for local development with no tokens. It is **never** for production. Every node that loads such a policy logs a loud `WARN` on startup and on every update. ::: ### `admin_role` — the privileged role @@ -62,7 +62,7 @@ Setting `default_role` equal to `admin_role` is permitted — it makes every una `admin_role` (default: `"admin"`) is the one role that bypasses the entire policy: - It is granted **full, unrestricted access** to every table and operation. An admin is **never** column-scoped or row-scoped, even by a policy entry that explicitly names the admin role — admin is an unconditional bypass, not a scoped grant. -- It is the gate for every admin surface: `/v1/admin/*` (raw SQL, policy CRUD, pipe CRUD), plus the schema and DLQ endpoints. The gate reads `admin_role` live from the policy, so changing it applies without a restart. +- It is the gate for the whole `/v1/ops/*` tree — raw SQL, policy CRUD, pipe CRUD, schema discovery, and DLQ stats. The gate reads `admin_role` live from the policy, so changing it applies without a restart. There is no separate `service` role. To reach an admin endpoint or to read data beyond what `default_role` grants, present a valid token whose `role_claim` is the admin role (or another granted role) — **or** present the non-JWT [operator key](#operator-key) described below. @@ -72,7 +72,7 @@ There is no separate `service` role. To reach an admin endpoint or to read data ### Operator key -`auth.operator_key` (config; presented in an `Authorization: Operator ` header, or the `X-Operator-Key` alias) is a static, role-free credential for the person running the deployment, meant for bootstrap and break-glass recovery. A request presenting it is authorized as a **full-access platform operator** — the entire data plane *and* the `/v1/admin/*` management surface — without minting a JWT, and independently of the JWT verifier (`jwt_secret`/`jwks_url`). +`auth.operator_key` (config; presented in an `Authorization: Operator ` header, or the `X-Operator-Key` alias) is a static, role-free credential for the person running the deployment, meant for bootstrap and break-glass recovery. A request presenting it is authorized as a **full-access platform operator** — the entire data plane *and* the `/v1/ops/*` management surface — without minting a JWT, and independently of the JWT verifier (`jwt_secret`/`jwks_url`). Unlike `admin_role`, the operator key is honored **even when the policy is `nil`** (deleted, or never seeded), so it is the one HTTP path that can restore a wiped policy — the break-glass case the caution above describes. It is matched in constant time, takes precedence over any Bearer token on the same request, and is disabled when empty (the default). Treat it as an admin secret: load it from a secret store, serve it only over TLS, and rotate it like any other credential. See [Configuration — Authentication](/configuration#authentication). @@ -87,7 +87,7 @@ head -c 32 /dev/urandom | base64 Set the result via `WH_AUTH_OPERATOR_KEY` (or `auth.operator_key`) and present it as `Authorization: Operator ` (or `X-Operator-Key: `): ```bash -curl -H "Authorization: Operator $(cat operator.key)" https://wavehouse.example.com/v1/admin/policy +curl -H "Authorization: Operator $(cat operator.key)" https://wavehouse.example.com/v1/ops/policy ``` **Monitor failed attempts.** A successful operator authentication is audit-logged at `INFO`; a request presenting a *wrong* operator key is logged at `WARN` (`operator key authentication failed`) and counted by the `wavehouse_auth_operator_key_failures_total` metric, then handled as an ordinary unauthenticated request (the middleware never rejects). A wrong operator key is never sent by accident, so a nonzero rate is a strong probing/brute-force signal against your most privileged credential — alert on it. @@ -99,7 +99,7 @@ A policy is a YAML or JSON document with three top-level fields: | Field | Type | Description | | ----- | ---- | ----------- | | `default_role` | string | Role that a roleless (tokenless or claimless) request assumes. Empty = no public access. | -| `admin_role` | string | Role granted full, unrestricted access and the `/v1/admin/*` gate. Optional; defaults to `"admin"`. | +| `admin_role` | string | Role granted full, unrestricted access and the `/v1/ops/*` gate. Optional; defaults to `"admin"`. | | `tables` | map | Per-table permissions, keyed by ClickHouse table name. | Each table entry holds permissions for two operations — `select` (reads) and `insert` (writes) — and each operation maps **role → permissions**: @@ -234,7 +234,7 @@ Values are always bound as SQL **parameters**, never concatenated into the query "Can't be resolved" means the claim path is **absent** from the token (or `null`) — or resolves to a JSON **object or array** rather than a scalar, which usually means a dropped path segment (`{{ jwt.app_metadata }}` where `{{ jwt.app_metadata.tenant_id }}` was meant); the one structured shape with defined semantics is the bare-claim `_in` array above. Scalar claims — strings, booleans, and numbers — resolve normally. A numeric claim binds in **canonical decimal form**, not the token's spelling: an integer id keeps every digit — up to a 100-digit bound, far past any real id — while `1.0` or `1e3` binds as `1` and `1000`, so spelling differences between issuers never change the bound value. What must fit the bound is the value's **exact decimal form**, exponent applied — roughly 100 digits (the exact-form gate allows 102 characters, whatever they are) — so `1e400` can't be resolved, but neither can `1e150` or `1e-150`, whose short spellings expand to 151- and 152-character exact forms even though a float64 could hold them; prefer issuing integer ids as integers. Type the scoped column to match: an integer column — or a `Decimal` whose scale covers the claim's fractional digits — keeps the comparison exact, while a `Float32`/`Float64` column rounds the stored value and quietly gives that exactness back (`col = '9007199254740993'` matches a stored `9007199254740992` there). A claim that is *present but empty* is a value the token vouches for: it resolves to `''` and binds normally, so `_neq` against an empty-string claim still emits `col != ''`. Make sure your identity provider actually issues the claims your policy templates reference — and omits unset claims rather than issuing them as empty strings. -Claim paths may contain only letters, digits, `_`, and `.` (the segment separator). Any `{{ … }}` fragment that is not a well-formed `{{ jwt. }}` template — a path outside that grammar (a hyphen: `{{ jwt.tenant-id }}`; a namespaced claim: `{{ jwt.https://app.example.com/tenant_id }}`), a missing or misspelled `jwt.` prefix, or an unterminated `{{` — is **not** recognized as a template, and left unchecked the resolver would bind the literal `{{ … }}` text as a value. (Policy values have no other placeholder syntax; a pipe's `{{param}}` placeholders are a different mechanism and are not accepted here.) That is *not* fail-closed: on a read filter `_neq`/`_lt` would then match essentially every row (a leak), and on a write `check` the literal text would be stamped into every inserted row (silent corruption). So such a policy is **rejected when it is written**: a bootstrap policy file carrying one makes WaveHouse refuse to start when the store is seeded from it (a populated KV store skips the file), and a `PUT` on `/v1/admin/policy` (or a `POST` to `/v1/admin/policy/validate`) returns `400`. A policy already stored in NATS KV is *not* re-validated when a node loads it ([#461](https://github.com/Wave-RF/WaveHouse/issues/461)), so after upgrading, re-`PUT` your policy once to surface a template written before this rule existed — until you do, that stored policy keeps binding the literal text, and the leak above stays live for it. Flatten hyphenated or namespaced claims into a supported path at your identity provider — on Auth0, an [Action can set a flat custom claim](https://auth0.com/docs/secure/tokens/json-web-tokens/create-custom-claims) outside the registered OIDC names (its legacy Rules required URL namespacing, which established tenants often still carry), and namespacing is a common OIDC convention elsewhere, so a namespaced claim is the shape you are most likely to meet first. +Claim paths may contain only letters, digits, `_`, and `.` (the segment separator). Any `{{ … }}` fragment that is not a well-formed `{{ jwt. }}` template — a path outside that grammar (a hyphen: `{{ jwt.tenant-id }}`; a namespaced claim: `{{ jwt.https://app.example.com/tenant_id }}`), a missing or misspelled `jwt.` prefix, or an unterminated `{{` — is **not** recognized as a template, and left unchecked the resolver would bind the literal `{{ … }}` text as a value. (Policy values have no other placeholder syntax; a pipe's `{{param}}` placeholders are a different mechanism and are not accepted here.) That is *not* fail-closed: on a read filter `_neq`/`_lt` would then match essentially every row (a leak), and on a write `check` the literal text would be stamped into every inserted row (silent corruption). So such a policy is **rejected when it is written**: a bootstrap policy file carrying one makes WaveHouse refuse to start when the store is seeded from it (a populated KV store skips the file), and a `PUT` on `/v1/ops/policy` (or a `POST` to `/v1/ops/policy/validate`) returns `400`. A policy already stored in NATS KV is *not* re-validated when a node loads it ([#461](https://github.com/Wave-RF/WaveHouse/issues/461)), so after upgrading, re-`PUT` your policy once to surface a template written before this rule existed — until you do, that stored policy keeps binding the literal text, and the leak above stays live for it. Flatten hyphenated or namespaced claims into a supported path at your identity provider — on Auth0, an [Action can set a flat custom claim](https://auth0.com/docs/secure/tokens/json-web-tokens/create-custom-claims) outside the registered OIDC names (its legacy Rules required URL namespacing, which established tenants often still carry), and namespacing is a common OIDC convention elsewhere, so a namespaced claim is the shape you are most likely to meet first. Row filters apply on the structured-query path and, per subscriber, on the live SSE stream — the stream evaluates the same resolved predicates in memory against each subscriber's token claims (see the [enforcement caution](#where-each-rule-is-enforced) for what its in-memory comparison can and cannot decide). Named pipes authorize by `allowed_roles` membership alone — scope a pipe's exposure in its SQL text, since neither the table policy's row `filter` nor its column allow/deny list is applied on the pipe path (see [Named Pipes](/pipes#authorizing-a-pipe)). @@ -379,7 +379,7 @@ The same policy drives every data path, but not every field is meaningful on eve | Structured read | `POST /v1/query?table={table}` | table+role `select` required, then `allow`/`deny_columns`, row `filter`, aggregation rules, and the per-role `max_rows` / `max_execution_time` / `max_rows_to_read` / `max_memory_usage` caps (over the [ClickHouse server-wide limits](/configuration#server-side-resource-limits)) | | Ingest (write) | `POST /v1/ingest?table={table}` | table+role `insert` required, then `allow`/`deny_columns` and `check` (enforced and auto-injected) | | Live stream | `GET /v1/stream` | table+role `select` required (a table the role can't read is skipped), then denied columns are masked from each event and the role's row `filter` is applied per subscriber against their JWT claims (see caution below) | -| Raw SQL | `POST /v1/admin/query` | `admin_role` only — no per-statement policy; the role gate is the entire authorization story | +| Raw SQL | `POST /v1/ops/query` | `admin_role` only — no per-statement policy; the role gate is the entire authorization story | | Named pipe | `GET/POST /v1/pipes/{name}` | per-pipe `allowed_roles` (not the policy engine; see [Named Pipes](/pipes)). Resource limits come from ClickHouse's [server-wide settings](/configuration#server-side-resource-limits), not per-role policy caps | :::caution[Live streams enforce column and row policy, but not resource limits] @@ -410,16 +410,16 @@ The policy lives behind three admin endpoints (all require `admin_role`): | Method | Endpoint | Purpose | | ------ | -------- | ------- | -| `GET` | `/v1/admin/policy` | Fetch the current policy. Returns `{"tables":{}}` when none is set. | -| `PUT` | `/v1/admin/policy` | Replace the **entire** policy. Validated before it is saved. | -| `POST` | `/v1/admin/policy/validate` | Dry-run validation — returns `{"valid": true}` or the error, without saving. | +| `GET` | `/v1/ops/policy` | Fetch the current policy. Returns `{"tables":{}}` when none is set. | +| `PUT` | `/v1/ops/policy` | Replace the **entire** policy. Validated before it is saved. | +| `POST` | `/v1/ops/policy/validate` | Dry-run validation — returns `{"valid": true}` or the error, without saving. | `PUT` is a full replace, not a merge: send the complete document every time. Validation rejects empty role names, negative limits, [malformed claim templates](#jwt-claim-templating), and unsupported `check` operators — `_neq`/`_gt`/`_lt`, or `_eq` and `_in` together on one column (but it intentionally allows `default_role == admin_role`, warning instead). Once accepted, the policy is written to NATS KV and propagated to every node via KV Watch — changes apply cluster-wide within moments, no restart. ```bash # Replace the policy (admin token required). POST the same body to -# /v1/admin/policy/validate first for a dry run that doesn't save. -curl -X PUT http://localhost:8080/v1/admin/policy \ +# /v1/ops/policy/validate first for a dry run that doesn't save. +curl -X PUT http://localhost:8080/v1/ops/policy \ -H "Authorization: Bearer $ADMIN_TOKEN" \ -H "Content-Type: application/json" \ -d @policy.json @@ -431,9 +431,9 @@ The full request and response shapes for these endpoints live in the [API Refere NATS KV is the **source of truth** for the live policy. A file is only ever a *seed* for an empty store: -- `policy.file_path` (config) points at a YAML or JSON policy file. On startup, **if and only if the KV store is empty**, the file is loaded, validated, and written to KV. On every subsequent boot the file is ignored — KV already has the authoritative copy, and runtime edits flow through `PUT /v1/admin/policy`. +- `policy.file_path` (config) points at a YAML or JSON policy file. On startup, **if and only if the KV store is empty**, the file is loaded, validated, and written to KV. On every subsequent boot the file is ignored — KV already has the authoritative copy, and runtime edits flow through `PUT /v1/ops/policy`. - **When `policy.file_path` is set, the file must exist, parse, and pass policy validation**, or WaveHouse refuses to boot. That turns a typo, a missing mount, or a policy invalid under a newly-tightened rule — such as the [malformed-template rejection](#jwt-claim-templating) above — into a loud failure instead of a silent fail-closed deployment that denies everything. -- **When `policy.file_path` is empty and KV is empty**, the store comes up with no policy — every **token-based** request is denied (logged loudly, admin included). Seed one via `PUT /v1/admin/policy` using the [operator key](#operator-key), the deliberate break-glass that works under a `nil` policy — or set `policy.file_path` and reboot. +- **When `policy.file_path` is empty and KV is empty**, the store comes up with no policy — every **token-based** request is denied (logged loudly, admin included). Seed one via `PUT /v1/ops/policy` using the [operator key](#operator-key), the deliberate break-glass that works under a `nil` policy — or set `policy.file_path` and reboot. :::caution Validation guards only the **seed** path. A policy already in KV is not re-validated when a node loads it ([#461](https://github.com/Wave-RF/WaveHouse/issues/461)), so upgrading WaveHouse never re-checks the stored policy — after an upgrade that tightens validation (like the [claim-template rules](#jwt-claim-templating) above), re-`PUT` your policy once; until you do, a policy the new rules would reject keeps its old runtime behavior. @@ -460,12 +460,12 @@ Because a deleted policy locks out every *token-based* caller (including admin), ## A complete example -A multi-tenant analytics deployment: anonymous callers get nothing, `viewer` reads its own tenant's rows with sensitive columns masked and expensive aggregations blocked, `writer` ingests events that are stamped with the caller's identity, and `admin` is unrestricted. Save it as your `policy.file_path` bootstrap file (`policy.yaml` or `policy.json` — both parse identically), or `PUT` it to `/v1/admin/policy`: +A multi-tenant analytics deployment: anonymous callers get nothing, `viewer` reads its own tenant's rows with sensitive columns masked and expensive aggregations blocked, `writer` ingests events that are stamped with the caller's identity, and `admin` is unrestricted. Save it as your `policy.file_path` bootstrap file (`policy.yaml` or `policy.json` — both parse identically), or `PUT` it to `/v1/ops/policy`: ```yaml -# admin: full access + the /v1/admin/* gate (this is also the default) +# admin: full access + the /v1/ops/* gate (this is also the default) admin_role: admin # "" = closed: a request with no/!valid token is denied default_role: "" diff --git a/docs/src/content/docs/api.md b/docs/src/content/docs/api.md index 79bb5e5e..52c6416f 100644 --- a/docs/src/content/docs/api.md +++ b/docs/src/content/docs/api.md @@ -5,7 +5,7 @@ sidebar: order: 7 --- -Every HTTP endpoint WaveHouse exposes — ingest, query, streaming, schema introspection, and admin — with request/response formats, error codes, and examples. The JWT middleware always runs; what a caller can do is driven by the policy; see [Configuration](/configuration#authentication) for the full auth config surface. +Every HTTP endpoint WaveHouse exposes — ingest, query, streaming, and the admin-gated `/v1/ops/*` surface (raw SQL, schema introspection, DLQ stats, policy and pipe CRUD) — with request/response formats, error codes, and examples. The JWT middleware always runs; what a caller can do is driven by the policy; see [Configuration](/configuration#authentication) for the full auth config surface. ## Authentication @@ -27,9 +27,9 @@ The `Authorization` header takes precedence when both are provided: the `?token= **Authentication is decoupled from authorization.** A request with **no token**, or an **invalid/expired/malformed** one, is *not* rejected outright — it falls back to an empty role that resolves to the policy `default_role`, and authorization is decided downstream. Because the bad-token reason is remembered, a request that is then denied for lacking permission fails loud (`401` "invalid/expired token") instead of a bare `403`. Elevated access requires a valid token whose role is granted (or equals the `admin_role`). A `403` body has two forms: a request that resolves to **no role at all** (no token and no `default_role` configured) returns `{"error":"forbidden: request has no role and no public default_role is configured"}`, while a request carrying a concrete-but-unauthorized role returns the bare `{"error":"forbidden"}` shown in the tables below. -**Public (unauthenticated) access is driven by the policy.** Define a usable `default_role` and no-token requests are evaluated as that role (see [Roles & Access Control](#roles--access-control)); remove it and roleless requests are denied. Setting `default_role` equal to the `admin_role` is allowed — it makes every unauthenticated request admin (including `/v1/admin/*`), handy for local/dev — but it is logged loudly on every node that loads such a policy and must not be used in production. `/v1/admin/*` **and** the schema/DLQ endpoints are admin-only, and a pipe with **no `allowed_roles` authorizes nobody but the admin role** — but a pipe *can* be reached by the public when its `allowed_roles` lists the role the `default_role` resolves to (pipe access is plain allowlist membership, the same as any other role). +**Public (unauthenticated) access is driven by the policy.** Define a usable `default_role` and no-token requests are evaluated as that role (see [Roles & Access Control](#roles--access-control)); remove it and roleless requests are denied. Setting `default_role` equal to the `admin_role` is allowed — it makes every unauthenticated request admin (including `/v1/ops/*`), handy for local/dev — but it is logged loudly on every node that loads such a policy and must not be used in production. `/v1/ops/*` (raw SQL, policy/pipe CRUD, schema, DLQ) is admin-only, and a pipe with **no `allowed_roles` authorizes nobody but the admin role** — but a pipe *can* be reached by the public when its `allowed_roles` lists the role the `default_role` resolves to (pipe access is plain allowlist membership, the same as any other role). -**Operator key (non-JWT, break-glass).** A separate, role-free credential — `auth.operator_key` — authorizes a caller as a **full-access platform operator**: the entire data plane *and* the `/v1/admin/*` surface, without a JWT and independently of the token verifier. Present it in the standard `Authorization` header with the `Operator` scheme (forwarded verbatim by proxies, no collision with Bearer JWTs), or via the `X-Operator-Key` alias: +**Operator key (non-JWT, break-glass).** A separate, role-free credential — `auth.operator_key` — authorizes a caller as a **full-access platform operator**: the entire data plane *and* the `/v1/ops/*` surface, without a JWT and independently of the token verifier. Present it in the standard `Authorization` header with the `Operator` scheme (forwarded verbatim by proxies, no collision with Bearer JWTs), or via the `X-Operator-Key` alias: ```text Authorization: Operator @@ -185,12 +185,12 @@ Accepts a single flat JSON object, a JSON array of objects, or a newline-delimit | a JSON array of objects (any length, even 1) | `application/json` | per-record summary — see [Batch Ingest](#batch-ingest) | | one JSON object per line (NDJSON) | `application/x-ndjson` | per-record summary — see [Batch Ingest](#batch-ingest) | -The inbound request body is capped at 16 MiB; a body over the cap is rejected with `413` (matching [`POST /v1/admin/query`](#post-v1adminquery--query-clickhouse)). For uploads larger than that, use the streaming NDJSON form below rather than one big body, and set your own outer limit at the [reverse proxy](/reverse-proxy#request-body-size-limits). +The inbound request body is capped at 16 MiB; a body over the cap is rejected with `413` (matching [`POST /v1/ops/query`](#post-v1opsquery--query-clickhouse)). For uploads larger than that, use the streaming NDJSON form below rather than one big body, and set your own outer limit at the [reverse proxy](/reverse-proxy#request-body-size-limits). The `{table}` URL query must match a table that exists in ClickHouse. WaveHouse discovers table schemas on startup and refreshes them periodically. :::note[Insert-only] -The ingest pipeline accepts only inserts. All other mutations — `DELETE`, `UPDATE`, `TRUNCATE`, `DROP`, `ALTER`, `REPLACE`, etc. — must be issued through [`POST /v1/admin/query`](#post-v1adminquery--query-clickhouse), which is restricted to the admin role (`admin_role`, the same gate as the rest of `/v1/admin/*`). +The ingest pipeline accepts only inserts. All other mutations — `DELETE`, `UPDATE`, `TRUNCATE`, `DROP`, `ALTER`, `REPLACE`, etc. — must be issued through [`POST /v1/ops/query`](#post-v1opsquery--query-clickhouse), which is restricted to the admin role (`admin_role`, the same gate as the rest of `/v1/ops/*`). The policy engine authorizes mutations by inspecting the columns being written. That works for inserts but not for predicate-driven mutations like `DELETE … WHERE` — there's no way to prove the predicate matches only rows the caller is allowed to touch. Routing those statements through the admin-gated raw-SQL surface keeps the policy contract honest. ::: @@ -281,7 +281,7 @@ curl -X POST "http://localhost:8080/v1/ingest?table=clicks" \ WaveHouse pins `date_time_input_format=best_effort` on its inserts — the ClickHouse server default since 26.5. On an older server whose default was `basic`, a plain `DateTime` column read an all-digit timestamp string of five or more digits as Unix seconds (shorter runs it rejected outright, where `best_effort` reads `"2026"` as a year); under `best_effort`, `"20260711"` stores 2026-07-11, not 1970-08-23, and some lengths (e.g. 12 digits) are rejected outright. (`DateTime64` columns diverge the same way on calendar-shaped runs — `"20260711"` is 1970-08-23 under `basic`, 2026-07-11 under `best_effort` — and additionally whenever an epoch run's unit doesn't match the column scale, e.g. a 16-digit microsecond epoch into a `DateTime64(3)`; an epoch run whose unit matches the column scale (a 13-digit millisecond epoch into a `DateTime64(3)`) reads identically too — only 9–10-digit Unix-seconds runs, with an optional fraction, agree at *every* scale.) The canonical form itself is what the pin rescues: under `basic` an RFC 3339 value's `Z` suffix is rejected outright (the row fails and lands in the DLQ), and the pin is what makes it insertable regardless of server version. Zone-less date-times and 9–10-digit Unix-seconds strings parse identically under both settings. ::: -**The canonical form, precisely.** This is the one strict timestamp spelling in WaveHouse — the same one `/v1/query` and `/v1/pipes/{name}` render for top-level timestamp columns and the SSE stream carries (the raw-SQL proxy `/v1/admin/query` instead renders server-side via `date_time_output_format=iso`, which keeps trailing fraction zeros): +**The canonical form, precisely.** This is the one strict timestamp spelling in WaveHouse — the same one `/v1/query` and `/v1/pipes/{name}` render for top-level timestamp columns and the SSE stream carries (the raw-SQL proxy `/v1/ops/query` instead renders server-side via `date_time_output_format=iso`, which keeps trailing fraction zeros): - `YYYY-MM-DDTHH:MM:SSZ`, or `YYYY-MM-DDTHH:MM:SS.FZ` when there is a sub-second part: uppercase `T` separator, uppercase `Z` suffix, always UTC — never a numeric offset — and seconds always present. - The fraction is **truncated** (never rounded) to the column's precision: a `DateTime` column (whole seconds) never carries a fraction; a `DateTime64(3)` column carries at most three digits. @@ -377,7 +377,7 @@ curl -X POST "http://localhost:8080/v1/ingest?table=clicks" \ --- -### `POST /v1/admin/query` — Query ClickHouse +### `POST /v1/ops/query` — Query ClickHouse Executes a SQL statement directly against ClickHouse. **WaveHouse proxies the SQL string verbatim to ClickHouse's HTTP interface** — any statement ClickHouse accepts works, including arbitrary DDL/DML/SYSTEM verbs and inline FORMAT directives. Multi-statement input (`SELECT 1; TRUNCATE t`) also works on recent ClickHouse versions where multi-query is enabled by default; older or restrictively-configured servers may reject the second statement with a clear error. Read queries return a JSON array of result rows; mutations/DDL return HTTP 200 with `[]` on success. DateTime columns are ISO-8601 formatted via the upstream `date_time_output_format=iso` setting — server-side rendering that keeps trailing fraction zeros, so a `DateTime64(3)` whole-second value returns `.000Z` here where `/v1/query` renders plain `Z`; other types are returned as ClickHouse renders them under `FORMAT JSON`. @@ -392,10 +392,10 @@ The proxy buffers the upstream response in memory before forwarding (no row-stre This endpoint **does not cache, does not singleflight, and emits `Cache-Control: no-store`** — every request goes straight to ClickHouse, mutation or read, and downstream HTTP caches are explicitly told not to store the response. Raw SQL is an admin escape hatch with infrequent, ad-hoc traffic, so the L1/singleflight machinery would only add complexity without a real hit-rate win. Use [`POST /v1/query?table={table}`](#post-v1querytabletable--structured-query) or [`GET/POST /v1/pipes/{name}`](#getpost-v1pipesname--execute-named-pipe) for the cached read paths (dashboards, high-QPS clients, etc.) — both share an in-process L1 (Ristretto) with singleflight coalescing. :::note[Admin only] -The route is mounted under `/v1/admin/*`, behind the `RequireAdmin` gate: only a caller whose JWT role equals the policy `admin_role` (`"admin"` by default) may use it. A request with no/invalid token resolves to the `default_role` (not the admin role unless `default_role` is deliberately set to it — a loudly-warned dev-only setting) and is rejected. Raw SQL has no per-statement scope check (a full SQL parser would be needed to authorize predicates), so the role gate is the entire authorization story, shared with the rest of `/v1/admin/*` (policy CRUD, pipes CRUD). The normal surfaces for non-admin callers are `POST /v1/ingest?table={table}` for writes, `POST /v1/query?table={table}` for structured reads, and `GET/POST /v1/pipes/{name}` for pre-defined queries — none of which expose raw SQL. +The route is mounted under `/v1/ops/*`, behind the `RequireAdmin` gate: only a caller whose JWT role equals the policy `admin_role` (`"admin"` by default) — or who presents the non-JWT [operator key](#authentication) — may use it. A tokenless request (or a valid token without a role claim) resolves to the `default_role` (not the admin role unless `default_role` is deliberately set to it — a loudly-warned dev-only setting) and is rejected with `403`; a present-but-invalid token — expired, malformed, bad signature — keeps its stashed verification error and fails loud with `401` instead. Raw SQL has no per-statement scope check (a full SQL parser would be needed to authorize predicates), so the role gate is the entire authorization story, shared with the rest of `/v1/ops/*` (see [Admin Endpoints](#admin-endpoints)). The normal surfaces for non-admin callers are `POST /v1/ingest?table={table}` for writes, `POST /v1/query?table={table}` for structured reads, and `GET/POST /v1/pipes/{name}` for pre-defined queries — none of which expose raw SQL. ::: -`/v1/admin/query` is the only sanctioned surface for non-insert mutations (the ingest pipeline is insert-only). Granting raw-SQL access to a non-admin role via the policy engine is no longer supported: authenticate with the admin role (`admin_role`). +`/v1/ops/query` is the only sanctioned surface for non-insert mutations (the ingest pipeline is insert-only). Granting raw-SQL access to a non-admin role via the policy engine is no longer supported: authenticate with the admin role (`admin_role`). **Request:** @@ -443,7 +443,7 @@ The earlier handler accepted a `params` array bound to `?` placeholders; the HTT ```bash # Requires an admin-role JWT — see "Generating a JWT for Testing" below. -curl -X POST http://localhost:8080/v1/admin/query \ +curl -X POST http://localhost:8080/v1/ops/query \ -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" \ -d '{"sql": "SELECT * FROM clicks LIMIT 10"}' @@ -498,7 +498,7 @@ Table, column, and alias names may contain any characters ClickHouse accepts — **Response:** -JSON array of result rows. Top-level `DateTime`/`DateTime64` values are returned in canonical RFC 3339 UTC (`2026-06-21T04:00:00.123Z`) — `Nullable` timestamp columns included (a SQL `NULL` renders as JSON `null`), while timestamps nested inside `Array`/`Map`/`Tuple` columns are rendered in the column's declared zone, else the ClickHouse server's, as the driver returns them — byte-identical to the [SSE stream](#get-v1stream--server-sent-events-stream) for values [canonicalized at ingest](#timestamp-canonicalization) (a fail-open pass-through that ClickHouse accepted still comes back canonical here, though it streamed in the producer's spelling). The response carries an `X-Cache: HIT` or `X-Cache: MISS` header — this endpoint shares the in-process L1 (Ristretto) + singleflight machinery (unlike `/v1/admin/query`, which always hits ClickHouse). +JSON array of result rows. Top-level `DateTime`/`DateTime64` values are returned in canonical RFC 3339 UTC (`2026-06-21T04:00:00.123Z`) — `Nullable` timestamp columns included (a SQL `NULL` renders as JSON `null`), while timestamps nested inside `Array`/`Map`/`Tuple` columns are rendered in the column's declared zone, else the ClickHouse server's, as the driver returns them — byte-identical to the [SSE stream](#get-v1stream--server-sent-events-stream) for values [canonicalized at ingest](#timestamp-canonicalization) (a fail-open pass-through that ClickHouse accepted still comes back canonical here, though it streamed in the producer's spelling). The response carries an `X-Cache: HIT` or `X-Cache: MISS` header — this endpoint shares the in-process L1 (Ristretto) + singleflight machinery (unlike `/v1/ops/query`, which always hits ClickHouse). The inbound request body is capped at 1 MiB; a body over the cap is rejected with `413`. A query AST is bounded by nature (far under 1 MiB even with a large `in`-list), and the cap blocks a single-request memory-exhaustion vector on this public endpoint. Set a tighter or higher outer limit at your [reverse proxy](/reverse-proxy#request-body-size-limits) — but it can only narrow the effective limit, not raise it past this cap. @@ -517,7 +517,7 @@ The inbound request body is capped at 1 MiB; a body over the cap is rejected wit ### `GET/POST /v1/pipes/{name}` — Execute Named Pipe -Executes a pre-defined named query (pipe) with parameter binding. Parameters can be supplied via query string and/or JSON body. Results are cached in the shared L1 (Ristretto) with singleflight coalescing — same machinery as the structured query endpoint, and again, unlike `/v1/admin/query`. +Executes a pre-defined named query (pipe) with parameter binding. Parameters can be supplied via query string and/or JSON body. Results are cached in the shared L1 (Ristretto) with singleflight coalescing — same machinery as the structured query endpoint, and again, unlike `/v1/ops/query`. **Query Parameters:** Any key matching a pipe parameter name. @@ -534,7 +534,7 @@ Executes a pre-defined named query (pipe) with parameter binding. Parameters can JSON array of result rows, with `X-Cache: HIT` or `X-Cache: MISS` indicating whether the row came from the in-process L1. -The POST parameter body is capped at 1 MiB; a body over the cap is rejected with `413` (the same control-plane cap as [`POST /v1/query`](#post-v1querytabletable--structured-query) — see [reverse proxy → body limits](/reverse-proxy#request-body-size-limits)). A malformed-but-within-cap body is ignored rather than rejected, since parameters may legitimately come from the query string alone. +The POST parameter body is capped at 1 MiB; a body over the cap is rejected with `413` (the same 1 MiB parameter/AST-body cap as [`POST /v1/query`](#post-v1querytabletable--structured-query) — see [reverse proxy → body limits](/reverse-proxy#request-body-size-limits)). A malformed-but-within-cap body is ignored rather than rejected, since parameters may legitimately come from the query string alone. **Error responses:** @@ -601,13 +601,15 @@ curl -N "http://localhost:8080/v1/stream?table=clicks&since=2026-03-24T11:00:00Z --- -### `GET /v1/schema` — List All Table Schemas +### Admin Endpoints -Returns all discovered ClickHouse table schemas. +Every admin-gated surface lives under the `/v1/ops/*` prefix, behind a single `RequireAdmin` gate: schema discovery, DLQ stats, and the policy and pipe CRUD below, plus the raw-SQL passthrough [`POST /v1/ops/query`](#post-v1opsquery--query-clickhouse) documented with the query endpoints above. They require the policy `admin_role` (`"admin"` by default, exact case-sensitive match) — or the non-JWT [operator key](#authentication), which reaches the same surface without a token; other callers get 401 (present-but-invalid token) / 403, and the quickstart's trial `public` role cannot call any of them. There is no separate `service` role. The JWT middleware always runs — a tokenless request (or a valid token without a role claim) resolves to the `default_role` (not the admin role unless `default_role` is deliberately set to it — a loudly-warned dev-only setting) and is denied `403`, while a present-but-invalid token keeps its stashed verification error and is denied `401`. -:::note[Admin only] -The schema and DLQ endpoints in this section require the `admin_role` (like [`/v1/admin/query`](#post-v1adminquery--query-clickhouse)); other callers get 401 (bad token) / 403. The quickstart's trial `public` role cannot call them. -::: +The admin endpoints in this section that accept a request body — `PUT /v1/ops/policy`, `POST /v1/ops/policy/validate`, and `PUT /v1/ops/pipes/{name}` — cap it at 1 MiB (the same 1 MiB parameter/AST-body cap as `POST /v1/query`); an over-cap body is rejected with `413 {"error":"request body exceeded 1048576 bytes"}`. A policy document or pipe definition is bounded, so this never binds legitimate use. The raw-SQL `POST /v1/ops/query` instead carries the 16 MiB bulk-payload cap documented with the query endpoints above. + +#### `GET /v1/ops/schema` — List All Table Schemas + +Returns all discovered ClickHouse table schemas. **Response:** @@ -626,7 +628,7 @@ The schema and DLQ endpoints in this section require the `admin_role` (like [`/v --- -### `GET /v1/schema?table={table}` — Get Table Schema +#### `GET /v1/ops/schema?table={table}` — Get Table Schema Returns the schema for a specific table. @@ -652,9 +654,9 @@ Returns the schema for a specific table. --- -### `POST /v1/schema/refresh` — Refresh Schemas +#### `POST /v1/ops/schema/refresh` — Refresh Schemas -Triggers an immediate re-discovery of ClickHouse table schemas, then returns the refreshed schema list (same array shape as `GET /v1/schema`). Admin-only, like the rest of this section. +Triggers an immediate re-discovery of ClickHouse table schemas, then returns the refreshed schema list (same array shape as `GET /v1/ops/schema`). Admin-only, like the rest of this section. **Error responses:** @@ -678,9 +680,9 @@ Triggers an immediate re-discovery of ClickHouse table schemas, then returns the --- -### `GET /v1/dlq/stats` — DLQ Statistics +#### `GET /v1/ops/dlq/stats` — DLQ Statistics -Returns per-table message counts in the Dead Letter Queue. Admin-only, like the rest of this section. Before any failure has ever occurred, the endpoint returns `200` with `{"tables":{},"total":0}`. +Returns per-table message counts in the Dead Letter Queue. Admin-only, like the rest of this section, and registered only while the DLQ is enabled (`dlq.enabled`, the default). Before any failure has ever occurred, the endpoint returns `200` with `{"tables":{},"total":0}`. **Error responses:** @@ -688,6 +690,7 @@ Returns per-table message counts in the Dead Letter Queue. Admin-only, like the | ------ | ---- | ----- | | 401 | `{"error":"invalid token"}` / `{"error":"token expired"}` | A present-but-invalid/expired token was supplied and denied (the gate surfaces the token reason) | | 403 | `{"error":"forbidden"}` | Caller's role is not the policy `admin_role` (`"admin"` by default) | +| 404 | `{"error":"not found"}` | The DLQ is disabled (`dlq.enabled: false`), so the route is not registered. Only observable as admin — a non-admin caller is denied by the tree-level gate first (403) | | 500 | `{"error":"stream info failed"}` | NATS JetStream stream-info lookup failed | **Query Parameters:** @@ -710,17 +713,11 @@ Returns per-table message counts in the Dead Letter Queue. Admin-only, like the --- -### Admin Endpoints - -Admin endpoints require the policy `admin_role` (`"admin"` by default, exact case-sensitive match). There is no separate `service` role. The JWT middleware always runs — a request with no/invalid token resolves to the `default_role` (not the admin role unless `default_role` is deliberately set to it — a loudly-warned dev-only setting) and is denied. - -The admin endpoints that accept a request body — `PUT /v1/admin/policy`, `POST /v1/admin/policy/validate`, and `PUT /v1/admin/pipes/{name}` — cap it at 1 MiB (the same control-plane backstop as the public read endpoints); an over-cap body is rejected with `413 {"error":"request body exceeded 1048576 bytes"}`. A policy document or pipe definition is bounded, so this never binds legitimate use. - -#### `GET /v1/admin/policy` — Get Access Control Policy +#### `GET /v1/ops/policy` — Get Access Control Policy Returns the current access control policy. -#### `PUT /v1/admin/policy` — Update Access Control Policy +#### `PUT /v1/ops/policy` — Update Access Control Policy Replaces the entire access control policy. Validated before saving. @@ -756,21 +753,21 @@ Replaces the entire access control policy. Validated before saving. } ``` -The `default_role` field (optional) is the role assigned to any request that reaches the policy engine **without** a role — a valid token carrying no role claim, a request with no token at all, or one whose token was invalid/expired. **Setting it enables unauthenticated access:** roleless requests are evaluated as that role and receive exactly its permissions (or are denied if it grants none on the table/operation). If `default_role` is unset, a roleless request is denied. Setting it equal to the `admin_role` is allowed — every roleless request then becomes admin (including `/v1/admin/*`), which is handy for local/dev — but each node that loads such a policy logs a loud warning, and it must not be used in production. +The `default_role` field (optional) is the role assigned to any request that reaches the policy engine **without** a role — a valid token carrying no role claim, a request with no token at all, or one whose token was invalid/expired. **Setting it enables unauthenticated access:** roleless requests are evaluated as that role and receive exactly its permissions (or are denied if it grants none on the table/operation). If `default_role` is unset, a roleless request is denied. Setting it equal to the `admin_role` is allowed — every roleless request then becomes admin (including `/v1/ops/*`), which is handy for local/dev — but each node that loads such a policy logs a loud warning, and it must not be used in production. -#### `POST /v1/admin/policy/validate` — Validate Policy (Dry Run) +#### `POST /v1/ops/policy/validate` — Validate Policy (Dry Run) Validates a policy without saving it. Returns `{"valid": true}` or an error. -#### `GET /v1/admin/pipes` — List Named Pipes +#### `GET /v1/ops/pipes` — List Named Pipes Returns all registered named query pipes. -#### `GET /v1/admin/pipes/{name}` — Get Named Pipe +#### `GET /v1/ops/pipes/{name}` — Get Named Pipe Returns a specific named pipe definition. -#### `PUT /v1/admin/pipes/{name}` — Create/Update Named Pipe +#### `PUT /v1/ops/pipes/{name}` — Create/Update Named Pipe ```json { @@ -786,7 +783,7 @@ Returns a specific named pipe definition. **`allowed_roles`** restricts execution: the caller's role (a tokenless or roleless request is first resolved to the policy `default_role`) must appear in the list. The admin role (`admin_role`) always passes. Matching is exact — there is no `"*"` wildcard — and empty-string entries are ignored. An empty or omitted list authorizes **nobody but the admin role**, and a request whose role is absent or unlisted is denied (fails closed). -#### `DELETE /v1/admin/pipes/{name}` — Delete Named Pipe +#### `DELETE /v1/ops/pipes/{name}` — Delete Named Pipe ## Event Message Format @@ -832,7 +829,7 @@ Same as the wire format — events are passed through directly: When a batch insert to ClickHouse fails (e.g., type errors, connection issues), the worker re-inserts the batch row by row: rows that succeed are acked, and only the rows that fail again are published to the DLQ NATS stream (`WAVEHOUSE_DLQ`) under subjects `dlq.{table}`. This prevents infinite retry loops — those messages are ACKed from the main stream and moved to the DLQ for inspection. The DLQ message body is the published `EventMessage` envelope (`{"table_name":…,"received_timestamp":…,"data":{…}}` — the failed row is under its `data` key, its `DateTime`/`DateTime64` values as published: canonicalized where WaveHouse could parse them, otherwise the producer's original spelling — see [timestamp canonicalization](#timestamp-canonicalization)); the failure reason, table, and time travel in the `X-DLQ-Table` / `X-DLQ-Error` / `X-DLQ-Timestamp` message headers. -Use `GET /v1/dlq/stats` to monitor DLQ depth. +Use `GET /v1/ops/dlq/stats` to monitor DLQ depth. ## Generating a JWT for Testing diff --git a/docs/src/content/docs/architecture.md b/docs/src/content/docs/architecture.md index 1d25cbf5..37736584 100644 --- a/docs/src/content/docs/architecture.md +++ b/docs/src/content/docs/architecture.md @@ -69,13 +69,13 @@ internal/ The API layer uses [Chi](https://github.com/go-chi/chi) for routing with RequestID, a CORS middleware, and a custom JSON recoverer (`jsonRecoverer`) that emits a JSON `500` on panic instead of chi's plain-text `middleware.Recoverer`. -- **router.go** — Route definitions. Public: `/livez`, `/readyz`, and the content-free `/v1/health` SDK ping (plus the permanent `/healthz` alias and the deprecated `/health`, `/ready` aliases). Policy-gated: `/v1/ingest?table={table}`, `/v1/query?table={table}` (structured), `/v1/pipes/{name}` (named pipes), `/v1/stream`. Admin-only (`RequireAdmin` — role == `policy.admin_role`, or a request bearing the operator key's operator bit, which passes even under a nil policy): `/v1/schema/*`, `/v1/dlq/stats`, `/v1/admin/policy`, `/v1/admin/pipes/*`, `/v1/admin/query` (raw SQL — same gate as the rest of `/v1/admin/*`). +- **router.go** — Route definitions. Public: `/livez`, `/readyz`, and the content-free `/v1/health` SDK ping (plus the permanent `/healthz` alias and the deprecated `/health`, `/ready` aliases). Policy-gated: `/v1/ingest?table={table}`, `/v1/query?table={table}` (structured), `/v1/pipes/{name}` (named pipes), `/v1/stream`. Admin-only (`RequireAdmin` — role == `policy.admin_role`, or a request bearing the operator key's operator bit, which passes even under a nil policy): `/v1/ops/schema/*`, `/v1/ops/dlq/stats`, `/v1/ops/policy`, `/v1/ops/pipes/*`, `/v1/ops/query` (raw SQL — same gate as the rest of `/v1/ops/*`). - **auth middleware** — the JWT/JWKS authentication middleware is its own package, [`auth/`](#auth--authentication); the router runs it on every `/v1/*` route. -- **policy.go** — CRUD handler for access control policies (`/v1/admin/policy`). +- **policy.go** — CRUD handler for access control policies (`/v1/ops/policy`). - **pipes.go** — Named query pipe handlers: admin CRUD and execution with parameter binding. - **structured_query.go** — Handler for `POST /v1/query?table={table}`: validates query AST, enforces permissions, builds and executes SQL. - **ingest.go** — Accepts flat JSON body for `POST /v1/ingest?table={table}`, validates against discovered schema, optional dedup, publishes to NATS subject `ingest.{table}`. When dedup is on, a row missing the configured `id_field` can't be deduped: it is logged at `WARN` and counted by `wavehouse_ingest_dedupe_missing_id_total` (labeled by `table`), then published un-deduped — or rejected when `dedupe.require_id` is set ([#219](https://github.com/Wave-RF/WaveHouse/issues/219)). -- **query.go** — Proxies raw SQL for `POST /v1/admin/query` straight to ClickHouse's HTTP interface. **Not cached** — sets `Cache-Control: no-store` so every request hits ClickHouse; DateTime is rendered ISO-8601 via `date_time_output_format=iso` (the Go-side type conversion lives in the structured-query / pipes path, not here). +- **query.go** — Proxies raw SQL for `POST /v1/ops/query` straight to ClickHouse's HTTP interface. **Not cached** — sets `Cache-Control: no-store` so every request hits ClickHouse; DateTime is rendered ISO-8601 via `date_time_output_format=iso` (the Go-side type conversion lives in the structured-query / pipes path, not here). - **stream.go** — Real-time streaming via SSE. Callers select a table with the `?table=` query parameter. Each connection registers one `Subscriber` (the `stream/` package) with both the event `Hub` (under its `(topic, role)`) and the shared keepalive wheel, then drains both from a single byte-pump — so idle streams keep emitting `:` keepalive comments (surviving reverse-proxy idle timeouts) while live events arrive already projected and serialized. Per-event projection/serialization happens **once per role** in the `Hub`, not once per subscriber ([#294](https://github.com/Wave-RF/WaveHouse/issues/294)); the handler also snapshots the connection's JWT claims onto the `Subscriber`, which the `Hub` evaluates per subscriber when the role carries a row-level `filter` ([#319](https://github.com/Wave-RF/WaveHouse/issues/319)). Gap-fill replay from NATS JetStream (`DeliverByStartTime`) stays per-connection (low-volume, one-time on connect). - **schema.go** — Schema discovery API: list all schemas, get one table, trigger refresh. - **dlq.go** — DLQ stats endpoint and `EnsureDLQStream` helper for creating the `WAVEHOUSE_DLQ` NATS stream. @@ -120,7 +120,7 @@ The SSE fan-out, factored out of `api/` so the delivery hot path ([#294](https:/ ### `ingest/` — Ingest Pipeline, DLQ & Sweeping -- **worker.go** — `StartIngestWorker` launches an ingest pipeline: a JetStream consumer reads from the `WAVEHOUSE` stream via a durable `buffer-consumer` pull subscription, batches events per table, and performs bulk INSERTs to ClickHouse. The pipeline is **insert-only**. The wire format `EventMessage` carries `{table_name, received_timestamp, data}` and nothing else; the worker accepts any table name now (the table name in the NATS subject is `query.SafeEncodeNATS(rawUnsafeTableName)`), then bulk-INSERTs. The embedded NATS server runs with `DontListen: true` (`internal/mq/embedded.go`), so the only Publishers reachable on the `ingest.>` subjects are in-process Go code — today, only the HTTP `/v1/ingest?table={table}` handler. Non-insert mutations (`DELETE`/`UPDATE`/`TRUNCATE`/…) must go through `POST /v1/admin/query` under the admin role (`policy.admin_role`) — see the Query Path section below; the `/v1/admin/*` `RequireAdmin` middleware enforces the check at the API layer, so a no/invalid-token request (resolved to `default_role`, not admin in a production config) never reaches the proxy. On a bulk-insert failure the batch is re-inserted row by row; rows that succeed are acked, and only the rows that fail again are routed to the DLQ (`sendToDLQ`), which republishes the as-published `EventMessage` envelope to `dlq.{table}` NATS subjects with the failure context in `X-DLQ-*` headers when DLQ is enabled — see [Ingest Pipeline](/ingest-pipeline) for the worker internals. +- **worker.go** — `StartIngestWorker` launches an ingest pipeline: a JetStream consumer reads from the `WAVEHOUSE` stream via a durable `buffer-consumer` pull subscription, batches events per table, and performs bulk INSERTs to ClickHouse. The pipeline is **insert-only**. The wire format `EventMessage` carries `{table_name, received_timestamp, data}` and nothing else; the worker accepts any table name now (the table name in the NATS subject is `query.SafeEncodeNATS(rawUnsafeTableName)`), then bulk-INSERTs. The embedded NATS server runs with `DontListen: true` (`internal/mq/embedded.go`), so the only Publishers reachable on the `ingest.>` subjects are in-process Go code — today, only the HTTP `/v1/ingest?table={table}` handler. Non-insert mutations (`DELETE`/`UPDATE`/`TRUNCATE`/…) must go through `POST /v1/ops/query` under the admin role (`policy.admin_role`) — see the Query Path section below; the `/v1/ops/*` `RequireAdmin` middleware enforces the check at the API layer, so a no/invalid-token request (resolved to `default_role`, not admin in a production config) never reaches the proxy. On a bulk-insert failure the batch is re-inserted row by row; rows that succeed are acked, and only the rows that fail again are routed to the DLQ (`sendToDLQ`), which republishes the as-published `EventMessage` envelope to `dlq.{table}` NATS subjects with the failure context in `X-DLQ-*` headers when DLQ is enabled — see [Ingest Pipeline](/ingest-pipeline) for the worker internals. - **types.go** — `EventMessage` struct (TableName, ReceivedTimestamp, Data) and `BufferConsumerName` constant, shared across API handlers and the ingest pipeline. - **sweeper.go** — `Sweeper` implements the Active Sweeper pattern. It runs every minute and purges NATS JetStream messages that are **both** ACKed by the buffer consumer (written to ClickHouse) **and** older than the configurable gap window. @@ -191,8 +191,8 @@ Ingest worker pipeline (StartIngestWorker): (Insert-only pipeline. The wire format `EventMessage` carries only {table_name, received_timestamp, data}; non-insert mutations - DELETE/UPDATE/TRUNCATE/DROP/etc. must go through POST /v1/admin/query — the - /v1/admin/* RequireAdmin gate rejects non-admin callers at the API layer, so + DELETE/UPDATE/TRUNCATE/DROP/etc. must go through POST /v1/ops/query — the + /v1/ops/* RequireAdmin gate rejects non-admin callers at the API layer, so a no/invalid-token request (resolved to default_role, not admin in a production config) cannot reach the proxy.) @@ -206,16 +206,21 @@ Active Sweeper (async goroutine, every 60s): ### Query Path ```text -Client POST /v1/admin/query - → JWT auth middleware (always runs; no/invalid token → empty role) - → /v1/admin RequireAdmin (role == policy.admin_role, or the operator-key bit) — single gate shared - with the rest of /v1/admin/* (policy CRUD, pipes CRUD). Raw SQL has - no per-statement scope check (a full SQL parser would be needed to - authorize predicates), so the role gate is the entire authorization - story. /v1/admin/query is the only sanctioned surface for non-SELECT - statements (DELETE/UPDATE/TRUNCATE/DROP/ALTER/…); non-admin callers - use `POST /v1/ingest?table={table}` for writes and the structured query - endpoint or named pipes for reads. +Client POST /v1/ops/query + → JWT auth middleware (always runs, never rejects; a bad token yields an + empty role and stashes its verification error for the denying gate) + → policy.ResolveRole (empty role → default_role — the one sanctioned + roleless exception) + → /v1/ops RequireAdmin (resolved role == policy.admin_role, or the + operator-key bit) — single gate shared with the rest of /v1/ops/* + (policy CRUD, pipes CRUD, schema discovery, DLQ stats). A denial is + 401 when a stashed error shows the caller presented an invalid token, + else 403. Raw SQL has no per-statement scope check (a full SQL parser + would be needed to authorize predicates), so the role gate is the + entire authorization story. /v1/ops/query is the only sanctioned + surface for non-SELECT statements (DELETE/UPDATE/TRUNCATE/DROP/ALTER/…); + non-admin callers use `POST /v1/ingest?table={table}` for writes and + the structured query endpoint or named pipes for reads. → Decode {"sql": "..."} from the request body. → POST the SQL verbatim to ClickHouse's HTTP interface at ://:/?default_format=JSON diff --git a/docs/src/content/docs/configuration.mdx b/docs/src/content/docs/configuration.mdx index 4a615484..3380a017 100644 --- a/docs/src/content/docs/configuration.mdx +++ b/docs/src/content/docs/configuration.mdx @@ -53,7 +53,7 @@ The server speaks **plain HTTP** — there is no inbound-TLS setting. Terminate | YAML Key | Env Var | Default | Description | | --- | --- | ------- | ----------- | | `clickhouse.addr` | `WH_CH_ADDR` | `localhost:9000` | ClickHouse native protocol address. | -| `clickhouse.http_port` | `WH_CH_HTTP_PORT` | `8123` | ClickHouse HTTP interface port. Used by the ingest worker (`internal/ingest`) for bulk INSERT and by the raw-SQL proxy (`POST /v1/admin/query`, `internal/api/query.go`) to forward SQL to ClickHouse. Schema discovery uses the native protocol on `addr` instead. | +| `clickhouse.http_port` | `WH_CH_HTTP_PORT` | `8123` | ClickHouse HTTP interface port. Used by the ingest worker (`internal/ingest`) for bulk INSERT and by the raw-SQL proxy (`POST /v1/ops/query`, `internal/api/query.go`) to forward SQL to ClickHouse. Schema discovery uses the native protocol on `addr` instead. | | `clickhouse.http_scheme` | `WH_CH_HTTP_SCHEME` | `http` | HTTP scheme for the ClickHouse HTTP interface (`http` or `https`). Set to `https` for TLS-encrypted ClickHouse connections. | | `clickhouse.database` | `WH_CH_DATABASE` | `default` | Database name. Tables are discovered from this database. | | `clickhouse.username` | `WH_CH_USERNAME` | `default` | Authentication username. | @@ -112,7 +112,7 @@ WaveHouse's per-role caps are sent as per-query `SETTINGS` on its connection, so | YAML Key | Env Var | Default | Description | | --- | --- | ------- | ----------- | -| `schema.refresh_interval` | `WH_SCHEMA_REFRESH_INTERVAL` | `60` | How often (in seconds) to re-discover ClickHouse table schemas. Also refreshable on-demand via `POST /v1/schema/refresh` (admin-only). | +| `schema.refresh_interval` | `WH_SCHEMA_REFRESH_INTERVAL` | `60` | How often (in seconds) to re-discover ClickHouse table schemas. Also refreshable on-demand via `POST /v1/ops/schema/refresh` (admin-only). | ### Message Queue (NATS) @@ -145,16 +145,16 @@ WaveHouse's per-role caps are sent as per-query `SETTINGS` on its connection, so | `auth.jwt_secret` | `WH_AUTH_JWT_SECRET` | *(empty)* | HMAC secret for JWT validation. Set this (or `jwks_url`) so presented tokens are verified; see [Access Control](/access-control). | | `auth.jwks_url` | `WH_AUTH_JWKS_URL` | *(empty)* | JWKS endpoint URL for public key validation (e.g., `https://auth.example.com/.well-known/jwks.json`). When set, JWKS is the **sole** verifier and `jwt_secret` is ignored (not a per-token fallback); the endpoint must be reachable at startup or the server fails to boot. | | `auth.role_claim` | `WH_AUTH_ROLE_CLAIM` | `role` | Dot-separated JWT claim path for role extraction (e.g., `app_metadata.role`). | -| `auth.operator_key` | `WH_AUTH_OPERATOR_KEY` | *(empty)* | Non-JWT operator credential. A request presenting it — in an `Authorization: Operator ` header, or the `X-Operator-Key` alias — is authorized as a full-access platform operator (the whole data plane *and* the `/v1/admin/*` management surface), independent of the JWT verifier, and it is honored even when the policy is missing/deleted (break-glass). Empty disables it. Treat it as an admin secret. See below and [Access Control](/access-control). | +| `auth.operator_key` | `WH_AUTH_OPERATOR_KEY` | *(empty)* | Non-JWT operator credential. A request presenting it — in an `Authorization: Operator ` header, or the `X-Operator-Key` alias — is authorized as a full-access platform operator (the whole data plane *and* the `/v1/ops/*` management surface), independent of the JWT verifier, and it is honored even when the policy is missing/deleted (break-glass). Empty disables it. Treat it as an admin secret. See below and [Access Control](/access-control). | WaveHouse accepts only the signing algorithms matching the active verifier — `HS256`/`HS384`/`HS512` for the HMAC secret, or the asymmetric family (`RS*`/`ES*`/`PS*`/`EdDSA`) for JWKS — and validates the token's `alg` before any key is used, so your IdP must sign with one of these and `alg: none` is always rejected. **There is no auth on/off switch.** The JWT middleware always runs. A request with no token, or an invalid/expired one, falls back to the policy `default_role`; elevated access needs a valid token whose role is granted (or equals the policy `admin_role`). The privileged role and public access are **policy** settings, not config flags: -- **`admin_role`** (policy field, `"admin"` by default, exact case-sensitive match): the role granted full access and the `/v1/admin/*` gate. There is no separate `service` role, though the non-JWT `operator_key` (below) reaches the same surface without a token. -- **`default_role`** (policy field): set it to open public (no-token) access — roleless requests are evaluated as that role; remove it to close public access. Setting it equal to `admin_role` is allowed and makes every roleless request admin (including `/v1/admin/*`) — handy for local/dev, logged loudly on every node that loads such a policy, and not for production use. `/v1/admin/*` and the schema/DLQ endpoints are admin-only, and a pipe with no `allowed_roles` authorizes nobody but the admin role. +- **`admin_role`** (policy field, `"admin"` by default, exact case-sensitive match): the role granted full access and the `/v1/ops/*` gate. There is no separate `service` role, though the non-JWT `operator_key` (below) reaches the same surface without a token. +- **`default_role`** (policy field): set it to open public (no-token) access — roleless requests are evaluated as that role; remove it to close public access. Setting it equal to `admin_role` is allowed and makes every roleless request admin (including `/v1/ops/*`) — handy for local/dev, logged loudly on every node that loads such a policy, and not for production use. `/v1/ops/*` — raw SQL, policy/pipe CRUD, schema, DLQ — is admin-only, and a pipe with no `allowed_roles` authorizes nobody but the admin role. -**Operator key (break-glass).** `auth.operator_key`, when set, is a non-JWT credential for the person running the deployment. A request presenting it — in the standard `Authorization: Operator ` header (forwarded verbatim by proxies, no collision with Bearer JWTs), or the `X-Operator-Key` alias — is authorized as a full-access platform operator (the whole data plane plus `/v1/admin/*`) without minting a JWT, and it is honored even when the policy is nil or deleted, so an operator can restore a wiped policy over HTTP (the one recovery path that otherwise requires SSH). It is matched with a constant-time comparison and is independent of `jwt_secret`/`jwks_url`; it takes precedence over any Bearer token on the same request. Because it is effectively an admin secret, load it from a secret store, serve it only over TLS, and leave it empty on deployments that do not need it. +**Operator key (break-glass).** `auth.operator_key`, when set, is a non-JWT credential for the person running the deployment. A request presenting it — in the standard `Authorization: Operator ` header (forwarded verbatim by proxies, no collision with Bearer JWTs), or the `X-Operator-Key` alias — is authorized as a full-access platform operator (the whole data plane plus `/v1/ops/*`) without minting a JWT, and it is honored even when the policy is nil or deleted, so an operator can restore a wiped policy over HTTP (the one recovery path that otherwise requires SSH). It is matched with a constant-time comparison and is independent of `jwt_secret`/`jwks_url`; it takes precedence over any Bearer token on the same request. Because it is effectively an admin secret, load it from a secret store, serve it only over TLS, and leave it empty on deployments that do not need it. See [API — Authentication](/api#authentication). @@ -168,7 +168,7 @@ See [API — Authentication](/api#authentication). | YAML Key | Env Var | Default | Description | | --- | --- | ------- | ----------- | -| `policy.file_path` | `WH_POLICY_FILE_PATH` | *(empty)* | Optional path to a YAML/JSON policy file used to seed the policy store on first startup (when NATS KV is empty). **When set, the file MUST exist, parse, and pass policy validation — WaveHouse refuses to boot otherwise**, so a typo, a missing mount, or an invalid policy surfaces immediately instead of silently denying every request (`Evaluate` fails closed on a `nil` policy, including the admin role). Validation includes the [claim-template grammar](/access-control#jwt-claim-templating), and it runs only when the store is actually **seeded** from the file — a fresh deployment, or one whose KV was cleared. An existing deployment keeps the policy already in NATS KV, which is *not* re-validated on load ([#461](https://github.com/Wave-RF/WaveHouse/issues/461)), so after upgrading, re-`PUT` your policy to surface anything the tightened rules reject. Empty default — no implicit `policy.yaml` lookup — so operators opt into the bootstrap file explicitly; without one, seed the policy via `PUT /v1/admin/policy`. Once KV is populated, the file is ignored on subsequent boots (KV is the source of truth; runtime updates flow through the API and KV Watch). | +| `policy.file_path` | `WH_POLICY_FILE_PATH` | *(empty)* | Optional path to a YAML/JSON policy file used to seed the policy store on first startup (when NATS KV is empty). **When set, the file MUST exist, parse, and pass policy validation — WaveHouse refuses to boot otherwise**, so a typo, a missing mount, or an invalid policy surfaces immediately instead of silently denying every request (`Evaluate` fails closed on a `nil` policy, including the admin role). Validation includes the [claim-template grammar](/access-control#jwt-claim-templating), and it runs only when the store is actually **seeded** from the file — a fresh deployment, or one whose KV was cleared. An existing deployment keeps the policy already in NATS KV, which is *not* re-validated on load ([#461](https://github.com/Wave-RF/WaveHouse/issues/461)), so after upgrading, re-`PUT` your policy to surface anything the tightened rules reject. Empty default — no implicit `policy.yaml` lookup — so operators opt into the bootstrap file explicitly; without one, seed the policy via `PUT /v1/ops/policy`. Once KV is populated, the file is ignored on subsequent boots (KV is the source of truth; runtime updates flow through the API and KV Watch). | ### Named Pipes @@ -266,7 +266,7 @@ dlq: enabled: true policy: - file_path: "" # empty = skip bootstrap (seed via PUT /v1/admin/policy); + file_path: "" # empty = skip bootstrap (seed via PUT /v1/ops/policy); # set to a path and the file MUST exist, parse, and # pass policy validation or boot fails diff --git a/docs/src/content/docs/deployment.md b/docs/src/content/docs/deployment.md index 60a2e9b3..cc497a0e 100644 --- a/docs/src/content/docs/deployment.md +++ b/docs/src/content/docs/deployment.md @@ -81,15 +81,25 @@ Production images are published to GitHub Container Registry via GoReleaser: ghcr.io/wave-rf/wavehouse: ``` -Published images carry a signed [Sigstore](https://www.sigstore.dev/) build-provenance attestation (stored in the registry). Verify one before deploying: +`:vX.Y.Z` and `:latest` track tagged releases; `:dev` is the rolling `main`-branch build, and `:dev-` (immutable, pruned after 30 days) captures a single commit. To pin (see the [alpha-stage caution](https://github.com/Wave-RF/WaveHouse#-project-status) in the README), use a `:dev-` tag — the full 40-character commit SHA, not the short form — or an image digest. + +Published images carry a signed [Sigstore](https://www.sigstore.dev/) build-provenance attestation (stored in the registry). Verify one before deploying, pinning the signer to the workflow that publishes the tag — `--repo` alone accepts an attestation from any workflow in the repo: ```bash -gh attestation verify oci://ghcr.io/wave-rf/wavehouse: --repo Wave-RF/WaveHouse +# :dev and :dev- images are published by publish-dev.yml +gh attestation verify oci://ghcr.io/wave-rf/wavehouse:dev \ + --repo Wave-RF/WaveHouse \ + --signer-workflow Wave-RF/WaveHouse/.github/workflows/publish-dev.yml + +# :vX.Y.Z and :latest release images are published by release.yml +gh attestation verify oci://ghcr.io/wave-rf/wavehouse:vX.Y.Z \ + --repo Wave-RF/WaveHouse \ + --signer-workflow Wave-RF/WaveHouse/.github/workflows/release.yml ``` ## Releases -Releases are built with [GoReleaser](https://goreleaser.com/). The configuration is in `.goreleaser.yaml`. The release archives attached to each GitHub Release carry a signed [Sigstore](https://www.sigstore.dev/) build-provenance attestation — verify a downloaded archive with `gh attestation verify --repo Wave-RF/WaveHouse`. (This covers the prebuilt archives, not `go install`, which compiles from source.) +Releases are built with [GoReleaser](https://goreleaser.com/). The configuration is in `.goreleaser.yaml`. The release archives attached to each GitHub Release carry a signed [Sigstore](https://www.sigstore.dev/) build-provenance attestation — verify a downloaded archive with `gh attestation verify --repo Wave-RF/WaveHouse --signer-workflow Wave-RF/WaveHouse/.github/workflows/release.yml`. (This covers the prebuilt archives, not `go install`, which compiles from source.) ### Supported Platforms @@ -118,7 +128,7 @@ Key variables for production: ```bash # Required WH_CH_ADDR=clickhouse:9000 -# Port for HTTP inserts + /v1/admin/query proxy (default: 8123) +# Port for HTTP inserts + /v1/ops/query proxy (default: 8123) WH_CH_HTTP_PORT=8123 WH_CH_HTTP_SCHEME=http # Scheme for the same (http/https) @@ -146,7 +156,7 @@ WH_AUTH_OPERATOR_KEY= # WH_POLICY_FILE_PATH is set, the file MUST exist, parse, and pass policy # validation (including the {{ jwt.… }} claim-path grammar) when the store is # seeded from it, or the process refuses to boot (silent fail-closed is the -# alternative). Leave unset to skip bootstrap and seed via PUT /v1/admin/policy. +# alternative). Leave unset to skip bootstrap and seed via PUT /v1/ops/policy. WH_POLICY_FILE_PATH=/etc/wavehouse/policy.yaml WH_PIPES_DIR=/etc/wavehouse/pipes @@ -252,7 +262,7 @@ services: - ./my-pipes:/app/pipes:ro # ← read-only seed ``` -The directory is a *seed*, not authoritative storage: after bootstrap, the API + KV are the source of truth. Runtime pipe edits go through `PUT /v1/admin/pipes/{name}`, not by editing the files. The `:ro` mount makes that contract explicit and prevents accidental writes from confusing future readers. Empty default (`WH_PIPES_DIR=""`) skips bootstrap entirely — most users will create pipes via the API. +The directory is a *seed*, not authoritative storage: after bootstrap, the API + KV are the source of truth. Runtime pipe edits go through `PUT /v1/ops/pipes/{name}`, not by editing the files. The `:ro` mount makes that contract explicit and prevents accidental writes from confusing future readers. Empty default (`WH_PIPES_DIR=""`) skips bootstrap entirely — most users will create pipes via the API. ## Health Checks @@ -356,11 +366,11 @@ CREATE TABLE IF NOT EXISTS clicks ( ORDER BY (page); ``` -WaveHouse discovers this schema on startup and refreshes it every `schema.refresh_interval` seconds (default: 60). You can also trigger an immediate refresh via `POST /v1/schema/refresh` (admin-only). +WaveHouse discovers this schema on startup and refreshes it every `schema.refresh_interval` seconds (default: 60). You can also trigger an immediate refresh via `POST /v1/ops/schema/refresh` (admin-only). ## Dead Letter Queue (DLQ) -When `dlq.enabled` is `true` (default), a failed batch insert is retried row by row and the rows that fail again are published to the `WAVEHOUSE_DLQ` NATS stream under subjects `dlq.{table}`. This prevents infinite retry loops. Monitor DLQ depth via `GET /v1/dlq/stats`. +When `dlq.enabled` is `true` (default), a failed batch insert is retried row by row and the rows that fail again are published to the `WAVEHOUSE_DLQ` NATS stream under subjects `dlq.{table}`. This prevents infinite retry loops. Monitor DLQ depth via `GET /v1/ops/dlq/stats`. ## Observability diff --git a/docs/src/content/docs/development.md b/docs/src/content/docs/development.md index 1899b8c1..c5aa9d21 100644 --- a/docs/src/content/docs/development.md +++ b/docs/src/content/docs/development.md @@ -119,13 +119,13 @@ curl http://localhost:8080/livez # → {"status":"ok"} curl http://localhost:8080/readyz # → {"status":"ready"} ``` -The admin surface — `/v1/schema`, `/v1/admin/query` (raw SQL), `/v1/dlq/stats` — needs the **admin** role, which the `public` trial role doesn't have. Mint an admin JWT (see [Validating tokens](#validating-tokens) below) and pass it: +The admin surface — `/v1/ops/schema`, `/v1/ops/query` (raw SQL), `/v1/ops/dlq/stats` — needs the **admin** role, which the `public` trial role doesn't have. Mint an admin JWT (see [Validating tokens](#validating-tokens) below) and pass it: ```bash -curl -s http://localhost:8080/v1/schema -H "Authorization: Bearer $TOKEN" | jq -curl -s -X POST http://localhost:8080/v1/admin/query -H "Authorization: Bearer $TOKEN" \ +curl -s http://localhost:8080/v1/ops/schema -H "Authorization: Bearer $TOKEN" | jq +curl -s -X POST http://localhost:8080/v1/ops/query -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" -d '{"sql": "SELECT * FROM clicks LIMIT 10"}' -curl -s http://localhost:8080/v1/dlq/stats -H "Authorization: Bearer $TOKEN" +curl -s http://localhost:8080/v1/ops/dlq/stats -H "Authorization: Bearer $TOKEN" ``` ### How `make dev` works @@ -211,9 +211,9 @@ The **operator key** is a non-JWT alternative: set one and send it in an `Author ```bash WH_AUTH_OPERATOR_KEY=dev-operator-key make dev # ...then, in another shell — the admin surface works even with no policy seeded: -curl -H "Authorization: Operator dev-operator-key" http://localhost:8080/v1/admin/policy +curl -H "Authorization: Operator dev-operator-key" http://localhost:8080/v1/ops/policy # the X-Operator-Key alias works too: -curl -H "X-Operator-Key: dev-operator-key" http://localhost:8080/v1/admin/policy +curl -H "X-Operator-Key: dev-operator-key" http://localhost:8080/v1/ops/policy ``` Then mint a token (role == the policy `admin_role`) and call an admin endpoint: @@ -222,7 +222,7 @@ Then mint a token (role == the policy `admin_role`) and call an admin endpoint: # Using jwt-cli (https://github.com/mike-engel/jwt-cli) export TOKEN=$(jwt encode --secret "my-secret" '{"role": "admin", "exp": 9999999999}') -curl -s -X POST http://localhost:8080/v1/admin/query \ +curl -s -X POST http://localhost:8080/v1/ops/query \ -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" \ -d '{"sql": "SELECT * FROM clicks LIMIT 10"}' diff --git a/docs/src/content/docs/getting-started.md b/docs/src/content/docs/getting-started.md index 5ae0e978..17d02c46 100644 --- a/docs/src/content/docs/getting-started.md +++ b/docs/src/content/docs/getting-started.md @@ -47,7 +47,7 @@ docker compose -f deployments/compose/standalone.yaml exec clickhouse \ " ``` -Schemas refresh every 60 seconds by default, or on demand via `POST /v1/schema/refresh` (admin-only). If the first ingest below returns `404 unknown table: clicks`, the refresh simply hasn't picked the new table up yet — wait and retry (worst case the next refresh is a full 60 seconds out). +Schemas refresh every 60 seconds by default, or on demand via `POST /v1/ops/schema/refresh` (admin-only). If the first ingest below returns `404 unknown table: clicks`, the refresh simply hasn't picked the new table up yet — wait and retry (worst case the next refresh is a full 60 seconds out). ## 3. Ingest an event @@ -73,7 +73,7 @@ curl -s -X POST "http://localhost:8080/v1/query?table=clicks" \ -d '{"columns": ["page", "button", "score"], "limit": 10}' ``` -`POST /v1/query?table={table}` and `GET/POST /v1/pipes/{name}` are cached in-process (L1 Ristretto) with singleflight coalescing — duplicate concurrent queries hit ClickHouse once. For raw SQL there's `POST /v1/admin/query` (an admin escape hatch that never caches, emitting `Cache-Control: no-store`), but it's **admin-only** — the trial `public` role can't reach it. To use it, swap the public default for real auth: configure a JWT secret and present a token whose role is the policy [`admin_role`](/access-control#admin_role--the-privileged-role). +`POST /v1/query?table={table}` and `GET/POST /v1/pipes/{name}` are cached in-process (L1 Ristretto) with singleflight coalescing — duplicate concurrent queries hit ClickHouse once. For raw SQL there's `POST /v1/ops/query` (an admin escape hatch that never caches, emitting `Cache-Control: no-store`), but it's **admin-only** — the trial `public` role can't reach it. To use it, swap the public default for real auth: configure a JWT secret and present a token whose role is the policy [`admin_role`](/access-control#admin_role--the-privileged-role). :::tip[Prefer a type-safe client?] The [TypeScript SDK](/sdk) wraps this endpoint in a chainable query builder with autocomplete on your table names and row types — plus live queries and streaming. The raw shapes are in the [structured query reference](/api#post-v1querytabletable--structured-query). @@ -95,7 +95,7 @@ curl -N "http://localhost:8080/v1/stream?table=clicks&since=2026-03-24T11:00:00Z The handful of things that most often trip up a first session — each is expected behavior with a quick fix: -- **`404 unknown table: clicks` on the first ingest.** Schema discovery refreshes every 60 seconds (`WH_SCHEMA_REFRESH_INTERVAL`), so a just-created table may not be visible yet. Wait and retry — worst case the next refresh is a full 60 seconds out. (`POST /v1/schema/refresh` forces it, but that endpoint is admin-only — the trial `public` role can't call it.) +- **`404 unknown table: clicks` on the first ingest.** Schema discovery refreshes every 60 seconds (`WH_SCHEMA_REFRESH_INTERVAL`), so a just-created table may not be visible yet. Wait and retry — worst case the next refresh is a full 60 seconds out. (`POST /v1/ops/schema/refresh` forces it, but that endpoint is admin-only — the trial `public` role can't call it.) - **The query returns `[]` right after an ingest succeeded.** Ingest acknowledges as soon as the event is durable in the WAL; the batch worker flushes to ClickHouse every few seconds. If you query within that window the rows simply aren't in ClickHouse yet — re-query after ~5 seconds. (The [SSE stream](#5-subscribe-to-real-time-updates) sees events *immediately* — it's broadcast before the flush.) - **`403` on a table you created yourself.** WaveHouse is fail-closed and the trial policy grants the `public` role access to the *named demo tables only* (`clicks`, `events`). A new table needs a policy entry — see [Access Control](/access-control) for granting roles per table. - **A port is already taken.** The stack binds `8080` (WaveHouse) and `8123`/`9000` (ClickHouse). Stop whatever holds the port or edit the `ports:` mappings in `deployments/compose/standalone.yaml`. diff --git a/docs/src/content/docs/ingest-pipeline.md b/docs/src/content/docs/ingest-pipeline.md index a4e6a9c9..b248ebeb 100644 --- a/docs/src/content/docs/ingest-pipeline.md +++ b/docs/src/content/docs/ingest-pipeline.md @@ -22,8 +22,8 @@ goroutine / channel / timer interplay is subtle. The pipeline is **insert-only**. The wire format carries `{table_name, received_timestamp, data}` and nothing else; the worker parses the envelope and bulk-`INSERT`s — schema validation already happened at the -HTTP ingest handler, before publish. Non-insert mutations go through a -different admin path. +HTTP ingest handler, before publish. Non-insert mutations go through +`POST /v1/ops/query` (admin-only). ## High-level shape diff --git a/docs/src/content/docs/pipes.mdx b/docs/src/content/docs/pipes.mdx index 5accbd3b..a3128e96 100644 --- a/docs/src/content/docs/pipes.mdx +++ b/docs/src/content/docs/pipes.mdx @@ -119,22 +119,22 @@ Pipe execution is gated by `allowed_roles` — a plain allowlist, evaluated inde To make a pipe public, list the role that `default_role` resolves to. For example, with `default_role: viewer`, a pipe with `"allowed_roles": ["viewer"]` is reachable with no token at all. -This is the *only* authorization check on the execute path — pipes deliberately sit outside the `/v1/admin/*` gate so non-admin callers can run them. The pipe's SQL is not re-checked against the table policy's column or row rules, so **scope the data in the pipe's SQL itself** rather than assuming policy column masking applies. A pipe's `{{param}}` placeholders bind caller-supplied values, never token claims — a parameterized predicate is a convenience for the caller, not an authorization boundary. +This is the *only* authorization check on the execute path — pipes deliberately sit outside the `/v1/ops/*` gate so non-admin callers can run them. The pipe's SQL is not re-checked against the table policy's column or row rules, so **scope the data in the pipe's SQL itself** rather than assuming policy column masking applies. A pipe's `{{param}}` template parameters interpolate values from the request — or a parameter's configured default when the caller omits one ([resolution order](#how-a-value-is-resolved)) — inlined as escaped SQL literals, never token claims; a parameterized predicate is a convenience for the caller, not an authorization boundary. ## Creating and managing pipes -Pipe CRUD is admin-only, under `/v1/admin/pipes`: +Pipe CRUD is admin-only, under `/v1/ops/pipes`: | Method | Endpoint | Purpose | | ------ | -------- | ------- | -| `GET` | `/v1/admin/pipes` | List all pipes. | -| `GET` | `/v1/admin/pipes/{name}` | Get one pipe's definition. | -| `PUT` | `/v1/admin/pipes/{name}` | Create or replace a pipe. | -| `DELETE` | `/v1/admin/pipes/{name}` | Delete a pipe. | +| `GET` | `/v1/ops/pipes` | List all pipes. | +| `GET` | `/v1/ops/pipes/{name}` | Get one pipe's definition. | +| `PUT` | `/v1/ops/pipes/{name}` | Create or replace a pipe. | +| `DELETE` | `/v1/ops/pipes/{name}` | Delete a pipe. | ```bash # Create / update a pipe (admin token required) -curl -X PUT http://localhost:8080/v1/admin/pipes/top_pages \ +curl -X PUT http://localhost:8080/v1/ops/pipes/top_pages \ -H "Authorization: Bearer $ADMIN_TOKEN" \ -H "Content-Type: application/json" \ -d '{ @@ -202,7 +202,7 @@ On startup, each top-level `*.sql` file is loaded as a pipe whose **name is the - After bootstrap, the API and KV are the source of truth; the directory is read-only at runtime. Mount it read-only in containers (e.g. `./my-pipes:/app/pipes:ro`). :::caution[Bootstrapped pipes are admin-only until you grant roles] -A `.sql` file carries only the SQL — there's no way to express `allowed_roles`, `parameters`, or a `description` in the file. A pipe seeded this way therefore has an empty `allowed_roles`, which (fails-closed) means **admin only**. To expose it to other roles or add formal parameters, `PUT` the full definition through `/v1/admin/pipes/{name}` after boot. Inline `{{name:default}}` placeholders still work in a bootstrapped file, so the SQL can be parameter-ready even without formal declarations. +A `.sql` file carries only the SQL — there's no way to express `allowed_roles`, `parameters`, or a `description` in the file. A pipe seeded this way therefore has an empty `allowed_roles`, which (fails-closed) means **admin only**. To expose it to other roles or add formal parameters, `PUT` the full definition through `/v1/ops/pipes/{name}` after boot. Inline `{{name:default}}` placeholders still work in a bootstrapped file, so the SQL can be parameter-ready even without formal declarations. ::: ## End-to-end example @@ -218,7 +218,7 @@ Ship a curated "top pages" endpoint that the public dashboard can call with no t 2. **Register the pipe**, allowing `viewer`: ```bash - curl -X PUT http://localhost:8080/v1/admin/pipes/top_pages \ + curl -X PUT http://localhost:8080/v1/ops/pipes/top_pages \ -H "Authorization: Bearer $ADMIN_TOKEN" -H "Content-Type: application/json" \ -d '{ "sql": "SELECT page, count() AS views FROM clicks WHERE received_timestamp >= {{since:2024-01-01}} GROUP BY page ORDER BY views DESC LIMIT {{limit:50}}", diff --git a/docs/src/content/docs/reverse-proxy.mdx b/docs/src/content/docs/reverse-proxy.mdx index ebd64903..08da9166 100644 --- a/docs/src/content/docs/reverse-proxy.mdx +++ b/docs/src/content/docs/reverse-proxy.mdx @@ -18,7 +18,7 @@ WaveHouse and your proxy are layers of one system, not substitutes. The proxy ow | TLS termination / certificates | ✗ (plain HTTP on `:8080`) | ✓ | | Per-IP rate limiting, connection caps | ✗ | ✓ | | Slow-loris / slow-body mitigation | ✗ | ✓ | -| Request-body size limit | Fixed internal backstop (1 MiB control / 16 MiB ingest) | Tunable outer limit | +| Request-body size limit | Fixed internal backstop (1 MiB parameter/AST bodies, 16 MiB bulk payload bodies) | Tunable outer limit | | Authentication (JWT) | ✓ (validates / resolves role) | Pass through | | CORS | ✓ (`server.cors_allowed_origins`) | Pass through (don't double it) | | Health probes | ✓ (serves `/livez`, `/readyz`, `/v1/health`) | Route / expose appropriately | @@ -92,10 +92,10 @@ WaveHouse caps the inbound request body it will decode, as a memory-safety backs | Endpoints | Cap | Over-cap response | | --- | --- | --- | -| `POST /v1/query`, `GET/POST /v1/pipes/{name}` (control plane) | **1 MiB** | `413 {"error":"request body exceeded 1048576 bytes"}` | -| `POST /v1/ingest`, `POST /v1/admin/query` (data plane) | **16 MiB** | `413 {"error":"request body exceeded 16777216 bytes"}` | +| `POST /v1/query`, `GET/POST /v1/pipes/{name}`, `PUT /v1/ops/policy`, `POST /v1/ops/policy/validate`, `PUT /v1/ops/pipes/{name}` (parameter / AST / definition bodies) | **1 MiB** | `413 {"error":"request body exceeded 1048576 bytes"}` | +| `POST /v1/ingest`, `POST /v1/ops/query` (bulk payload bodies) | **16 MiB** | `413 {"error":"request body exceeded 16777216 bytes"}` | -These caps are **fixed and not configurable** — they aren't a tuning knob, they're an invariant. A JSON request body amplifies roughly an order of magnitude when decoded into memory (a large array of small values explodes into Go's in-memory representation), so an *uncapped* decoder on a public endpoint is a single-request out-of-memory vector. A query or pipe-parameter body is bounded by nature — a real one is far under 1 MiB even with a large `in`-list — so the control-plane cap is generous headroom that never binds legitimate use. +These caps are **fixed and not configurable** — they aren't a tuning knob, they're an invariant. A JSON request body amplifies roughly an order of magnitude when decoded into memory (a large array of small values explodes into Go's in-memory representation), so an *uncapped* decoder on a public endpoint is a single-request out-of-memory vector. A query or pipe-parameter body is bounded by nature — a real one is far under 1 MiB even with a large `in`-list — so the 1 MiB cap is generous headroom that never binds legitimate use. Set your own **outer** limit at the proxy, sized to your real needs: @@ -197,6 +197,10 @@ WaveHouse does **not** derive a client IP from forwarded headers — it does no - **CORS** — WaveHouse applies its own CORS from `server.cors_allowed_origins`. Let one layer own CORS: either pass it through the proxy untouched (recommended), or strip it from WaveHouse and do it at the proxy — not both, or browsers see duplicate `Access-Control-Allow-Origin` headers and reject the response. +## Fencing the admin surface + +Every admin-gated endpoint — raw SQL, policy CRUD, pipe CRUD, schema discovery, DLQ stats — lives under the single `/v1/ops/` prefix, so one proxy rule covers the entire management surface: an nginx `location /v1/ops/ { deny all; }` on the public vhost (serving it only on an internal listener), or an IP allowlist on that prefix. If you deny the prefix outright, keep an internal listener that still serves it — `PUT /v1/ops/policy` with the [operator key](/access-control#operator-key) is the one HTTP path that can restore a wiped policy, and a blanket fence with no internal route removes that break-glass too. Under a [path prefix](#path-prefixes), fence `/v1/ops/` instead. This is belt-and-braces, not a requirement — the server's `RequireAdmin` gate remains the authorization boundary and denies non-admin callers on its own — but keeping the management surface off the public vhost removes it from unauthenticated reach entirely. + ## Health probes WaveHouse serves Kubernetes-convention probes on `:8080` (full behavior in [Deployment → Health Checks](/deployment#health-checks)): @@ -229,8 +233,8 @@ server { # ssl_certificate /etc/ssl/wavehouse.crt; # ssl_certificate_key /etc/ssl/wavehouse.key; - # Outer body limit. WaveHouse's own backstop is 1 MiB (query/pipes) and - # 16 MiB (ingest); this can only make the effective limit tighter. + # Outer body limit. WaveHouse's own backstop: 1 MiB parameter/AST bodies, + # 16 MiB bulk payload bodies; this can only make the effective limit tighter. client_max_body_size 16m; # SSE: stream events immediately (buffering off). The long read timeout is @@ -264,7 +268,7 @@ server { wavehouse.example.com { # TLS is automatic. - # Outer body limit (WaveHouse backstop is 1 MiB control / 16 MiB ingest). + # Outer body limit (WaveHouse backstop: 1 MiB parameter/AST, 16 MiB bulk payload). request_body { max_size 16MB } diff --git a/docs/src/content/docs/sdk/admin.md b/docs/src/content/docs/sdk/admin.md index 2a4971b5..1484ceb3 100644 --- a/docs/src/content/docs/sdk/admin.md +++ b/docs/src/content/docs/sdk/admin.md @@ -3,8 +3,13 @@ title: "SDK Admin & System" description: "Schema introspection, access-control policy, DLQ stats, and health checks in @wavehouse/sdk." --- -Operational surfaces of `@wavehouse/sdk`. Everything here except -`wh.sys.health()` requires the admin role (`policy.admin_role`) — see +Operational surfaces of `@wavehouse/sdk`. With one exception, everything on +this page sits behind the server's admin gate: the caller must resolve to +the admin role (`policy.admin_role`) or present the non-JWT +[operator key](/api#authentication) — the SDK has no first-class +operator-key option, but [`options.headers`](/sdk#custom-headers) can carry +the `X-Operator-Key` header. The exception is `wh.sys.health()`, which calls +the public, content-free `/v1/health` route and needs no credentials. See [Access Control](/access-control) for how roles resolve. Examples import from `@wavehouse/sdk`; using the CDN instead, import from `https://esm.sh/@wavehouse/sdk` (see [Imports & Runtimes](/sdk#imports--runtimes)). @@ -24,13 +29,13 @@ await wh.schema.refresh(); Individual table schema is also available via `wh.from('clicks').schema()`. -> `wh.schema.list()`, `wh.schema.refresh()`, and `wh.from(t).schema()` hit `/v1/schema*`, which are **admin-only** endpoints. Against any non-dev policy (anything but `default_role: admin`), construct the client with an admin-role token or these calls return `403`. +> `wh.schema.list()`, `wh.schema.refresh()`, and `wh.from(t).schema()` hit `/v1/ops/schema*`, which are **admin-only** endpoints: the caller must pass the admin gate — resolve to the policy admin role (`admin_role`, `"admin"` by default) or present the non-JWT [operator key](/api#authentication). Unless the deployment deliberately sets `default_role` to the admin role (the loudly-warned dev-only setting), construct the client with an admin-role token — or send the operator key via [`options.headers`](/sdk#custom-headers) — or these calls return `403`. --- ## Policy — `wh.policy` -Manage Hasura-style access control policies. Requires the admin role (`policy.admin_role`). +Manage Hasura-style access control policies. Requires the admin gate — the admin role (`policy.admin_role`) or the [operator key](/api#authentication). ```ts // Get current policy @@ -61,7 +66,7 @@ const { data } = await wh.policy.validate(policyDraft); ## DLQ — `wh.dlq` -Dead Letter Queue operations. Requires the admin role (`policy.admin_role`). +Dead Letter Queue operations. Requires the admin gate — the admin role (`policy.admin_role`) or the [operator key](/api#authentication). ```ts // Get DLQ statistics diff --git a/docs/src/content/docs/sdk/pipes.md b/docs/src/content/docs/sdk/pipes.md index f0e5b8ed..2a923160 100644 --- a/docs/src/content/docs/sdk/pipes.md +++ b/docs/src/content/docs/sdk/pipes.md @@ -47,7 +47,8 @@ Open a live stream. See [Streaming](/sdk/streaming). ## Pipes Admin — `wh.pipes` -Manage named query pipes. Requires the admin role (`policy.admin_role`). +Manage named query pipes. Requires the admin gate — the admin role +(`policy.admin_role`) or the [operator key](/api#authentication). ```ts // List all pipes diff --git a/docs/src/content/docs/sdk/queries.md b/docs/src/content/docs/sdk/queries.md index 58a6506d..5d086479 100644 --- a/docs/src/content/docs/sdk/queries.md +++ b/docs/src/content/docs/sdk/queries.md @@ -69,7 +69,11 @@ await clicks.insertNDJSON(await openAsBlob('events.ndjson')); ### `.schema(opts?)` -Fetch the table's column definitions from ClickHouse. +Fetch the table's column definitions from ClickHouse. `.schema()` hits +`/v1/ops/schema`, an **admin-only** endpoint: the caller must pass the admin +gate — resolve to the policy admin role or present the non-JWT +[operator key](/api#authentication) via [`options.headers`](/sdk#custom-headers) +— or this returns `403`. ```ts const { data } = await clicks.schema(); @@ -255,7 +259,7 @@ while (result.hasMore && result.next) { ## Raw SQL — `wh.sql(query, opts?)` -Execute a raw SQL query. `/v1/admin/query` is admin-only: the caller's JWT must resolve to the policy admin role (`admin_role`, `"admin"` by default). A request with no token, or an invalid/expired one, falls back to the `default_role` and is rejected. +Execute a raw SQL query. `/v1/ops/query` is admin-only: the caller must resolve to the policy admin role (`admin_role`, `"admin"` by default). A tokenless request falls back to the `default_role`, so it is rejected with `403` on any policy that doesn't deliberately set `default_role` to the admin role (a loudly-warned dev-only setting); an invalid or expired token is rejected with `401`. The SDK has no first-class option for the server's non-JWT [operator key](/api#authentication) — an operator can send its `X-Operator-Key` header via [`options.headers`](/sdk#custom-headers). ```ts const { data, error } = await wh.sql('SELECT page, count() FROM clicks GROUP BY page LIMIT 10'); diff --git a/docs/src/content/docs/sdk/reference.md b/docs/src/content/docs/sdk/reference.md index 71d1727e..97806c4c 100644 --- a/docs/src/content/docs/sdk/reference.md +++ b/docs/src/content/docs/sdk/reference.md @@ -60,7 +60,7 @@ createClient(config) → WaveHouseClient │ ├── .selectAll() → QueryBuilder (PromiseLike) │ ├── .insert(data) → Promise> │ ├── .insertNDJSON(source) → Promise> -│ ├── .schema() → Promise> +│ ├── .schema() → Promise> (admin) │ └── .stream(opts?) → StreamController ├── .pipe(name, params?) → PipeRef (PromiseLike) │ ├── .fetch(opts?) → Promise> // { signal } only — no limit @@ -70,15 +70,15 @@ createClient(config) → WaveHouseClient │ ├── .get(name) → Promise> │ ├── .set(name, def) → Promise> │ └── .delete(name) → Promise> -├── .sql(query, opts?) → Promise> -├── .schema +├── .sql(query, opts?) → Promise> (admin) +├── .schema (admin) │ ├── .list() → Promise> │ └── .refresh() → Promise> ├── .policy (admin) │ ├── .get() → Promise> │ ├── .set(policy) → Promise> │ └── .validate(policy) → Promise> -├── .dlq +├── .dlq (admin) │ ├── .list() → Promise> │ ├── .table(name) → Promise> │ └── .stream() → StreamController // not yet functional server-side — #197 @@ -103,7 +103,7 @@ npx wavehouse-codegen --url http://localhost:8080 --out ./src/db.d.ts pnpm codegen --url http://localhost:8080 --out ./src/db.d.ts ``` -Codegen reads `/v1/schema`, which is **admin-only**. Against a non-dev server, pass an admin-role token with `--auth ` or the request is denied with `403`. +Codegen reads `/v1/ops/schema`, which is **admin-only**. Against a non-dev server, pass an admin-role token with `--auth ` or the request is denied with `403`. **Options:** diff --git a/docs/src/content/docs/why-wavehouse.md b/docs/src/content/docs/why-wavehouse.md index 9cb5b63d..70323229 100644 --- a/docs/src/content/docs/why-wavehouse.md +++ b/docs/src/content/docs/why-wavehouse.md @@ -50,7 +50,7 @@ Even if you remember to batch client-side, a naive ingest path has no safe way t - **No backpressure channel.** If the merger falls behind, ClickHouse raises an error at the *next* insert. The client has already left. - **No DLQ.** Bad events that fail to insert are either lost or logged into ClickHouse's error log. Good luck replaying yesterday's dropped rows. -WaveHouse fixes all three at the gateway: validates every payload against the real `system.columns` schema before accepting, returns `503 Service Unavailable` with a `Retry-After` header when the NATS WAL fills, and routes failed batch inserts to a dedicated `WAVEHOUSE_DLQ` stream you can inspect via `GET /v1/dlq/stats`. +WaveHouse fixes all three at the gateway: validates every payload against the real `system.columns` schema before accepting, returns `503 Service Unavailable` with a `Retry-After` header when the NATS WAL fills, and routes failed batch inserts to a dedicated `WAVEHOUSE_DLQ` stream you can inspect via `GET /v1/ops/dlq/stats`. ### No real-time push diff --git a/internal/api/cache_key.go b/internal/api/cache_key.go index e8675971..ecc40769 100644 --- a/internal/api/cache_key.go +++ b/internal/api/cache_key.go @@ -9,7 +9,7 @@ import ( ) // queryCacheKey produces a deterministic L1/L2 cache key for a (sql, params) -// pair. The raw-SQL endpoint (`POST /v1/admin/query`) does not cache, but the +// pair. The raw-SQL endpoint (`POST /v1/ops/query`) does not cache, but the // structured query (`POST /v1/query?table={table}`) and named pipes // (`GET/POST /v1/pipes/{name}`) handlers do — they share this helper so a // key change in one place propagates to every cached read path. diff --git a/internal/api/cache_key_test.go b/internal/api/cache_key_test.go index f4233968..11c36e5a 100644 --- a/internal/api/cache_key_test.go +++ b/internal/api/cache_key_test.go @@ -7,7 +7,7 @@ import ( ) // queryCacheKey is consumed by structured_query.go and pipes.go, so its -// contract has to stay stable even though /v1/admin/query no longer caches. +// contract has to stay stable even though /v1/ops/query no longer caches. // Table-driven so future collision regressions drop in as additional rows // without growing the assertion flow. func TestQueryCacheKey(t *testing.T) { diff --git a/internal/api/clickhouse_exec.go b/internal/api/clickhouse_exec.go index d190291c..83258993 100644 --- a/internal/api/clickhouse_exec.go +++ b/internal/api/clickhouse_exec.go @@ -20,7 +20,7 @@ import ( // // Used by the structured-query and pipes handlers — those are the cached // read paths that need explicit Query/Exec dispatch and per-row scanning. -// The raw-SQL endpoint (/v1/admin/query) proxies straight to ClickHouse +// The raw-SQL endpoint (/v1/ops/query) proxies straight to ClickHouse // over HTTP and never calls this; see internal/api/query.go. func executeCHQuery(ctx context.Context, conn driver.Conn, sql string, params []any) ([]map[string]any, error) { if isMutation(sql) { diff --git a/internal/api/dlq_test.go b/internal/api/dlq_test.go index cac04093..6362b4a9 100644 --- a/internal/api/dlq_test.go +++ b/internal/api/dlq_test.go @@ -22,7 +22,7 @@ func TestDLQStats_EmptyWhenNoStream(t *testing.T) { handler := NewDLQHandler(emb.JetStream(), slog.Default()) - req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/dlq/stats", nil) + req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/ops/dlq/stats", nil) rec := httptest.NewRecorder() handler.Stats(rec, req) @@ -61,7 +61,7 @@ func TestDLQStats_ReturnsCorrectCounts(t *testing.T) { } handler := NewDLQHandler(js, slog.Default()) - req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/dlq/stats", nil) + req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/ops/dlq/stats", nil) rec := httptest.NewRecorder() handler.Stats(rec, req) @@ -93,7 +93,7 @@ func TestDLQStats_SingleTable(t *testing.T) { require.NoError(t, err) handler := NewDLQHandler(js, slog.Default()) - req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/dlq/stats", nil) + req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/ops/dlq/stats", nil) rec := httptest.NewRecorder() handler.Stats(rec, req) diff --git a/internal/api/errors.go b/internal/api/errors.go index ec83363e..876b397b 100644 --- a/internal/api/errors.go +++ b/internal/api/errors.go @@ -35,12 +35,12 @@ func writeJSONError(w http.ResponseWriter, status int, message string) { // // Pass the role AFTER default-role resolution so forbiddenForRole's empty-role // message is accurate. allowedRoles is the set the gate would have accepted (a -// pipe's allowed_roles); the gates with no flat role list — the /v1/admin gate +// pipe's allowed_roles); the gates with no flat role list — the /v1/ops gate // and the policy-evaluator paths (ingest, structured query) — pass nil. attrs // are gate-specific structured fields appended to the WARN: each gate tags a // "gate" (admin / policy / pipe) so a denial is attributable to the check that -// raised it (the route pattern alone can't — /v1/schema runs the admin gate, -// not a policy one), and the policy paths add the table + action they evaluated. +// raised it without parsing the route pattern, and the policy paths add the +// table + action they evaluated. // logger is the calling gate's injected logger (each handler holds one; main // wires it, tests pass their own) — the denial WARN goes there, not to a // package global. @@ -74,7 +74,7 @@ func writeAuthzDenied(w http.ResponseWriter, r *http.Request, logger *slog.Logge // would have accepted, and the matched route + method. role_observed empty with // a non-empty role_resolved means the caller presented no role and was mapped to // default_role; roles_allowed is populated only by the pipe gate (a pipe's -// allowed_roles) and is empty for the /v1/admin gate and the policy-evaluator +// allowed_roles) and is empty for the /v1/ops gate and the policy-evaluator // paths (ingest, structured query). attrs carry each gate's own fields (the // "gate" tag, plus table + action on the policy paths) so the records stay // distinguishable beyond the route. diff --git a/internal/api/errors_test.go b/internal/api/errors_test.go index af4a2439..9b555575 100644 --- a/internal/api/errors_test.go +++ b/internal/api/errors_test.go @@ -65,7 +65,7 @@ func TestRequireAdmin_DenialLogsStructuredWarn(t *testing.T) { })) ctx := auth.WithRole(context.Background(), "viewer") - req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/v1/admin/query", nil) + req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/v1/ops/query", nil) handler.ServeHTTP(httptest.NewRecorder(), req) out := buf.String() @@ -75,7 +75,7 @@ func TestRequireAdmin_DenialLogsStructuredWarn(t *testing.T) { assert.Contains(t, out, `"role_observed":"viewer"`) assert.Contains(t, out, `"role_resolved":"viewer"`) assert.Contains(t, out, `"roles_allowed":null`, "the admin gate logs no explicit allowlist") - assert.Contains(t, out, `"route":"/v1/admin/query"`) + assert.Contains(t, out, `"route":"/v1/ops/query"`) assert.Contains(t, out, `"method":"GET"`) assert.Contains(t, out, `"status":403`) assert.Contains(t, out, `"gate":"admin"`) @@ -93,7 +93,7 @@ func TestRequireAdmin_EmptyRoleDenialLogsResolvedRole(t *testing.T) { t.Fatal("handler must not run on a denied request") })) - req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/admin/query", nil) + req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/ops/query", nil) handler.ServeHTTP(httptest.NewRecorder(), req) out := buf.String() @@ -115,7 +115,7 @@ func TestRequireAdmin_InvalidTokenDenialLogsFailLoudReason(t *testing.T) { })) ctx := auth.WithAuthError(context.Background(), errors.New("token expired")) - req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/v1/admin/query", nil) + req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/v1/ops/query", nil) handler.ServeHTTP(httptest.NewRecorder(), req) out := buf.String() @@ -184,8 +184,9 @@ func TestIngest_DenialLogsPolicyGate(t *testing.T) { // TestAuthzDenied_LogsChiRoutePattern: routed through the real mux, the WARN's // route is the matched route template, not the raw path — low-cardinality and -// free of concrete path params. /v1/schema runs the admin gate (no /admin -// prefix), so gate=admin is what tells the operator which check denied it. +// free of concrete path params. The /v1/ops gate is tree-level middleware, so +// it denies before sub-route matching and the template is /v1/ops/*; the +// gate=admin attribute tells the operator which check denied it. func TestAuthzDenied_LogsChiRoutePattern(t *testing.T) { t.Parallel() logger, buf := warnBufLogger() @@ -202,12 +203,12 @@ func TestAuthzDenied_LogsChiRoutePattern(t *testing.T) { }) ctx := auth.WithRole(context.Background(), "viewer") - req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/v1/schema", nil) + req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/v1/ops/schema", nil) rec := httptest.NewRecorder() router.ServeHTTP(rec, req) require.Equal(t, http.StatusForbidden, rec.Code) out := buf.String() - assert.Contains(t, out, `"route":"/v1/schema"`, "route should be the chi pattern") - assert.Contains(t, out, `"gate":"admin"`, "/v1/schema runs the admin gate, not a policy one") + assert.Contains(t, out, `"route":"/v1/ops/*"`, "route should be the chi pattern") + assert.Contains(t, out, `"gate":"admin"`, "/v1/ops/schema runs the admin gate, not a policy one") } diff --git a/internal/api/ingest.go b/internal/api/ingest.go index ecf34530..0e90a404 100644 --- a/internal/api/ingest.go +++ b/internal/api/ingest.go @@ -164,7 +164,7 @@ func (h *IngestHandler) Handle(w http.ResponseWriter, r *http.Request) { // TODO: set a scope (e.g., "org_id:123") – but scope requires us to know if a table is globally shared or scoped fully by roles/org/tenant. Currently we have no real way to set this, so scopes will be empty. scope := "" - // Bound the inbound body (parity with /v1/admin/query; also caps the + // Bound the inbound body (parity with /v1/ops/query; also caps the // array/stream decode vectors). See query.go for maxRequestBodyBytes. reqCap := int64(maxRequestBodyBytes) if h.maxRequestBytes > 0 { diff --git a/internal/api/pipes.go b/internal/api/pipes.go index eac7b853..06e8f133 100644 --- a/internal/api/pipes.go +++ b/internal/api/pipes.go @@ -115,7 +115,7 @@ func (h *PipesHandler) Execute(w http.ResponseWriter, r *http.Request) { // "*" any-role wildcard and empty entries are ignored, so a stray "" can't // authorize an empty role. The admin role bypasses every pipe's allowlist by // design, not oversight: admins author pipes and can run arbitrary SQL via - // /v1/admin/query, so allowed_roles is never a confidentiality boundary + // /v1/ops/query, so allowed_roles is never a confidentiality boundary // against them (mirrors Evaluate's admin bypass). A pipe with no // allowed_roles therefore authorizes nobody but admin (fails closed). var p *policy.Policy diff --git a/internal/api/pipes_test.go b/internal/api/pipes_test.go index a171b6e0..fc97c6cf 100644 --- a/internal/api/pipes_test.go +++ b/internal/api/pipes_test.go @@ -45,7 +45,7 @@ func TestPipesHandler_List(t *testing.T) { h := NewPipesHandler(store, nil, nil, nil, 0, testutil.NopLogger()) w := httptest.NewRecorder() - r := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/pipes", nil) + r := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/ops/pipes", nil) h.List(w, r) assert.Equal(t, http.StatusOK, w.Code) @@ -62,7 +62,7 @@ func TestPipesHandler_Get_Found(t *testing.T) { h := NewPipesHandler(store, nil, nil, nil, 0, testutil.NopLogger()) w := httptest.NewRecorder() - r := pipesRequest(t, http.MethodGet, "/v1/pipes/top_pages", "top_pages", nil) + r := pipesRequest(t, http.MethodGet, "/v1/ops/pipes/top_pages", "top_pages", nil) h.Get(w, r) assert.Equal(t, http.StatusOK, w.Code) @@ -77,7 +77,7 @@ func TestPipesHandler_Get_NotFound(t *testing.T) { h := NewPipesHandler(store, nil, nil, nil, 0, testutil.NopLogger()) w := httptest.NewRecorder() - r := pipesRequest(t, http.MethodGet, "/v1/pipes/nope", "nope", nil) + r := pipesRequest(t, http.MethodGet, "/v1/ops/pipes/nope", "nope", nil) h.Get(w, r) assert.Equal(t, http.StatusNotFound, w.Code) @@ -91,7 +91,7 @@ func TestPipesHandler_List_Empty(t *testing.T) { h := NewPipesHandler(store, nil, nil, nil, 0, testutil.NopLogger()) w := httptest.NewRecorder() - r := pipesRequest(t, http.MethodGet, "/v1/pipes", "", nil) + r := pipesRequest(t, http.MethodGet, "/v1/ops/pipes", "", nil) h.List(w, r) assert.Equal(t, http.StatusOK, w.Code) @@ -297,7 +297,7 @@ func TestPipesHandler_Put_InvalidJSON(t *testing.T) { h := NewPipesHandler(store, nil, nil, nil, 0, testutil.NopLogger()) w := httptest.NewRecorder() - r := httptest.NewRequestWithContext(context.Background(), http.MethodPut, "/v1/pipes/test", bytes.NewReader([]byte(`{bad}`))) + r := httptest.NewRequestWithContext(context.Background(), http.MethodPut, "/v1/ops/pipes/test", bytes.NewReader([]byte(`{bad}`))) rctx := chi.NewRouteContext() rctx.URLParams.Add("name", "test") r = r.WithContext(context.WithValue(r.Context(), chi.RouteCtxKey, rctx)) @@ -318,7 +318,7 @@ func TestPipesHandler_Put_RequestBodyCap(t *testing.T) { h.maxRequestBytes = 64 w := httptest.NewRecorder() - r := pipesRequest(t, http.MethodPut, "/v1/pipes/big", "big", map[string]any{ + r := pipesRequest(t, http.MethodPut, "/v1/ops/pipes/big", "big", map[string]any{ "sql": "SELECT " + strings.Repeat("a", 200), }) h.Put(w, r) @@ -334,7 +334,7 @@ func TestPipesHandler_Put_Success(t *testing.T) { h := NewPipesHandler(store, nil, nil, nil, 0, testutil.NopLogger()) w := httptest.NewRecorder() - r := pipesRequest(t, http.MethodPut, "/v1/pipes/new_pipe", "new_pipe", map[string]any{ + r := pipesRequest(t, http.MethodPut, "/v1/ops/pipes/new_pipe", "new_pipe", map[string]any{ "sql": "SELECT count(*) FROM clicks", "description": "counts", }) @@ -353,7 +353,7 @@ func TestPipesHandler_Put_MissingSQL(t *testing.T) { h := NewPipesHandler(store, nil, nil, nil, 0, testutil.NopLogger()) w := httptest.NewRecorder() - r := pipesRequest(t, http.MethodPut, "/v1/pipes/bad", "bad", map[string]any{ + r := pipesRequest(t, http.MethodPut, "/v1/ops/pipes/bad", "bad", map[string]any{ "description": "no sql", }) h.Put(w, r) @@ -370,7 +370,7 @@ func TestPipesHandler_Delete_Success(t *testing.T) { h := NewPipesHandler(store, nil, nil, nil, 0, testutil.NopLogger()) w := httptest.NewRecorder() - r := pipesRequest(t, http.MethodDelete, "/v1/pipes/to_delete", "to_delete", nil) + r := pipesRequest(t, http.MethodDelete, "/v1/ops/pipes/to_delete", "to_delete", nil) h.Delete(w, r) assert.Equal(t, http.StatusOK, w.Code) diff --git a/internal/api/policy_test.go b/internal/api/policy_test.go index 605fede5..03c836c9 100644 --- a/internal/api/policy_test.go +++ b/internal/api/policy_test.go @@ -20,7 +20,7 @@ func TestPolicyHandler_Get_NilPolicy(t *testing.T) { h := NewPolicyHandler(store) w := httptest.NewRecorder() - r := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/policy", nil) + r := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/ops/policy", nil) h.Get(w, r) assert.Equal(t, http.StatusOK, w.Code) @@ -42,7 +42,7 @@ func TestPolicyHandler_Get_WithPolicy(t *testing.T) { h := NewPolicyHandler(store) w := httptest.NewRecorder() - r := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/policy", nil) + r := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/ops/policy", nil) h.Get(w, r) assert.Equal(t, http.StatusOK, w.Code) @@ -67,7 +67,7 @@ func TestPolicyHandler_Validate_Valid(t *testing.T) { } body, _ := json.Marshal(p) w := httptest.NewRecorder() - r := httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/v1/policy/validate", bytes.NewReader(body)) + r := httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/v1/ops/policy/validate", bytes.NewReader(body)) h.Validate(w, r) assert.Equal(t, http.StatusOK, w.Code) @@ -80,7 +80,7 @@ func TestPolicyHandler_Validate_InvalidJSON(t *testing.T) { h := NewPolicyHandler(store) w := httptest.NewRecorder() - r := httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/v1/policy/validate", bytes.NewReader([]byte(`not json`))) + r := httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/v1/ops/policy/validate", bytes.NewReader([]byte(`not json`))) h.Validate(w, r) assert.Equal(t, http.StatusBadRequest, w.Code) @@ -94,7 +94,7 @@ func TestPolicyHandler_Put_InvalidJSON(t *testing.T) { h := NewPolicyHandler(store) w := httptest.NewRecorder() - r := httptest.NewRequestWithContext(context.Background(), http.MethodPut, "/v1/policy", bytes.NewReader([]byte(`{bad}`))) + r := httptest.NewRequestWithContext(context.Background(), http.MethodPut, "/v1/ops/policy", bytes.NewReader([]byte(`{bad}`))) h.Put(w, r) assert.Equal(t, http.StatusBadRequest, w.Code) @@ -129,8 +129,8 @@ func TestPolicyHandler_RequestBodyCap(t *testing.T) { path string call func(*PolicyHandler, http.ResponseWriter, *http.Request) }{ - {"put", http.MethodPut, "/v1/policy", (*PolicyHandler).Put}, - {"validate", http.MethodPost, "/v1/policy/validate", (*PolicyHandler).Validate}, + {"put", http.MethodPut, "/v1/ops/policy", (*PolicyHandler).Put}, + {"validate", http.MethodPost, "/v1/ops/policy/validate", (*PolicyHandler).Validate}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -164,7 +164,7 @@ func TestPolicyHandler_Validate_InvalidPolicy(t *testing.T) { } body, _ := json.Marshal(p) w := httptest.NewRecorder() - r := httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/v1/policy/validate", bytes.NewReader(body)) + r := httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/v1/ops/policy/validate", bytes.NewReader(body)) h.Validate(w, r) assert.Equal(t, http.StatusBadRequest, w.Code) diff --git a/internal/api/query.go b/internal/api/query.go index e24f1abc..c7c97965 100644 --- a/internal/api/query.go +++ b/internal/api/query.go @@ -12,9 +12,9 @@ import ( "time" ) -// QueryHandler handles POST /v1/admin/query. +// QueryHandler handles POST /v1/ops/query. // -// Authorization is enforced at the router (the /v1/admin/* RequireAdmin gate +// Authorization is enforced at the router (the /v1/ops/* RequireAdmin gate // in NewRouter). The handler trusts any caller that reaches it. See // internal/api/router.go for the role-gate rationale. // @@ -160,7 +160,7 @@ func (h *QueryHandler) Handle(w http.ResponseWriter, r *http.Request) { // dropped `params` array (or any other deprecated/typo'd field) // get a clear 400 instead of silently having the field ignored. // The pre-proxy /v1/admin/query handler accepted positional `?` params - // via a `params` array; the new /v1/admin/query HTTP proxy doesn't + // via a `params` array; the new /v1/ops/query HTTP proxy doesn't // forward query-string params at all, so a request that still ships // `params` is broken at the contract level — fail loudly. dec.DisallowUnknownFields() diff --git a/internal/api/query_test.go b/internal/api/query_test.go index b1134654..0a97294c 100644 --- a/internal/api/query_test.go +++ b/internal/api/query_test.go @@ -35,7 +35,7 @@ func newProxyHandler(t *testing.T, fakeCH http.Handler) *QueryHandler { func postQuery(h *QueryHandler, body []byte) *httptest.ResponseRecorder { w := httptest.NewRecorder() - r := httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/v1/admin/query", bytes.NewReader(body)) + r := httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/v1/ops/query", bytes.NewReader(body)) h.Handle(w, r) return w } @@ -55,7 +55,7 @@ func assertSecurityHeaders(t *testing.T, w *httptest.ResponseRecorder) { } // TestQueryHandler_RejectsMalformedRequests pins every contract-rejection -// path on /v1/admin/query: the handler must surface a 400 with a JSON +// path on /v1/ops/query: the handler must surface a 400 with a JSON // error envelope, the standard security headers, AND a specific error // message that lets clients tell the failure modes apart. None of these // cases involve an upstream call — the handler should reject before the @@ -109,7 +109,7 @@ func TestQueryHandler_RejectsMalformedRequests(t *testing.T) { t.Parallel() h := NewQueryHandler("http://unused.invalid", "", "", "", time.Second*time.Duration(30)) w := httptest.NewRecorder() - r := httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/v1/admin/query", bytes.NewReader([]byte(tt.body))) + r := httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/v1/ops/query", bytes.NewReader([]byte(tt.body))) h.Handle(w, r) testutil.AssertJSONContains(t, w, http.StatusBadRequest, map[string]any{"error": tt.wantErr}) @@ -476,7 +476,7 @@ func TestQueryHandler_ContextCancelPropagates(t *testing.T) { w := httptest.NewRecorder() ctx, cancel := context.WithCancel(context.Background()) - r := httptest.NewRequestWithContext(ctx, http.MethodPost, "/v1/admin/query", bytes.NewReader(body)) + r := httptest.NewRequestWithContext(ctx, http.MethodPost, "/v1/ops/query", bytes.NewReader(body)) done := make(chan struct{}) go func() { diff --git a/internal/api/router.go b/internal/api/router.go index be88c58f..275b2673 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -137,10 +137,6 @@ func NewRouter(deps Dependencies) http.Handler { r.Post("/ingest", deps.Ingest.Handle) r.Get("/stream", deps.SSE.Handle) - // Schema discovery — admin-only (no policy gate of its own). - r.With(requireAdmin).Get("/schema", deps.Schema.Get) - r.With(requireAdmin).Post("/schema/refresh", deps.Schema.Refresh) - // Structured query endpoint. if deps.StructuredQuery != nil { r.Post("/query", deps.StructuredQuery.Handle) @@ -152,21 +148,26 @@ func NewRouter(deps Dependencies) http.Handler { r.Post("/pipes/{name}", deps.Pipes.Execute) } - // DLQ stats — admin-only (no policy gate of its own). - if deps.DLQ != nil { - r.With(requireAdmin).Get("/dlq/stats", deps.DLQ.Stats) - } - - // Admin routes. The requireAdmin gate covers the whole tree; every - // surface below — including raw-SQL passthrough — shares the same admin - // principal set (policy.AdminRole). - r.Route("/admin", func(r chi.Router) { + // Ops routes — every admin-gated surface lives under /v1/ops. The + // requireAdmin gate covers the whole tree; every surface below — + // including raw-SQL passthrough — shares the same admin principal + // set (policy.AdminRole). + r.Route("/ops", func(r chi.Router) { r.Use(requireAdmin) + // Schema discovery. + r.Get("/schema", deps.Schema.Get) + r.Post("/schema/refresh", deps.Schema.Refresh) + + // DLQ stats. + if deps.DLQ != nil { + r.Get("/dlq/stats", deps.DLQ.Stats) + } + // Raw-SQL passthrough. The only sanctioned surface for // non-insert mutations (DELETE/UPDATE/TRUNCATE/DROP/ALTER/…) // and for ad-hoc SELECTs that don't fit the structured - // query AST. Authorization is the /v1/admin/* gate above: + // query AST. Authorization is the /v1/ops/* gate above: // raw SQL has no per-statement scope check (we can't // authorize predicates without a full SQL parser), so the // role gate is the entire authorization story. Non-admin diff --git a/internal/api/router_test.go b/internal/api/router_test.go index 4615602d..4d9462a7 100644 --- a/internal/api/router_test.go +++ b/internal/api/router_test.go @@ -9,10 +9,12 @@ import ( "github.com/Wave-RF/WaveHouse/internal/auth" "github.com/Wave-RF/WaveHouse/internal/discovery" + "github.com/Wave-RF/WaveHouse/internal/mq" "github.com/Wave-RF/WaveHouse/internal/policy" "github.com/Wave-RF/WaveHouse/internal/stream" "github.com/Wave-RF/WaveHouse/internal/testutil" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestRequireAdmin_AdminAllowed(t *testing.T) { @@ -285,15 +287,21 @@ func TestNewRouter_RoutesRegistered(t *testing.T) { pub := &testutil.MockPublisher{} hub := stream.NewHub(nil, nil, nil) + emb, err := mq.NewEmbedded(t.TempDir(), 1024*1024, testutil.NopLogger()) + require.NoError(t, err) + t.Cleanup(func() { _ = emb.Close() }) + deps := Dependencies{ - Ingest: NewIngestHandler(reg, pub, testutil.NopLogger()), - Query: &QueryHandler{}, - SSE: NewStreamHandler(hub, nil), - Health: &HealthHandler{}, - Version: NewVersionHandler("test", "test", "test"), - Schema: NewSchemaHandler(reg), - AuthMW: func(next http.Handler) http.Handler { return next }, - Logger: testutil.NopLogger(), + Ingest: NewIngestHandler(reg, pub, testutil.NopLogger()), + Query: &QueryHandler{}, + SSE: NewStreamHandler(hub, nil), + Health: &HealthHandler{}, + Version: NewVersionHandler("test", "test", "test"), + Schema: NewSchemaHandler(reg), + DLQ: NewDLQHandler(emb.JetStream(), testutil.NopLogger()), + AuthMW: func(next http.Handler) http.Handler { return next }, + PolicyStore: policy.NewMemoryStore(&policy.Policy{}), + Logger: testutil.NopLogger(), } router := NewRouter(deps) @@ -301,34 +309,50 @@ func TestNewRouter_RoutesRegistered(t *testing.T) { tests := []struct { method string path string - expect int // expected status (not 404/405) + role string // "" = roleless request (proves the row needs no gate) + expect int }{ - // Canonical K8s-convention probes. - {http.MethodGet, "/livez", http.StatusOK}, - {http.MethodGet, "/readyz", http.StatusOK}, + // Canonical K8s-convention probes. Roleless: a 200 proves ungated. + {http.MethodGet, "/livez", "", http.StatusOK}, + {http.MethodGet, "/readyz", "", http.StatusOK}, // Deprecated aliases (kept for v0.1.x, removed in v0.2.0). - {http.MethodGet, "/healthz", http.StatusOK}, - {http.MethodGet, "/health", http.StatusOK}, - {http.MethodGet, "/ready", http.StatusOK}, - {http.MethodGet, "/version", http.StatusOK}, + {http.MethodGet, "/healthz", "", http.StatusOK}, + {http.MethodGet, "/health", "", http.StatusOK}, + {http.MethodGet, "/ready", "", http.StatusOK}, + {http.MethodGet, "/version", "", http.StatusOK}, // Public content-free liveness ping for the SDK (under /v1, no auth gate). - {http.MethodGet, "/v1/health", http.StatusOK}, - // Schema is admin-only (see TestNewRouter_SchemaAdminOnly); a roleless - // request is denied 403 — the route still exists, which is what this - // registration test asserts (not 404/405). - {http.MethodGet, "/v1/schema", http.StatusForbidden}, - {http.MethodGet, "/v1/schema?table=events", http.StatusForbidden}, + {http.MethodGet, "/v1/health", "", http.StatusOK}, + // Admin-gated /v1/ops routes. The tree-level gate denies before + // sub-route matching, so a 403 would NOT prove registration (every + // /v1/ops/ 403s rolelessly) — these rows therefore run as + // admin and require the handler's 200 (denial is pinned separately by + // TestNewRouter_SchemaAdminOnly and TestNewRouter_RawSQLAdminGate). + {http.MethodGet, "/v1/ops/schema", "admin", http.StatusOK}, + {http.MethodGet, "/v1/ops/schema?table=events", "admin", http.StatusOK}, + {http.MethodPost, "/v1/ops/schema/refresh", "admin", http.StatusOK}, + {http.MethodGet, "/v1/ops/dlq/stats", "admin", http.StatusOK}, + // Pre-/v1/ops paths, removed with no aliases: they must stay 404. + // Roleless, so re-registering an alias fails this row whether the + // alias is gated (403) or — the real hazard — ungated (200). + {http.MethodGet, "/v1/schema", "", http.StatusNotFound}, + {http.MethodPost, "/v1/schema/refresh", "", http.StatusNotFound}, + {http.MethodGet, "/v1/dlq/stats", "", http.StatusNotFound}, + {http.MethodPost, "/v1/admin/query", "", http.StatusNotFound}, + {http.MethodGet, "/v1/admin/policy", "", http.StatusNotFound}, + {http.MethodGet, "/v1/admin/pipes", "", http.StatusNotFound}, } for _, tt := range tests { t.Run(tt.method+" "+tt.path, func(t *testing.T) { t.Parallel() - req := httptest.NewRequestWithContext(context.Background(), tt.method, tt.path, nil) + ctx := context.Background() + if tt.role != "" { + ctx = auth.WithRole(ctx, tt.role) + } + req := httptest.NewRequestWithContext(ctx, tt.method, tt.path, nil) rec := httptest.NewRecorder() router.ServeHTTP(rec, req) assert.Equal(t, tt.expect, rec.Code, "unexpected route status") - assert.NotEqual(t, http.StatusNotFound, rec.Code, "route should exist") - assert.NotEqual(t, http.StatusMethodNotAllowed, rec.Code, "method should be allowed") }) } } @@ -396,7 +420,7 @@ func TestNewRouter_CORSOnStream(t *testing.T) { }) } -// TestNewRouter_RawSQLAdminGate pins the contract for POST /v1/admin/query: +// TestNewRouter_RawSQLAdminGate pins the contract for POST /v1/ops/query: // // admin role → reaches handler // service role → 403 (no longer privileged) @@ -429,7 +453,7 @@ func TestNewRouter_RawSQLAdminGate(t *testing.T) { if role != "" { ctx = auth.WithRole(ctx, role) } - req := httptest.NewRequestWithContext(ctx, http.MethodPost, "/v1/admin/query", nil) + req := httptest.NewRequestWithContext(ctx, http.MethodPost, "/v1/ops/query", nil) rec := httptest.NewRecorder() router.ServeHTTP(rec, req) return rec @@ -489,16 +513,17 @@ func TestNewRouter_OptionalDepsNil(t *testing.T) { router := NewRouter(deps) // Admin pipes route should 404 when pipes is nil. Send it as admin so the - // /v1/admin gate passes and we observe the route being absent (the gate + // /v1/ops gate passes and we observe the route being absent (the gate // runs before sub-route matching, so a roleless request would 403 first). - req := httptest.NewRequestWithContext(auth.WithRole(context.Background(), "admin"), http.MethodGet, "/v1/admin/pipes", nil) + req := httptest.NewRequestWithContext(auth.WithRole(context.Background(), "admin"), http.MethodGet, "/v1/ops/pipes", nil) rec := httptest.NewRecorder() router.ServeHTTP(rec, req) assert.Equal(t, http.StatusNotFound, rec.Code) - // DLQ stats should 404 when DLQ is nil (the route — and its admin gate — - // is never registered, so no role is needed to observe the 404). - req = httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/dlq/stats", nil) + // DLQ stats should 404 when DLQ is nil. As with pipes above, send it as + // admin: the /v1/ops gate covers the whole tree and runs before sub-route + // matching, so a roleless request would 403 before the absent route 404s. + req = httptest.NewRequestWithContext(auth.WithRole(context.Background(), "admin"), http.MethodGet, "/v1/ops/dlq/stats", nil) rec = httptest.NewRecorder() router.ServeHTTP(rec, req) assert.Equal(t, http.StatusNotFound, rec.Code) @@ -643,7 +668,9 @@ func TestJSONRecoverer_PanicAfterPartialWriteDoesNotCorrupt(t *testing.T) { // gate is its entire authorization story. func TestNewRouter_SchemaAdminOnly(t *testing.T) { t.Parallel() - reg := testutil.NewTestSchemaRegistry(t, nil) + reg := testutil.NewTestSchemaRegistry(t, []*discovery.TableSchema{ + {Name: "events", Columns: []discovery.Column{{Name: "id", Type: "String"}}}, + }) pub := &testutil.MockPublisher{} hub := stream.NewHub(nil, nil, nil) @@ -669,7 +696,7 @@ func TestNewRouter_SchemaAdminOnly(t *testing.T) { return rec } - for _, path := range []string{"/v1/schema", "/v1/schema?table=events"} { + for _, path := range []string{"/v1/ops/schema", "/v1/ops/schema?table=events"} { t.Run(path+" tokenless 403", func(t *testing.T) { t.Parallel() rec := get(path, "") @@ -680,8 +707,10 @@ func TestNewRouter_SchemaAdminOnly(t *testing.T) { t.Run(path+" admin reaches handler", func(t *testing.T) { t.Parallel() rec := get(path, "admin") - assert.NotEqual(t, http.StatusForbidden, rec.Code, "admin must reach schema") - assert.NotEqual(t, http.StatusUnauthorized, rec.Code) + // Require the handler's 200, not merely "not denied": under the + // tree-level /v1/ops gate a 404 would also be "not 403", so only + // a success status proves the admin reached the schema handler. + assert.Equal(t, http.StatusOK, rec.Code, "admin must reach schema") }) } } diff --git a/internal/api/schema_test.go b/internal/api/schema_test.go index dd50be10..39e65ca8 100644 --- a/internal/api/schema_test.go +++ b/internal/api/schema_test.go @@ -22,7 +22,7 @@ func TestSchema_List(t *testing.T) { h := NewSchemaHandler(reg) w := httptest.NewRecorder() - r := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/schema", nil) + r := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/ops/schema", nil) h.List(w, r) assert.Equal(t, http.StatusOK, w.Code) @@ -43,7 +43,7 @@ func TestSchema_Get_Exists(t *testing.T) { h := NewSchemaHandler(reg) w := httptest.NewRecorder() - r := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/schema?table=clicks", nil) + r := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/ops/schema?table=clicks", nil) h.Get(w, r) @@ -61,7 +61,7 @@ func TestSchema_Get_NotFound(t *testing.T) { h := NewSchemaHandler(reg) w := httptest.NewRecorder() - r := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/schema?table=nonexistent", nil) + r := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/ops/schema?table=nonexistent", nil) h.Get(w, r) diff --git a/internal/config/config.go b/internal/config/config.go index 556b319c..4c37261d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -169,7 +169,7 @@ type Auth struct { // cleanly — policy.NewStore returns a fatal error otherwise, so a typo or a // missing mount surfaces as a refused boot instead of a silent fail-closed // (every request 403s, including admin). When left empty, the store comes up -// with no cached policy and the operator seeds via PUT /v1/admin/policy. +// with no cached policy and the operator seeds via PUT /v1/ops/policy. // // A baked-in default like "policy.yaml" would re-introduce the silent-lockout // failure mode for any deployment that didn't ship that exact file at CWD, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index bc9c9da5..168b7beb 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -445,7 +445,7 @@ func TestValidate_PrometheusPathReservedConflicts(t *testing.T) { {"health probe", "/health", 0, "reserved endpoint"}, {"ready probe", "/ready", 0, "reserved endpoint"}, {"v1 root same-port", "/v1", 0, "authenticated /v1"}, - {"v1 subpath same-port", "/v1/admin/metrics", 0, "authenticated /v1"}, + {"v1 subpath same-port", "/v1/ops/metrics", 0, "authenticated /v1"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/ingest/worker_test.go b/internal/ingest/worker_test.go index 0799525b..c4ff6633 100644 --- a/internal/ingest/worker_test.go +++ b/internal/ingest/worker_test.go @@ -996,9 +996,12 @@ func TestDispatchLoop_PerTableBatching_NoCrossTableContamination(t *testing.T) { t.Parallel() const ( - // worker.go config values + // worker.go config values. maxWait is deliberately far longer than the + // assertion window below (production's defaultMaxWait), so a + // size-trigger flush and a batch stranded waiting out the timer (the + // regression pinned here) can't blur together on a slow CI runner. maxBatch = 100 - maxWait = 2 * time.Second + maxWait = 30 * time.Second // test values batchA = 5 // intentionally under maxBatch @@ -1076,13 +1079,16 @@ func TestDispatchLoop_PerTableBatching_NoCrossTableContamination(t *testing.T) { require.NoError(t, err) } - // All B rows must reach ClickHouse within maxWait − epsilon. + // All B rows must reach ClickHouse within production's defaultMaxWait — + // proof they flushed on B's own size trigger faster than the real 5s + // timer bound, not on this test's (30s) timer. require.Eventually(t, func() bool { mu.Lock() defer mu.Unlock() return rowsByTable["tableB"] >= batchB - }, maxWait-(250*time.Millisecond), 25*time.Millisecond, - "table B should flush "+fmt.Sprint(batchB)+" rows within "+fmt.Sprint(maxWait)+" of being published "+ + }, defaultMaxWait, 25*time.Millisecond, + "table B hit maxBatch and should flush on its own size trigger "+ + "before production's defaultMaxWait timer bound "+ "(table A's prior events must not strand B rows in a batch "+ "that waits for the maxWait timer)", ) diff --git a/internal/policy/roles.go b/internal/policy/roles.go index 38256b6a..4b72309c 100644 --- a/internal/policy/roles.go +++ b/internal/policy/roles.go @@ -33,7 +33,7 @@ func AdminRole(p *Policy) string { // role is never admin either, regardless of how admin_role is configured — a // roleless request must never inherit admin via an empty-string match. This is // the single source of truth for the admin check; Evaluate, ResolveRole, -// Validate, the /v1/admin gate, and pipe authorization all route through it. +// Validate, the /v1/ops gate, and pipe authorization all route through it. func IsAdmin(p *Policy, role string) bool { if p == nil { return false @@ -44,7 +44,7 @@ func IsAdmin(p *Policy, role string) bool { // DefaultRoleGrantsAdmin reports whether the policy's default_role resolves to // the admin role. When true, ResolveRole maps every roleless request (no token, // or a token without a role claim) to the admin role, so unauthenticated callers -// receive full admin access — including /v1/admin/*. This is permitted as a +// receive full admin access — including /v1/ops/*. This is permitted as a // local/dev convenience (no token needed to exercise admin surfaces), but is // unsafe in production, so the policy store warns loudly whenever a policy with // this setting is adopted. An empty default_role (no public access) is never diff --git a/internal/policy/scalars.go b/internal/policy/scalars.go index 5ff3ca44..09af6f94 100644 --- a/internal/policy/scalars.go +++ b/internal/policy/scalars.go @@ -16,7 +16,7 @@ import ( // Millis and ByteSize are the human-friendly-in / number-out value types for the // policy's resource caps. On the way IN (a config file or a hand-crafted API // body), they accept either a readable string ("10s", "4GiB") or a bare number -// in the canonical unit. On the way OUT (GET /v1/admin/policy and any read-back) +// in the canonical unit. On the way OUT (GET /v1/ops/policy and any read-back) // they marshal as that bare number — they implement no Marshaler, so the default // integer encoding applies — so SDKs consume a plain int and never reimplement // the humanization. The canonical units are milliseconds (time) and bytes diff --git a/internal/policy/store.go b/internal/policy/store.go index ebdd21ae..80c25093 100644 --- a/internal/policy/store.go +++ b/internal/policy/store.go @@ -122,7 +122,7 @@ func (s *Store) Put(ctx context.Context, p *Policy) error { // Watch update from a peer node. func (s *Store) warnIfDefaultRoleGrantsAdmin(p *Policy) { if DefaultRoleGrantsAdmin(p) { - s.logger.Warn("default_role equals admin_role: every unauthenticated/roleless request is granted full admin access, including /v1/admin/* — intended for local/dev only, do NOT use in production", + s.logger.Warn("default_role equals admin_role: every unauthenticated/roleless request is granted full admin access, including /v1/ops/* — intended for local/dev only, do NOT use in production", "default_role", p.DefaultRole) } } diff --git a/tests/e2e/sdk/query.test.ts b/tests/e2e/sdk/query.test.ts index 9c98fbaf..c57a9b03 100644 --- a/tests/e2e/sdk/query.test.ts +++ b/tests/e2e/sdk/query.test.ts @@ -152,7 +152,7 @@ describe("Query", () => { }); it("raw SQL query", async () => { - // Scope to seededIds so the SQL string is unique per run. /v1/admin/query + // Scope to seededIds so the SQL string is unique per run. /v1/ops/query // itself never caches (Cache-Control: no-store on every response) — the // uniqueness here avoids confusing test output when this and admin.test.ts // each independently SELECT count() from the same table and the suites diff --git a/tests/e2e/sdk/setup.ts b/tests/e2e/sdk/setup.ts index 22862253..a66e8864 100644 --- a/tests/e2e/sdk/setup.ts +++ b/tests/e2e/sdk/setup.ts @@ -48,17 +48,17 @@ async function createTables(): Promise { } } -// /v1/schema returns an array of { name, columns, ... } — wait until every +// /v1/ops/schema returns an array of { name, columns, ... } — wait until every // generated table is present so we don't burn the full timeout on the happy // path. async function refreshSchema(): Promise { const headers = { Authorization: setupAuth() }; - await fetch(`${WH_URL}/v1/schema/refresh`, { method: "POST", headers }); + await fetch(`${WH_URL}/v1/ops/schema/refresh`, { method: "POST", headers }); const expected = allTableSpecs().map((t) => t.name); const start = Date.now(); while (Date.now() - start < 30_000) { - const res = await fetch(`${WH_URL}/v1/schema`, { headers }); + const res = await fetch(`${WH_URL}/v1/ops/schema`, { headers }); if (res.ok) { const schema = (await res.json()) as Array<{ name?: string }>; const present = new Set(schema.map((t) => t?.name).filter(Boolean)); @@ -85,7 +85,7 @@ async function bootstrapTestPolicy(): Promise { } const policy = { tables }; - const res = await fetch(`${WH_URL}/v1/admin/policy`, { + const res = await fetch(`${WH_URL}/v1/ops/policy`, { method: "PUT", headers: { "Content-Type": "application/json", diff --git a/tests/integration/dlq_test.go b/tests/integration/dlq_test.go index 96bfa697..b5fe8a7e 100644 --- a/tests/integration/dlq_test.go +++ b/tests/integration/dlq_test.go @@ -23,7 +23,7 @@ import ( func TestDLQ_StatsEmptyOnFreshStart(t *testing.T) { e := env(t) - resp, err := http.Get(e.server.URL + "/v1/dlq/stats") + resp, err := http.Get(e.server.URL + "/v1/ops/dlq/stats") require.NoError(t, err) defer resp.Body.Close() @@ -67,7 +67,7 @@ func TestDLQ_PopulatedOnIngestWorkerFailure(t *testing.T) { // loaded CI runner. The condition polls the API rather than the // stream so this also exercises the read path. assert.Eventually(t, func() bool { - resp, err := http.Get(e.server.URL + "/v1/dlq/stats") + resp, err := http.Get(e.server.URL + "/v1/ops/dlq/stats") if err != nil { return false } @@ -111,7 +111,7 @@ func TestDLQ_PopulatedOnIngestWorkerFailureWithBadName(t *testing.T) { // loaded CI runner. The condition polls the API rather than the // stream so this also exercises the read path. assert.Eventually(t, func() bool { - resp, err := http.Get(e.server.URL + "/v1/dlq/stats") + resp, err := http.Get(e.server.URL + "/v1/ops/dlq/stats") if err != nil { return false } diff --git a/tests/integration/ingest_test.go b/tests/integration/ingest_test.go index f9e983fa..c66bb44c 100644 --- a/tests/integration/ingest_test.go +++ b/tests/integration/ingest_test.go @@ -54,7 +54,7 @@ func TestIngest_FlowsToClickHouseWithoutDLQ(t *testing.T) { // Confirm the success path didn't tee anything into the DLQ for this // table — that's the actual contract we're asserting (no silent // duplicate writes to dlq.
alongside the real INSERT). - dlqResp, err := http.Get(e.server.URL + "/v1/dlq/stats") + dlqResp, err := http.Get(e.server.URL + "/v1/ops/dlq/stats") require.NoError(t, err) defer dlqResp.Body.Close() diff --git a/tests/integration/query_test.go b/tests/integration/query_test.go index 2a4556e6..a9ea66f4 100644 --- a/tests/integration/query_test.go +++ b/tests/integration/query_test.go @@ -19,7 +19,7 @@ import ( ) // TestQuery_MutationsReturnEmptyArray pins the response-shape contract for -// non-insert mutations through `/v1/admin/query`: HTTP 200 with body `[]`, +// non-insert mutations through `/v1/ops/query`: HTTP 200 with body `[]`, // never HTTP 500 or `null`. The endpoint proxies SQL to ClickHouse's HTTP // interface, which returns no body for mutations; WaveHouse normalises that // to the JSON empty-array shape callers can do `result.length` on. @@ -33,7 +33,7 @@ func TestQuery_MutationsReturnEmptyArray(t *testing.T) { tests := []struct { name string rowIDs []string // IDs to seed via /v1/ingest?table={table} before the mutation - mutationSQL func(table string) string // SQL to POST to /v1/admin/query + mutationSQL func(table string) string // SQL to POST to /v1/ops/query postCheck func(t *testing.T, ctx context.Context, e *testEnv, table string) // verify the mutation actually ran }{ { @@ -54,14 +54,14 @@ func TestQuery_MutationsReturnEmptyArray(t *testing.T) { }, { // Predicate-driven DELETE — the canonical example of why - // non-insert mutations route through /v1/admin/query rather + // non-insert mutations route through /v1/ops/query rather // than the policy-authorized ingest path: we can't prove a // WHERE predicate matches only rows the caller is allowed // to touch. ClickHouse lightweight DELETE marks matching // rows invisible synchronously, so the post-check can poll // for the row to disappear. // - // /v1/admin/query proxies to ClickHouse's HTTP interface + // /v1/ops/query proxies to ClickHouse's HTTP interface // (named-param syntax, not positional `?`), so the dropID // is inlined into the SQL literal — test-controlled, safe. name: "DELETE removes only the targeted row", @@ -86,7 +86,7 @@ func TestQuery_MutationsReturnEmptyArray(t *testing.T) { } return kept == 1 && dropped == 0 }, 30*time.Second, 500*time.Millisecond, - "DELETE through /v1/admin/query must mutate only the targeted row") + "DELETE through /v1/ops/query must mutate only the targeted row") }, }, } @@ -134,7 +134,7 @@ func TestQuery_MutationsReturnEmptyArray(t *testing.T) { mutationBody, _ := json.Marshal(map[string]string{"sql": tt.mutationSQL(table)}) qResp, err := http.Post( - e.server.URL+"/v1/admin/query", + e.server.URL+"/v1/ops/query", "application/json", bytes.NewReader(mutationBody), ) @@ -144,9 +144,9 @@ func TestQuery_MutationsReturnEmptyArray(t *testing.T) { respBytes, err := io.ReadAll(qResp.Body) require.NoError(t, err) require.Equal(t, http.StatusOK, qResp.StatusCode, - "mutations through /v1/admin/query must return 200, not 500; body: %s", respBytes) + "mutations through /v1/ops/query must return 200, not 500; body: %s", respBytes) assert.JSONEq(t, "[]", string(respBytes), - "mutations through /v1/admin/query must marshal to [] (empty result set), not null or {}") + "mutations through /v1/ops/query must marshal to [] (empty result set), not null or {}") postCtx, postCancel := context.WithTimeout(context.Background(), 60*time.Second) defer postCancel() diff --git a/tests/integration/setup_test.go b/tests/integration/setup_test.go index 348cabda..40f64c8c 100644 --- a/tests/integration/setup_test.go +++ b/tests/integration/setup_test.go @@ -316,7 +316,7 @@ func buildServer(ch *chInstance, embeddedMQ *mq.EmbeddedNATS, registry *discover deps := api.Dependencies{ Ingest: api.NewIngestHandler(registry, embeddedMQ, logger), - // /v1/admin/query proxies straight to ClickHouse's HTTP interface, + // /v1/ops/query proxies straight to ClickHouse's HTTP interface, // so the handler needs the HTTP URL + creds rather than the // native-protocol driver.Conn other handlers use. Query: api.NewQueryHandler(ch.httpURL(), testCHUser, testCHPassword, testCHDatabase, time.Second*time.Duration(30)),