Add ClickHouse destination

.config/nextest.toml

@@ -10,5 +10,5 @@ test-threads = "num-cpus"
 max-threads = 1
 
 [[profile.default.overrides]]
-filter = "test(exclusive_) | binary_id(etl::main) | (binary_id(etl-destinations::main) & test(/^(bigquery_pipeline|ducklake_destination|ducklake_pipeline|iceberg_destination)::/)) | (binary_id(etl-destinations) & test(/ducklake::core::tests::postgres_backed::/))"
+filter = "test(exclusive_) | binary_id(etl::main) | (binary_id(etl-destinations::main) & test(/^(bigquery_pipeline|clickhouse_pipeline|ducklake_destination|ducklake_pipeline|iceberg_destination)::/)) | (binary_id(etl-destinations) & test(/ducklake::core::tests::postgres_backed::/))"
 test-group = "shared-pg"

.github/workflows/ci.yml

@@ -177,6 +177,9 @@ jobs:
       NUM_LOCAL_DATABASES: 4
       TESTS_DATABASE_USERNAME: postgres
       TESTS_DATABASE_PASSWORD: postgres
+      TESTS_CLICKHOUSE_URL: http://localhost:8123
+      TESTS_CLICKHOUSE_USER: etl
+      TESTS_CLICKHOUSE_PASSWORD: etl
       ETL_DUCKDB_EXTENSION_ROOT: ${{ github.workspace }}/vendor/duckdb/extensions
     steps:
       - name: Checkout

Cargo.toml

@@ -74,6 +74,7 @@ byteorder = { version = "1.5.0", default-features = false }
 bytes = { version = "1.10.1" }
 chrono = { version = "0.4.41", default-features = false }
 clap = { version = "4.5.42", default-features = false }
+clickhouse = { version = "0.14", default-features = false }
 config = { version = "0.14", default-features = false }
 configcat = { version = "0.1.3", default-features = false }
 const-oid = { version = "0.9.6", default-features = false }
@@ -132,7 +133,7 @@ tracing-actix-web = { version = "0.7.19", default-features = false }
 tracing-appender = { version = "0.2.3", default-features = false }
 tracing-log = { version = "0.2.0", default-features = false }
 tracing-subscriber = { version = "0.3.19", default-features = false }
-url = { version = "2.5.8" }
+url = { version = "2.5.8", features = ["serde"] }
 utoipa = { version = "5.4.0", default-features = false }
 utoipa-swagger-ui = { version = "9.0.2", default-features = false, features = [
     "vendored",

DEVELOPMENT.md

@@ -66,10 +66,10 @@ The fastest way to get started is using the setup script:
 ```
 
 This script will:
-1. Start PostgreSQL via Docker Compose
-2. Run etl-api migrations
-3. Seed the default replicator image
-4. Configure the Kubernetes environment (OrbStack)
+1. Start PostgreSQL, ClickHouse, and the local Iceberg dependencies via Docker Compose.
+2. Run etl-api migrations.
+3. Seed the default replicator image.
+4. Configure the Kubernetes environment (OrbStack).
 
 ## Database Setup
 
@@ -100,12 +100,19 @@ POSTGRES_DATA_VOLUME=/path/to/data ./scripts/init.sh
 | `POSTGRES_DB` | `postgres` | Database name |
 | `POSTGRES_PORT` | `5430` | Database port |
 | `POSTGRES_HOST` | `localhost` | Database host |
+| `CLICKHOUSE_HTTP_PORT` | `8123` | ClickHouse HTTP port |
+| `CLICKHOUSE_NATIVE_PORT` | `9000` | ClickHouse native TCP port |
+| `CLICKHOUSE_USER` | `etl` | ClickHouse user for the local Docker Compose setup |
+| `CLICKHOUSE_PASSWORD` | `etl` | ClickHouse password for the local Docker Compose setup |
 | `SKIP_DOCKER` | (empty) | Skip Docker Compose if set |
-| `POSTGRES_DATA_VOLUME` | (empty) | Path for persistent storage |
+| `POSTGRES_DATA_VOLUME` | (empty) | Path for PostgreSQL persistent storage |
+| `CLICKHOUSE_DATA_VOLUME` | (empty) | Path for ClickHouse persistent storage |
 | `REPLICATOR_IMAGE` | `ramsup/etl-replicator:latest` | Default replicator image |
 
 PostgreSQL 18+ containers store data under `/var/lib/postgresql/<major>/data`, so the Docker Compose setup mounts the parent `/var/lib/postgresql` directory to keep upgrades compatible.
 
+The same Docker Compose stack also starts ClickHouse on `http://localhost:8123` by default, which is enough for local destination development and ClickHouse integration tests.
+
 ### Manual Setup
 
 If you prefer manual setup or have an existing PostgreSQL instance:
@@ -369,6 +376,18 @@ Iceberg destination tests use local MinIO and Lakekeeper instances. The followin
 
 **Note:** Iceberg tests are only run when the `iceberg` and `test-utils` features are enabled. These use hardcoded local URLs and do not require environment variables.
 
+#### ClickHouse Test Variables
+
+ClickHouse destination tests require a reachable ClickHouse HTTP endpoint:
+
+| Variable | Required | Description |
+|----------|----------|-------------|
+| `TESTS_CLICKHOUSE_URL` | **Yes** | ClickHouse HTTP URL (for example, `http://localhost:8123`) |
+| `TESTS_CLICKHOUSE_USER` | **Yes** | ClickHouse user name (for the local Docker Compose setup, use `etl`) |
+| `TESTS_CLICKHOUSE_PASSWORD` | No | ClickHouse password; for the local Docker Compose setup, use `etl` |
+
+**Note:** ClickHouse tests are only run when the `clickhouse` and `test-utils` features are enabled. Each test creates a unique database in ClickHouse and drops it automatically when the test finishes. The Docker Compose setup started by `./scripts/init.sh` is sufficient for these tests.
+
 #### Test Output and Logging
 
 | Variable | Description |
@@ -407,6 +426,11 @@ export TESTS_DATABASE_PASSWORD=postgres
 export TESTS_BIGQUERY_PROJECT_ID=your-gcp-project-id
 export TESTS_BIGQUERY_SA_KEY_PATH=/path/to/service-account-key.json
 
+# ClickHouse test configuration (optional - only needed for ClickHouse tests)
+export TESTS_CLICKHOUSE_URL=http://localhost:8123
+export TESTS_CLICKHOUSE_USER=etl
+export TESTS_CLICKHOUSE_PASSWORD=etl
+
 # Enable test output (optional)
 export ENABLE_TRACING=1
 export RUST_LOG=info
@@ -432,6 +456,11 @@ TESTS_DATABASE_PASSWORD=postgres
 TESTS_BIGQUERY_PROJECT_ID=your-gcp-project-id
 TESTS_BIGQUERY_SA_KEY_PATH=/path/to/service-account-key.json
 
+# ClickHouse (optional - only for ClickHouse tests)
+TESTS_CLICKHOUSE_URL=http://localhost:8123
+TESTS_CLICKHOUSE_USER=etl
+TESTS_CLICKHOUSE_PASSWORD=etl
+
 # Test output (optional)
 ENABLE_TRACING=1
 RUST_LOG=info
@@ -462,6 +491,9 @@ TESTS_DATABASE_HOST=localhost TESTS_DATABASE_PORT=5430 TESTS_DATABASE_USERNAME=p
 
 # Run tests with tracing output for debugging
 TESTS_DATABASE_HOST=localhost TESTS_DATABASE_PORT=5430 TESTS_DATABASE_USERNAME=postgres TESTS_DATABASE_PASSWORD=postgres ENABLE_TRACING=1 RUST_LOG=info cargo test -p etl-api --test tenants tenant_can_be_created -- --nocapture
+
+# Run the ClickHouse destination integration test against the local Docker Compose service
+TESTS_DATABASE_HOST=localhost TESTS_DATABASE_PORT=5430 TESTS_DATABASE_USERNAME=postgres TESTS_DATABASE_PASSWORD=postgres TESTS_CLICKHOUSE_URL=http://localhost:8123 TESTS_CLICKHOUSE_USER=etl TESTS_CLICKHOUSE_PASSWORD=etl cargo test -p etl-destinations --features clickhouse,test-utils clickhouse_pipeline -- --nocapture
 ```
 
 **Packages requiring `--features test-utils`:**

etl-api/Cargo.toml

@@ -31,7 +31,7 @@ configcat = { workspace = true }
 constant_time_eq = { workspace = true }
 etl = { workspace = true }
 etl-config = { workspace = true, features = ["utoipa", "supabase"] }
-etl-destinations = { workspace = true, features = ["bigquery", "iceberg", "ducklake"] }
+etl-destinations = { workspace = true, features = ["bigquery", "clickhouse", "ducklake", "iceberg"] }
 etl-postgres = { workspace = true, features = ["replication"] }
 etl-telemetry = { workspace = true }
 k8s-openapi = { workspace = true, features = ["latest"] }
@@ -49,11 +49,12 @@ thiserror = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 tracing = { workspace = true, default-features = false }
 tracing-actix-web = { workspace = true, features = ["emit_event_on_error"] }
+url = { workspace = true }
 utoipa = { workspace = true, features = ["actix_extras"] }
 utoipa-swagger-ui = { workspace = true, features = ["actix-web"] }
 
 [dev-dependencies]
-etl-destinations = { workspace = true, features = ["test-utils", "iceberg", "bigquery", "ducklake"] }
+etl-destinations = { workspace = true, features = ["test-utils", "bigquery", "clickhouse", "ducklake", "iceberg"] }
 etl-postgres = { workspace = true, features = ["test-utils", "sqlx"] }
 
 insta = { workspace = true, features = ["json", "redactions"] }