Load Generation for Plateau

.github/workflows/bench-image.yml

@@ -0,0 +1,54 @@
+name: Bench Image
+
+on:
+  workflow_dispatch: {}
+
+env:
+  BENCH_REGISTRY_IMAGE: ghcr.io/wallaroolabs/plateau-bench
+  RUST_VERSION: 1.84.1
+
+jobs:
+  build-and-push:
+    runs-on:
+      - ubuntu-latest
+    steps:
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.BENCH_REGISTRY_IMAGE }}
+          tags: |
+            type=sha
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+          labels: |
+            org.opencontainers.image.vendor="Wallaroo Labs"
+            org.opencontainers.image.source="https://github.com/WallarooLabs/plateau/Dockerfile"
+            org.opencontainers.image.title="plateau-bench"
+
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.PUSH_CONTAINER_TOKEN }}
+
+      - name: Login to us-docker.pkg.dev/wallaroo-dev-253816/docker-hub-us
+        uses: docker/login-action@v3
+        with:
+          registry: us-docker.pkg.dev
+          username: _json_key
+          password: ${{ secrets.US_PKG_DEV_CACHE_JSON_KEY }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          target: batch-load
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          push: true
+          build-args: |
+            RUST_VERSION=${{ env.RUST_VERSION }}

.github/workflows/rust.yml

@@ -76,6 +76,7 @@ jobs:
         id: build
         uses: docker/build-push-action@v6
         with:
+          target: plateau
           labels: ${{ steps.meta.outputs.labels }}
           outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
           build-args: |
@@ -134,6 +135,7 @@ jobs:
         id: build
         uses: docker/build-push-action@v6
         with:
+          target: plateau
           labels: ${{ steps.meta.outputs.labels }}
           outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
           build-args: |
@@ -199,3 +201,4 @@ jobs:
       - name: Inspect image
         run: |
           docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
+

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 ARG RUST_VERSION
 FROM ghcr.io/wallaroolabs/rust:${RUST_VERSION} AS base
 
 FROM base AS build
 ARG BUILD_COMMIT
@@ -11,14 +11,24 @@
 RUN \
     if [ "${TARGETARCH}" = "amd64" ]; then ARCH=x86_64; elif [ "${TARGETARCH}" = "arm64" ]; then ARCH=aarch64; else exit 1; fi && \
     rustup target add ${ARCH}-unknown-linux-musl && \
-    cargo build --release --target ${ARCH}-unknown-linux-musl -p plateau && \
-    cp target/${ARCH}-unknown-linux-musl/release/plateau target/release/plateau
+    cargo build --release --target ${ARCH}-unknown-linux-musl -p plateau -p bench && \
+    cp target/${ARCH}-unknown-linux-musl/release/plateau target/release/plateau && \
+    cp target/${ARCH}-unknown-linux-musl/release/batch-load target/release/batch-load
 
-FROM scratch
+FROM scratch AS plateau
 
 LABEL org.opencontainers.image.vendor="Wallaroo Labs"
 LABEL org.opencontainers.image.source="https://github.com/WallarooLabs/plateau/Dockerfile"
 LABEL org.opencontainers.image.title="plateau"
 
 COPY --from=build /usr/src/plateau/target/release/plateau .
 CMD ["./plateau"]
+
+FROM scratch AS batch-load
+
+LABEL org.opencontainers.image.vendor="Wallaroo Labs"
+LABEL org.opencontainers.image.source="https://github.com/WallarooLabs/plateau/Dockerfile"
+LABEL org.opencontainers.image.title="plateau-bench"
+
+COPY --from=build /usr/src/plateau/target/release/batch-load .
+CMD ["./batch-load"]

bench/Cargo.toml

@@ -10,8 +10,11 @@ authors.workspace = true
 [dependencies]
 anyhow = "1"
 async-trait = "0.1"
+clap = { version = "4", features = ["derive"] }
+toml = "0.8"
 serde = { version = "1", features = ["derive"] }
 humantime-serde = "1"
+humantime = "2"
 
 sample-std = "0.2.1"
 sample-arrow-rs = "55.2.0"
@@ -38,3 +41,10 @@ plateau-client = { workspace = true, features = ["health"] }
 
 [[bin]]
 name = "cv"
+
+[[bin]]
+name = "load"
+
+[[bin]]
+name = "batch-load"
+path = "src/bin/batch_load.rs"

bench/k8s/README.md

@@ -0,0 +1,53 @@
+# Running batch-load in-cluster
+
+Running through `kubectl port-forward` causes spurious
+`Failed to buffer the request body` (400) errors: the tunnel is a single
+apiserver-proxied stream that stalls under sustained load, truncating request
+bodies. Running in-cluster talks straight to the Service and avoids this.
+
+## Build and push the image (outside CI)
+
+The `batch-load` target in the repo-root `Dockerfile` produces a static musl
+binary in a `scratch` image. Build it directly with buildx — no CI required:
+
+```sh
+# from the repo root
+REGISTRY=ghcr.io/wallaroolabs/plateau-bench
+TAG=dev
+
+docker buildx build \
+  --target batch-load \
+  --platform linux/amd64 \
+  --build-arg RUST_VERSION=1.84.1 \
+  -t $REGISTRY:$TAG \
+  --push \
+  .
+```
+
+Notes:
+- `--target batch-load` selects the bench stage (the default target builds the
+  plateau server).
+- `--platform linux/amd64` matches a typical cluster; add/replace with
+  `linux/arm64` if your nodes are arm.
+- `--push` uploads straight to the registry. Log in first
+  (`docker login ghcr.io`). Use any registry your cluster can pull from.
+
+## Deploy the Job
+
+Edit `job.yaml`:
+- `image:` → the tag you just pushed
+- `PLATEAU_URL` → `http://<plateau-service>.<namespace>.svc.cluster.local:3030`
+
+Then:
+
+```sh
+kubectl apply -n <namespace> -f bench/k8s/job.yaml
+kubectl logs -n <namespace> -f job/batch-load
+```
+
+The schemas dir and state file live on a PVC, so the Job resumes the same
+topic pool and schedule if it restarts. For a throwaway run, replace the
+`persistentVolumeClaim` volume with `emptyDir: {}`.
+
+To restart with a clean slate, delete the PVC (`kubectl delete pvc
+batch-load-data`) before re-applying.

bench/k8s/batch-config.toml

@@ -0,0 +1,20 @@
+# Sample batch-load configuration. Mounted into the Job via a ConfigMap.
+#
+# All range values (columns / partitions / rows) are drawn from a clamped
+# normal distribution. See `batch-load --help` for full field docs.
+
+speed = 200.0                 # compress time: 1h intervals fire every minute
+schemas_dir = "/data/batch-schemas"
+state_file = "/data/batch-state.json"
+
+[topics]
+count = 200                  # total topic pool
+active = 8                   # topics writing at once
+rotation_interval = "1h"     # how often the active window advances
+columns_min = 3
+columns_max = 35
+partitions_min = 1
+partitions_max = 8
+rows_min = 1000
+rows_max = 50000
+batch_interval = "1h"        # interval between batches per partition

bench/k8s/job.yaml

@@ -0,0 +1,94 @@
+# In-cluster batch-load runner.
+#
+# Talks directly to the plateau Service (no port-forward), so the spurious
+# "Failed to buffer the request body" errors from a choked tunnel go away.
+#
+# Edit before applying:
+#   - image:        the tag you built and pushed
+#   - PLATEAU_URL:  http://<plateau-service>.<namespace>.svc.cluster.local:3030
+#   - namespace:    via `kubectl apply -n <ns> -f job.yaml`
+#
+# The schemas dir and state file live on a PVC so a restarted Job resumes the
+# same topic pool and schedule. For a throwaway run, swap the PVC for an
+# `emptyDir: {}` volume.
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: batch-load-config
+data:
+  batch-config.toml: |
+    speed = 60.0
+    schemas_dir = "/data/batch-schemas"
+    state_file = "/data/batch-state.json"
+
+    [topics]
+    count = 200
+    active = 8
+    rotation_interval = "1h"
+    columns_min = 3
+    columns_max = 35
+    partitions_min = 1
+    partitions_max = 2
+    rows_min = 1000
+    rows_max = 50000
+    batch_interval = "15m"
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: batch-load-data
+spec:
+  accessModes: ["ReadWriteOnce"]
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: batch-load
+spec:
+  backoffLimit: 6
+  template:
+    metadata:
+      labels:
+        app: batch-load
+    spec:
+      restartPolicy: OnFailure
+      # Reuse the same ghcr.io pull secret the plateau pods use. Find its name
+      # with `kubectl get secrets -n <ns> --field-selector
+      # type=kubernetes.io/dockerconfigjson` and set it here.
+      imagePullSecrets:
+        - name: regcred
+      containers:
+        - name: batch-load
+          image: ghcr.io/wallaroolabs/plateau-bench:dev
+          command: ["./batch-load"]
+          args:
+            - "--config=/config/batch-config.toml"
+            - "--url=$(PLATEAU_URL)"
+          env:
+            - name: PLATEAU_URL
+              value: "http://plateau.wallaroo.svc.cluster.local:3030"
+            - name: RUST_LOG
+              value: "warn,bench=info"
+          volumeMounts:
+            - name: config
+              mountPath: /config
+            - name: data
+              mountPath: /data
+          resources:
+            requests:
+              cpu: "500m"
+              memory: "512Mi"
+            limits:
+              cpu: "2"
+              memory: "2Gi"
+      volumes:
+        - name: config
+          configMap:
+            name: batch-load-config
+        - name: data
+          persistentVolumeClaim:
+            claimName: batch-load-data