Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion .github/workflows/build-and-push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,4 +65,26 @@ jobs:
run: gradle test

- name: Build fat JAR
run: gradle shadowJar
run: gradle shadowJar

# Validates the GraalVM native build (metadata hints stay correct).
native-build:
runs-on: ubuntu-latest
needs: [changes, test-helm-chart]
if: always() && (needs.test-helm-chart.result == 'success' || needs.test-helm-chart.result == 'skipped')
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Set up GraalVM (JDK 21)
uses: graalvm/setup-graalvm@v1
with:
java-version: '21'
distribution: 'graalvm-community'
github-token: ${{ secrets.GITHUB_TOKEN }}

- name: Setup Gradle
uses: gradle/actions/setup-gradle@v4

- name: Compile native image
run: gradle nativeCompile --no-daemon
102 changes: 102 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ permissions:
jobs:
release:
runs-on: ubuntu-latest
outputs:
version: ${{ steps.version.outputs.version }}
steps:
- name: Checkout code
uses: actions/checkout@v4
Expand Down Expand Up @@ -95,3 +97,103 @@ jobs:
tag_name: ${{ steps.version.outputs.version }}
generate_release_notes: true
files: build/libs/klag-${{ steps.version.outputs.version }}-fat.jar

# GraalVM native image. native-image cannot cross-compile, so each arch is
# built on a matching native runner and pushed by digest, then merged into a
# single multi-arch manifest tagged `:<version>-native` and `:native`.
release-native:
needs: release
strategy:
fail-fast: false
matrix:
include:
- platform: linux/amd64
runner: ubuntu-latest
- platform: linux/arm64
runner: ubuntu-24.04-arm
runs-on: ${{ matrix.runner }}
outputs:
version: ${{ needs.release.outputs.version }}
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}

- name: Log in to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Build and push native image by digest
id: build
uses: docker/build-push-action@v6
with:
context: .
file: Dockerfile.native
platforms: ${{ matrix.platform }}
outputs: type=image,name=themoah/klag,push-by-digest=true,name-canonical=true,push=true

Comment thread
qodo-code-review[bot] marked this conversation as resolved.
- name: Export digest
run: |
mkdir -p /tmp/digests
digest="${{ steps.build.outputs.digest }}"
touch "/tmp/digests/${digest#sha256:}"

- name: Upload digest
uses: actions/upload-artifact@v4
with:
name: digest-${{ strategy.job-index }}
path: /tmp/digests/*
retention-days: 1

release-native-manifest:
needs: release-native
runs-on: ubuntu-latest
env:
VERSION: ${{ needs.release-native.outputs.version }}
steps:
- name: Download digests
uses: actions/download-artifact@v4
with:
path: /tmp/digests
pattern: digest-*
merge-multiple: true

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}

- name: Log in to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Create and push manifests
working-directory: /tmp/digests
run: |
# Per-arch digests were pushed to Docker Hub only. imagetools reads those
# source digests and copies the manifest to each target registry.
sources=$(printf "themoah/klag@sha256:%s " *)
for repo in themoah/klag ghcr.io/themoah/klag; do
docker buildx imagetools create \
-t "${repo}:${VERSION}-native" \
-t "${repo}:native" \
$sources
done
21 changes: 21 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,27 @@ Klag is a Kafka Lag Exporter built with Vert.x 4.5.22. Monitors consumer lag and
./scripts/e2e-strimzi-matrix.sh # Strimzi e2e across supported Kafka versions
```

### GraalVM Native Image (startup/memory optimized)

Requires a GraalVM JDK 21 (LTS) with `native-image` (e.g. `sdk install java 21.0.2-graalce`).
Run Gradle with that JDK as `JAVA_HOME`/`GRAALVM_HOME`.

```bash
gradle nativeCompile # -> build/native/nativeCompile/klag (standalone binary)
docker build -f Dockerfile.native -t klag:native . # distroless runtime image
scripts/benchmark-startup.sh native - build/native/nativeCompile/klag # startup/RSS bench
```

Native config lives in `build.gradle.kts` (`graalvmNative` block) plus reachability
hints in `src/main/resources/META-INF/native-image/`. Reflection metadata for Netty,
kafka-clients, logback and micrometer comes from the GraalVM Reachability Metadata
Repository (auto-enabled). Entry point is `KlagLauncher` (direct `new MainVerticle()`,
no reflective Vert.x launcher).

**Measured (macOS arm64, prometheus reporter, Kafka up):** native ≈ 70-100 ms startup /
44 MB RSS vs JVM 21 ≈ 470-520 ms / 119 MB. JVM 25 (LTS) showed no startup/memory gain
over 21 for this workload (slightly higher RSS), so the runtime stays on JDK 21.

## Architecture

Vert.x reactive framework with `Future<T>`-based async API.
Expand Down
37 changes: 37 additions & 0 deletions Dockerfile.native
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# syntax=docker/dockerfile:1
# ---- Native build stage ----
# GraalVM CE community image with native-image, JDK 21 (LTS).
FROM ghcr.io/graalvm/native-image-community:21-ol9 AS builder

ARG GRADLE_VERSION=8.14.3
ARG GRADLE_SHA256=bd71102213493060956ec229d946beee57158dbd89d0e62b91bca0fa2c5f3531
WORKDIR /app

# Install Gradle (GraalVM image ships no build tool). Checksum-verified download.
# Image already ships the C toolchain; native-image needs xargs (findutils) and
# zlib-static for the mostly-static link (-PnativeStatic).
RUN microdnf install -y unzip findutils zlib-static && \
curl -fsSL "https://services.gradle.org/distributions/gradle-${GRADLE_VERSION}-bin.zip" -o /tmp/gradle.zip && \
echo "${GRADLE_SHA256} /tmp/gradle.zip" | sha256sum -c - && \
unzip -d /opt/gradle /tmp/gradle.zip && \
ln -s "/opt/gradle/gradle-${GRADLE_VERSION}/bin/gradle" /usr/bin/gradle && \
rm -f /tmp/gradle.zip

# Cache dependencies (fail fast on resolution/config errors)
COPY build.gradle.kts settings.gradle.kts ./
RUN gradle dependencies --no-daemon

# Build the native binary (mostly-static: only libc dynamic -> runs on distroless/base)
COPY src src
RUN gradle nativeCompile --no-daemon -PnativeStatic

# ---- Runtime stage ----
# distroless/base provides glibc only (no JVM, minimal attack surface). The binary
# is mostly-static (-PnativeStatic) so it needs no libz.so.1 / libstdc++ at runtime.
FROM gcr.io/distroless/base-debian12

WORKDIR /app
COPY --from=builder /app/build/native/nativeCompile/klag /app/klag

EXPOSE 8888
ENTRYPOINT ["/app/klag"]
17 changes: 17 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,23 @@ docker run -e KAFKA_BOOTSTRAP_SERVERS=kafka:9092 \

Metrics available at `http://localhost:8888/metrics`

### Native image (faster startup, lower memory)

A GraalVM native build is published alongside the JVM image, tagged `:native` and
`:<version>-native`:

```bash
docker run -e KAFKA_BOOTSTRAP_SERVERS=kafka:9092 \
-e METRICS_REPORTER=prometheus \
-p 8888:8888 \
themoah/klag:native
```

The native binary starts in ~70-100 ms using ~44 MB RSS, versus ~500 ms / ~119 MB for
the JVM image — ideal for fast scaling and low-footprint deployments. Same config,
endpoints, and metrics. Build locally with `gradle nativeCompile` (needs a GraalVM
JDK 21) or `docker build -f Dockerfile.native -t klag:native .`.

## Metrics Exposed

| Metric | Description |
Expand Down
30 changes: 30 additions & 0 deletions build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ plugins {
java
application
id("com.gradleup.shadow") version "9.2.2"
id("org.graalvm.buildtools.native") version "0.10.6"
}

group = "io.github.themoah"
Expand Down Expand Up @@ -95,3 +96,32 @@ tasks.withType<ProcessResources> {
)
}
}

// GraalVM native image configuration.
// Entry point is KlagLauncher (direct `new MainVerticle()` - no reflective Vert.x launcher).
// Reachability metadata for Netty, kafka-clients, logback, micrometer is pulled
// from the GraalVM Reachability Metadata Repository; project-specific hints live
// in src/main/resources/META-INF/native-image/.
graalvmNative {
binaries {
named("main") {
imageName.set("klag")
mainClass.set(launcherClassName)
buildArgs.add("--no-fallback")
buildArgs.add("-H:+ReportExceptionStackTraces")
buildArgs.add("--enable-url-protocols=http,https")
// Vert.x/Netty/logback are not safe to initialize at build time.
buildArgs.add("--initialize-at-run-time=io.netty")
// -PnativeStatic (Linux/CI): statically link everything except libc so the
// binary runs on a distroless/base image with no libz.so.1 etc. Not used on
// macOS where static linking is unsupported.
if (project.hasProperty("nativeStatic")) {
buildArgs.add("-H:+StaticExecutableWithDynamicLibC")
}
}
}
metadataRepository {
enabled.set(true)
}
toolchainDetection.set(false)
}
65 changes: 65 additions & 0 deletions scripts/benchmark-startup.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
#!/usr/bin/env bash
# Benchmark startup time and memory footprint for klag variants.
#
# - startup time : launch -> "Klag started successfully" log line
# - RSS : steady state, ~3s after ready
#
# Requires a reachable Kafka (startup blocks on describeCluster):
# docker compose up -d kafka
#
# Usage:
# scripts/benchmark-startup.sh <label> <java-home|-> <jar-or-binary> [base-port]
# java-home "-" => run target as a native binary
set -euo pipefail

LABEL="${1:?label}"
JAVA_HOME_ARG="${2:?java-home or -}"
TARGET="${3:?target}"
BASE_PORT="${4:-8900}"
RUNS="${RUNS:-3}"

export KAFKA_BOOTSTRAP_SERVERS="${KAFKA_BOOTSTRAP_SERVERS:-localhost:9092}"
export METRICS_REPORTER="${METRICS_REPORTER:-prometheus}"
export LOG_LEVEL_KLAG=INFO
marker="Klag started successfully"

run_once() {
local port="$1" logf pid start end rss
logf="$(mktemp)"
export HTTP_PORT="$port"
if [[ "$JAVA_HOME_ARG" == "-" ]]; then
"$TARGET" >"$logf" 2>&1 &
else
"$JAVA_HOME_ARG/bin/java" ${JAVA_OPTS:-} -jar "$TARGET" >"$logf" 2>&1 &
fi
pid=$!
start=$(python3 -c 'import time;print(int(time.time()*1000))')
local i=0 max_iters=$(( ${STARTUP_TIMEOUT_S:-120} * 20 )) # 0.05s per iteration
while :; do
grep -q "$marker" "$logf" && break
if ! kill -0 "$pid" 2>/dev/null; then
echo "DIED" >&2; tail -5 "$logf" >&2; rm -f "$logf"; return 1
fi
if (( ++i > max_iters )); then
echo "TIMEOUT after ${STARTUP_TIMEOUT_S:-120}s" >&2; tail -5 "$logf" >&2
kill "$pid" 2>/dev/null || true; rm -f "$logf"; return 1
fi
sleep 0.05
done
end=$(python3 -c 'import time;print(int(time.time()*1000))')
sleep 3
rss=$(ps -o rss= -p "$pid" | tr -d ' ')
kill "$pid" 2>/dev/null || true; wait "$pid" 2>/dev/null || true
rm -f "$logf"
echo "$((end - start)) ${rss:-0}"
}

echo "== $LABEL ($RUNS runs) =="
tms=0; trss=0
for i in $(seq 1 "$RUNS"); do
res=$(run_once "$((BASE_PORT + i))")
ms=${res% *}; rss=${res#* }
printf " run %d: startup=%5d ms rss=%6d KB (%d MB)\n" "$i" "$ms" "$rss" "$((rss/1024))"
tms=$((tms+ms)); trss=$((trss+rss))
done
printf " AVG : startup=%5d ms rss=%6d KB (%d MB)\n" "$((tms/RUNS))" "$((trss/RUNS))" "$((trss/RUNS/1024))"
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"resources": {
"includes": [
{"pattern": "logback.xml"},
{"pattern": "version.properties"},
{"pattern": "application.properties.template"},
{"pattern": "application.properties"}
]
},
"bundles": []
}
Loading