From 095e83cae6cc673e3153ca2749aa8acac4dd7cfa Mon Sep 17 00:00:00 2001 From: obchain Date: Wed, 27 May 2026 15:27:42 +0530 Subject: [PATCH 1/2] feat(scripts): one-shot local dev launcher with frontend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sibling of e2e-smoke.sh. Boots the same M1 pipeline plus the Next.js dev server, prints a single ready URL, and blocks on Ctrl-C. No assertions — this is a developer ergonomics script for the visual demo + day-to-day inner loop. Volumes are preserved across runs (ClickHouse history persists). LOGS=1 multiplexes all four service logs to stdout with a [service] prefix. SKIP_BUILD=1 skips cargo + go compile when binaries are already fresh. PID polling is used in place of wait -n so the script runs on macOS bash 3.2. Teardown reaps rust + go + node processes by name and brings the compose stack down (without -v) so the next run starts hot. --- scripts/dev-stack.sh | 212 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100755 scripts/dev-stack.sh diff --git a/scripts/dev-stack.sh b/scripts/dev-stack.sh new file mode 100755 index 0000000..e555603 --- /dev/null +++ b/scripts/dev-stack.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# Local dev stack launcher (issue #70). +# +# Sibling of `e2e-smoke.sh`: brings the same M1 pipeline online plus the +# Next.js dev server, prints a single ready URL, then blocks until the +# operator hits Ctrl-C. No assertions — this is a developer ergonomics +# script, not a pass/fail harness. +# +# Pipeline: +# +# Deribit WS → ingestion → normalizer → ClickHouse + Redis +# ↓ +# engine (60 s) +# ↓ +# API REST + WS +# ↓ +# Next.js dev server → browser +# +# Tears down cleanly on Ctrl-C / SIGTERM / EXIT. Volumes are preserved by +# default — re-running keeps yesterday's ClickHouse history. +# +# Requirements on PATH: docker, cargo, go, pnpm, curl. +# +# Usage: ./scripts/dev-stack.sh +# LOGS=1 ./scripts/dev-stack.sh # multiplex service logs to stdout +# SKIP_BUILD=1 ./scripts/dev-stack.sh + +set -euo pipefail + +# ------- config -------------------------------------------------------------- + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +COMPOSE_FILE="${ROOT_DIR}/docker/docker-compose.yml" +COMPOSE_PROJECT="volx-local" + +CLICKHOUSE_HOST="${CLICKHOUSE_HOST:-127.0.0.1}" +CLICKHOUSE_HTTP_PORT="${CLICKHOUSE_HTTP_PORT:-8123}" + +API_HOST="${API_HOST:-localhost}" +API_PORT="${API_PORT:-8080}" +API_BASE="http://${API_HOST}:${API_PORT}" + +FRONTEND_HOST="${FRONTEND_HOST:-localhost}" +FRONTEND_PORT="${FRONTEND_PORT:-3000}" +FRONTEND_URL="http://${FRONTEND_HOST}:${FRONTEND_PORT}" + +LOGS="${LOGS:-0}" +SKIP_BUILD="${SKIP_BUILD:-0}" + +LOG_DIR="$(mktemp -d -t volx-dev-XXXXXX)" +INGESTION_LOG="${LOG_DIR}/ingestion.log" +ENGINE_LOG="${LOG_DIR}/engine.log" +API_LOG="${LOG_DIR}/api.log" +FRONTEND_LOG="${LOG_DIR}/frontend.log" + +declare -a CHILD_PIDS=() +declare -a TAIL_PIDS=() + +ING_BIN="${ROOT_DIR}/target/release/volx-ingestion" +ENG_BIN="${ROOT_DIR}/target/release/volx-engine" +API_BIN="${ROOT_DIR}/api/api-bin" + +# ------- helpers ------------------------------------------------------------- + +t_start_total=$(date +%s) +elapsed_total() { echo $(( $(date +%s) - t_start_total )); } + +stage() { echo "==> $1" >&2; } + +teardown() { + local rc=$? + set +e + echo "" >&2 + stage "teardown" + # Reap by binary name first — `cargo run` parents fork-exec the compiled + # binary, and Next.js spawns child workers that don't get the SIGTERM + # from killing the pnpm wrapper alone. + pkill -f "target/release/volx-ingestion" 2>/dev/null + pkill -f "target/release/volx-engine" 2>/dev/null + pkill -f "api/api-bin" 2>/dev/null + pkill -f "next dev" 2>/dev/null + pkill -f "next-server" 2>/dev/null + for pid in "${CHILD_PIDS[@]:-}"; do + [ -n "${pid:-}" ] && kill "${pid}" 2>/dev/null + done + for pid in "${TAIL_PIDS[@]:-}"; do + [ -n "${pid:-}" ] && kill "${pid}" 2>/dev/null + done + # Free the well-known ports in case anything escaped. + lsof -ti ":${API_PORT}" 2>/dev/null | xargs -r kill -9 2>/dev/null + lsof -ti ":${FRONTEND_PORT}" 2>/dev/null | xargs -r kill -9 2>/dev/null + # Volumes preserved on purpose — re-runs keep ClickHouse history. + docker compose -p "${COMPOSE_PROJECT}" -f "${COMPOSE_FILE}" down >/dev/null 2>&1 + echo "logs preserved in: ${LOG_DIR}" >&2 + echo "total runtime: $(elapsed_total)s" >&2 + exit "${rc}" +} +trap teardown EXIT INT TERM + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +wait_until() { + local label="$1"; shift + local timeout="$1"; shift + local deadline=$(( $(date +%s) + timeout )) + while [ "$(date +%s)" -lt "${deadline}" ]; do + if eval "$@" >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + fail "${label} did not become ready within ${timeout}s" +} + +start_service() { + local label="$1" logfile="$2"; shift 2 + stage "spawn ${label} → ${logfile}" + ( cd "${ROOT_DIR}" && "$@" ) >"${logfile}" 2>&1 & + CHILD_PIDS+=("$!") +} + +# Optional log multiplexer — tail every service log with a [service] prefix. +maybe_tail_logs() { + [ "${LOGS}" = "1" ] || return 0 + stage "streaming logs to stdout (LOGS=1)" + local files=("ingestion:${INGESTION_LOG}" "engine:${ENGINE_LOG}" "api:${API_LOG}" "frontend:${FRONTEND_LOG}") + for entry in "${files[@]}"; do + local name="${entry%%:*}" + local path="${entry#*:}" + # `tail -F` survives the file being recreated on service restart. + ( tail -n 0 -F "${path}" 2>/dev/null | sed -u "s/^/[${name}] /" ) & + TAIL_PIDS+=("$!") + done +} + +# ------- stages -------------------------------------------------------------- + +stage "compose-up (volumes preserved)" +docker compose -p "${COMPOSE_PROJECT}" -f "${COMPOSE_FILE}" up -d + +stage "compose-healthy" +wait_until "clickhouse healthy" 60 \ + "curl -sS --fail http://${CLICKHOUSE_HOST}:${CLICKHOUSE_HTTP_PORT}/ping" +wait_until "redis healthy" 30 \ + "docker exec volx-redis redis-cli ping | grep -q PONG" + +if [ "${SKIP_BUILD}" = "1" ]; then + stage "build skipped (SKIP_BUILD=1)" + [ -x "${ING_BIN}" ] || fail "SKIP_BUILD=1 but ${ING_BIN} not present" + [ -x "${ENG_BIN}" ] || fail "SKIP_BUILD=1 but ${ENG_BIN} not present" + [ -x "${API_BIN}" ] || fail "SKIP_BUILD=1 but ${API_BIN} not present" +else + stage "build-rust" + ( cd "${ROOT_DIR}" && cargo build --release -p volx-ingestion -p volx-engine ) \ + || fail "cargo build --release failed" + + stage "build-go" + ( cd "${ROOT_DIR}/api" && go build -o "${API_BIN}" ./cmd/api ) \ + || fail "go build ./cmd/api failed" +fi + +start_service "ingestion" "${INGESTION_LOG}" "${ING_BIN}" +start_service "engine" "${ENGINE_LOG}" "${ENG_BIN}" +start_service "api" "${API_LOG}" "${API_BIN}" + +# Next.js inherits PORT from env; pass through whatever FRONTEND_PORT is. +stage "spawn frontend (pnpm dev) → ${FRONTEND_LOG}" +( cd "${ROOT_DIR}/frontend" && PORT="${FRONTEND_PORT}" pnpm dev ) \ + >"${FRONTEND_LOG}" 2>&1 & +CHILD_PIDS+=("$!") + +stage "api-ready" +wait_until "api /v1/health" 60 "curl -sS --fail ${API_BASE}/v1/health" + +stage "frontend-ready" +# Next dev needs ~2 s on warm cache, ~15 s cold (first-run TS compile). +wait_until "frontend root" 60 "curl -sS --fail ${FRONTEND_URL}" + +maybe_tail_logs + +# ------- ready -------------------------------------------------------------- + +cat <&2 + +================================ READY ($(elapsed_total)s) ================================ + landing ${FRONTEND_URL} + chart ${FRONTEND_URL}/chart/bvol + api health ${API_BASE}/v1/health + api latest ${API_BASE}/v1/index/bvol/latest + + logs dir ${LOG_DIR} + ${LOGS:+ (streaming above with [service] prefix)} + + Ctrl-C to tear down. Volumes are preserved — re-run keeps ClickHouse history. +================================================================================== + +EOF + +# Block until any spawned service dies. `wait -n` is bash 4+, macOS ships +# 3.2, so we poll PIDs instead. 5 s cadence is plenty — the operator +# notices a missing chart long before the script does. +while true; do + for pid in "${CHILD_PIDS[@]}"; do + if ! kill -0 "${pid}" 2>/dev/null; then + fail "background service (pid ${pid}) exited unexpectedly (check ${LOG_DIR})" + fi + done + sleep 5 +done From c195e05677136d7090f1f181c3b654824606b057 Mon Sep 17 00:00:00 2001 From: obchain Date: Wed, 27 May 2026 15:32:45 +0530 Subject: [PATCH 2/2] fix(scripts): address review on dev-stack launcher HIGH-1: Next.js worker grandchildren survived teardown. The recorded PID is the pnpm wrapper; pkill -f "next dev" / "next-server" misses the worker tree whose argv doesn't contain either string. Teardown now walks two layers of children under the tracked FRONTEND_PID before falling back to pkill + the port sweep. HIGH-2: A background service crashing during the readiness wait was silently swallowed (bash pipefail doesn't apply to backgrounded jobs). Added check_services_alive between every wait_until so a dead ingestion or engine fails the boot loud instead of greeting the operator with READY plus a dark chart. MED-1: Added pre-flight check_port_free for API_PORT + FRONTEND_PORT. A stale process on the same port can return 200 to the health probe and let our fresh binary die on EADDRINUSE unnoticed. MED-2: Guarded teardown with _TEARING_DOWN so the trap can't fire twice when SIGINT delivers and the in-trap `exit` triggers EXIT. Previously re-entered, ran compose down twice, and clobbered the propagated rc. MED-3: Teardown now prunes /tmp/volx-dev-* older than 2 days. The launcher runs many times per day; /tmp would grow unbounded otherwise. LOW findings either match the e2e-smoke sibling for consistency (redundant CHILD_PIDS expansion, narrow lsof sweep) or do not apply (volx-normalizer is library, not binary). --- scripts/dev-stack.sh | 63 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/scripts/dev-stack.sh b/scripts/dev-stack.sh index e555603..7db85d2 100755 --- a/scripts/dev-stack.sh +++ b/scripts/dev-stack.sh @@ -55,6 +55,8 @@ FRONTEND_LOG="${LOG_DIR}/frontend.log" declare -a CHILD_PIDS=() declare -a TAIL_PIDS=() +FRONTEND_PID="" # tracked separately so teardown can pgrep -P its tree +_TEARING_DOWN=0 # guard so the trap can't double-fire on Ctrl-C ING_BIN="${ROOT_DIR}/target/release/volx-ingestion" ENG_BIN="${ROOT_DIR}/target/release/volx-engine" @@ -69,17 +71,32 @@ stage() { echo "==> $1" >&2; } teardown() { local rc=$? + # Guard so the trap fires once. SIGINT delivers to teardown, teardown's + # `exit` then fires the EXIT pseudo-signal which would re-enter without + # this latch — leading to double `compose down` and a clobbered rc. + [ "${_TEARING_DOWN}" = "1" ] && return + _TEARING_DOWN=1 set +e echo "" >&2 stage "teardown" - # Reap by binary name first — `cargo run` parents fork-exec the compiled - # binary, and Next.js spawns child workers that don't get the SIGTERM - # from killing the pnpm wrapper alone. + # Next.js spawns worker children whose argv doesn't contain "next dev" + # or "next-server", so neither pkill pattern catches them. Walk the + # frontend PID's child tree first, then kill the parent. Repeat for + # any deeper grandchild layer Next ever adds. + if [ -n "${FRONTEND_PID}" ]; then + pgrep -P "${FRONTEND_PID}" 2>/dev/null | while read -r child; do + pgrep -P "${child}" 2>/dev/null | xargs -r kill 2>/dev/null + kill "${child}" 2>/dev/null + done + kill "${FRONTEND_PID}" 2>/dev/null + fi + # Reap rust + go by binary name — `cargo run` parents fork-exec the + # compiled binary, so killing the recorded PID alone may not reap it. pkill -f "target/release/volx-ingestion" 2>/dev/null pkill -f "target/release/volx-engine" 2>/dev/null pkill -f "api/api-bin" 2>/dev/null - pkill -f "next dev" 2>/dev/null pkill -f "next-server" 2>/dev/null + pkill -f "next dev" 2>/dev/null for pid in "${CHILD_PIDS[@]:-}"; do [ -n "${pid:-}" ] && kill "${pid}" 2>/dev/null done @@ -91,6 +108,9 @@ teardown() { lsof -ti ":${FRONTEND_PORT}" 2>/dev/null | xargs -r kill -9 2>/dev/null # Volumes preserved on purpose — re-runs keep ClickHouse history. docker compose -p "${COMPOSE_PROJECT}" -f "${COMPOSE_FILE}" down >/dev/null 2>&1 + # Prune stale log dirs from prior runs (older than 2 days). This script + # is a daily-driver launcher; /tmp would otherwise grow unbounded. + find /tmp -maxdepth 1 -name "volx-dev-*" -mtime +2 -exec rm -rf {} + 2>/dev/null echo "logs preserved in: ${LOG_DIR}" >&2 echo "total runtime: $(elapsed_total)s" >&2 exit "${rc}" @@ -122,6 +142,30 @@ start_service() { CHILD_PIDS+=("$!") } +# Pre-flight: bail loud if the port we're about to bind is already taken. +# A stale process bound to the same port can let `curl --fail /v1/health` +# return 200 from the wrong server, masking a "real" service that died +# on startup with EADDRINUSE. +check_port_free() { + local port="$1" label="$2" + if lsof -ti ":${port}" 2>/dev/null | grep -q .; then + local owner + owner=$(lsof -nP -iTCP:"${port}" -sTCP:LISTEN 2>/dev/null | awk 'NR==2{print $1" pid="$2}') + fail "port ${port} (${label}) already in use → ${owner}. Free it or override the port env var." + fi +} + +# Fail-fast if any background service has died. Background process exit +# does NOT trigger pipefail, so without this check a crashed ingestion +# or engine wouldn't surface until well after READY is printed. +check_services_alive() { + for pid in "${CHILD_PIDS[@]:-}"; do + if [ -n "${pid:-}" ] && ! kill -0 "${pid}" 2>/dev/null; then + fail "background service (pid ${pid}) died before stack was ready (check ${LOG_DIR})" + fi + done +} + # Optional log multiplexer — tail every service log with a [service] prefix. maybe_tail_logs() { [ "${LOGS}" = "1" ] || return 0 @@ -147,6 +191,12 @@ wait_until "clickhouse healthy" 60 \ wait_until "redis healthy" 30 \ "docker exec volx-redis redis-cli ping | grep -q PONG" +stage "port-preflight" +# Compose already owns 8123 + 6379 by this point; only check the two ports +# our own binaries will try to claim. +check_port_free "${API_PORT}" "api" +check_port_free "${FRONTEND_PORT}" "frontend" + if [ "${SKIP_BUILD}" = "1" ]; then stage "build skipped (SKIP_BUILD=1)" [ -x "${ING_BIN}" ] || fail "SKIP_BUILD=1 but ${ING_BIN} not present" @@ -170,12 +220,15 @@ start_service "api" "${API_LOG}" "${API_BIN}" stage "spawn frontend (pnpm dev) → ${FRONTEND_LOG}" ( cd "${ROOT_DIR}/frontend" && PORT="${FRONTEND_PORT}" pnpm dev ) \ >"${FRONTEND_LOG}" 2>&1 & -CHILD_PIDS+=("$!") +FRONTEND_PID="$!" +CHILD_PIDS+=("${FRONTEND_PID}") stage "api-ready" +check_services_alive wait_until "api /v1/health" 60 "curl -sS --fail ${API_BASE}/v1/health" stage "frontend-ready" +check_services_alive # Next dev needs ~2 s on warm cache, ~15 s cold (first-run TS compile). wait_until "frontend root" 60 "curl -sS --fail ${FRONTEND_URL}"