From 2c079d4a4784b119d43dbe1218a0d7ae07fff170 Mon Sep 17 00:00:00 2001 From: flamboh Date: Fri, 25 Sep 2026 03:43:22 -0700 Subject: [PATCH 1/5] feat(infra): add self-hosted container stack Run the SQLite dashboard as a Docker container on a self-hosted host, managed by the atlantis-campus Alchemy stack. The container publishes its port on the host loopback only; users connect through SSH port forwarding. The stack creates no Cloudflare resources and keeps Cloudflare only as the state store. - ATLANTIS_DB_DRIVER=sqlite builds use @sveltejs/adapter-node. - apps/web/Dockerfile builds with Bun and runs on Node 24.18.1, which avoids the Node 24.19+ better-sqlite3 GC abort (nodejs/node#65446). - The image tag is a content hash of the build inputs, so a source change replaces the container and an unchanged deploy is a no-op. - The data mount is writable because WAL-mode databases need their -shm/-wal sidecars; the dashboard still opens them read-only. - LOCAL_DATA_DIR selects the dataset directory the dashboard scans. --- .env.example | 2 + AGENTS.md | 2 +- apps/web/Dockerfile | 58 ++++++++++ apps/web/Dockerfile.dockerignore | 18 +++ apps/web/package.json | 1 + apps/web/src/env.ts | 5 + apps/web/src/lib/server/db/local-files.ts | 6 +- .../tests/lib/server/db/local-files.test.ts | 42 +++++++ apps/web/vite.config.ts | 3 + bun.lock | 39 ++++++- docs/code/architecture.md | 8 ++ docs/code/development.md | 16 ++- docs/user/operations.md | 96 +++++++++++++++- docs/user/setup-web.md | 2 +- infra/campus.ts | 103 ++++++++++++++++++ infra/shared.ts | 4 + package.json | 6 +- patches/alchemy@2.0.0-beta.79.patch | 68 ++++++++++++ 18 files changed, 465 insertions(+), 14 deletions(-) create mode 100644 apps/web/Dockerfile create mode 100644 apps/web/Dockerfile.dockerignore create mode 100644 apps/web/tests/lib/server/db/local-files.test.ts create mode 100644 infra/campus.ts create mode 100644 patches/alchemy@2.0.0-beta.79.patch diff --git a/.env.example b/.env.example index 1fbb814f..714f2fbd 100644 --- a/.env.example +++ b/.env.example @@ -14,3 +14,5 @@ DEFAULT_DATASET=uoregon # Set one database path or use automatic discovery at data/*/netflow.sqlite. # LOCAL_SQLITE_PATH=/absolute/path/to/netflow.sqlite # DATABASE_PATH=/absolute/path/to/netflow.sqlite +# Or discover */netflow.sqlite under another directory instead of data/. +# LOCAL_DATA_DIR=/absolute/path/to/data diff --git a/AGENTS.md b/AGENTS.md index 5464e753..307488da 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,7 +7,7 @@ ATLANTIS turns NetFlow captures and CSV imports into queryable aggregate databas - `tools/netflow-db`: Rust pipeline (`atlantis-netflow-db` crate) for ingestion, aggregation, verification, and analysis-window exports. Native `nfcapd` ingestion uses the pinned `nfdump` fork in `vendor/nfdump`. - `apps/web`: Svelte 5/SvelteKit 3 dashboard and API routes. It reads local SQLite during development and Cloudflare D1 in deployment. - `apps/landing`: Astro marketing and SEO site. -- `infra`: Alchemy v2 stacks (Effect programs) that deploy the dashboard. `cloudflare.ts` deploys the worker and its D1 database; `shared.ts` holds names and paths common to all stacks. +- `infra`: Alchemy v2 stacks (Effect programs) that deploy the dashboard. `cloudflare.ts` deploys the worker and its D1 database; `campus.ts` runs the SQLite dashboard as a Docker container on a self-hosted host that users reach through SSH port forwarding; `shared.ts` holds names and paths common to all stacks. - `vendor/*`: Third-party analysis submodules. Treat these as read-only; build repo-local binaries through the scripts in `vendor/scripts/`. - `data/`, `.env`, and `datasets.json`: Machine-local inputs and generated databases. Keep paths and dataset contents out of commits. - `docs/user`: User setup and operation documentation. diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile new file mode 100644 index 00000000..07ec8865 --- /dev/null +++ b/apps/web/Dockerfile @@ -0,0 +1,58 @@ +# syntax=docker/dockerfile:1 + +ARG BUN_VERSION=1.3.11 +ARG NODE_VERSION=24.18.1 + +FROM oven/bun:${BUN_VERSION}-slim AS bun + +FROM node:${NODE_VERSION}-bookworm-slim AS workspace + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install --yes --no-install-recommends g++ make python3 \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=bun /usr/local/bin/bun /usr/local/bin/bun + +WORKDIR /app +COPY package.json bun.lock ./ +COPY patches patches +COPY apps/web/package.json apps/web/package.json +COPY apps/landing/package.json apps/landing/package.json +COPY infra/package.json infra/package.json + +FROM workspace AS build + +RUN bun install --frozen-lockfile --ignore-scripts --filter @atlantis/web + +COPY apps/web apps/web + +RUN cd apps/web && ATLANTIS_DB_DRIVER=sqlite bun run build + +FROM workspace AS production-deps + +RUN bun install --frozen-lockfile --ignore-scripts --production --filter @atlantis/web \ + && npm run build-release --prefix node_modules/better-sqlite3 \ + && rm -rf node_modules/better-sqlite3/build/Release/obj node_modules/better-sqlite3/deps \ + && node -e "new (require('better-sqlite3'))(':memory:').prepare('select 1').get()" + +FROM node:${NODE_VERSION}-bookworm-slim AS runtime + +ARG RUNTIME_UID=1000 +ARG RUNTIME_GID=1000 + +ENV NODE_ENV=production \ + HOST=0.0.0.0 \ + PORT=3000 \ + LOCAL_DATA_DIR=/data + +WORKDIR /app +COPY --from=production-deps /app/node_modules node_modules +COPY --from=build /app/apps/web/package.json package.json +COPY --from=build /app/apps/web/build build + +RUN mkdir -p /data + +USER ${RUNTIME_UID}:${RUNTIME_GID} +EXPOSE 3000 +CMD ["node", "build"] diff --git a/apps/web/Dockerfile.dockerignore b/apps/web/Dockerfile.dockerignore new file mode 100644 index 00000000..a5d348bb --- /dev/null +++ b/apps/web/Dockerfile.dockerignore @@ -0,0 +1,18 @@ +* +!package.json +!bun.lock +!patches +!infra/package.json +!apps/landing/package.json +!apps/web +apps/web/node_modules +apps/web/build +apps/web/dist +apps/web/.svelte-kit +apps/web/test-results +apps/web/playwright-report +apps/web/tests +apps/web/Dockerfile +apps/web/Dockerfile.dockerignore +**/.env +**/.env.* diff --git a/apps/web/package.json b/apps/web/package.json index e98774e1..f38c9723 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -26,6 +26,7 @@ "@eslint/js": "^9.18.0", "@internationalized/date": "^3.12.0", "@playwright/test": "1.52.0", + "@sveltejs/adapter-node": "6.0.0-next.12", "@sveltejs/kit": "3.0.0-next.27", "@sveltejs/load-config": "^0.2.3", "@sveltejs/vite-plugin-svelte": "^7.3.1", diff --git a/apps/web/src/env.ts b/apps/web/src/env.ts index 49025812..2c5afd51 100644 --- a/apps/web/src/env.ts +++ b/apps/web/src/env.ts @@ -11,6 +11,11 @@ export const variables = defineEnvVars({ description: 'Fallback for LOCAL_SQLITE_PATH', schema: optional }, + LOCAL_DATA_DIR: { + description: + 'Directory scanned for /netflow.sqlite products; replaces the data/ and ../../data defaults', + schema: optional + }, DEFAULT_DATASET: { description: 'Dataset ID selected when a request does not name one', schema: optional diff --git a/apps/web/src/lib/server/db/local-files.ts b/apps/web/src/lib/server/db/local-files.ts index a9ffaccb..9e576274 100644 --- a/apps/web/src/lib/server/db/local-files.ts +++ b/apps/web/src/lib/server/db/local-files.ts @@ -1,4 +1,4 @@ -import { DATABASE_PATH, LOCAL_SQLITE_PATH } from '$app/env/private'; +import { DATABASE_PATH, LOCAL_DATA_DIR, LOCAL_SQLITE_PATH } from '$app/env/private'; async function resolvePath(value: string): Promise { if (value === ':memory:') { @@ -17,7 +17,9 @@ export async function discoverLocalSqlitePaths(): Promise { const fs = await import('node:fs/promises'); const path = await import('node:path'); - const roots = [path.resolve(process.cwd(), 'data'), path.resolve(process.cwd(), '../../data')]; + const roots = LOCAL_DATA_DIR + ? [await resolvePath(LOCAL_DATA_DIR)] + : [path.resolve(process.cwd(), 'data'), path.resolve(process.cwd(), '../../data')]; const dbPaths = new Set(); for (const root of roots) { diff --git a/apps/web/tests/lib/server/db/local-files.test.ts b/apps/web/tests/lib/server/db/local-files.test.ts new file mode 100644 index 00000000..bb1b4888 --- /dev/null +++ b/apps/web/tests/lib/server/db/local-files.test.ts @@ -0,0 +1,42 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { discoverLocalSqlitePaths } from '../../../../src/lib/server/db/local-files'; + +describe('local SQLite discovery', () => { + let dataDir: string; + + beforeEach(() => { + dataDir = mkdtempSync(join(tmpdir(), 'atlantis-local-files-')); + for (const dataset of ['beta', 'alpha']) { + mkdirSync(join(dataDir, dataset)); + writeFileSync(join(dataDir, dataset, 'netflow.sqlite'), ''); + } + mkdirSync(join(dataDir, 'backups')); + writeFileSync(join(dataDir, 'backups', 'old.sqlite'), ''); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + rmSync(dataDir, { recursive: true, force: true }); + }); + + it('scans LOCAL_DATA_DIR for dataset products', async () => { + vi.stubEnv('LOCAL_DATA_DIR', dataDir); + + await expect(discoverLocalSqlitePaths()).resolves.toEqual([ + join(dataDir, 'alpha', 'netflow.sqlite'), + join(dataDir, 'beta', 'netflow.sqlite') + ]); + }); + + it('prefers LOCAL_SQLITE_PATH over LOCAL_DATA_DIR', async () => { + vi.stubEnv('LOCAL_DATA_DIR', dataDir); + vi.stubEnv('LOCAL_SQLITE_PATH', join(dataDir, 'beta', 'netflow.sqlite')); + + await expect(discoverLocalSqlitePaths()).resolves.toEqual([ + join(dataDir, 'beta', 'netflow.sqlite') + ]); + }); +}); diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index 9d35e4c6..517e4536 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -1,4 +1,5 @@ import { fileURLToPath } from 'node:url'; +import adapterNode from '@sveltejs/adapter-node'; import tailwindcss from '@tailwindcss/vite'; import { sveltekit } from '@sveltejs/kit/vite'; import { vitePreprocess } from '@sveltejs/vite-plugin-svelte'; @@ -56,6 +57,7 @@ function databaseDriver(driver: DatabaseDriver): Plugin { export default defineConfig(({ command, mode, isPreview = false }) => { const driver = resolveDatabaseDriver(command, mode, isPreview); + const nodeBuild = command === 'build' && driver === 'sqlite'; return { plugins: [ @@ -63,6 +65,7 @@ export default defineConfig(({ command, mode, isPreview = false }) => { databaseDriver(driver), sveltekit({ preprocess: vitePreprocess(), + adapter: nodeBuild ? adapterNode() : undefined, env: { dir: '../..' } diff --git a/bun.lock b/bun.lock index adf22851..efabbd45 100644 --- a/bun.lock +++ b/bun.lock @@ -1,6 +1,5 @@ { "lockfileVersion": 1, - "configVersion": 0, "workspaces": { "": { "name": "atlantis", @@ -44,6 +43,7 @@ "@eslint/js": "^9.18.0", "@internationalized/date": "^3.12.0", "@playwright/test": "1.52.0", + "@sveltejs/adapter-node": "6.0.0-next.12", "@sveltejs/kit": "3.0.0-next.27", "@sveltejs/load-config": "^0.2.3", "@sveltejs/vite-plugin-svelte": "^7.3.1", @@ -94,6 +94,7 @@ }, }, "patchedDependencies": { + "alchemy@2.0.0-beta.79": "patches/alchemy@2.0.0-beta.79.patch", "@alchemy.run/frontend-frameworks@2.0.0-beta.79": "patches/@alchemy.run%2Ffrontend-frameworks@2.0.0-beta.79.patch", }, "packages": { @@ -617,6 +618,8 @@ "@sveltejs/acorn-typescript": ["@sveltejs/acorn-typescript@1.0.13", "", { "peerDependencies": { "acorn": "^8.9.0" } }, "sha512-wgKggnhZVL9Bfx1OaKKTrYY9BFRk6C8UAkQNUcIv1+llzYrIqy+RZm5HPKzn0NpEBvTVhTqB4kQyllZywsRBRQ=="], + "@sveltejs/adapter-node": ["@sveltejs/adapter-node@6.0.0-next.12", "", { "dependencies": { "rolldown": "^1.2.3" }, "peerDependencies": { "@sveltejs/kit": "^3.0.0-next.0" } }, "sha512-wheNAOlAqLoXvGeSxJChM4jMEAKoMIcBCE18jGbTUdDDENPLefTmX55ACeUOkFqaVeZbAwKVHKifKZeyVNR9Yg=="], + "@sveltejs/kit": ["@sveltejs/kit@3.0.0-next.27", "", { "dependencies": { "@rolldown/pluginutils": "^1.0.1", "@standard-schema/spec": "^1.1.0", "cookie": "^2.0.1", "devalue": "^5.9.0", "esm-env": "^1.2.2", "magic-string": "^1.1.0", "mrmime": "^2.0.1", "sirv": "^3.0.2" }, "peerDependencies": { "@opentelemetry/api": "^1.0.0", "@sveltejs/vite-plugin-svelte": "^7.0.0", "svelte": "^5.56.4", "typescript": "^6.0.0", "vite": "^8.0.12" }, "optionalPeers": ["@opentelemetry/api", "typescript"], "bin": { "svelte-kit": "svelte-kit.js" } }, "sha512-m2R6GvcMhiAbYVKhzRf5mzLSvcXEuy/H0Va72XvxfEl1ZglrSC/321GKYa6idDHPojuRzkyh9EB0jBZmAwf6+Q=="], "@sveltejs/load-config": ["@sveltejs/load-config@0.2.3", "", {}, "sha512-VT3qmUb8pRV2QrZjd8iAmtg8lf4W0TIjZbvXtz5MKei/q96teWZgGJyyidJzOjzZzvdq616eSRVeMYIQChUTAQ=="], @@ -1717,6 +1720,8 @@ "@rollup/pluginutils/picomatch": ["picomatch@4.0.4", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="], + "@sveltejs/adapter-node/rolldown": ["rolldown@1.2.10", "", { "dependencies": { "@oxc-project/types": "=0.151.0", "@rolldown/pluginutils": "^1.0.0" }, "optionalDependencies": { "@rolldown/binding-android-arm-eabi": "1.2.10", "@rolldown/binding-android-arm64": "1.2.10", "@rolldown/binding-darwin-arm64": "1.2.10", "@rolldown/binding-darwin-x64": "1.2.10", "@rolldown/binding-freebsd-x64": "1.2.10", "@rolldown/binding-linux-arm-gnueabihf": "1.2.10", "@rolldown/binding-linux-arm64-gnu": "1.2.10", "@rolldown/binding-linux-arm64-musl": "1.2.10", "@rolldown/binding-linux-ppc64-gnu": "1.2.10", "@rolldown/binding-linux-s390x-gnu": "1.2.10", "@rolldown/binding-linux-x64-gnu": "1.2.10", "@rolldown/binding-linux-x64-musl": "1.2.10", "@rolldown/binding-openharmony-arm64": "1.2.10", "@rolldown/binding-win32-arm64-msvc": "1.2.10", "@rolldown/binding-win32-x64-msvc": "1.2.10" }, "bin": { "rolldown": "./bin/cli.mjs" } }, "sha512-OxkA08pSryMK7B3XiFA09B4OJ1xJMPgIYCBMY2xchzpqgBGsV1o0DetPAE+Sl3N3L4oCPiEzmHVSOj7iR04Zog=="], + "@sveltejs/kit/cookie": ["cookie@2.0.1", "", {}, "sha512-yuToqVvRrj6pfDXREyQAAv8SkAEk/8GS3jQRTiUMm66TVtBYmqQeoEjL2Lmq8Rpo6271vH76InTChTitEAm65w=="], "@sveltejs/kit/devalue": ["devalue@5.9.4", "", {}, "sha512-sPAT4pztbu6586/hrhOnMKS17IJrvg12mXiSPSS3W5qDeN2RGgvZ0diZCm31dBbnevfVmujNO3IM2wrS4Y2Rhg=="], @@ -1979,6 +1984,38 @@ "@puppeteer/browsers/yargs/string-width": ["string-width@8.3.0", "", { "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" } }, "sha512-ZbmZM0JCihQN91dWnxoipT2KOEyHqEyfRXUyjuRhW8b/xnqPDoq4gWEVApTVa9db2wN8mmoikgFBbjh71+cGeQ=="], + "@sveltejs/adapter-node/rolldown/@oxc-project/types": ["@oxc-project/types@0.151.0", "", {}, "sha512-J1yXrIlNDZVzE3ada310xeAw7nH8yCAyLPuUIsjKatFPmfn5bS1oW+cM+QsGOtVWd5nhSpbwZWx/rue+r5Z+PA=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-android-arm-eabi": ["@rolldown/binding-android-arm-eabi@1.2.10", "", { "os": "android", "cpu": "arm" }, "sha512-bp9svZb+QurZeh+8H4BhrZkifEB0YBNvTVzNSJnJQkj4NrRwmQoDUCGP0vSN7PbvLeM7l1tK6GXL8mrTiH2myg=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-android-arm64": ["@rolldown/binding-android-arm64@1.2.10", "", { "os": "android", "cpu": "arm64" }, "sha512-wm6Dld3RXUAZ/gRWKyUy+4W1B5CB5UeFaOzsSWJWEdxZXHH8rCYiZ5dGe6oJmhsunAPWzL7FZV+VtvmN5Ye2eA=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-darwin-arm64": ["@rolldown/binding-darwin-arm64@1.2.10", "", { "os": "darwin", "cpu": "arm64" }, "sha512-UbEfXq/AqGNgRTV3ik+X/iR6mUxu2QdYAadwRxJWquUGnW6gDqdP1FtLtFXRow7RJx0ssRwi80XAPr4r+4DtsA=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-darwin-x64": ["@rolldown/binding-darwin-x64@1.2.10", "", { "os": "darwin", "cpu": "x64" }, "sha512-7f5h17q5KZVx/ji1vb8OTq31ch1O2I7K8NPIr44GkyWTApXMIsmhWqZfgpOH10xeauqghDAvGlZktasCkcF6Eg=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-freebsd-x64": ["@rolldown/binding-freebsd-x64@1.2.10", "", { "os": "freebsd", "cpu": "x64" }, "sha512-ynOk/eEYhC6ZB2xCGvKrEOwE58oBy9LnrAqtkrDF9Fz1VTaNdGZTsV0VarJdhPwb+sOJTGjCLwcuyRJZ1dnMcQ=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-linux-arm-gnueabihf": ["@rolldown/binding-linux-arm-gnueabihf@1.2.10", "", { "os": "linux", "cpu": "arm" }, "sha512-ERrAs185meZZhGan7a4l3RiiJK1ArSDlHdST++uvSxe+FDbR4TwUPahT/cbZJvaG6fIpDpF78surN+tX708Y4Q=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-linux-arm64-gnu": ["@rolldown/binding-linux-arm64-gnu@1.2.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-KN7OHKD0J3jy1UzBwZWPxpwhODf9IARUIJcrH+yLYKOcmegZ8luEUM38lDP1bDVj40yP6PsSzCqOJF76vljFnQ=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-linux-arm64-musl": ["@rolldown/binding-linux-arm64-musl@1.2.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-8l9wP8O+wa8zD6iw6egSfzVtu7oZVfH3hlUsMM4MwbLMhxleqeoXbZzjddyK3YyNlwLhqznq3tF7PkNJ8T/V2w=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-linux-ppc64-gnu": ["@rolldown/binding-linux-ppc64-gnu@1.2.10", "", { "os": "linux", "cpu": "ppc64" }, "sha512-SeXNKeQzA5kLhz/J0CH6ZP0/HJ3v1xm/0YbiYpE0kK7emfRC2OIGGIaE14xzkISEGv2aYuUSpiLiU5Gbq+OI0A=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-linux-s390x-gnu": ["@rolldown/binding-linux-s390x-gnu@1.2.10", "", { "os": "linux", "cpu": "s390x" }, "sha512-mtht0nR+y8/hart4175Ll15w7lY8dg7CtQ+j2FDNTsDRspOWTK/2V3l0aj9sIj7XmvqxT8Yli/wq22e7feTTWg=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-linux-x64-gnu": ["@rolldown/binding-linux-x64-gnu@1.2.10", "", { "os": "linux", "cpu": "x64" }, "sha512-FSM94nGd55NYo48usCyM/nHfUKRnqc9+b0vJNuKV0oCCpIp/OGims7rO1Nv/DkFkt0S/s2rxsJ2kkS8J3HcpeA=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-linux-x64-musl": ["@rolldown/binding-linux-x64-musl@1.2.10", "", { "os": "linux", "cpu": "x64" }, "sha512-C3YxNB16myRLs7o+B+6PnQ6jBsdIS4+AE4Ah8glVGhDpEv9AOvxhZ/1duAb4B0UGczEK/lBbccksd8VI+p6zfw=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-openharmony-arm64": ["@rolldown/binding-openharmony-arm64@1.2.10", "", { "os": "none", "cpu": "arm64" }, "sha512-571TlE/F1eeTjjdjYAMMMPs1Mfv3MtX6s3+ZKVU6HiUjZ5Njc6c/qzNy/8K3zALTZnaw3JQVYrHxvNfjm43KAg=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-win32-arm64-msvc": ["@rolldown/binding-win32-arm64-msvc@1.2.10", "", { "os": "win32", "cpu": "arm64" }, "sha512-QXW+ZWaiqs2c7Fi++D/SsW07LTPcUrncxcskJGfGNBoaLik1IU6fJymz4HsqwEO0u5Iq11yTO0B/mc4cPk7jrQ=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-win32-x64-msvc": ["@rolldown/binding-win32-x64-msvc@1.2.10", "", { "os": "win32", "cpu": "x64" }, "sha512-5FQFGgah17YeMtG1Yd5a+rMxQpTksyNXxRtKz06FVTaQw3RKYUJQbUoKk0/5jrXBpDo+7makNP7UHA2LQyH64A=="], + "@tailwindcss/node/lightningcss/lightningcss-android-arm64": ["lightningcss-android-arm64@1.32.0", "", { "os": "android", "cpu": "arm64" }, "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg=="], "@tailwindcss/node/lightningcss/lightningcss-darwin-arm64": ["lightningcss-darwin-arm64@1.32.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ=="], diff --git a/docs/code/architecture.md b/docs/code/architecture.md index 540d3d45..9587a430 100644 --- a/docs/code/architecture.md +++ b/docs/code/architecture.md @@ -37,6 +37,14 @@ The local dashboard reads SQLite databases with `better-sqlite3`. It opens these The deployed dashboard reads the `DB` Cloudflare D1 binding from `cloudflare:workers`. `infra/cloudflare.ts` defines the worker and the D1 database as an Alchemy stack. Alchemy builds the worker with its own SvelteKit adapter and applies the D1 migrations during a deploy. +The campus dashboard runs the SQLite driver on a self-hosted Docker host. `infra/campus.ts` builds `apps/web/Dockerfile` on that host through an SSH Docker context and runs it as a container. The image holds the adapter-node build. The container mounts the host data directory at `/data` and publishes its port on the host loopback only. Users reach it through SSH port forwarding. The image tag is a content hash of the build inputs, so a changed input replaces the container and an unchanged deploy leaves it running. + +The data mount is writable because the pipeline publishes WAL-mode databases. SQLite can read a WAL database only if it can create the `-shm` and `-wal` files next to it, even for a read-only connection. The dashboard still opens every database read-only with `query_only`, so it never writes data. + +The container names its image as a plain `:` string. Alchemy compares a container's properties at plan time only when every property is resolved, and an output of an image that is being rebuilt stays unresolved until apply, so the plan would record an in-place update that never swaps the image. The container instead binds to the image's `imageId` output. The binding orders the container after the image on create and before it on destroy, and it is not one of the properties that the container compares. The plan also builds the new image while it compares the image, so the tagged image exists before apply replaces the container. + +The image pins Node.js 24.18.1. Node.js 24.19.0 and later abort the process when the garbage collector frees a `better-sqlite3` statement ([nodejs/node#65446](https://github.com/nodejs/node/issues/65446)). + The build selects one database driver. `apps/web/src/lib/server/db/d1.ts` reads D1, and `apps/web/src/lib/server/db/sqlite.ts` reads the pipeline SQLite files. Server code imports the driver as `#db`. [Development](development.md#choose-the-database-driver) explains the selection. The landing site has no database. It builds static files in `apps/landing/dist`. diff --git a/docs/code/development.md b/docs/code/development.md index c4e2acf3..52c0dd82 100644 --- a/docs/code/development.md +++ b/docs/code/development.md @@ -47,10 +47,10 @@ The root `dev` command starts both applications. It does not start the pipeline. The dashboard has two database drivers. Each build and each development server includes only one of them. -| `ATLANTIS_DB_DRIVER` | Driver | Default for | -| -------------------- | --------------------------------------------------------------------------------------------- | ----------------------------------- | -| `sqlite` | `src/lib/server/db/sqlite.ts` reads `data//netflow.sqlite` or `LOCAL_SQLITE_PATH` | `vite dev`, `vite preview` (always) | -| `d1` | `src/lib/server/db/d1.ts` reads the `DB` binding from `cloudflare:workers` | `vite build` | +| `ATLANTIS_DB_DRIVER` | Driver | Default for | +| -------------------- | ---------------------------------------------------------------------------------------------------------------- | ----------------------------------- | +| `sqlite` | `src/lib/server/db/sqlite.ts` reads `data//netflow.sqlite`, `LOCAL_DATA_DIR`, or `LOCAL_SQLITE_PATH` | `vite dev`, `vite preview` (always) | +| `d1` | `src/lib/server/db/d1.ts` reads the `DB` binding from `cloudflare:workers` | `vite build` | Server code imports the driver as `#db`. The `imports` field in `apps/web/package.json` maps `#db` to the D1 driver under the `atlantis-d1` export condition, and to the SQLite driver otherwise. `apps/web/vite.config.ts` adds that condition to the server environments and keeps `cloudflare:workers` external when the driver is `d1`. Both drivers implement `DatabaseDriver` in `src/lib/server/db/driver.ts`. @@ -59,11 +59,15 @@ Set `ATLANTIS_DB_DRIVER` in the shell. The value in `.env` does not select the d ```bash bun run dev:web # SQLite bun run build:web # D1 server bundle, no adapter -ATLANTIS_DB_DRIVER=sqlite bun run build:web # SQLite server bundle, no adapter +ATLANTIS_DB_DRIVER=sqlite bun run build:web # Node server in apps/web/build bun run --cwd apps/web preview # SQLite build, then vite preview ``` -The web app has no SvelteKit adapter. `bun run build:web` checks that the D1 bundle compiles. The Cloudflare worker is built by Alchemy during a deploy, which injects its own adapter into the `sveltekit()` call. [Operations](../user/operations.md#deploy-the-dashboard) describes the deploy. +A D1 build has no SvelteKit adapter. `bun run build:web` checks that the D1 bundle compiles. The Cloudflare worker is built by Alchemy during a deploy, which injects its own adapter into the `sveltekit()` call. [Operations](../user/operations.md#deploy-the-dashboard) describes the deploy. + +A SQLite build uses `@sveltejs/adapter-node` and writes a Node server to `apps/web/build`. Start it with `node build` from `apps/web`. The campus deployment runs this build in a container. [Operations](../user/operations.md#deploy-the-campus-dashboard) describes it. + +The build leaves `paths.origin` unset. SvelteKit 3 replaced adapter-node's runtime `ORIGIN` variable with this build-time option, and a fixed origin would break SSH port forwarding, where each user picks a local port. Adapter-node then builds the request URL from the `Host` header and the `https` protocol, so a request to `http://localhost:8080` has the origin `https://localhost:8080`. The dashboard has no form actions, remote functions, or mutating endpoints, so SvelteKit's CSRF origin check never runs. Before you add a `POST` form, set `PROTOCOL_HEADER` or `paths.origin` so that the origin check sees the browser's real origin. `preview` always rebuilds with SQLite before it serves, because `vite preview` runs the server in Node and cannot load the D1 bundle. diff --git a/docs/user/operations.md b/docs/user/operations.md index 249b1dc8..58a82b7b 100644 --- a/docs/user/operations.md +++ b/docs/user/operations.md @@ -1,8 +1,8 @@ # Operations -Use these procedures to verify and publish a database. This document also gives the available Cloudflare deployment commands. +Use these procedures to verify and publish a database. This document also gives the Cloudflare and campus deployment commands. -The D1 and deployment sections apply to the hosted ATLANTIS deployment and need Cloudflare access. A local installation does not use them. +The D1 and Cloudflare deployment sections apply to the hosted ATLANTIS deployment and need Cloudflare access. A local installation does not use them. ## Verify a SQLite database @@ -155,6 +155,98 @@ CAUTION: A restore overwrites remote D1 data. Record the current bookmark first. bunx wrangler@4.141.0 d1 time-travel restore atlantis-db --bookmark= ``` +## Deploy the campus dashboard + +`infra/campus.ts` runs the dashboard as a Docker container on a self-hosted machine. It is an [Alchemy](https://alchemy.run) stack named `atlantis-campus`. The container reads the pipeline SQLite databases from a directory on that machine. It is not publicly reachable: it listens on the host loopback only, and users connect through SSH port forwarding. + +The stack has three resources: + +- A Docker context that reaches the host's Docker daemon over SSH. +- The web image, built from `apps/web/Dockerfile` on the host. The image holds the Node build of the dashboard with the SQLite driver. +- The web container. It mounts the data directory at `/data`, has a 1 GB memory limit, restarts unless stopped, and reports health from `/api/datasets`. + +The image tag is a hash of the build inputs: the files that `apps/web/Dockerfile.dockerignore` admits, the Dockerfile, and the build arguments. A deploy after a source change builds a new image and replaces the container. A deploy without changes rebuilds from the Docker cache and leaves the container running. + +Each stage names its image and container `atlantis-campus-web-`. The `prod` stage uses `atlantis-campus-web`. + +### Prepare the host + +The host needs: + +- Docker Engine. The operator's account must be able to run `docker` without `sudo`, for example through the `docker` group. +- SSH key access for the operator. `ssh docker info` must succeed without a password prompt. Docker uses the operator's SSH configuration, so a host alias from `~/.ssh/config` works. +- A data directory that holds `/netflow.sqlite` for each dataset. + +The operator's machine needs Docker's command-line client and the repository dependencies from `bun install`. It does not need a local Docker daemon, because builds run on the host. + +### Configure the stack + +Set these variables in the shell: + +| Variable | Required | Meaning | +| ----------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ATLANTIS_CAMPUS_DOCKER_HOST` | yes | Docker host as an SSH URL, for example `ssh://barbera` or `ssh://user@host.example.edu`. | +| `ATLANTIS_CAMPUS_DATA_DIR` | yes | Absolute data directory on the host. | +| `ATLANTIS_CAMPUS_PORT` | no | Host loopback port for the dashboard. The default is `8080`. Pick a free port: `ssh ss -ltn` lists the ports in use. | +| `ATLANTIS_CAMPUS_DATA_USER` | no | `:` that the container runs as. The default is `1000:1000`. Use the owner of the data files, for example `$(id -u):$(id -g)` of that account on the host. | + +Alchemy stores the stack state in Cloudflare, like the [Cloudflare stack](#set-up-cloudflare-access). Export `CLOUDFLARE_API_TOKEN` and `CLOUDFLARE_ACCOUNT_ID` too. The stack creates no Cloudflare resources. + +### Deploy and remove a stage + +```bash +bun run plan:campus --stage +bun run deploy:campus --stage +bun run destroy:campus --stage +``` + +`plan:campus` builds the image on the host to compare it with the deployed image. It does not change the container. `deploy:campus` shows the plan and asks for approval. Add `--yes` to skip the prompt. The deploy prints the SSH command that users need. + +Check the container on the host: + +```bash +ssh docker ps --filter name=atlantis-campus-web +``` + +The status reads `healthy` after the first health check passes. + +`destroy:campus` removes the container, the current image, and the Docker context. It does not remove the data directory or its files. Images from earlier deploys keep their hash tags. Remove them on the host with `docker image rm atlantis-campus-web-:`. + +### Load data + +Copy each dataset to `//netflow.sqlite` on the host. The dashboard discovers every dataset directory. It picks up a new or replaced database on the next request, so the container does not need a restart. + +Copy only a database that no process is writing. To replace a live database, use `sqlite-maintenance`, which replaces the file atomically. See [Publish a local SQLite database](#publish-a-local-sqlite-database). + +The dashboard opens each database read-only. The mount is still writable, because SQLite creates the `netflow.sqlite-shm` and `netflow.sqlite-wal` files next to a WAL-mode database when it reads it. Set `ATLANTIS_CAMPUS_DATA_USER` to the owner of the data files, so that the container can create those files and the pipeline can still write them. + +### Run the pipeline on the host + +The stack does not build the pipeline image. The pipeline is a batch command, not a service, and its image build compiles nfdump and the Rust toolchain. + +Run the pipeline from a checkout of this repository on the host with the [Docker wrapper](setup-pipeline.md#docker-setup). The wrapper writes to the checkout's `data/` directory, so set `ATLANTIS_CAMPUS_DATA_DIR` to that directory: + +```bash +ssh +git clone https://gitlab.com/onrg/netflow-analysis.git atlantis +cd atlantis +./scripts/netflow-db-docker.sh --capture-root /absolute/path/to/captures pipeline ... +``` + +The wrapper builds the `atlantis-netflow-db:local` image on the host when it is missing or out of date. Follow [Set up the data pipeline](setup-pipeline.md) for the `datasets.json` configuration and the pipeline commands. + +### Connect as a user + +Users need an SSH account on the host. Forward a local port to the dashboard port on the host loopback: + +```bash +ssh -N -L 8080:127.0.0.1: +``` + +Then open `http://localhost:8080`. Replace the first `8080` with any free local port, and open that port instead. The tunnel stays open until you stop `ssh`. + +A connection to `:` from another machine fails, because the container publishes its port on `127.0.0.1` only. + ## Deploy the landing site 1. Build the site with its public URL. diff --git a/docs/user/setup-web.md b/docs/user/setup-web.md index c7156cce..e72984cd 100644 --- a/docs/user/setup-web.md +++ b/docs/user/setup-web.md @@ -37,7 +37,7 @@ First, [configure a dataset](datasets.md). Then, [set up the data pipeline](setu 2. In `.env`, set `DEFAULT_DATASET` to your dataset ID. This step is optional. Without it, the dashboard uses the first discovered dataset. - The dashboard reads `DEFAULT_DATASET`, `LOCAL_SQLITE_PATH`, and `DATABASE_PATH` from `.env`. `apps/web/src/env.ts` lists these variables. + The dashboard reads `DEFAULT_DATASET`, `LOCAL_SQLITE_PATH`, `DATABASE_PATH`, and `LOCAL_DATA_DIR` from `.env`. `LOCAL_DATA_DIR` replaces `data/` as the directory that holds `/netflow.sqlite`. `apps/web/src/env.ts` lists these variables. 3. Start the dashboard. diff --git a/infra/campus.ts b/infra/campus.ts new file mode 100644 index 00000000..b676cc17 --- /dev/null +++ b/infra/campus.ts @@ -0,0 +1,103 @@ +import * as Alchemy from 'alchemy'; +import * as Cloudflare from 'alchemy/Cloudflare'; +import * as Docker from 'alchemy/Docker'; +import { hashDockerBuildInputs } from 'alchemy/Docker/BuildHash'; +import * as Config from 'effect/Config'; +import * as Effect from 'effect/Effect'; +import { appName, repoRoot, stageName, webDockerfile } from './shared.ts'; + +const campusName = `${appName}-campus`; + +const webPort = 3000; + +const platform = 'linux/amd64'; + +const parseDataUser = (value: string) => { + const match = /^(\d+):(\d+)$/.exec(value.trim()); + if (!match) { + throw new Error(`ATLANTIS_CAMPUS_DATA_USER must be ':', got '${value}'`); + } + return { uid: match[1], gid: match[2] }; +}; + +const webHealthcheck = `node -e "fetch('http://127.0.0.1:${webPort}/api/datasets').then((response) => process.exit(response.ok ? 0 : 1), () => process.exit(1))"`; + +export default Alchemy.Stack( + campusName, + { + providers: Docker.providers(), + state: Cloudflare.state() + }, + Effect.gen(function* () { + const { stage } = yield* Alchemy.Stack; + + const dockerHost = yield* Config.String('ATLANTIS_CAMPUS_DOCKER_HOST'); + const dataDir = yield* Config.String('ATLANTIS_CAMPUS_DATA_DIR'); + const port = yield* Config.Port('ATLANTIS_CAMPUS_PORT').pipe(Config.withDefault(8080)); + const dataUser = yield* Config.String('ATLANTIS_CAMPUS_DATA_USER').pipe( + Config.withDefault('1000:1000'), + Config.map(parseDataUser) + ); + + if (!dataDir.startsWith('/') || dataDir === '/') { + return yield* Effect.die( + new Error(`ATLANTIS_CAMPUS_DATA_DIR must be an absolute host directory, got '${dataDir}'`) + ); + } + + const webName = stageName(`${campusName}-web`, stage); + const buildArgs = { RUNTIME_UID: dataUser.uid, RUNTIME_GID: dataUser.gid }; + const buildHash = yield* hashDockerBuildInputs( + { context: repoRoot, dockerfile: webDockerfile, platform, buildArgs }, + 'effective' + ).pipe(Effect.orDie); + + const context = yield* Docker.Context('DockerHost', { + name: stageName(campusName, stage), + docker: `host=${dockerHost}`, + description: `ATLANTIS campus deployment (${stage})` + }); + + const webImage = yield* Docker.Image('WebImage', { + name: webName, + tag: buildHash, + context, + build: { + context: repoRoot, + dockerfile: webDockerfile, + platform, + args: buildArgs + } + }); + + const web = yield* Docker.Container('Web', { + name: webName, + context, + image: `${webName}:${buildHash}`, + environment: { + NODE_OPTIONS: '--max-old-space-size=768' + }, + volumes: [{ hostPath: dataDir, containerPath: '/data' }], + ports: [{ external: `127.0.0.1:${port}`, internal: webPort }], + memory: '1g', + restart: 'unless-stopped', + stopTimeout: '30 seconds', + healthcheck: { + cmd: webHealthcheck, + interval: '30 seconds', + timeout: '10 seconds', + retries: 3, + startPeriod: '30 seconds' + }, + start: true + }); + yield* web.bind('WebImage', webImage.imageId as never); + + return { + container: web.name, + image: webImage.imageRef, + dataDir, + tunnel: `ssh -N -L ${port}:127.0.0.1:${port} ${dockerHost.replace(/^ssh:\/\//, '')}` + }; + }) +); diff --git a/infra/shared.ts b/infra/shared.ts index 5c44c50e..1a24e75a 100644 --- a/infra/shared.ts +++ b/infra/shared.ts @@ -20,3 +20,7 @@ export const isProduction = (stage: string) => stage === productionStage; export const stageName = (base: string, stage: string) => isProduction(stage) ? base : `${base}-${stage}`; + +export const repoRoot = '.'; + +export const webDockerfile = `${webRoot}/Dockerfile`; diff --git a/package.json b/package.json index 6f029536..f988bf94 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,9 @@ "deploy:cloudflare": "ATLANTIS_DB_DRIVER=d1 alchemy deploy infra/cloudflare.ts", "plan:cloudflare": "ATLANTIS_DB_DRIVER=d1 alchemy plan infra/cloudflare.ts", "destroy:cloudflare": "alchemy destroy infra/cloudflare.ts", + "deploy:campus": "alchemy deploy infra/campus.ts", + "plan:campus": "alchemy plan infra/campus.ts", + "destroy:campus": "alchemy destroy infra/campus.ts", "check:web": "bun run --cwd apps/web check", "lint": "bun run --cwd apps/web lint && bun run --cwd infra lint && cargo clippy --workspace --all-targets --all-features --locked -- -D warnings", "typecheck": "bun run --cwd apps/web typecheck && bun run --cwd infra typecheck && cargo check --workspace --all-targets --locked", @@ -41,6 +44,7 @@ "infra" ], "patchedDependencies": { - "@alchemy.run/frontend-frameworks@2.0.0-beta.79": "patches/@alchemy.run%2Ffrontend-frameworks@2.0.0-beta.79.patch" + "@alchemy.run/frontend-frameworks@2.0.0-beta.79": "patches/@alchemy.run%2Ffrontend-frameworks@2.0.0-beta.79.patch", + "alchemy@2.0.0-beta.79": "patches/alchemy@2.0.0-beta.79.patch" } } diff --git a/patches/alchemy@2.0.0-beta.79.patch b/patches/alchemy@2.0.0-beta.79.patch new file mode 100644 index 00000000..f95d3b3e --- /dev/null +++ b/patches/alchemy@2.0.0-beta.79.patch @@ -0,0 +1,68 @@ +diff --git a/lib/Docker/Container.d.ts b/lib/Docker/Container.d.ts +index 1132b60513040b72a1c51cfe2342c9c31a3c4031..4d34e38f9226ed14227914ebd3b8c3260a9700f8 100644 +--- a/lib/Docker/Container.d.ts ++++ b/lib/Docker/Container.d.ts +@@ -57,6 +57,7 @@ export interface ContainerProps { + start?: boolean; + /** Docker healthcheck configuration. */ + healthcheck?: Container.Healthcheck; ++ memory?: string; + } + export declare namespace Container { + type Status = "created" | "running" | "paused" | "restarting" | "removing" | "exited" | "dead"; +diff --git a/lib/Docker/Container.js b/lib/Docker/Container.js +index eda60bf2183cfda4a47c410f00a92c32ed5afbd2..c8d09e19d588b64e5e35a1d1959538231588bd73 100644 +--- a/lib/Docker/Container.js ++++ b/lib/Docker/Container.js +@@ -289,6 +289,7 @@ const makeCreateArgs = (id, news, instanceId) => dockerPhysicalName(id, news, in + label: news.labels, + "stop-timeout": toSeconds(news.stopTimeout)?.toString(), + rm: news.removeOnExit ?? false, ++ memory: news.memory, + ...(news.healthcheck + ? { + "health-cmd": Array.isArray(news.healthcheck.cmd) +diff --git a/lib/Docker/Docker.d.ts b/lib/Docker/Docker.d.ts +index 932a35f17b74317e3d9b431aaca5b72923c02120..895bcd7c6298dd9dd151707f387d5f4300c552ef 100644 +--- a/lib/Docker/Docker.d.ts ++++ b/lib/Docker/Docker.d.ts +@@ -48,6 +48,7 @@ declare const Docker_base: Context.ServiceClass | undefined; + /** `--add-host` entries, each `hostname:address`. */ + "add-host"?: Array | undefined; ++ memory?: string | undefined; + command: Array | undefined; + label?: Record; + context?: string; +diff --git a/src/Docker/Container.ts b/src/Docker/Container.ts +index 366e65915c9405bd12f629a1dc3cce64b5f64c38..30bb7bebe96b24a5e11e773b6005ead6c0a76a03 100644 +--- a/src/Docker/Container.ts ++++ b/src/Docker/Container.ts +@@ -63,6 +63,7 @@ export interface ContainerProps { + start?: boolean; + /** Docker healthcheck configuration. */ + healthcheck?: Container.Healthcheck; ++ memory?: string; + } + + export declare namespace Container { +@@ -502,6 +503,7 @@ const makeCreateArgs = (id: string, news: ContainerProps, instanceId: string) => + label: news.labels, + "stop-timeout": toSeconds(news.stopTimeout)?.toString(), + rm: news.removeOnExit ?? false, ++ memory: news.memory, + ...(news.healthcheck + ? { + "health-cmd": Array.isArray(news.healthcheck.cmd) +diff --git a/src/Docker/Docker.ts b/src/Docker/Docker.ts +index ef0f9ffa1641f8a4c2003d19030dd9ec6cacc852..45930c0a576ad5854cf2e7e83420e8d27df038a6 100644 +--- a/src/Docker/Docker.ts ++++ b/src/Docker/Docker.ts +@@ -89,6 +89,7 @@ export class Docker extends Context.Service< + p: Array | undefined; + /** `--add-host` entries, each `hostname:address`. */ + "add-host"?: Array | undefined; ++ memory?: string | undefined; + command: Array | undefined; + label?: Record; + context?: string; From dff26e13a2d59beeab91b280e1c610363ed6541b Mon Sep 17 00:00:00 2001 From: flamboh Date: Sat, 26 Sep 2026 13:28:31 -0700 Subject: [PATCH 2/5] fix(infra): rename self-hosted stack, bound build hashing, align toolchain - Rename the campus stack, scripts, env vars, and docs to self-hosted. - Hash only the Dockerfile.dockerignore allowlist instead of walking the whole checkout before applying ignores. - Emit `-p ` and quote arguments in the printed SSH tunnel command, and require an ssh:// Docker host URL. - Pin Bun 1.3.11 and Node 24.18.1 in package.json, .node-version files, shell.nix, and CI. - Add infra unit tests and run them in CI. - Lead the operations docs with setup steps; move internals to architecture docs and drop real host names. --- .github/workflows/ci.yaml | 10 ++- .node-version | 2 +- AGENTS.md | 2 +- apps/landing/.node-version | 2 +- bun.lock | 1 + docs/code/architecture.md | 8 ++- docs/code/development.md | 2 +- docs/user/datasets.md | 12 ++-- docs/user/operations.md | 102 +++++++++++++-------------- docs/user/requirements.md | 6 +- docs/user/setup-pipeline.md | 4 +- infra/build-context.ts | 103 ++++++++++++++++++++++++++++ infra/{campus.ts => self-hosted.ts} | 52 +++++++++----- infra/ssh.ts | 31 +++++++++ infra/tests/build-context.test.ts | 73 ++++++++++++++++++++ infra/tests/ssh.test.ts | 40 +++++++++++ package.json | 8 +-- shell.nix | 51 +++++++++++++- 18 files changed, 410 insertions(+), 99 deletions(-) create mode 100644 infra/build-context.ts rename infra/{campus.ts => self-hosted.ts} (56%) create mode 100644 infra/ssh.ts create mode 100644 infra/tests/build-context.test.ts create mode 100644 infra/tests/ssh.test.ts diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e86aedd7..a735e9db 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,6 +23,10 @@ jobs: - run: cp datasets.json.example datasets.json + - uses: actions/setup-node@v6 + with: + node-version-file: .node-version + - uses: oven-sh/setup-bun@v2 with: bun-version-file: package.json @@ -49,7 +53,7 @@ jobs: run: bun run --cwd apps/web typecheck && bun run --cwd infra typecheck - name: Test - run: bun run test:web + run: bun run test:web && bun run test:infra - name: Build run: bun run build:web @@ -96,6 +100,10 @@ jobs: - run: cp datasets.json.example datasets.json + - uses: actions/setup-node@v6 + with: + node-version-file: .node-version + - uses: oven-sh/setup-bun@v2 with: bun-version-file: package.json diff --git a/.node-version b/.node-version index 9d11232a..8dfc5cb1 100644 --- a/.node-version +++ b/.node-version @@ -1 +1 @@ -24.4.1 +24.18.1 diff --git a/AGENTS.md b/AGENTS.md index 307488da..d06f06b9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,7 +7,7 @@ ATLANTIS turns NetFlow captures and CSV imports into queryable aggregate databas - `tools/netflow-db`: Rust pipeline (`atlantis-netflow-db` crate) for ingestion, aggregation, verification, and analysis-window exports. Native `nfcapd` ingestion uses the pinned `nfdump` fork in `vendor/nfdump`. - `apps/web`: Svelte 5/SvelteKit 3 dashboard and API routes. It reads local SQLite during development and Cloudflare D1 in deployment. - `apps/landing`: Astro marketing and SEO site. -- `infra`: Alchemy v2 stacks (Effect programs) that deploy the dashboard. `cloudflare.ts` deploys the worker and its D1 database; `campus.ts` runs the SQLite dashboard as a Docker container on a self-hosted host that users reach through SSH port forwarding; `shared.ts` holds names and paths common to all stacks. +- `infra`: Alchemy v2 stacks (Effect programs) that deploy the dashboard. `cloudflare.ts` deploys the worker and its D1 database; `self-hosted.ts` runs the SQLite dashboard as a Docker container on a self-hosted machine that users reach through SSH port forwarding; `shared.ts` holds names and paths common to all stacks. - `vendor/*`: Third-party analysis submodules. Treat these as read-only; build repo-local binaries through the scripts in `vendor/scripts/`. - `data/`, `.env`, and `datasets.json`: Machine-local inputs and generated databases. Keep paths and dataset contents out of commits. - `docs/user`: User setup and operation documentation. diff --git a/apps/landing/.node-version b/apps/landing/.node-version index 5b540673..8dfc5cb1 100644 --- a/apps/landing/.node-version +++ b/apps/landing/.node-version @@ -1 +1 @@ -22.16.0 +24.18.1 diff --git a/bun.lock b/bun.lock index efabbd45..fbd429f1 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "atlantis", diff --git a/docs/code/architecture.md b/docs/code/architecture.md index 9587a430..159ee7db 100644 --- a/docs/code/architecture.md +++ b/docs/code/architecture.md @@ -37,13 +37,15 @@ The local dashboard reads SQLite databases with `better-sqlite3`. It opens these The deployed dashboard reads the `DB` Cloudflare D1 binding from `cloudflare:workers`. `infra/cloudflare.ts` defines the worker and the D1 database as an Alchemy stack. Alchemy builds the worker with its own SvelteKit adapter and applies the D1 migrations during a deploy. -The campus dashboard runs the SQLite driver on a self-hosted Docker host. `infra/campus.ts` builds `apps/web/Dockerfile` on that host through an SSH Docker context and runs it as a container. The image holds the adapter-node build. The container mounts the host data directory at `/data` and publishes its port on the host loopback only. Users reach it through SSH port forwarding. The image tag is a content hash of the build inputs, so a changed input replaces the container and an unchanged deploy leaves it running. +The self-hosted dashboard runs the SQLite driver on a Docker host. `infra/self-hosted.ts` defines the `atlantis-self-hosted` stack: a Docker context that reaches the host over SSH, an image built from `apps/web/Dockerfile` on that host, and a container named `atlantis-self-hosted-web-` (`atlantis-self-hosted-web` for `prod`). The image holds the adapter-node build. The container mounts the host data directory at `/data`, publishes its port on the host loopback only, and has a 1 GB memory limit. Users reach it through SSH port forwarding. + +The image tag is a content hash of the platform, build arguments, Dockerfile, and build context. `Dockerfile.dockerignore` is an allowlist, and `infra/build-context.ts` walks only the allowlisted paths, so large ignored trees such as captures, `target`, and `node_modules` are never read. A changed input replaces the container; an unchanged deploy leaves it running. The data mount is writable because the pipeline publishes WAL-mode databases. SQLite can read a WAL database only if it can create the `-shm` and `-wal` files next to it, even for a read-only connection. The dashboard still opens every database read-only with `query_only`, so it never writes data. The container names its image as a plain `:` string. Alchemy compares a container's properties at plan time only when every property is resolved, and an output of an image that is being rebuilt stays unresolved until apply, so the plan would record an in-place update that never swaps the image. The container instead binds to the image's `imageId` output. The binding orders the container after the image on create and before it on destroy, and it is not one of the properties that the container compares. The plan also builds the new image while it compares the image, so the tagged image exists before apply replaces the container. -The image pins Node.js 24.18.1. Node.js 24.19.0 and later abort the process when the garbage collector frees a `better-sqlite3` statement ([nodejs/node#65446](https://github.com/nodejs/node/issues/65446)). +The repository, the image, and CI pin Node.js 24.18.1. Node.js 24.19.0 and later abort the process when the garbage collector frees a `better-sqlite3` statement ([nodejs/node#65446](https://github.com/nodejs/node/issues/65446)). The build selects one database driver. `apps/web/src/lib/server/db/d1.ts` reads D1, and `apps/web/src/lib/server/db/sqlite.ts` reads the pipeline SQLite files. Server code imports the driver as `#db`. [Development](development.md#choose-the-database-driver) explains the selection. @@ -68,4 +70,4 @@ Both implementations must keep compatible table and column contracts. No automat - Chart.js - Rust 1.97.1 - SQLite and Cloudflare D1 -- Bun 1.2.16 +- Bun 1.3.11 diff --git a/docs/code/development.md b/docs/code/development.md index 52c0dd82..17d36c10 100644 --- a/docs/code/development.md +++ b/docs/code/development.md @@ -65,7 +65,7 @@ bun run --cwd apps/web preview # SQLite build, then vite preview A D1 build has no SvelteKit adapter. `bun run build:web` checks that the D1 bundle compiles. The Cloudflare worker is built by Alchemy during a deploy, which injects its own adapter into the `sveltekit()` call. [Operations](../user/operations.md#deploy-the-dashboard) describes the deploy. -A SQLite build uses `@sveltejs/adapter-node` and writes a Node server to `apps/web/build`. Start it with `node build` from `apps/web`. The campus deployment runs this build in a container. [Operations](../user/operations.md#deploy-the-campus-dashboard) describes it. +A SQLite build uses `@sveltejs/adapter-node` and writes a Node server to `apps/web/build`. Start it with `node build` from `apps/web`. The self-hosted deployment runs this build in a container. [Operations](../user/operations.md#deploy-the-self-hosted-dashboard) describes it. The build leaves `paths.origin` unset. SvelteKit 3 replaced adapter-node's runtime `ORIGIN` variable with this build-time option, and a fixed origin would break SSH port forwarding, where each user picks a local port. Adapter-node then builds the request URL from the `Host` header and the `https` protocol, so a request to `http://localhost:8080` has the origin `https://localhost:8080`. The dashboard has no form actions, remote functions, or mutating endpoints, so SvelteKit's CSRF origin check never runs. Before you add a `POST` form, set `PROTOCOL_HEADER` or `paths.origin` so that the origin check sees the browser's real origin. diff --git a/docs/user/datasets.md b/docs/user/datasets.md index df8297c6..557f0634 100644 --- a/docs/user/datasets.md +++ b/docs/user/datasets.md @@ -122,8 +122,8 @@ defines its own logical sources and `daily_active_sources` selection. ```json [ { - "dataset_id": "campus-a", - "root_path": "/data/netflow/campus", + "dataset_id": "dataset-a", + "root_path": "/data/netflow/example", "source_ids": ["router-a"], "locality": [ { "type": "prefixes", "prefixes": ["198.18.0.0/16", "198.19.0.0/16"] } @@ -132,11 +132,11 @@ defines its own logical sources and `daily_active_sources` selection. "kind": "daily_active_sources", "ip_prefix": "198.18.0.0/16" }, - "db_path": "data/campus-a/netflow.sqlite" + "db_path": "data/dataset-a/netflow.sqlite" }, { - "dataset_id": "campus-b", - "root_path": "/data/netflow/campus", + "dataset_id": "dataset-b", + "root_path": "/data/netflow/example", "source_ids": ["router-a"], "locality": [ { "type": "prefixes", "prefixes": ["198.18.0.0/16", "198.19.0.0/16"] } @@ -145,7 +145,7 @@ defines its own logical sources and `daily_active_sources` selection. "kind": "daily_active_sources", "ip_prefix": "198.19.0.0/16" }, - "db_path": "data/campus-b/netflow.sqlite" + "db_path": "data/dataset-b/netflow.sqlite" } ] ``` diff --git a/docs/user/operations.md b/docs/user/operations.md index 58a82b7b..d638533f 100644 --- a/docs/user/operations.md +++ b/docs/user/operations.md @@ -1,6 +1,6 @@ # Operations -Use these procedures to verify and publish a database. This document also gives the Cloudflare and campus deployment commands. +Use these procedures to verify and publish a database. This document also gives the Cloudflare and self-hosted deployment commands. The D1 and Cloudflare deployment sections apply to the hosted ATLANTIS deployment and need Cloudflare access. A local installation does not use them. @@ -155,97 +155,89 @@ CAUTION: A restore overwrites remote D1 data. Record the current bookmark first. bunx wrangler@4.141.0 d1 time-travel restore atlantis-db --bookmark= ``` -## Deploy the campus dashboard +## Deploy the self-hosted dashboard -`infra/campus.ts` runs the dashboard as a Docker container on a self-hosted machine. It is an [Alchemy](https://alchemy.run) stack named `atlantis-campus`. The container reads the pipeline SQLite databases from a directory on that machine. It is not publicly reachable: it listens on the host loopback only, and users connect through SSH port forwarding. +This runs the dashboard as a Docker container on a machine you control. Users reach it through an SSH tunnel; it is not exposed to the network. -The stack has three resources: +### Prerequisites -- A Docker context that reaches the host's Docker daemon over SSH. -- The web image, built from `apps/web/Dockerfile` on the host. The image holds the Node build of the dashboard with the SQLite driver. -- The web container. It mounts the data directory at `/data`, has a 1 GB memory limit, restarts unless stopped, and reports health from `/api/datasets`. +On the host: -The image tag is a hash of the build inputs: the files that `apps/web/Dockerfile.dockerignore` admits, the Dockerfile, and the build arguments. A deploy after a source change builds a new image and replaces the container. A deploy without changes rebuilds from the Docker cache and leaves the container running. +- Docker Engine, usable by your account without `sudo` (for example through the `docker` group). +- A data directory with `/netflow.sqlite` for each dataset. -Each stage names its image and container `atlantis-campus-web-`. The `prod` stage uses `atlantis-campus-web`. +On your machine: -### Prepare the host +- The repository with `bun install` done. +- Docker's command-line client. A local Docker daemon is not needed; the image builds on the host. +- SSH key access to the host: `ssh docker info` must succeed without a password prompt. Host aliases from `~/.ssh/config` work. +- `CLOUDFLARE_API_TOKEN` and `CLOUDFLARE_ACCOUNT_ID`, as for the [Cloudflare stack](#set-up-cloudflare-access). Alchemy keeps the deploy state in Cloudflare; no Cloudflare resources are created. -The host needs: +### Configure -- Docker Engine. The operator's account must be able to run `docker` without `sudo`, for example through the `docker` group. -- SSH key access for the operator. `ssh docker info` must succeed without a password prompt. Docker uses the operator's SSH configuration, so a host alias from `~/.ssh/config` works. -- A data directory that holds `/netflow.sqlite` for each dataset. +Export these variables: -The operator's machine needs Docker's command-line client and the repository dependencies from `bun install`. It does not need a local Docker daemon, because builds run on the host. +| Variable | Required | Meaning | +| ---------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `ATLANTIS_SELF_HOSTED_DOCKER_HOST` | yes | The host as an SSH URL, for example `ssh://user@host.example.com`, `ssh://user@host.example.com:2222`, or an `~/.ssh/config` alias as `ssh://my-host`. | +| `ATLANTIS_SELF_HOSTED_DATA_DIR` | yes | Absolute path of the data directory on the host. | +| `ATLANTIS_SELF_HOSTED_PORT` | no | Port on the host's loopback interface. Default `8080`. `ssh ss -ltn` lists ports already in use. | +| `ATLANTIS_SELF_HOSTED_DATA_USER` | no | `:` the container runs as. Default `1000:1000`. Use the owner of the data files (`id -u` and `id -g` on the host). | -### Configure the stack +The container must run as the data owner because SQLite creates `-shm` and `-wal` files next to each database even when it only reads it. -Set these variables in the shell: +### Deploy -| Variable | Required | Meaning | -| ----------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `ATLANTIS_CAMPUS_DOCKER_HOST` | yes | Docker host as an SSH URL, for example `ssh://barbera` or `ssh://user@host.example.edu`. | -| `ATLANTIS_CAMPUS_DATA_DIR` | yes | Absolute data directory on the host. | -| `ATLANTIS_CAMPUS_PORT` | no | Host loopback port for the dashboard. The default is `8080`. Pick a free port: `ssh ss -ltn` lists the ports in use. | -| `ATLANTIS_CAMPUS_DATA_USER` | no | `:` that the container runs as. The default is `1000:1000`. Use the owner of the data files, for example `$(id -u):$(id -g)` of that account on the host. | +```bash +bun run deploy:self-hosted --stage +``` -Alchemy stores the stack state in Cloudflare, like the [Cloudflare stack](#set-up-cloudflare-access). Export `CLOUDFLARE_API_TOKEN` and `CLOUDFLARE_ACCOUNT_ID` too. The stack creates no Cloudflare resources. +The deploy shows its plan and asks for approval; add `--yes` to skip the prompt. It prints the tunnel command for users. `bun run plan:self-hosted --stage ` previews the change; it builds the image on the host but leaves the container alone. -### Deploy and remove a stage +Deploy again after pulling new code. An unchanged checkout leaves the running container alone. + +Check that the container is healthy: ```bash -bun run plan:campus --stage -bun run deploy:campus --stage -bun run destroy:campus --stage +ssh docker ps --filter name=atlantis-self-hosted-web ``` -`plan:campus` builds the image on the host to compare it with the deployed image. It does not change the container. `deploy:campus` shows the plan and asks for approval. Add `--yes` to skip the prompt. The deploy prints the SSH command that users need. +### Connect -Check the container on the host: +Users need an SSH account on the host. Run the tunnel command that the deploy printed, for example: ```bash -ssh docker ps --filter name=atlantis-campus-web +ssh -N -L 8080:127.0.0.1:8080 user@host.example.com ``` -The status reads `healthy` after the first health check passes. +Then open `http://localhost:8080`. To use a different local port, change the first number and open that port instead. The tunnel stays open until you stop `ssh`. -`destroy:campus` removes the container, the current image, and the Docker context. It does not remove the data directory or its files. Images from earlier deploys keep their hash tags. Remove them on the host with `docker image rm atlantis-campus-web-:`. +### Tear down -### Load data +```bash +bun run destroy:self-hosted --stage +``` -Copy each dataset to `//netflow.sqlite` on the host. The dashboard discovers every dataset directory. It picks up a new or replaced database on the next request, so the container does not need a restart. +This removes the container, its current image, and the Docker context. The data directory is untouched. Images from earlier deploys remain; remove them on the host with `docker image rm atlantis-self-hosted-web-:`. -Copy only a database that no process is writing. To replace a live database, use `sqlite-maintenance`, which replaces the file atomically. See [Publish a local SQLite database](#publish-a-local-sqlite-database). +### Add or replace datasets -The dashboard opens each database read-only. The mount is still writable, because SQLite creates the `netflow.sqlite-shm` and `netflow.sqlite-wal` files next to a WAL-mode database when it reads it. Set `ATLANTIS_CAMPUS_DATA_USER` to the owner of the data files, so that the container can create those files and the pipeline can still write them. +Copy each database to `//netflow.sqlite` on the host. The dashboard picks up new and replaced databases on the next request, without a restart. -### Run the pipeline on the host +Copy only a database that no process is writing. To replace a live database, use `sqlite-maintenance`; see [Publish a local SQLite database](#publish-a-local-sqlite-database). -The stack does not build the pipeline image. The pipeline is a batch command, not a service, and its image build compiles nfdump and the Rust toolchain. +### Run the pipeline on the host -Run the pipeline from a checkout of this repository on the host with the [Docker wrapper](setup-pipeline.md#docker-setup). The wrapper writes to the checkout's `data/` directory, so set `ATLANTIS_CAMPUS_DATA_DIR` to that directory: +Use the [Docker wrapper](setup-pipeline.md#docker-setup) from a checkout on the host. It writes to the checkout's `data/` directory, so point `ATLANTIS_SELF_HOSTED_DATA_DIR` there: ```bash ssh -git clone https://gitlab.com/onrg/netflow-analysis.git atlantis +git clone https://github.com/flamboh/atlantis.git atlantis cd atlantis -./scripts/netflow-db-docker.sh --capture-root /absolute/path/to/captures pipeline ... +./scripts/netflow-db-docker.sh --capture-root /data/netflow/example pipeline ... ``` -The wrapper builds the `atlantis-netflow-db:local` image on the host when it is missing or out of date. Follow [Set up the data pipeline](setup-pipeline.md) for the `datasets.json` configuration and the pipeline commands. - -### Connect as a user - -Users need an SSH account on the host. Forward a local port to the dashboard port on the host loopback: - -```bash -ssh -N -L 8080:127.0.0.1: -``` - -Then open `http://localhost:8080`. Replace the first `8080` with any free local port, and open that port instead. The tunnel stays open until you stop `ssh`. - -A connection to `:` from another machine fails, because the container publishes its port on `127.0.0.1` only. +See [Set up the data pipeline](setup-pipeline.md) for `datasets.json` and the pipeline commands. ## Deploy the landing site diff --git a/docs/user/requirements.md b/docs/user/requirements.md index 8264b9e0..474c1753 100644 --- a/docs/user/requirements.md +++ b/docs/user/requirements.md @@ -10,7 +10,7 @@ On NixOS, run `nix-shell` and skip the manual installation. Running the pipeline with Docker needs only Git and Docker on the host. The image build supplies the Rust toolchain, the nfdump build tools, and the pinned fork, and it does not need initialized Git submodules. -Docker covers the pipeline only; the dashboard runs natively. +To run the dashboard in Docker on a server, see [Deploy the self-hosted dashboard](operations.md#deploy-the-self-hosted-dashboard). Otherwise the dashboard runs natively. ## Dashboard @@ -19,8 +19,8 @@ The dashboard and all `bun run` commands need these tools: | Tool | Version | Source of truth | | ------- | ------------------------- | --------------- | | Git | Current supported version | Git releases | -| Bun | 1.2.16 | `package.json` | -| Node.js | 24.4.1 | `.node-version` | +| Bun | 1.3.11 | `package.json` | +| Node.js | 24.18.1 | `.node-version` | Node.js is necessary even though Bun installs the packages. The development server runs under Node.js, and `bun install` needs Node.js on `PATH` to download the prebuilt SQLite driver. Without it, the install prints a `better-sqlite3` warning and the dashboard cannot open a database (see [Troubleshooting](troubleshooting.md)). diff --git a/docs/user/setup-pipeline.md b/docs/user/setup-pipeline.md index 7853525c..628cec68 100644 --- a/docs/user/setup-pipeline.md +++ b/docs/user/setup-pipeline.md @@ -72,8 +72,8 @@ Native runs must name the pinned ATLANTIS nfdump fork explicitly: ```bash ./scripts/netflow-db.sh pipeline \ --nfdump target/nfdump/libexec/nfdump \ - --dataset campus-a \ - --dataset campus-b \ + --dataset dataset-a \ + --dataset dataset-b \ --start-date \ --end-date ``` diff --git a/infra/build-context.ts b/infra/build-context.ts new file mode 100644 index 00000000..2985615e --- /dev/null +++ b/infra/build-context.ts @@ -0,0 +1,103 @@ +import { selectDockerBuildContext, type DockerBuildSource } from 'alchemy/Docker/BuildHash'; +import * as Effect from 'effect/Effect'; +import * as FileSystem from 'effect/FileSystem'; +import * as Path from 'effect/Path'; +import * as Result from 'effect/Result'; +import * as Stream from 'effect/Stream'; +import * as crypto from 'node:crypto'; + +interface ContextEntry { + path: string; + fullPath: string; + type: string; + mode?: number; + size?: string; + target?: string; +} + +const allowlistedRoots = (dockerignore: string, ignoreFile: string) => { + const rules = dockerignore + .replace(/^\uFEFF/, '') + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line.length > 0 && !line.startsWith('#')); + if (rules[0] !== '*') { + return Effect.fail( + new Error(`${ignoreFile} must start with '*' and allowlist the build inputs`) + ); + } + return Effect.succeed( + rules + .filter((rule) => rule.startsWith('!')) + .map((rule) => + rule + .slice(1) + .trim() + .replace(/^\.?\/+/, '') + .replace(/\/+$/, '') + ) + ); +}; + +export const hashAllowlistedBuildContext = Effect.fn(function* (source: DockerBuildSource) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const selection = yield* selectDockerBuildContext(source); + const ignoreFile = `${selection.dockerfile}.dockerignore`; + const ignoreContent = yield* fs.readFileString(ignoreFile); + const roots = yield* allowlistedRoots(ignoreContent, ignoreFile); + + const entries: ContextEntry[] = []; + const pending = [...roots]; + while (pending.length > 0) { + const relativePath = pending.pop()!; + if (!selection.includes(relativePath)) { + continue; + } + const fullPath = path.join(selection.context, relativePath); + const link = yield* Effect.result(fs.readLink(fullPath)); + if (Result.isSuccess(link)) { + entries.push({ path: relativePath, fullPath, type: 'SymbolicLink', target: link.success }); + continue; + } + const info = yield* fs.stat(fullPath); + entries.push({ + path: relativePath, + fullPath, + type: info.type, + mode: info.mode & 0o7777, + size: info.type === 'File' ? String(info.size) : undefined + }); + if (info.type === 'Directory') { + for (const child of yield* fs.readDirectory(fullPath)) { + pending.push(`${relativePath}/${child}`); + } + } + } + + const hasher = crypto.createHash('sha256'); + hasher.update( + JSON.stringify({ + platform: source.platform, + buildArgs: Object.entries(source.buildArgs ?? {}).sort(([a], [b]) => + a < b ? -1 : a > b ? 1 : 0 + ), + dockerignore: ignoreContent + }) + ); + hasher.update('Dockerfile\0'); + hasher.update(yield* fs.readFile(selection.dockerfile)); + + for (const { fullPath, ...entry } of entries.sort((a, b) => + a.path < b.path ? -1 : a.path > b.path ? 1 : 0 + )) { + hasher.update(`${JSON.stringify(entry)}\0`); + if (entry.type === 'File') { + yield* fs + .stream(fullPath) + .pipe(Stream.runForEach((chunk) => Effect.sync(() => hasher.update(chunk)))); + } + } + + return hasher.digest('hex').slice(0, 32); +}); diff --git a/infra/campus.ts b/infra/self-hosted.ts similarity index 56% rename from infra/campus.ts rename to infra/self-hosted.ts index b676cc17..c547d962 100644 --- a/infra/campus.ts +++ b/infra/self-hosted.ts @@ -1,12 +1,14 @@ import * as Alchemy from 'alchemy'; import * as Cloudflare from 'alchemy/Cloudflare'; import * as Docker from 'alchemy/Docker'; -import { hashDockerBuildInputs } from 'alchemy/Docker/BuildHash'; import * as Config from 'effect/Config'; +import * as ConfigProvider from 'effect/ConfigProvider'; import * as Effect from 'effect/Effect'; -import { appName, repoRoot, stageName, webDockerfile } from './shared.ts'; +import { hashAllowlistedBuildContext } from './build-context.ts'; +import { appName, invalidStageMessage, repoRoot, stageName, webDockerfile } from './shared.ts'; +import { parseSshDockerHost, sshTunnelCommand } from './ssh.ts'; -const campusName = `${appName}-campus`; +const selfHostedName = `${appName}-self-hosted`; const webPort = 3000; @@ -15,7 +17,7 @@ const platform = 'linux/amd64'; const parseDataUser = (value: string) => { const match = /^(\d+):(\d+)$/.exec(value.trim()); if (!match) { - throw new Error(`ATLANTIS_CAMPUS_DATA_USER must be ':', got '${value}'`); + throw new Error(`ATLANTIS_SELF_HOSTED_DATA_USER must be ':', got '${value}'`); } return { uid: match[1], gid: match[2] }; }; @@ -23,39 +25,51 @@ const parseDataUser = (value: string) => { const webHealthcheck = `node -e "fetch('http://127.0.0.1:${webPort}/api/datasets').then((response) => process.exit(response.ok ? 0 : 1), () => process.exit(1))"`; export default Alchemy.Stack( - campusName, + selfHostedName, { providers: Docker.providers(), state: Cloudflare.state() }, Effect.gen(function* () { const { stage } = yield* Alchemy.Stack; + const invalidStage = invalidStageMessage(stage); + if (invalidStage) { + return yield* Effect.fail( + new Config.ConfigError(new ConfigProvider.SourceError({ message: invalidStage })) + ); + } - const dockerHost = yield* Config.String('ATLANTIS_CAMPUS_DOCKER_HOST'); - const dataDir = yield* Config.String('ATLANTIS_CAMPUS_DATA_DIR'); - const port = yield* Config.Port('ATLANTIS_CAMPUS_PORT').pipe(Config.withDefault(8080)); - const dataUser = yield* Config.String('ATLANTIS_CAMPUS_DATA_USER').pipe( + const dockerHost = yield* Config.String('ATLANTIS_SELF_HOSTED_DOCKER_HOST').pipe( + Config.map(parseSshDockerHost) + ); + const dataDir = yield* Config.String('ATLANTIS_SELF_HOSTED_DATA_DIR'); + const port = yield* Config.Port('ATLANTIS_SELF_HOSTED_PORT').pipe(Config.withDefault(8080)); + const dataUser = yield* Config.String('ATLANTIS_SELF_HOSTED_DATA_USER').pipe( Config.withDefault('1000:1000'), Config.map(parseDataUser) ); if (!dataDir.startsWith('/') || dataDir === '/') { return yield* Effect.die( - new Error(`ATLANTIS_CAMPUS_DATA_DIR must be an absolute host directory, got '${dataDir}'`) + new Error( + `ATLANTIS_SELF_HOSTED_DATA_DIR must be an absolute host directory, got '${dataDir}'` + ) ); } - const webName = stageName(`${campusName}-web`, stage); + const webName = stageName(`${selfHostedName}-web`, stage); const buildArgs = { RUNTIME_UID: dataUser.uid, RUNTIME_GID: dataUser.gid }; - const buildHash = yield* hashDockerBuildInputs( - { context: repoRoot, dockerfile: webDockerfile, platform, buildArgs }, - 'effective' - ).pipe(Effect.orDie); + const buildHash = yield* hashAllowlistedBuildContext({ + context: repoRoot, + dockerfile: webDockerfile, + platform, + buildArgs + }).pipe(Effect.orDie); const context = yield* Docker.Context('DockerHost', { - name: stageName(campusName, stage), - docker: `host=${dockerHost}`, - description: `ATLANTIS campus deployment (${stage})` + name: stageName(selfHostedName, stage), + docker: `host=${dockerHost.url}`, + description: `ATLANTIS self-hosted deployment (${stage})` }); const webImage = yield* Docker.Image('WebImage', { @@ -97,7 +111,7 @@ export default Alchemy.Stack( container: web.name, image: webImage.imageRef, dataDir, - tunnel: `ssh -N -L ${port}:127.0.0.1:${port} ${dockerHost.replace(/^ssh:\/\//, '')}` + tunnel: sshTunnelCommand(dockerHost, port) }; }) ); diff --git a/infra/ssh.ts b/infra/ssh.ts new file mode 100644 index 00000000..a5dab2bc --- /dev/null +++ b/infra/ssh.ts @@ -0,0 +1,31 @@ +export interface SshDockerHost { + url: string; + user: string | undefined; + host: string; + port: string | undefined; +} + +export const parseSshDockerHost = (value: string): SshDockerHost => { + const url = URL.parse(value.trim()); + if (url === null || url.protocol !== 'ssh:' || url.hostname === '') { + throw new Error(`ATLANTIS_SELF_HOSTED_DOCKER_HOST must be an ssh:// URL, got '${value}'`); + } + return { + url: value.trim(), + user: url.username === '' ? undefined : decodeURIComponent(url.username), + host: url.hostname.replace(/^\[(.*)\]$/, '$1'), + port: url.port === '' ? undefined : url.port + }; +}; + +const shellQuote = (value: string) => + /^[\w@%+=:,./-]+$/.test(value) ? value : `'${value.replaceAll("'", `'\\''`)}'`; + +export const sshTunnelCommand = (dockerHost: SshDockerHost, port: number) => { + const destination = + dockerHost.user === undefined ? dockerHost.host : `${dockerHost.user}@${dockerHost.host}`; + const portArgs = dockerHost.port === undefined ? [] : ['-p', dockerHost.port]; + return ['ssh', '-N', '-L', `${port}:127.0.0.1:${port}`, ...portArgs, destination] + .map(shellQuote) + .join(' '); +}; diff --git a/infra/tests/build-context.test.ts b/infra/tests/build-context.test.ts new file mode 100644 index 00000000..92f8ec37 --- /dev/null +++ b/infra/tests/build-context.test.ts @@ -0,0 +1,73 @@ +import * as BunFileSystem from '@effect/platform-bun/BunFileSystem'; +import * as BunPath from '@effect/platform-bun/BunPath'; +import * as Effect from 'effect/Effect'; +import * as Layer from 'effect/Layer'; +import { chmod, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { hashAllowlistedBuildContext } from '../build-context.ts'; + +const dockerignore = ['*', '!package.json', '!apps/web', 'apps/web/node_modules', ''].join('\n'); + +let root: string; + +const write = async (relativePath: string, content: string) => { + await mkdir(join(root, relativePath, '..'), { recursive: true }); + await writeFile(join(root, relativePath), content); +}; + +const hash = (buildArgs: Record = {}) => + Effect.runPromise( + hashAllowlistedBuildContext({ + context: root, + dockerfile: 'apps/web/Dockerfile', + platform: 'linux/amd64', + buildArgs + }).pipe(Effect.provide(Layer.mergeAll(BunFileSystem.layer, BunPath.layer))) + ); + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'atlantis-build-context-')); + await write('package.json', '{}'); + await write('apps/web/Dockerfile', 'FROM scratch\n'); + await write('apps/web/Dockerfile.dockerignore', dockerignore); + await write('apps/web/src/index.ts', 'export {};\n'); + await write('apps/web/node_modules/dep/index.js', ''); + await write('data/captures/nfcapd.202601010000', 'capture'); +}); + +afterEach(async () => { + await chmod(join(root, 'data'), 0o755); + await rm(root, { recursive: true, force: true }); +}); + +describe('hashAllowlistedBuildContext', () => { + it('changes when an allowlisted input changes', async () => { + const before = await hash(); + await write('apps/web/src/index.ts', 'export const changed = true;\n'); + expect(await hash()).not.toBe(before); + }); + + it('changes when a build argument changes', async () => { + expect(await hash({ RUNTIME_UID: '1000' })).not.toBe(await hash({ RUNTIME_UID: '1001' })); + }); + + it('ignores excluded subtrees and files outside the allowlist', async () => { + const before = await hash(); + await write('apps/web/node_modules/dep/index.js', 'changed'); + await write('data/captures/nfcapd.202601010005', 'capture'); + await write('README.md', 'changed'); + expect(await hash()).toBe(before); + }); + + it('does not walk directories outside the allowlist', async () => { + await chmod(join(root, 'data'), 0o000); + await expect(hash()).resolves.toMatch(/^[0-9a-f]{32}$/); + }); + + it('rejects an ignore file that is not an allowlist', async () => { + await write('apps/web/Dockerfile.dockerignore', 'node_modules\n'); + await expect(hash()).rejects.toThrow(/must start with '\*'/); + }); +}); diff --git a/infra/tests/ssh.test.ts b/infra/tests/ssh.test.ts new file mode 100644 index 00000000..c509104b --- /dev/null +++ b/infra/tests/ssh.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest'; +import { parseSshDockerHost, sshTunnelCommand } from '../ssh.ts'; + +describe('sshTunnelCommand', () => { + it('uses an SSH config alias as the destination', () => { + expect(sshTunnelCommand(parseSshDockerHost('ssh://docker-host'), 8080)).toBe( + 'ssh -N -L 8080:127.0.0.1:8080 docker-host' + ); + }); + + it('passes a non-default SSH port with -p', () => { + expect(sshTunnelCommand(parseSshDockerHost('ssh://user@host.example.com:2222'), 8081)).toBe( + 'ssh -N -L 8081:127.0.0.1:8081 -p 2222 user@host.example.com' + ); + }); + + it('unwraps bracketed IPv6 hosts', () => { + expect(sshTunnelCommand(parseSshDockerHost('ssh://user@[2001:db8::1]:2222'), 8080)).toBe( + 'ssh -N -L 8080:127.0.0.1:8080 -p 2222 user@2001:db8::1' + ); + }); + + it('quotes arguments that the shell would reinterpret', () => { + expect(sshTunnelCommand(parseSshDockerHost("ssh://o'brien%20x@host.example.com"), 8080)).toBe( + `ssh -N -L 8080:127.0.0.1:8080 'o'\\''brien x@host.example.com'` + ); + }); +}); + +describe('parseSshDockerHost', () => { + it('keeps the configured URL for the Docker context', () => { + expect(parseSshDockerHost(' ssh://user@host.example.com:2222 ').url).toBe( + 'ssh://user@host.example.com:2222' + ); + }); + + it.each(['host.example.com', 'tcp://host.example.com:2375', 'ssh://'])('rejects %s', (value) => { + expect(() => parseSshDockerHost(value)).toThrow(/must be an ssh:\/\/ URL/); + }); +}); diff --git a/package.json b/package.json index f988bf94..b4c5a281 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "drizzle-kit": "1.0.0-rc.5-ab785fc", "husky": "^9.1.7" }, - "packageManager": "bun@1.2.16", + "packageManager": "bun@1.3.11", "private": true, "scripts": { "prepare": "husky", @@ -18,9 +18,9 @@ "deploy:cloudflare": "ATLANTIS_DB_DRIVER=d1 alchemy deploy infra/cloudflare.ts", "plan:cloudflare": "ATLANTIS_DB_DRIVER=d1 alchemy plan infra/cloudflare.ts", "destroy:cloudflare": "alchemy destroy infra/cloudflare.ts", - "deploy:campus": "alchemy deploy infra/campus.ts", - "plan:campus": "alchemy plan infra/campus.ts", - "destroy:campus": "alchemy destroy infra/campus.ts", + "deploy:self-hosted": "alchemy deploy infra/self-hosted.ts", + "plan:self-hosted": "alchemy plan infra/self-hosted.ts", + "destroy:self-hosted": "alchemy destroy infra/self-hosted.ts", "check:web": "bun run --cwd apps/web check", "lint": "bun run --cwd apps/web lint && bun run --cwd infra lint && cargo clippy --workspace --all-targets --all-features --locked -- -D warnings", "typecheck": "bun run --cwd apps/web typecheck && bun run --cwd infra typecheck && cargo check --workspace --all-targets --locked", diff --git a/shell.nix b/shell.nix index 1bc919ef..dcf0c0c5 100644 --- a/shell.nix +++ b/shell.nix @@ -4,19 +4,66 @@ with (import (builtins.fetchTarball { url = "https://github.com/nixos/nixpkgs/archive/ce01daebf8489ba97bd1609d185ea276efdeb121.tar.gz"; sha256 = "10cqhkqkifcgyibj9nwxrnq424crfl40kwr3daky83m2fisb4f6p"; }) {}); +let + system = stdenv.hostPlatform.system; + + nodeVersion = "24.18.1"; + nodePlatform = + { + x86_64-linux = { name = "linux-x64"; hash = "sha256-1sZk3z8/YUWOjCd1hVcTKFItcFFmcjp8eCOpJTpNFaA="; }; + aarch64-linux = { name = "linux-arm64"; hash = "sha256-cgHjoJ3IJbrFeGfIGRPiuPDvh9BMuQgq9M2oL2/z2Iw="; }; + x86_64-darwin = { name = "darwin-x64"; hash = "sha256-+JLHiVcg9A03UL3iTzVUJC028jYCtRZ7W3PsTROTiu8="; }; + aarch64-darwin = { name = "darwin-arm64"; hash = "sha256-HWC3A/5dfnBySJvoGH9DDxoJWmWMMeXh4oEzGlhz+sM="; }; + } + .${system} or (throw "Unsupported system: ${system}"); + nodejs = stdenv.mkDerivation { + pname = "nodejs"; + version = nodeVersion; + src = fetchurl { + url = "https://nodejs.org/dist/v${nodeVersion}/node-v${nodeVersion}-${nodePlatform.name}.tar.xz"; + inherit (nodePlatform) hash; + }; + nativeBuildInputs = lib.optionals stdenv.hostPlatform.isLinux [ autoPatchelfHook ]; + buildInputs = lib.optionals stdenv.hostPlatform.isLinux [ stdenv.cc.cc.lib ]; + dontConfigure = true; + dontBuild = true; + installPhase = '' + mkdir -p $out + cp -r bin include lib share $out/ + ''; + }; + + bunVersion = "1.3.11"; + bunPlatform = + { + x86_64-linux = { name = "linux-x64"; hash = "sha256-hhG6k1r4hvBabzh0ChUWAybBXl1dB63vlmEwtEk2B+0="; }; + aarch64-linux = { name = "linux-aarch64"; hash = "sha256-0TlE2hKlPsx0v2pyC9HQTEVVwDjf5CI2U1anvkdpH98="; }; + x86_64-darwin = { name = "darwin-x64"; hash = "sha256-xP4rkkchiwKV8k6JWq7I/uYudEUmeakCa2fqy9YRooY="; }; + aarch64-darwin = { name = "darwin-aarch64"; hash = "sha256-b1o0Z+2crsR5W/eM1HZQfZ+HDH1XuGyUX8szgSZ3L/w="; }; + } + .${system} or (throw "Unsupported system: ${system}"); + bun = pkgs.bun.overrideAttrs { + version = bunVersion; + src = fetchurl { + url = "https://github.com/oven-sh/bun/releases/download/bun-v${bunVersion}/bun-${bunPlatform.name}.zip"; + inherit (bunPlatform) hash; + }; + sourceRoot = "bun-${bunPlatform.name}"; + }; +in mkShell { buildInputs = [ pkgs.autoconf pkgs.automake pkgs.bison - pkgs.bun + bun pkgs.flex pkgs.gcc pkgs.git pkgs.gnumake pkgs.gnutar pkgs.libtool - pkgs.nodejs_24 + nodejs pkgs.pkg-config pkgs.python3 pkgs.rustup From d723b283d547d22ec3dbeac15ec5d93a247ecc1f Mon Sep 17 00:00:00 2001 From: flamboh Date: Sat, 26 Sep 2026 14:54:22 -0700 Subject: [PATCH 3/5] fix(ci): keep e2e paths on the prior Node until Playwright is upgraded Playwright 1.52 hangs while loading its config on Node 24.18.1. Keep shell.nix and the CI e2e job on the earlier Node 24 and document why; Bun stays pinned to 1.3.11 everywhere. --- .github/workflows/ci.yaml | 4 ---- docs/code/architecture.md | 2 +- shell.nix | 28 +--------------------------- 3 files changed, 2 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a735e9db..a851a25f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -100,10 +100,6 @@ jobs: - run: cp datasets.json.example datasets.json - - uses: actions/setup-node@v6 - with: - node-version-file: .node-version - - uses: oven-sh/setup-bun@v2 with: bun-version-file: package.json diff --git a/docs/code/architecture.md b/docs/code/architecture.md index 159ee7db..8d5bafca 100644 --- a/docs/code/architecture.md +++ b/docs/code/architecture.md @@ -45,7 +45,7 @@ The data mount is writable because the pipeline publishes WAL-mode databases. SQ The container names its image as a plain `:` string. Alchemy compares a container's properties at plan time only when every property is resolved, and an output of an image that is being rebuilt stays unresolved until apply, so the plan would record an in-place update that never swaps the image. The container instead binds to the image's `imageId` output. The binding orders the container after the image on create and before it on destroy, and it is not one of the properties that the container compares. The plan also builds the new image while it compares the image, so the tagged image exists before apply replaces the container. -The repository, the image, and CI pin Node.js 24.18.1. Node.js 24.19.0 and later abort the process when the garbage collector frees a `better-sqlite3` statement ([nodejs/node#65446](https://github.com/nodejs/node/issues/65446)). +`.node-version`, the image, and the CI web job pin Node.js 24.18.1. Node.js 24.19.0 and later abort the process when the garbage collector frees a `better-sqlite3` statement ([nodejs/node#65446](https://github.com/nodejs/node/issues/65446)). `shell.nix` and the CI e2e job stay on an earlier Node.js 24, because Playwright 1.52 hangs while loading its config on Node.js 24.18.1; Playwright 1.53 fixes this. The build selects one database driver. `apps/web/src/lib/server/db/d1.ts` reads D1, and `apps/web/src/lib/server/db/sqlite.ts` reads the pipeline SQLite files. Server code imports the driver as `#db`. [Development](development.md#choose-the-database-driver) explains the selection. diff --git a/shell.nix b/shell.nix index dcf0c0c5..5250b79d 100644 --- a/shell.nix +++ b/shell.nix @@ -7,32 +7,6 @@ with (import (builtins.fetchTarball { let system = stdenv.hostPlatform.system; - nodeVersion = "24.18.1"; - nodePlatform = - { - x86_64-linux = { name = "linux-x64"; hash = "sha256-1sZk3z8/YUWOjCd1hVcTKFItcFFmcjp8eCOpJTpNFaA="; }; - aarch64-linux = { name = "linux-arm64"; hash = "sha256-cgHjoJ3IJbrFeGfIGRPiuPDvh9BMuQgq9M2oL2/z2Iw="; }; - x86_64-darwin = { name = "darwin-x64"; hash = "sha256-+JLHiVcg9A03UL3iTzVUJC028jYCtRZ7W3PsTROTiu8="; }; - aarch64-darwin = { name = "darwin-arm64"; hash = "sha256-HWC3A/5dfnBySJvoGH9DDxoJWmWMMeXh4oEzGlhz+sM="; }; - } - .${system} or (throw "Unsupported system: ${system}"); - nodejs = stdenv.mkDerivation { - pname = "nodejs"; - version = nodeVersion; - src = fetchurl { - url = "https://nodejs.org/dist/v${nodeVersion}/node-v${nodeVersion}-${nodePlatform.name}.tar.xz"; - inherit (nodePlatform) hash; - }; - nativeBuildInputs = lib.optionals stdenv.hostPlatform.isLinux [ autoPatchelfHook ]; - buildInputs = lib.optionals stdenv.hostPlatform.isLinux [ stdenv.cc.cc.lib ]; - dontConfigure = true; - dontBuild = true; - installPhase = '' - mkdir -p $out - cp -r bin include lib share $out/ - ''; - }; - bunVersion = "1.3.11"; bunPlatform = { @@ -63,7 +37,7 @@ mkShell { pkgs.gnumake pkgs.gnutar pkgs.libtool - nodejs + pkgs.nodejs_24 pkgs.pkg-config pkgs.python3 pkgs.rustup From 7282da0b6b37511035780f76476790a2ea96de6b Mon Sep 17 00:00:00 2001 From: flamboh Date: Mon, 28 Sep 2026 02:47:37 -0700 Subject: [PATCH 4/5] test(web): skip obsolete products found under LOCAL_DATA_DIR --- apps/web/tests/lib/server/datasets.test.ts | 32 ++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/apps/web/tests/lib/server/datasets.test.ts b/apps/web/tests/lib/server/datasets.test.ts index 29cd3192..cb8de0d2 100644 --- a/apps/web/tests/lib/server/datasets.test.ts +++ b/apps/web/tests/lib/server/datasets.test.ts @@ -548,6 +548,38 @@ describe('dataset server helpers', () => { await expect(datasets.getDatasetConfig('legacy')).rejects.toThrow(/Unknown dataset 'legacy'/); }); + it('skips pre-locality databases found under LOCAL_DATA_DIR', async () => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'datasets-local-data-dir-')); + const dataDir = path.join(workspace, 'mounted-data'); + const legacyDir = path.join(dataDir, 'legacy'); + const currentDir = path.join(dataDir, 'current'); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.mkdirSync(currentDir, { recursive: true }); + seedPreLocalityDatasetDb( + path.join(legacyDir, 'netflow.sqlite'), + 'legacy', + 'Legacy Only', + 'router-legacy' + ); + seedDatasetDb(path.join(currentDir, 'netflow.sqlite'), 'current', 'Current', 'router-current'); + process.chdir(os.tmpdir()); + vi.stubEnv('LOCAL_DATA_DIR', dataDir); + + const datasets = await loadDatasetsModule(); + + await expect(datasets.listDatasetSummaries()).resolves.toEqual([ + { + datasetId: 'current', + label: 'Current', + defaultStartDate: '2025-03-01', + discoveryMode: 'static', + hasLocality: false, + isDefault: true + } + ]); + await expect(datasets.getDatasetConfig('legacy')).rejects.toThrow(/Unknown dataset 'legacy'/); + }); + it('refreshes local dataset discovery after files move', async () => { const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'datasets-refresh-')); const alphaDir = path.join(workspace, 'data', 'alpha'); From a8f21a42c57c42ecbdc9b3aa183989446d5e7180 Mon Sep 17 00:00:00 2001 From: flamboh Date: Mon, 28 Sep 2026 12:50:38 -0700 Subject: [PATCH 5/5] docs: use neutral dataset IDs in locality examples --- docs/user/datasets.md | 4 ++-- tools/netflow-db/src/registry.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/user/datasets.md b/docs/user/datasets.md index 557f0634..dd4696bf 100644 --- a/docs/user/datasets.md +++ b/docs/user/datasets.md @@ -68,8 +68,8 @@ A logical source combines the captures from more than one collector directory. E ```json { - "dataset_id": "campus", - "root_path": "/data/netflow/campus", + "dataset_id": "dataset-a", + "root_path": "/data/netflow/dataset-a", "sources": [{ "source_id": "router-a", "members": ["router-a"] }], "locality": [ { "type": "prefixes", "prefixes": ["192.0.2.0/24", "2001:db8::/32"] }, diff --git a/tools/netflow-db/src/registry.rs b/tools/netflow-db/src/registry.rs index eb975451..dc59a0cb 100644 --- a/tools/netflow-db/src/registry.rs +++ b/tools/netflow-db/src/registry.rs @@ -448,7 +448,7 @@ mod tests { fs::write( &list, serde_json::json!([{ - "dataset_id": "campus", + "dataset_id": "dataset-a", "root_path": "/captures", "locality": [ {"type": "tos_anonymized"}, @@ -462,7 +462,7 @@ mod tests { .unwrap(); let registry = DatasetRegistry::load(&list, root.path()).unwrap(); - let dataset = registry.get("campus").unwrap(); + let dataset = registry.get("dataset-a").unwrap(); let rules = dataset.locality_rules().unwrap(); assert_eq!(