diff --git a/.env.example b/.env.example index 1fbb814f..714f2fbd 100644 --- a/.env.example +++ b/.env.example @@ -14,3 +14,5 @@ DEFAULT_DATASET=uoregon # Set one database path or use automatic discovery at data/*/netflow.sqlite. # LOCAL_SQLITE_PATH=/absolute/path/to/netflow.sqlite # DATABASE_PATH=/absolute/path/to/netflow.sqlite +# Or discover */netflow.sqlite under another directory instead of data/. +# LOCAL_DATA_DIR=/absolute/path/to/data diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e86aedd7..a851a25f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,6 +23,10 @@ jobs: - run: cp datasets.json.example datasets.json + - uses: actions/setup-node@v6 + with: + node-version-file: .node-version + - uses: oven-sh/setup-bun@v2 with: bun-version-file: package.json @@ -49,7 +53,7 @@ jobs: run: bun run --cwd apps/web typecheck && bun run --cwd infra typecheck - name: Test - run: bun run test:web + run: bun run test:web && bun run test:infra - name: Build run: bun run build:web diff --git a/.node-version b/.node-version index 9d11232a..8dfc5cb1 100644 --- a/.node-version +++ b/.node-version @@ -1 +1 @@ -24.4.1 +24.18.1 diff --git a/AGENTS.md b/AGENTS.md index 5464e753..d06f06b9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,7 +7,7 @@ ATLANTIS turns NetFlow captures and CSV imports into queryable aggregate databas - `tools/netflow-db`: Rust pipeline (`atlantis-netflow-db` crate) for ingestion, aggregation, verification, and analysis-window exports. Native `nfcapd` ingestion uses the pinned `nfdump` fork in `vendor/nfdump`. - `apps/web`: Svelte 5/SvelteKit 3 dashboard and API routes. It reads local SQLite during development and Cloudflare D1 in deployment. - `apps/landing`: Astro marketing and SEO site. -- `infra`: Alchemy v2 stacks (Effect programs) that deploy the dashboard. `cloudflare.ts` deploys the worker and its D1 database; `shared.ts` holds names and paths common to all stacks. +- `infra`: Alchemy v2 stacks (Effect programs) that deploy the dashboard. `cloudflare.ts` deploys the worker and its D1 database; `self-hosted.ts` runs the SQLite dashboard as a Docker container on a self-hosted machine that users reach through SSH port forwarding; `shared.ts` holds names and paths common to all stacks. - `vendor/*`: Third-party analysis submodules. Treat these as read-only; build repo-local binaries through the scripts in `vendor/scripts/`. - `data/`, `.env`, and `datasets.json`: Machine-local inputs and generated databases. Keep paths and dataset contents out of commits. - `docs/user`: User setup and operation documentation. diff --git a/apps/landing/.node-version b/apps/landing/.node-version index 5b540673..8dfc5cb1 100644 --- a/apps/landing/.node-version +++ b/apps/landing/.node-version @@ -1 +1 @@ -22.16.0 +24.18.1 diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile new file mode 100644 index 00000000..07ec8865 --- /dev/null +++ b/apps/web/Dockerfile @@ -0,0 +1,58 @@ +# syntax=docker/dockerfile:1 + +ARG BUN_VERSION=1.3.11 +ARG NODE_VERSION=24.18.1 + +FROM oven/bun:${BUN_VERSION}-slim AS bun + +FROM node:${NODE_VERSION}-bookworm-slim AS workspace + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install --yes --no-install-recommends g++ make python3 \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=bun /usr/local/bin/bun /usr/local/bin/bun + +WORKDIR /app +COPY package.json bun.lock ./ +COPY patches patches +COPY apps/web/package.json apps/web/package.json +COPY apps/landing/package.json apps/landing/package.json +COPY infra/package.json infra/package.json + +FROM workspace AS build + +RUN bun install --frozen-lockfile --ignore-scripts --filter @atlantis/web + +COPY apps/web apps/web + +RUN cd apps/web && ATLANTIS_DB_DRIVER=sqlite bun run build + +FROM workspace AS production-deps + +RUN bun install --frozen-lockfile --ignore-scripts --production --filter @atlantis/web \ + && npm run build-release --prefix node_modules/better-sqlite3 \ + && rm -rf node_modules/better-sqlite3/build/Release/obj node_modules/better-sqlite3/deps \ + && node -e "new (require('better-sqlite3'))(':memory:').prepare('select 1').get()" + +FROM node:${NODE_VERSION}-bookworm-slim AS runtime + +ARG RUNTIME_UID=1000 +ARG RUNTIME_GID=1000 + +ENV NODE_ENV=production \ + HOST=0.0.0.0 \ + PORT=3000 \ + LOCAL_DATA_DIR=/data + +WORKDIR /app +COPY --from=production-deps /app/node_modules node_modules +COPY --from=build /app/apps/web/package.json package.json +COPY --from=build /app/apps/web/build build + +RUN mkdir -p /data + +USER ${RUNTIME_UID}:${RUNTIME_GID} +EXPOSE 3000 +CMD ["node", "build"] diff --git a/apps/web/Dockerfile.dockerignore b/apps/web/Dockerfile.dockerignore new file mode 100644 index 00000000..a5d348bb --- /dev/null +++ b/apps/web/Dockerfile.dockerignore @@ -0,0 +1,18 @@ +* +!package.json +!bun.lock +!patches +!infra/package.json +!apps/landing/package.json +!apps/web +apps/web/node_modules +apps/web/build +apps/web/dist +apps/web/.svelte-kit +apps/web/test-results +apps/web/playwright-report +apps/web/tests +apps/web/Dockerfile +apps/web/Dockerfile.dockerignore +**/.env +**/.env.* diff --git a/apps/web/package.json b/apps/web/package.json index e98774e1..f38c9723 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -26,6 +26,7 @@ "@eslint/js": "^9.18.0", "@internationalized/date": "^3.12.0", "@playwright/test": "1.52.0", + "@sveltejs/adapter-node": "6.0.0-next.12", "@sveltejs/kit": "3.0.0-next.27", "@sveltejs/load-config": "^0.2.3", "@sveltejs/vite-plugin-svelte": "^7.3.1", diff --git a/apps/web/src/env.ts b/apps/web/src/env.ts index 49025812..2c5afd51 100644 --- a/apps/web/src/env.ts +++ b/apps/web/src/env.ts @@ -11,6 +11,11 @@ export const variables = defineEnvVars({ description: 'Fallback for LOCAL_SQLITE_PATH', schema: optional }, + LOCAL_DATA_DIR: { + description: + 'Directory scanned for /netflow.sqlite products; replaces the data/ and ../../data defaults', + schema: optional + }, DEFAULT_DATASET: { description: 'Dataset ID selected when a request does not name one', schema: optional diff --git a/apps/web/src/lib/server/db/local-files.ts b/apps/web/src/lib/server/db/local-files.ts index a9ffaccb..9e576274 100644 --- a/apps/web/src/lib/server/db/local-files.ts +++ b/apps/web/src/lib/server/db/local-files.ts @@ -1,4 +1,4 @@ -import { DATABASE_PATH, LOCAL_SQLITE_PATH } from '$app/env/private'; +import { DATABASE_PATH, LOCAL_DATA_DIR, LOCAL_SQLITE_PATH } from '$app/env/private'; async function resolvePath(value: string): Promise { if (value === ':memory:') { @@ -17,7 +17,9 @@ export async function discoverLocalSqlitePaths(): Promise { const fs = await import('node:fs/promises'); const path = await import('node:path'); - const roots = [path.resolve(process.cwd(), 'data'), path.resolve(process.cwd(), '../../data')]; + const roots = LOCAL_DATA_DIR + ? [await resolvePath(LOCAL_DATA_DIR)] + : [path.resolve(process.cwd(), 'data'), path.resolve(process.cwd(), '../../data')]; const dbPaths = new Set(); for (const root of roots) { diff --git a/apps/web/tests/lib/server/datasets.test.ts b/apps/web/tests/lib/server/datasets.test.ts index 29cd3192..cb8de0d2 100644 --- a/apps/web/tests/lib/server/datasets.test.ts +++ b/apps/web/tests/lib/server/datasets.test.ts @@ -548,6 +548,38 @@ describe('dataset server helpers', () => { await expect(datasets.getDatasetConfig('legacy')).rejects.toThrow(/Unknown dataset 'legacy'/); }); + it('skips pre-locality databases found under LOCAL_DATA_DIR', async () => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'datasets-local-data-dir-')); + const dataDir = path.join(workspace, 'mounted-data'); + const legacyDir = path.join(dataDir, 'legacy'); + const currentDir = path.join(dataDir, 'current'); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.mkdirSync(currentDir, { recursive: true }); + seedPreLocalityDatasetDb( + path.join(legacyDir, 'netflow.sqlite'), + 'legacy', + 'Legacy Only', + 'router-legacy' + ); + seedDatasetDb(path.join(currentDir, 'netflow.sqlite'), 'current', 'Current', 'router-current'); + process.chdir(os.tmpdir()); + vi.stubEnv('LOCAL_DATA_DIR', dataDir); + + const datasets = await loadDatasetsModule(); + + await expect(datasets.listDatasetSummaries()).resolves.toEqual([ + { + datasetId: 'current', + label: 'Current', + defaultStartDate: '2025-03-01', + discoveryMode: 'static', + hasLocality: false, + isDefault: true + } + ]); + await expect(datasets.getDatasetConfig('legacy')).rejects.toThrow(/Unknown dataset 'legacy'/); + }); + it('refreshes local dataset discovery after files move', async () => { const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'datasets-refresh-')); const alphaDir = path.join(workspace, 'data', 'alpha'); diff --git a/apps/web/tests/lib/server/db/local-files.test.ts b/apps/web/tests/lib/server/db/local-files.test.ts new file mode 100644 index 00000000..bb1b4888 --- /dev/null +++ b/apps/web/tests/lib/server/db/local-files.test.ts @@ -0,0 +1,42 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { discoverLocalSqlitePaths } from '../../../../src/lib/server/db/local-files'; + +describe('local SQLite discovery', () => { + let dataDir: string; + + beforeEach(() => { + dataDir = mkdtempSync(join(tmpdir(), 'atlantis-local-files-')); + for (const dataset of ['beta', 'alpha']) { + mkdirSync(join(dataDir, dataset)); + writeFileSync(join(dataDir, dataset, 'netflow.sqlite'), ''); + } + mkdirSync(join(dataDir, 'backups')); + writeFileSync(join(dataDir, 'backups', 'old.sqlite'), ''); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + rmSync(dataDir, { recursive: true, force: true }); + }); + + it('scans LOCAL_DATA_DIR for dataset products', async () => { + vi.stubEnv('LOCAL_DATA_DIR', dataDir); + + await expect(discoverLocalSqlitePaths()).resolves.toEqual([ + join(dataDir, 'alpha', 'netflow.sqlite'), + join(dataDir, 'beta', 'netflow.sqlite') + ]); + }); + + it('prefers LOCAL_SQLITE_PATH over LOCAL_DATA_DIR', async () => { + vi.stubEnv('LOCAL_DATA_DIR', dataDir); + vi.stubEnv('LOCAL_SQLITE_PATH', join(dataDir, 'beta', 'netflow.sqlite')); + + await expect(discoverLocalSqlitePaths()).resolves.toEqual([ + join(dataDir, 'beta', 'netflow.sqlite') + ]); + }); +}); diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index 9d35e4c6..517e4536 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -1,4 +1,5 @@ import { fileURLToPath } from 'node:url'; +import adapterNode from '@sveltejs/adapter-node'; import tailwindcss from '@tailwindcss/vite'; import { sveltekit } from '@sveltejs/kit/vite'; import { vitePreprocess } from '@sveltejs/vite-plugin-svelte'; @@ -56,6 +57,7 @@ function databaseDriver(driver: DatabaseDriver): Plugin { export default defineConfig(({ command, mode, isPreview = false }) => { const driver = resolveDatabaseDriver(command, mode, isPreview); + const nodeBuild = command === 'build' && driver === 'sqlite'; return { plugins: [ @@ -63,6 +65,7 @@ export default defineConfig(({ command, mode, isPreview = false }) => { databaseDriver(driver), sveltekit({ preprocess: vitePreprocess(), + adapter: nodeBuild ? adapterNode() : undefined, env: { dir: '../..' } diff --git a/bun.lock b/bun.lock index adf22851..fbd429f1 100644 --- a/bun.lock +++ b/bun.lock @@ -44,6 +44,7 @@ "@eslint/js": "^9.18.0", "@internationalized/date": "^3.12.0", "@playwright/test": "1.52.0", + "@sveltejs/adapter-node": "6.0.0-next.12", "@sveltejs/kit": "3.0.0-next.27", "@sveltejs/load-config": "^0.2.3", "@sveltejs/vite-plugin-svelte": "^7.3.1", @@ -94,6 +95,7 @@ }, }, "patchedDependencies": { + "alchemy@2.0.0-beta.79": "patches/alchemy@2.0.0-beta.79.patch", "@alchemy.run/frontend-frameworks@2.0.0-beta.79": "patches/@alchemy.run%2Ffrontend-frameworks@2.0.0-beta.79.patch", }, "packages": { @@ -617,6 +619,8 @@ "@sveltejs/acorn-typescript": ["@sveltejs/acorn-typescript@1.0.13", "", { "peerDependencies": { "acorn": "^8.9.0" } }, "sha512-wgKggnhZVL9Bfx1OaKKTrYY9BFRk6C8UAkQNUcIv1+llzYrIqy+RZm5HPKzn0NpEBvTVhTqB4kQyllZywsRBRQ=="], + "@sveltejs/adapter-node": ["@sveltejs/adapter-node@6.0.0-next.12", "", { "dependencies": { "rolldown": "^1.2.3" }, "peerDependencies": { "@sveltejs/kit": "^3.0.0-next.0" } }, "sha512-wheNAOlAqLoXvGeSxJChM4jMEAKoMIcBCE18jGbTUdDDENPLefTmX55ACeUOkFqaVeZbAwKVHKifKZeyVNR9Yg=="], + "@sveltejs/kit": ["@sveltejs/kit@3.0.0-next.27", "", { "dependencies": { "@rolldown/pluginutils": "^1.0.1", "@standard-schema/spec": "^1.1.0", "cookie": "^2.0.1", "devalue": "^5.9.0", "esm-env": "^1.2.2", "magic-string": "^1.1.0", "mrmime": "^2.0.1", "sirv": "^3.0.2" }, "peerDependencies": { "@opentelemetry/api": "^1.0.0", "@sveltejs/vite-plugin-svelte": "^7.0.0", "svelte": "^5.56.4", "typescript": "^6.0.0", "vite": "^8.0.12" }, "optionalPeers": ["@opentelemetry/api", "typescript"], "bin": { "svelte-kit": "svelte-kit.js" } }, "sha512-m2R6GvcMhiAbYVKhzRf5mzLSvcXEuy/H0Va72XvxfEl1ZglrSC/321GKYa6idDHPojuRzkyh9EB0jBZmAwf6+Q=="], "@sveltejs/load-config": ["@sveltejs/load-config@0.2.3", "", {}, "sha512-VT3qmUb8pRV2QrZjd8iAmtg8lf4W0TIjZbvXtz5MKei/q96teWZgGJyyidJzOjzZzvdq616eSRVeMYIQChUTAQ=="], @@ -1717,6 +1721,8 @@ "@rollup/pluginutils/picomatch": ["picomatch@4.0.4", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="], + "@sveltejs/adapter-node/rolldown": ["rolldown@1.2.10", "", { "dependencies": { "@oxc-project/types": "=0.151.0", "@rolldown/pluginutils": "^1.0.0" }, "optionalDependencies": { "@rolldown/binding-android-arm-eabi": "1.2.10", "@rolldown/binding-android-arm64": "1.2.10", "@rolldown/binding-darwin-arm64": "1.2.10", "@rolldown/binding-darwin-x64": "1.2.10", "@rolldown/binding-freebsd-x64": "1.2.10", "@rolldown/binding-linux-arm-gnueabihf": "1.2.10", "@rolldown/binding-linux-arm64-gnu": "1.2.10", "@rolldown/binding-linux-arm64-musl": "1.2.10", "@rolldown/binding-linux-ppc64-gnu": "1.2.10", "@rolldown/binding-linux-s390x-gnu": "1.2.10", "@rolldown/binding-linux-x64-gnu": "1.2.10", "@rolldown/binding-linux-x64-musl": "1.2.10", "@rolldown/binding-openharmony-arm64": "1.2.10", "@rolldown/binding-win32-arm64-msvc": "1.2.10", "@rolldown/binding-win32-x64-msvc": "1.2.10" }, "bin": { "rolldown": "./bin/cli.mjs" } }, "sha512-OxkA08pSryMK7B3XiFA09B4OJ1xJMPgIYCBMY2xchzpqgBGsV1o0DetPAE+Sl3N3L4oCPiEzmHVSOj7iR04Zog=="], + "@sveltejs/kit/cookie": ["cookie@2.0.1", "", {}, "sha512-yuToqVvRrj6pfDXREyQAAv8SkAEk/8GS3jQRTiUMm66TVtBYmqQeoEjL2Lmq8Rpo6271vH76InTChTitEAm65w=="], "@sveltejs/kit/devalue": ["devalue@5.9.4", "", {}, "sha512-sPAT4pztbu6586/hrhOnMKS17IJrvg12mXiSPSS3W5qDeN2RGgvZ0diZCm31dBbnevfVmujNO3IM2wrS4Y2Rhg=="], @@ -1979,6 +1985,38 @@ "@puppeteer/browsers/yargs/string-width": ["string-width@8.3.0", "", { "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" } }, "sha512-ZbmZM0JCihQN91dWnxoipT2KOEyHqEyfRXUyjuRhW8b/xnqPDoq4gWEVApTVa9db2wN8mmoikgFBbjh71+cGeQ=="], + "@sveltejs/adapter-node/rolldown/@oxc-project/types": ["@oxc-project/types@0.151.0", "", {}, "sha512-J1yXrIlNDZVzE3ada310xeAw7nH8yCAyLPuUIsjKatFPmfn5bS1oW+cM+QsGOtVWd5nhSpbwZWx/rue+r5Z+PA=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-android-arm-eabi": ["@rolldown/binding-android-arm-eabi@1.2.10", "", { "os": "android", "cpu": "arm" }, "sha512-bp9svZb+QurZeh+8H4BhrZkifEB0YBNvTVzNSJnJQkj4NrRwmQoDUCGP0vSN7PbvLeM7l1tK6GXL8mrTiH2myg=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-android-arm64": ["@rolldown/binding-android-arm64@1.2.10", "", { "os": "android", "cpu": "arm64" }, "sha512-wm6Dld3RXUAZ/gRWKyUy+4W1B5CB5UeFaOzsSWJWEdxZXHH8rCYiZ5dGe6oJmhsunAPWzL7FZV+VtvmN5Ye2eA=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-darwin-arm64": ["@rolldown/binding-darwin-arm64@1.2.10", "", { "os": "darwin", "cpu": "arm64" }, "sha512-UbEfXq/AqGNgRTV3ik+X/iR6mUxu2QdYAadwRxJWquUGnW6gDqdP1FtLtFXRow7RJx0ssRwi80XAPr4r+4DtsA=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-darwin-x64": ["@rolldown/binding-darwin-x64@1.2.10", "", { "os": "darwin", "cpu": "x64" }, "sha512-7f5h17q5KZVx/ji1vb8OTq31ch1O2I7K8NPIr44GkyWTApXMIsmhWqZfgpOH10xeauqghDAvGlZktasCkcF6Eg=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-freebsd-x64": ["@rolldown/binding-freebsd-x64@1.2.10", "", { "os": "freebsd", "cpu": "x64" }, "sha512-ynOk/eEYhC6ZB2xCGvKrEOwE58oBy9LnrAqtkrDF9Fz1VTaNdGZTsV0VarJdhPwb+sOJTGjCLwcuyRJZ1dnMcQ=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-linux-arm-gnueabihf": ["@rolldown/binding-linux-arm-gnueabihf@1.2.10", "", { "os": "linux", "cpu": "arm" }, "sha512-ERrAs185meZZhGan7a4l3RiiJK1ArSDlHdST++uvSxe+FDbR4TwUPahT/cbZJvaG6fIpDpF78surN+tX708Y4Q=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-linux-arm64-gnu": ["@rolldown/binding-linux-arm64-gnu@1.2.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-KN7OHKD0J3jy1UzBwZWPxpwhODf9IARUIJcrH+yLYKOcmegZ8luEUM38lDP1bDVj40yP6PsSzCqOJF76vljFnQ=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-linux-arm64-musl": ["@rolldown/binding-linux-arm64-musl@1.2.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-8l9wP8O+wa8zD6iw6egSfzVtu7oZVfH3hlUsMM4MwbLMhxleqeoXbZzjddyK3YyNlwLhqznq3tF7PkNJ8T/V2w=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-linux-ppc64-gnu": ["@rolldown/binding-linux-ppc64-gnu@1.2.10", "", { "os": "linux", "cpu": "ppc64" }, "sha512-SeXNKeQzA5kLhz/J0CH6ZP0/HJ3v1xm/0YbiYpE0kK7emfRC2OIGGIaE14xzkISEGv2aYuUSpiLiU5Gbq+OI0A=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-linux-s390x-gnu": ["@rolldown/binding-linux-s390x-gnu@1.2.10", "", { "os": "linux", "cpu": "s390x" }, "sha512-mtht0nR+y8/hart4175Ll15w7lY8dg7CtQ+j2FDNTsDRspOWTK/2V3l0aj9sIj7XmvqxT8Yli/wq22e7feTTWg=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-linux-x64-gnu": ["@rolldown/binding-linux-x64-gnu@1.2.10", "", { "os": "linux", "cpu": "x64" }, "sha512-FSM94nGd55NYo48usCyM/nHfUKRnqc9+b0vJNuKV0oCCpIp/OGims7rO1Nv/DkFkt0S/s2rxsJ2kkS8J3HcpeA=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-linux-x64-musl": ["@rolldown/binding-linux-x64-musl@1.2.10", "", { "os": "linux", "cpu": "x64" }, "sha512-C3YxNB16myRLs7o+B+6PnQ6jBsdIS4+AE4Ah8glVGhDpEv9AOvxhZ/1duAb4B0UGczEK/lBbccksd8VI+p6zfw=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-openharmony-arm64": ["@rolldown/binding-openharmony-arm64@1.2.10", "", { "os": "none", "cpu": "arm64" }, "sha512-571TlE/F1eeTjjdjYAMMMPs1Mfv3MtX6s3+ZKVU6HiUjZ5Njc6c/qzNy/8K3zALTZnaw3JQVYrHxvNfjm43KAg=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-win32-arm64-msvc": ["@rolldown/binding-win32-arm64-msvc@1.2.10", "", { "os": "win32", "cpu": "arm64" }, "sha512-QXW+ZWaiqs2c7Fi++D/SsW07LTPcUrncxcskJGfGNBoaLik1IU6fJymz4HsqwEO0u5Iq11yTO0B/mc4cPk7jrQ=="], + + "@sveltejs/adapter-node/rolldown/@rolldown/binding-win32-x64-msvc": ["@rolldown/binding-win32-x64-msvc@1.2.10", "", { "os": "win32", "cpu": "x64" }, "sha512-5FQFGgah17YeMtG1Yd5a+rMxQpTksyNXxRtKz06FVTaQw3RKYUJQbUoKk0/5jrXBpDo+7makNP7UHA2LQyH64A=="], + "@tailwindcss/node/lightningcss/lightningcss-android-arm64": ["lightningcss-android-arm64@1.32.0", "", { "os": "android", "cpu": "arm64" }, "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg=="], "@tailwindcss/node/lightningcss/lightningcss-darwin-arm64": ["lightningcss-darwin-arm64@1.32.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ=="], diff --git a/docs/code/architecture.md b/docs/code/architecture.md index 540d3d45..8d5bafca 100644 --- a/docs/code/architecture.md +++ b/docs/code/architecture.md @@ -37,6 +37,16 @@ The local dashboard reads SQLite databases with `better-sqlite3`. It opens these The deployed dashboard reads the `DB` Cloudflare D1 binding from `cloudflare:workers`. `infra/cloudflare.ts` defines the worker and the D1 database as an Alchemy stack. Alchemy builds the worker with its own SvelteKit adapter and applies the D1 migrations during a deploy. +The self-hosted dashboard runs the SQLite driver on a Docker host. `infra/self-hosted.ts` defines the `atlantis-self-hosted` stack: a Docker context that reaches the host over SSH, an image built from `apps/web/Dockerfile` on that host, and a container named `atlantis-self-hosted-web-` (`atlantis-self-hosted-web` for `prod`). The image holds the adapter-node build. The container mounts the host data directory at `/data`, publishes its port on the host loopback only, and has a 1 GB memory limit. Users reach it through SSH port forwarding. + +The image tag is a content hash of the platform, build arguments, Dockerfile, and build context. `Dockerfile.dockerignore` is an allowlist, and `infra/build-context.ts` walks only the allowlisted paths, so large ignored trees such as captures, `target`, and `node_modules` are never read. A changed input replaces the container; an unchanged deploy leaves it running. + +The data mount is writable because the pipeline publishes WAL-mode databases. SQLite can read a WAL database only if it can create the `-shm` and `-wal` files next to it, even for a read-only connection. The dashboard still opens every database read-only with `query_only`, so it never writes data. + +The container names its image as a plain `:` string. Alchemy compares a container's properties at plan time only when every property is resolved, and an output of an image that is being rebuilt stays unresolved until apply, so the plan would record an in-place update that never swaps the image. The container instead binds to the image's `imageId` output. The binding orders the container after the image on create and before it on destroy, and it is not one of the properties that the container compares. The plan also builds the new image while it compares the image, so the tagged image exists before apply replaces the container. + +`.node-version`, the image, and the CI web job pin Node.js 24.18.1. Node.js 24.19.0 and later abort the process when the garbage collector frees a `better-sqlite3` statement ([nodejs/node#65446](https://github.com/nodejs/node/issues/65446)). `shell.nix` and the CI e2e job stay on an earlier Node.js 24, because Playwright 1.52 hangs while loading its config on Node.js 24.18.1; Playwright 1.53 fixes this. + The build selects one database driver. `apps/web/src/lib/server/db/d1.ts` reads D1, and `apps/web/src/lib/server/db/sqlite.ts` reads the pipeline SQLite files. Server code imports the driver as `#db`. [Development](development.md#choose-the-database-driver) explains the selection. The landing site has no database. It builds static files in `apps/landing/dist`. @@ -60,4 +70,4 @@ Both implementations must keep compatible table and column contracts. No automat - Chart.js - Rust 1.97.1 - SQLite and Cloudflare D1 -- Bun 1.2.16 +- Bun 1.3.11 diff --git a/docs/code/development.md b/docs/code/development.md index c4e2acf3..17d36c10 100644 --- a/docs/code/development.md +++ b/docs/code/development.md @@ -47,10 +47,10 @@ The root `dev` command starts both applications. It does not start the pipeline. The dashboard has two database drivers. Each build and each development server includes only one of them. -| `ATLANTIS_DB_DRIVER` | Driver | Default for | -| -------------------- | --------------------------------------------------------------------------------------------- | ----------------------------------- | -| `sqlite` | `src/lib/server/db/sqlite.ts` reads `data//netflow.sqlite` or `LOCAL_SQLITE_PATH` | `vite dev`, `vite preview` (always) | -| `d1` | `src/lib/server/db/d1.ts` reads the `DB` binding from `cloudflare:workers` | `vite build` | +| `ATLANTIS_DB_DRIVER` | Driver | Default for | +| -------------------- | ---------------------------------------------------------------------------------------------------------------- | ----------------------------------- | +| `sqlite` | `src/lib/server/db/sqlite.ts` reads `data//netflow.sqlite`, `LOCAL_DATA_DIR`, or `LOCAL_SQLITE_PATH` | `vite dev`, `vite preview` (always) | +| `d1` | `src/lib/server/db/d1.ts` reads the `DB` binding from `cloudflare:workers` | `vite build` | Server code imports the driver as `#db`. The `imports` field in `apps/web/package.json` maps `#db` to the D1 driver under the `atlantis-d1` export condition, and to the SQLite driver otherwise. `apps/web/vite.config.ts` adds that condition to the server environments and keeps `cloudflare:workers` external when the driver is `d1`. Both drivers implement `DatabaseDriver` in `src/lib/server/db/driver.ts`. @@ -59,11 +59,15 @@ Set `ATLANTIS_DB_DRIVER` in the shell. The value in `.env` does not select the d ```bash bun run dev:web # SQLite bun run build:web # D1 server bundle, no adapter -ATLANTIS_DB_DRIVER=sqlite bun run build:web # SQLite server bundle, no adapter +ATLANTIS_DB_DRIVER=sqlite bun run build:web # Node server in apps/web/build bun run --cwd apps/web preview # SQLite build, then vite preview ``` -The web app has no SvelteKit adapter. `bun run build:web` checks that the D1 bundle compiles. The Cloudflare worker is built by Alchemy during a deploy, which injects its own adapter into the `sveltekit()` call. [Operations](../user/operations.md#deploy-the-dashboard) describes the deploy. +A D1 build has no SvelteKit adapter. `bun run build:web` checks that the D1 bundle compiles. The Cloudflare worker is built by Alchemy during a deploy, which injects its own adapter into the `sveltekit()` call. [Operations](../user/operations.md#deploy-the-dashboard) describes the deploy. + +A SQLite build uses `@sveltejs/adapter-node` and writes a Node server to `apps/web/build`. Start it with `node build` from `apps/web`. The self-hosted deployment runs this build in a container. [Operations](../user/operations.md#deploy-the-self-hosted-dashboard) describes it. + +The build leaves `paths.origin` unset. SvelteKit 3 replaced adapter-node's runtime `ORIGIN` variable with this build-time option, and a fixed origin would break SSH port forwarding, where each user picks a local port. Adapter-node then builds the request URL from the `Host` header and the `https` protocol, so a request to `http://localhost:8080` has the origin `https://localhost:8080`. The dashboard has no form actions, remote functions, or mutating endpoints, so SvelteKit's CSRF origin check never runs. Before you add a `POST` form, set `PROTOCOL_HEADER` or `paths.origin` so that the origin check sees the browser's real origin. `preview` always rebuilds with SQLite before it serves, because `vite preview` runs the server in Node and cannot load the D1 bundle. diff --git a/docs/user/datasets.md b/docs/user/datasets.md index df8297c6..dd4696bf 100644 --- a/docs/user/datasets.md +++ b/docs/user/datasets.md @@ -68,8 +68,8 @@ A logical source combines the captures from more than one collector directory. E ```json { - "dataset_id": "campus", - "root_path": "/data/netflow/campus", + "dataset_id": "dataset-a", + "root_path": "/data/netflow/dataset-a", "sources": [{ "source_id": "router-a", "members": ["router-a"] }], "locality": [ { "type": "prefixes", "prefixes": ["192.0.2.0/24", "2001:db8::/32"] }, @@ -122,8 +122,8 @@ defines its own logical sources and `daily_active_sources` selection. ```json [ { - "dataset_id": "campus-a", - "root_path": "/data/netflow/campus", + "dataset_id": "dataset-a", + "root_path": "/data/netflow/example", "source_ids": ["router-a"], "locality": [ { "type": "prefixes", "prefixes": ["198.18.0.0/16", "198.19.0.0/16"] } @@ -132,11 +132,11 @@ defines its own logical sources and `daily_active_sources` selection. "kind": "daily_active_sources", "ip_prefix": "198.18.0.0/16" }, - "db_path": "data/campus-a/netflow.sqlite" + "db_path": "data/dataset-a/netflow.sqlite" }, { - "dataset_id": "campus-b", - "root_path": "/data/netflow/campus", + "dataset_id": "dataset-b", + "root_path": "/data/netflow/example", "source_ids": ["router-a"], "locality": [ { "type": "prefixes", "prefixes": ["198.18.0.0/16", "198.19.0.0/16"] } @@ -145,7 +145,7 @@ defines its own logical sources and `daily_active_sources` selection. "kind": "daily_active_sources", "ip_prefix": "198.19.0.0/16" }, - "db_path": "data/campus-b/netflow.sqlite" + "db_path": "data/dataset-b/netflow.sqlite" } ] ``` diff --git a/docs/user/operations.md b/docs/user/operations.md index 249b1dc8..d638533f 100644 --- a/docs/user/operations.md +++ b/docs/user/operations.md @@ -1,8 +1,8 @@ # Operations -Use these procedures to verify and publish a database. This document also gives the available Cloudflare deployment commands. +Use these procedures to verify and publish a database. This document also gives the Cloudflare and self-hosted deployment commands. -The D1 and deployment sections apply to the hosted ATLANTIS deployment and need Cloudflare access. A local installation does not use them. +The D1 and Cloudflare deployment sections apply to the hosted ATLANTIS deployment and need Cloudflare access. A local installation does not use them. ## Verify a SQLite database @@ -155,6 +155,90 @@ CAUTION: A restore overwrites remote D1 data. Record the current bookmark first. bunx wrangler@4.141.0 d1 time-travel restore atlantis-db --bookmark= ``` +## Deploy the self-hosted dashboard + +This runs the dashboard as a Docker container on a machine you control. Users reach it through an SSH tunnel; it is not exposed to the network. + +### Prerequisites + +On the host: + +- Docker Engine, usable by your account without `sudo` (for example through the `docker` group). +- A data directory with `/netflow.sqlite` for each dataset. + +On your machine: + +- The repository with `bun install` done. +- Docker's command-line client. A local Docker daemon is not needed; the image builds on the host. +- SSH key access to the host: `ssh docker info` must succeed without a password prompt. Host aliases from `~/.ssh/config` work. +- `CLOUDFLARE_API_TOKEN` and `CLOUDFLARE_ACCOUNT_ID`, as for the [Cloudflare stack](#set-up-cloudflare-access). Alchemy keeps the deploy state in Cloudflare; no Cloudflare resources are created. + +### Configure + +Export these variables: + +| Variable | Required | Meaning | +| ---------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `ATLANTIS_SELF_HOSTED_DOCKER_HOST` | yes | The host as an SSH URL, for example `ssh://user@host.example.com`, `ssh://user@host.example.com:2222`, or an `~/.ssh/config` alias as `ssh://my-host`. | +| `ATLANTIS_SELF_HOSTED_DATA_DIR` | yes | Absolute path of the data directory on the host. | +| `ATLANTIS_SELF_HOSTED_PORT` | no | Port on the host's loopback interface. Default `8080`. `ssh ss -ltn` lists ports already in use. | +| `ATLANTIS_SELF_HOSTED_DATA_USER` | no | `:` the container runs as. Default `1000:1000`. Use the owner of the data files (`id -u` and `id -g` on the host). | + +The container must run as the data owner because SQLite creates `-shm` and `-wal` files next to each database even when it only reads it. + +### Deploy + +```bash +bun run deploy:self-hosted --stage +``` + +The deploy shows its plan and asks for approval; add `--yes` to skip the prompt. It prints the tunnel command for users. `bun run plan:self-hosted --stage ` previews the change; it builds the image on the host but leaves the container alone. + +Deploy again after pulling new code. An unchanged checkout leaves the running container alone. + +Check that the container is healthy: + +```bash +ssh docker ps --filter name=atlantis-self-hosted-web +``` + +### Connect + +Users need an SSH account on the host. Run the tunnel command that the deploy printed, for example: + +```bash +ssh -N -L 8080:127.0.0.1:8080 user@host.example.com +``` + +Then open `http://localhost:8080`. To use a different local port, change the first number and open that port instead. The tunnel stays open until you stop `ssh`. + +### Tear down + +```bash +bun run destroy:self-hosted --stage +``` + +This removes the container, its current image, and the Docker context. The data directory is untouched. Images from earlier deploys remain; remove them on the host with `docker image rm atlantis-self-hosted-web-:`. + +### Add or replace datasets + +Copy each database to `//netflow.sqlite` on the host. The dashboard picks up new and replaced databases on the next request, without a restart. + +Copy only a database that no process is writing. To replace a live database, use `sqlite-maintenance`; see [Publish a local SQLite database](#publish-a-local-sqlite-database). + +### Run the pipeline on the host + +Use the [Docker wrapper](setup-pipeline.md#docker-setup) from a checkout on the host. It writes to the checkout's `data/` directory, so point `ATLANTIS_SELF_HOSTED_DATA_DIR` there: + +```bash +ssh +git clone https://github.com/flamboh/atlantis.git atlantis +cd atlantis +./scripts/netflow-db-docker.sh --capture-root /data/netflow/example pipeline ... +``` + +See [Set up the data pipeline](setup-pipeline.md) for `datasets.json` and the pipeline commands. + ## Deploy the landing site 1. Build the site with its public URL. diff --git a/docs/user/requirements.md b/docs/user/requirements.md index 8264b9e0..474c1753 100644 --- a/docs/user/requirements.md +++ b/docs/user/requirements.md @@ -10,7 +10,7 @@ On NixOS, run `nix-shell` and skip the manual installation. Running the pipeline with Docker needs only Git and Docker on the host. The image build supplies the Rust toolchain, the nfdump build tools, and the pinned fork, and it does not need initialized Git submodules. -Docker covers the pipeline only; the dashboard runs natively. +To run the dashboard in Docker on a server, see [Deploy the self-hosted dashboard](operations.md#deploy-the-self-hosted-dashboard). Otherwise the dashboard runs natively. ## Dashboard @@ -19,8 +19,8 @@ The dashboard and all `bun run` commands need these tools: | Tool | Version | Source of truth | | ------- | ------------------------- | --------------- | | Git | Current supported version | Git releases | -| Bun | 1.2.16 | `package.json` | -| Node.js | 24.4.1 | `.node-version` | +| Bun | 1.3.11 | `package.json` | +| Node.js | 24.18.1 | `.node-version` | Node.js is necessary even though Bun installs the packages. The development server runs under Node.js, and `bun install` needs Node.js on `PATH` to download the prebuilt SQLite driver. Without it, the install prints a `better-sqlite3` warning and the dashboard cannot open a database (see [Troubleshooting](troubleshooting.md)). diff --git a/docs/user/setup-pipeline.md b/docs/user/setup-pipeline.md index 7853525c..628cec68 100644 --- a/docs/user/setup-pipeline.md +++ b/docs/user/setup-pipeline.md @@ -72,8 +72,8 @@ Native runs must name the pinned ATLANTIS nfdump fork explicitly: ```bash ./scripts/netflow-db.sh pipeline \ --nfdump target/nfdump/libexec/nfdump \ - --dataset campus-a \ - --dataset campus-b \ + --dataset dataset-a \ + --dataset dataset-b \ --start-date \ --end-date ``` diff --git a/docs/user/setup-web.md b/docs/user/setup-web.md index c7156cce..e72984cd 100644 --- a/docs/user/setup-web.md +++ b/docs/user/setup-web.md @@ -37,7 +37,7 @@ First, [configure a dataset](datasets.md). Then, [set up the data pipeline](setu 2. In `.env`, set `DEFAULT_DATASET` to your dataset ID. This step is optional. Without it, the dashboard uses the first discovered dataset. - The dashboard reads `DEFAULT_DATASET`, `LOCAL_SQLITE_PATH`, and `DATABASE_PATH` from `.env`. `apps/web/src/env.ts` lists these variables. + The dashboard reads `DEFAULT_DATASET`, `LOCAL_SQLITE_PATH`, `DATABASE_PATH`, and `LOCAL_DATA_DIR` from `.env`. `LOCAL_DATA_DIR` replaces `data/` as the directory that holds `/netflow.sqlite`. `apps/web/src/env.ts` lists these variables. 3. Start the dashboard. diff --git a/infra/build-context.ts b/infra/build-context.ts new file mode 100644 index 00000000..2985615e --- /dev/null +++ b/infra/build-context.ts @@ -0,0 +1,103 @@ +import { selectDockerBuildContext, type DockerBuildSource } from 'alchemy/Docker/BuildHash'; +import * as Effect from 'effect/Effect'; +import * as FileSystem from 'effect/FileSystem'; +import * as Path from 'effect/Path'; +import * as Result from 'effect/Result'; +import * as Stream from 'effect/Stream'; +import * as crypto from 'node:crypto'; + +interface ContextEntry { + path: string; + fullPath: string; + type: string; + mode?: number; + size?: string; + target?: string; +} + +const allowlistedRoots = (dockerignore: string, ignoreFile: string) => { + const rules = dockerignore + .replace(/^\uFEFF/, '') + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line.length > 0 && !line.startsWith('#')); + if (rules[0] !== '*') { + return Effect.fail( + new Error(`${ignoreFile} must start with '*' and allowlist the build inputs`) + ); + } + return Effect.succeed( + rules + .filter((rule) => rule.startsWith('!')) + .map((rule) => + rule + .slice(1) + .trim() + .replace(/^\.?\/+/, '') + .replace(/\/+$/, '') + ) + ); +}; + +export const hashAllowlistedBuildContext = Effect.fn(function* (source: DockerBuildSource) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const selection = yield* selectDockerBuildContext(source); + const ignoreFile = `${selection.dockerfile}.dockerignore`; + const ignoreContent = yield* fs.readFileString(ignoreFile); + const roots = yield* allowlistedRoots(ignoreContent, ignoreFile); + + const entries: ContextEntry[] = []; + const pending = [...roots]; + while (pending.length > 0) { + const relativePath = pending.pop()!; + if (!selection.includes(relativePath)) { + continue; + } + const fullPath = path.join(selection.context, relativePath); + const link = yield* Effect.result(fs.readLink(fullPath)); + if (Result.isSuccess(link)) { + entries.push({ path: relativePath, fullPath, type: 'SymbolicLink', target: link.success }); + continue; + } + const info = yield* fs.stat(fullPath); + entries.push({ + path: relativePath, + fullPath, + type: info.type, + mode: info.mode & 0o7777, + size: info.type === 'File' ? String(info.size) : undefined + }); + if (info.type === 'Directory') { + for (const child of yield* fs.readDirectory(fullPath)) { + pending.push(`${relativePath}/${child}`); + } + } + } + + const hasher = crypto.createHash('sha256'); + hasher.update( + JSON.stringify({ + platform: source.platform, + buildArgs: Object.entries(source.buildArgs ?? {}).sort(([a], [b]) => + a < b ? -1 : a > b ? 1 : 0 + ), + dockerignore: ignoreContent + }) + ); + hasher.update('Dockerfile\0'); + hasher.update(yield* fs.readFile(selection.dockerfile)); + + for (const { fullPath, ...entry } of entries.sort((a, b) => + a.path < b.path ? -1 : a.path > b.path ? 1 : 0 + )) { + hasher.update(`${JSON.stringify(entry)}\0`); + if (entry.type === 'File') { + yield* fs + .stream(fullPath) + .pipe(Stream.runForEach((chunk) => Effect.sync(() => hasher.update(chunk)))); + } + } + + return hasher.digest('hex').slice(0, 32); +}); diff --git a/infra/self-hosted.ts b/infra/self-hosted.ts new file mode 100644 index 00000000..c547d962 --- /dev/null +++ b/infra/self-hosted.ts @@ -0,0 +1,117 @@ +import * as Alchemy from 'alchemy'; +import * as Cloudflare from 'alchemy/Cloudflare'; +import * as Docker from 'alchemy/Docker'; +import * as Config from 'effect/Config'; +import * as ConfigProvider from 'effect/ConfigProvider'; +import * as Effect from 'effect/Effect'; +import { hashAllowlistedBuildContext } from './build-context.ts'; +import { appName, invalidStageMessage, repoRoot, stageName, webDockerfile } from './shared.ts'; +import { parseSshDockerHost, sshTunnelCommand } from './ssh.ts'; + +const selfHostedName = `${appName}-self-hosted`; + +const webPort = 3000; + +const platform = 'linux/amd64'; + +const parseDataUser = (value: string) => { + const match = /^(\d+):(\d+)$/.exec(value.trim()); + if (!match) { + throw new Error(`ATLANTIS_SELF_HOSTED_DATA_USER must be ':', got '${value}'`); + } + return { uid: match[1], gid: match[2] }; +}; + +const webHealthcheck = `node -e "fetch('http://127.0.0.1:${webPort}/api/datasets').then((response) => process.exit(response.ok ? 0 : 1), () => process.exit(1))"`; + +export default Alchemy.Stack( + selfHostedName, + { + providers: Docker.providers(), + state: Cloudflare.state() + }, + Effect.gen(function* () { + const { stage } = yield* Alchemy.Stack; + const invalidStage = invalidStageMessage(stage); + if (invalidStage) { + return yield* Effect.fail( + new Config.ConfigError(new ConfigProvider.SourceError({ message: invalidStage })) + ); + } + + const dockerHost = yield* Config.String('ATLANTIS_SELF_HOSTED_DOCKER_HOST').pipe( + Config.map(parseSshDockerHost) + ); + const dataDir = yield* Config.String('ATLANTIS_SELF_HOSTED_DATA_DIR'); + const port = yield* Config.Port('ATLANTIS_SELF_HOSTED_PORT').pipe(Config.withDefault(8080)); + const dataUser = yield* Config.String('ATLANTIS_SELF_HOSTED_DATA_USER').pipe( + Config.withDefault('1000:1000'), + Config.map(parseDataUser) + ); + + if (!dataDir.startsWith('/') || dataDir === '/') { + return yield* Effect.die( + new Error( + `ATLANTIS_SELF_HOSTED_DATA_DIR must be an absolute host directory, got '${dataDir}'` + ) + ); + } + + const webName = stageName(`${selfHostedName}-web`, stage); + const buildArgs = { RUNTIME_UID: dataUser.uid, RUNTIME_GID: dataUser.gid }; + const buildHash = yield* hashAllowlistedBuildContext({ + context: repoRoot, + dockerfile: webDockerfile, + platform, + buildArgs + }).pipe(Effect.orDie); + + const context = yield* Docker.Context('DockerHost', { + name: stageName(selfHostedName, stage), + docker: `host=${dockerHost.url}`, + description: `ATLANTIS self-hosted deployment (${stage})` + }); + + const webImage = yield* Docker.Image('WebImage', { + name: webName, + tag: buildHash, + context, + build: { + context: repoRoot, + dockerfile: webDockerfile, + platform, + args: buildArgs + } + }); + + const web = yield* Docker.Container('Web', { + name: webName, + context, + image: `${webName}:${buildHash}`, + environment: { + NODE_OPTIONS: '--max-old-space-size=768' + }, + volumes: [{ hostPath: dataDir, containerPath: '/data' }], + ports: [{ external: `127.0.0.1:${port}`, internal: webPort }], + memory: '1g', + restart: 'unless-stopped', + stopTimeout: '30 seconds', + healthcheck: { + cmd: webHealthcheck, + interval: '30 seconds', + timeout: '10 seconds', + retries: 3, + startPeriod: '30 seconds' + }, + start: true + }); + yield* web.bind('WebImage', webImage.imageId as never); + + return { + container: web.name, + image: webImage.imageRef, + dataDir, + tunnel: sshTunnelCommand(dockerHost, port) + }; + }) +); diff --git a/infra/shared.ts b/infra/shared.ts index 5c44c50e..1a24e75a 100644 --- a/infra/shared.ts +++ b/infra/shared.ts @@ -20,3 +20,7 @@ export const isProduction = (stage: string) => stage === productionStage; export const stageName = (base: string, stage: string) => isProduction(stage) ? base : `${base}-${stage}`; + +export const repoRoot = '.'; + +export const webDockerfile = `${webRoot}/Dockerfile`; diff --git a/infra/ssh.ts b/infra/ssh.ts new file mode 100644 index 00000000..a5dab2bc --- /dev/null +++ b/infra/ssh.ts @@ -0,0 +1,31 @@ +export interface SshDockerHost { + url: string; + user: string | undefined; + host: string; + port: string | undefined; +} + +export const parseSshDockerHost = (value: string): SshDockerHost => { + const url = URL.parse(value.trim()); + if (url === null || url.protocol !== 'ssh:' || url.hostname === '') { + throw new Error(`ATLANTIS_SELF_HOSTED_DOCKER_HOST must be an ssh:// URL, got '${value}'`); + } + return { + url: value.trim(), + user: url.username === '' ? undefined : decodeURIComponent(url.username), + host: url.hostname.replace(/^\[(.*)\]$/, '$1'), + port: url.port === '' ? undefined : url.port + }; +}; + +const shellQuote = (value: string) => + /^[\w@%+=:,./-]+$/.test(value) ? value : `'${value.replaceAll("'", `'\\''`)}'`; + +export const sshTunnelCommand = (dockerHost: SshDockerHost, port: number) => { + const destination = + dockerHost.user === undefined ? dockerHost.host : `${dockerHost.user}@${dockerHost.host}`; + const portArgs = dockerHost.port === undefined ? [] : ['-p', dockerHost.port]; + return ['ssh', '-N', '-L', `${port}:127.0.0.1:${port}`, ...portArgs, destination] + .map(shellQuote) + .join(' '); +}; diff --git a/infra/tests/build-context.test.ts b/infra/tests/build-context.test.ts new file mode 100644 index 00000000..92f8ec37 --- /dev/null +++ b/infra/tests/build-context.test.ts @@ -0,0 +1,73 @@ +import * as BunFileSystem from '@effect/platform-bun/BunFileSystem'; +import * as BunPath from '@effect/platform-bun/BunPath'; +import * as Effect from 'effect/Effect'; +import * as Layer from 'effect/Layer'; +import { chmod, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { hashAllowlistedBuildContext } from '../build-context.ts'; + +const dockerignore = ['*', '!package.json', '!apps/web', 'apps/web/node_modules', ''].join('\n'); + +let root: string; + +const write = async (relativePath: string, content: string) => { + await mkdir(join(root, relativePath, '..'), { recursive: true }); + await writeFile(join(root, relativePath), content); +}; + +const hash = (buildArgs: Record = {}) => + Effect.runPromise( + hashAllowlistedBuildContext({ + context: root, + dockerfile: 'apps/web/Dockerfile', + platform: 'linux/amd64', + buildArgs + }).pipe(Effect.provide(Layer.mergeAll(BunFileSystem.layer, BunPath.layer))) + ); + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'atlantis-build-context-')); + await write('package.json', '{}'); + await write('apps/web/Dockerfile', 'FROM scratch\n'); + await write('apps/web/Dockerfile.dockerignore', dockerignore); + await write('apps/web/src/index.ts', 'export {};\n'); + await write('apps/web/node_modules/dep/index.js', ''); + await write('data/captures/nfcapd.202601010000', 'capture'); +}); + +afterEach(async () => { + await chmod(join(root, 'data'), 0o755); + await rm(root, { recursive: true, force: true }); +}); + +describe('hashAllowlistedBuildContext', () => { + it('changes when an allowlisted input changes', async () => { + const before = await hash(); + await write('apps/web/src/index.ts', 'export const changed = true;\n'); + expect(await hash()).not.toBe(before); + }); + + it('changes when a build argument changes', async () => { + expect(await hash({ RUNTIME_UID: '1000' })).not.toBe(await hash({ RUNTIME_UID: '1001' })); + }); + + it('ignores excluded subtrees and files outside the allowlist', async () => { + const before = await hash(); + await write('apps/web/node_modules/dep/index.js', 'changed'); + await write('data/captures/nfcapd.202601010005', 'capture'); + await write('README.md', 'changed'); + expect(await hash()).toBe(before); + }); + + it('does not walk directories outside the allowlist', async () => { + await chmod(join(root, 'data'), 0o000); + await expect(hash()).resolves.toMatch(/^[0-9a-f]{32}$/); + }); + + it('rejects an ignore file that is not an allowlist', async () => { + await write('apps/web/Dockerfile.dockerignore', 'node_modules\n'); + await expect(hash()).rejects.toThrow(/must start with '\*'/); + }); +}); diff --git a/infra/tests/ssh.test.ts b/infra/tests/ssh.test.ts new file mode 100644 index 00000000..c509104b --- /dev/null +++ b/infra/tests/ssh.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest'; +import { parseSshDockerHost, sshTunnelCommand } from '../ssh.ts'; + +describe('sshTunnelCommand', () => { + it('uses an SSH config alias as the destination', () => { + expect(sshTunnelCommand(parseSshDockerHost('ssh://docker-host'), 8080)).toBe( + 'ssh -N -L 8080:127.0.0.1:8080 docker-host' + ); + }); + + it('passes a non-default SSH port with -p', () => { + expect(sshTunnelCommand(parseSshDockerHost('ssh://user@host.example.com:2222'), 8081)).toBe( + 'ssh -N -L 8081:127.0.0.1:8081 -p 2222 user@host.example.com' + ); + }); + + it('unwraps bracketed IPv6 hosts', () => { + expect(sshTunnelCommand(parseSshDockerHost('ssh://user@[2001:db8::1]:2222'), 8080)).toBe( + 'ssh -N -L 8080:127.0.0.1:8080 -p 2222 user@2001:db8::1' + ); + }); + + it('quotes arguments that the shell would reinterpret', () => { + expect(sshTunnelCommand(parseSshDockerHost("ssh://o'brien%20x@host.example.com"), 8080)).toBe( + `ssh -N -L 8080:127.0.0.1:8080 'o'\\''brien x@host.example.com'` + ); + }); +}); + +describe('parseSshDockerHost', () => { + it('keeps the configured URL for the Docker context', () => { + expect(parseSshDockerHost(' ssh://user@host.example.com:2222 ').url).toBe( + 'ssh://user@host.example.com:2222' + ); + }); + + it.each(['host.example.com', 'tcp://host.example.com:2375', 'ssh://'])('rejects %s', (value) => { + expect(() => parseSshDockerHost(value)).toThrow(/must be an ssh:\/\/ URL/); + }); +}); diff --git a/package.json b/package.json index 6f029536..b4c5a281 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "drizzle-kit": "1.0.0-rc.5-ab785fc", "husky": "^9.1.7" }, - "packageManager": "bun@1.2.16", + "packageManager": "bun@1.3.11", "private": true, "scripts": { "prepare": "husky", @@ -18,6 +18,9 @@ "deploy:cloudflare": "ATLANTIS_DB_DRIVER=d1 alchemy deploy infra/cloudflare.ts", "plan:cloudflare": "ATLANTIS_DB_DRIVER=d1 alchemy plan infra/cloudflare.ts", "destroy:cloudflare": "alchemy destroy infra/cloudflare.ts", + "deploy:self-hosted": "alchemy deploy infra/self-hosted.ts", + "plan:self-hosted": "alchemy plan infra/self-hosted.ts", + "destroy:self-hosted": "alchemy destroy infra/self-hosted.ts", "check:web": "bun run --cwd apps/web check", "lint": "bun run --cwd apps/web lint && bun run --cwd infra lint && cargo clippy --workspace --all-targets --all-features --locked -- -D warnings", "typecheck": "bun run --cwd apps/web typecheck && bun run --cwd infra typecheck && cargo check --workspace --all-targets --locked", @@ -41,6 +44,7 @@ "infra" ], "patchedDependencies": { - "@alchemy.run/frontend-frameworks@2.0.0-beta.79": "patches/@alchemy.run%2Ffrontend-frameworks@2.0.0-beta.79.patch" + "@alchemy.run/frontend-frameworks@2.0.0-beta.79": "patches/@alchemy.run%2Ffrontend-frameworks@2.0.0-beta.79.patch", + "alchemy@2.0.0-beta.79": "patches/alchemy@2.0.0-beta.79.patch" } } diff --git a/patches/alchemy@2.0.0-beta.79.patch b/patches/alchemy@2.0.0-beta.79.patch new file mode 100644 index 00000000..f95d3b3e --- /dev/null +++ b/patches/alchemy@2.0.0-beta.79.patch @@ -0,0 +1,68 @@ +diff --git a/lib/Docker/Container.d.ts b/lib/Docker/Container.d.ts +index 1132b60513040b72a1c51cfe2342c9c31a3c4031..4d34e38f9226ed14227914ebd3b8c3260a9700f8 100644 +--- a/lib/Docker/Container.d.ts ++++ b/lib/Docker/Container.d.ts +@@ -57,6 +57,7 @@ export interface ContainerProps { + start?: boolean; + /** Docker healthcheck configuration. */ + healthcheck?: Container.Healthcheck; ++ memory?: string; + } + export declare namespace Container { + type Status = "created" | "running" | "paused" | "restarting" | "removing" | "exited" | "dead"; +diff --git a/lib/Docker/Container.js b/lib/Docker/Container.js +index eda60bf2183cfda4a47c410f00a92c32ed5afbd2..c8d09e19d588b64e5e35a1d1959538231588bd73 100644 +--- a/lib/Docker/Container.js ++++ b/lib/Docker/Container.js +@@ -289,6 +289,7 @@ const makeCreateArgs = (id, news, instanceId) => dockerPhysicalName(id, news, in + label: news.labels, + "stop-timeout": toSeconds(news.stopTimeout)?.toString(), + rm: news.removeOnExit ?? false, ++ memory: news.memory, + ...(news.healthcheck + ? { + "health-cmd": Array.isArray(news.healthcheck.cmd) +diff --git a/lib/Docker/Docker.d.ts b/lib/Docker/Docker.d.ts +index 932a35f17b74317e3d9b431aaca5b72923c02120..895bcd7c6298dd9dd151707f387d5f4300c552ef 100644 +--- a/lib/Docker/Docker.d.ts ++++ b/lib/Docker/Docker.d.ts +@@ -48,6 +48,7 @@ declare const Docker_base: Context.ServiceClass | undefined; + /** `--add-host` entries, each `hostname:address`. */ + "add-host"?: Array | undefined; ++ memory?: string | undefined; + command: Array | undefined; + label?: Record; + context?: string; +diff --git a/src/Docker/Container.ts b/src/Docker/Container.ts +index 366e65915c9405bd12f629a1dc3cce64b5f64c38..30bb7bebe96b24a5e11e773b6005ead6c0a76a03 100644 +--- a/src/Docker/Container.ts ++++ b/src/Docker/Container.ts +@@ -63,6 +63,7 @@ export interface ContainerProps { + start?: boolean; + /** Docker healthcheck configuration. */ + healthcheck?: Container.Healthcheck; ++ memory?: string; + } + + export declare namespace Container { +@@ -502,6 +503,7 @@ const makeCreateArgs = (id: string, news: ContainerProps, instanceId: string) => + label: news.labels, + "stop-timeout": toSeconds(news.stopTimeout)?.toString(), + rm: news.removeOnExit ?? false, ++ memory: news.memory, + ...(news.healthcheck + ? { + "health-cmd": Array.isArray(news.healthcheck.cmd) +diff --git a/src/Docker/Docker.ts b/src/Docker/Docker.ts +index ef0f9ffa1641f8a4c2003d19030dd9ec6cacc852..45930c0a576ad5854cf2e7e83420e8d27df038a6 100644 +--- a/src/Docker/Docker.ts ++++ b/src/Docker/Docker.ts +@@ -89,6 +89,7 @@ export class Docker extends Context.Service< + p: Array | undefined; + /** `--add-host` entries, each `hostname:address`. */ + "add-host"?: Array | undefined; ++ memory?: string | undefined; + command: Array | undefined; + label?: Record; + context?: string; diff --git a/shell.nix b/shell.nix index 1bc919ef..5250b79d 100644 --- a/shell.nix +++ b/shell.nix @@ -4,12 +4,33 @@ with (import (builtins.fetchTarball { url = "https://github.com/nixos/nixpkgs/archive/ce01daebf8489ba97bd1609d185ea276efdeb121.tar.gz"; sha256 = "10cqhkqkifcgyibj9nwxrnq424crfl40kwr3daky83m2fisb4f6p"; }) {}); +let + system = stdenv.hostPlatform.system; + + bunVersion = "1.3.11"; + bunPlatform = + { + x86_64-linux = { name = "linux-x64"; hash = "sha256-hhG6k1r4hvBabzh0ChUWAybBXl1dB63vlmEwtEk2B+0="; }; + aarch64-linux = { name = "linux-aarch64"; hash = "sha256-0TlE2hKlPsx0v2pyC9HQTEVVwDjf5CI2U1anvkdpH98="; }; + x86_64-darwin = { name = "darwin-x64"; hash = "sha256-xP4rkkchiwKV8k6JWq7I/uYudEUmeakCa2fqy9YRooY="; }; + aarch64-darwin = { name = "darwin-aarch64"; hash = "sha256-b1o0Z+2crsR5W/eM1HZQfZ+HDH1XuGyUX8szgSZ3L/w="; }; + } + .${system} or (throw "Unsupported system: ${system}"); + bun = pkgs.bun.overrideAttrs { + version = bunVersion; + src = fetchurl { + url = "https://github.com/oven-sh/bun/releases/download/bun-v${bunVersion}/bun-${bunPlatform.name}.zip"; + inherit (bunPlatform) hash; + }; + sourceRoot = "bun-${bunPlatform.name}"; + }; +in mkShell { buildInputs = [ pkgs.autoconf pkgs.automake pkgs.bison - pkgs.bun + bun pkgs.flex pkgs.gcc pkgs.git diff --git a/tools/netflow-db/src/registry.rs b/tools/netflow-db/src/registry.rs index eb975451..dc59a0cb 100644 --- a/tools/netflow-db/src/registry.rs +++ b/tools/netflow-db/src/registry.rs @@ -448,7 +448,7 @@ mod tests { fs::write( &list, serde_json::json!([{ - "dataset_id": "campus", + "dataset_id": "dataset-a", "root_path": "/captures", "locality": [ {"type": "tos_anonymized"}, @@ -462,7 +462,7 @@ mod tests { .unwrap(); let registry = DatasetRegistry::load(&list, root.path()).unwrap(); - let dataset = registry.get("campus").unwrap(); + let dataset = registry.get("dataset-a").unwrap(); let rules = dataset.locality_rules().unwrap(); assert_eq!(