From 0073b16706a5e772f20674ada63fa824f31ece26 Mon Sep 17 00:00:00 2001 From: Xore Date: Tue, 25 Aug 2026 14:46:08 +0200 Subject: [PATCH 1/3] feat(design-lab): serve variants against real data, without being able to write The lab's whole premise is that a theme is reviewed against real captured data rather than fixtures -- both previous design refreshes were run that way and that is what shipped. The harness that made it possible went with the Go dashboard in cb77cdf8, so the pattern branding/design-lab/ documents has not been executable since. The Go version got its safety from constructing the server with nil write-services. frontend-next has no such handle: it reaches data over HTTP through two bases, and one of them is write-capable by definition. So the guarantee is made at that seam instead -- BACKEND_URL a gate forwarding GET/HEAD, refusing the rest with 405 BACKEND_MOUNTED_URL nothing at all; every request 503s -- which is stronger than what it replaces, because it is enforced per request rather than by remembering to pass nil. That is not a formality. The Rust tier really exposes stores.rs's generic_delete, preferences.rs's put, the honeyfs implant writer and the canarytoken minter, so a reviewer clicking through a variant would otherwise delete real documents and mint real tokens against live infrastructure. lab.test.mjs drives the actual harness against a recording stand-in backend and asserts a write never arrives; it runs in CI, because the guarantee is the point of the tool and a silent regression would be indistinguishable from it working. Variant stylesheets are symlinked into the dev server's public/ tree, so a variant layers on the vendored theme.css with no rebuild and no change to the shipped vite config -- and saving the source file is enough to see it. Ports are the ones the lab has always used, so the review log's links keep resolving. Verified end to end against the homeserver's real backend: the page renders cowrie, dionaea, wordpot, galah and suricata with live states, carries both stylesheets in the right order, and the gate refuses a delete without it reaching Elasticsearch. Booting it also surfaced that routeShape.test.ts was being scanned as a route file and warned on every dev-server start; renamed to the generator's ignore prefix, which vitest's own include pattern is unaffected by. Closes #1828 --- .github/workflows/quality.yml | 14 ++ .../frontend-next/.gitignore | 4 + ...routeShape.test.ts => -routeShape.test.ts} | 4 + .../frontend-next/src/routes/__root.tsx | 17 +- branding/design-lab/README.md | 43 +++- branding/design-lab/lab.mjs | 216 ++++++++++++++++++ branding/design-lab/lab.test.mjs | 79 +++++++ 7 files changed, 369 insertions(+), 8 deletions(-) rename arcane/home/honeypot-dashboard/frontend-next/src/routes/{routeShape.test.ts => -routeShape.test.ts} (91%) create mode 100644 branding/design-lab/lab.mjs create mode 100644 branding/design-lab/lab.test.mjs diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 0f92350b8..c108255c4 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -49,6 +49,20 @@ jobs: - run: python scripts/check-public-leaks.py - run: python scripts/validate-oidc-redirects.py + design-lab-readonly: + # #1828: the design lab serves variants against the real captured-data + # Elasticsearch, so its read-only guarantee is a safety property, not a + # convenience. The test drives the actual harness against a recording + # stand-in backend and fails if a write ever reaches it. + name: Design lab is read-only + runs-on: ${{ (github.event_name == 'workflow_dispatch' && inputs.use_self_hosted_runner) && 'self-hosted' || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: "22" + - run: node --test branding/design-lab/lab.test.mjs + go-fmt: name: Go formatting runs-on: ${{ (github.event_name == 'workflow_dispatch' && inputs.use_self_hosted_runner) && 'self-hosted' || 'ubuntu-latest' }} diff --git a/arcane/home/honeypot-dashboard/frontend-next/.gitignore b/arcane/home/honeypot-dashboard/frontend-next/.gitignore index 713db202c..549515269 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/.gitignore +++ b/arcane/home/honeypot-dashboard/frontend-next/.gitignore @@ -4,3 +4,7 @@ node_modules/ .tanstack/ dist/ .env + +# #1828: the design lab symlinks variant stylesheets in here at start-up +# and clears it on exit. Never content, always disposable. +public/static/lab/ diff --git a/arcane/home/honeypot-dashboard/frontend-next/src/routes/routeShape.test.ts b/arcane/home/honeypot-dashboard/frontend-next/src/routes/-routeShape.test.ts similarity index 91% rename from arcane/home/honeypot-dashboard/frontend-next/src/routes/routeShape.test.ts rename to arcane/home/honeypot-dashboard/frontend-next/src/routes/-routeShape.test.ts index ebb038118..2fc4f988a 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/src/routes/routeShape.test.ts +++ b/arcane/home/honeypot-dashboard/frontend-next/src/routes/-routeShape.test.ts @@ -1,3 +1,7 @@ +// Named with a leading "-" so TanStack's route generator skips it: without +// the prefix it scans this file for a Route export, finds none, and warns on +// every dev-server start (routeFileIgnorePrefix, see the generator's own +// message). vitest still collects it -- its include is src/**/*.test.ts. // A route file that has children is a layout, and a layout that redirects // is an infinite loop. // diff --git a/arcane/home/honeypot-dashboard/frontend-next/src/routes/__root.tsx b/arcane/home/honeypot-dashboard/frontend-next/src/routes/__root.tsx index 9f212f123..5a5b50fe2 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/src/routes/__root.tsx +++ b/arcane/home/honeypot-dashboard/frontend-next/src/routes/__root.tsx @@ -60,6 +60,11 @@ const getAppearance = createServerFn({ method: 'GET' }).handler(async (): Promis } }) +// #1828: an optional stylesheet layered after theme.css, for the design +// lab's variants. Read once at module scope -- it is a property of the +// process, not of a request, and production never sets it. +const variantCSS = process.env.VARIANT_CSS ?? '' + export const Route = createRootRoute({ // BFF-owned auth: every navigation resolves the redis session on the // server; unauthenticated requests bounce to the Keycloak flow. The @@ -95,7 +100,17 @@ export const Route = createRootRoute({ { name: 'robots', content: 'noindex, nofollow' }, { title: 'APIARY' }, ], - links: [{ rel: 'stylesheet', href: '/static/theme.css' }], + links: [ + { rel: 'stylesheet', href: '/static/theme.css' }, + // #1828: a design-lab variant layers its token overrides on the + // vendored stylesheet instead of replacing it, so a variant is a diff + // against what ships rather than a fork of it — the authoring pattern + // branding/design-lab/v5-picks-override.css already documents. + // + // Absent unless VARIANT_CSS names one, so production renders exactly + // one stylesheet and this costs nothing. + ...(variantCSS ? [{ rel: 'stylesheet' as const, href: variantCSS }] : []), + ], scripts: [ { // Pre-paint theme + palette boot, byte-compatible with the Go diff --git a/branding/design-lab/README.md b/branding/design-lab/README.md index 03d6a9c46..a5a6a81fa 100644 --- a/branding/design-lab/README.md +++ b/branding/design-lab/README.md @@ -81,13 +81,42 @@ session, a `STATIC_DIR` override and nil write-services so real Elasticsearch stayed read-only. It went with the Go dashboard in `cb77cdf8` and is not recoverable from here. -The `frontend-next` equivalent — a dev server with the same read-only -guarantees — still needs building. That is the remaining part of #1763 and is -worth its own issue once #1753's approach is settled; `gen_palettes.py` has -meanwhile been superseded upstream by `scripts/theme-tokens.mjs` and -`check-contrast.mjs` in Xore/theme, which do the same job in CI over 422 -pairs. It is kept here as the record of how the palettes were derived, not as -something to run. +`lab.mjs` is the `frontend-next` equivalent (#1828). Run it from the repo +root: + +``` +node branding/design-lab/lab.mjs # v5-picks-override.css on 19201 +node branding/design-lab/lab.mjs a.css b.css # two variants, side by side +APIARY_BACKEND=http://10.8.0.2:8081 node branding/design-lab/lab.mjs +``` + +Variants take 19201-19205 and the elements playground is on 19300, the ports +this lab has always used. A variant stylesheet is symlinked into the dev +server's `public/static/lab/`, so saving the file and reloading the page is +enough — no rebuild, and no change to the shipped vite config. + +The read-only guarantee is made at the one seam `frontend-next` has, since +there is no service handle to pass nil to: + +| | in the lab | +|---|---| +| `BACKEND_URL` | a gate that forwards `GET`/`HEAD` and refuses everything else with 405 | +| `BACKEND_MOUNTED_URL` | absent — every request 503s | + +That is not decoration. The Rust tier really exposes `generic_delete`, +`preferences::put`, the honeyfs implant writer and the canarytoken minter, so +a reviewer clicking around a variant would otherwise delete real documents and +mint real tokens against live infrastructure. `lab.test.mjs` drives the actual +harness against a recording stand-in backend and fails if a write reaches it; +it runs in CI as `Design lab is read-only`. + +`OIDC_DISABLED=1` supplies the stubbed session, so there is no login +round-trip per variant per page. + +`gen_palettes.py` has meanwhile been superseded upstream by +`scripts/theme-tokens.mjs` and `check-contrast.mjs` in Xore/theme, which do the +same job in CI over 422 pairs. It is kept here as the record of how the +palettes were derived, not as something to run. ## Related diff --git a/branding/design-lab/lab.mjs b/branding/design-lab/lab.mjs new file mode 100644 index 000000000..f98cbd906 --- /dev/null +++ b/branding/design-lab/lab.mjs @@ -0,0 +1,216 @@ +#!/usr/bin/env node +// The design lab's variant harness (#1828). +// +// Reviewing a theme against fixtures tells you the theme looks fine against +// fixtures. Both previous design refreshes were run against real captured +// data and that is what shipped, so #1753 wants the nine themes drafted the +// same way -- which needs a dashboard pointed at the real Elasticsearch that +// structurally cannot write to it. +// +// The Go harness this replaces (dev_serve_test.go, lost with cb77cdf8) got +// that by constructing the server with nil write-services. frontend-next has +// no such handle: it reaches data over HTTP through two bases, and the +// mounted one is write-capable by definition (see backend.server.ts). So the +// same guarantee is made at that seam instead -- +// +// BACKEND_URL -> a gate that forwards GET/HEAD and refuses the +// rest with 405, so a delete is a visible failure +// BACKEND_MOUNTED_URL -> nothing at all; every request 503s +// +// -- which is stronger than the original rather than weaker: it is enforced +// per request at runtime, not by remembering to pass nil. +// +// This matters concretely. The Rust tier really does expose stores.rs's +// generic_delete, preferences.rs's put, the honeyfs implant writer and the +// canarytoken minter. A reviewer clicking around a variant would otherwise +// delete real documents and mint real tokens against live infrastructure. +// +// Usage: +// node branding/design-lab/lab.mjs # v5-picks-override.css +// node branding/design-lab/lab.mjs a.css b.css # two variants, side by side +// APIARY_BACKEND=http://10.8.0.2:8081 node .../lab.mjs # against the homeserver +// +// Variants land on 19201+, the elements playground on 19300 -- the ports the +// lab has always used, so the review log's links keep resolving. + +import { spawn } from 'node:child_process' +import { createServer } from 'node:http' +import { createReadStream, existsSync, mkdirSync, rmSync, symlinkSync } from 'node:fs' +import { dirname, extname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const LAB = dirname(fileURLToPath(import.meta.url)) +const FRONTEND = resolve(LAB, '../../arcane/home/honeypot-dashboard/frontend-next') +const PLAYGROUND = join(LAB, 'playground') + +// Where variant stylesheets are exposed to the browser. vite serves public/ +// at the root, so a symlink here is reachable at /static/lab/.css with +// no rebuild and no vite config change -- and because it is a symlink, saving +// the source file and reloading is enough to see the change. The directory is +// disposable and gitignored; the lab clears it on every start. +const LINK_DIR = join(FRONTEND, 'public/static/lab') + +const GATE_PORT = 19199 // read-only door to the real backend +const ABSENT_PORT = 19198 // the write-capable backend, deliberately not here +const FIRST_VARIANT_PORT = 19201 +const MAX_VARIANTS = 5 // 19201-19205, the documented range +const PLAYGROUND_PORT = 19300 + +const REAL_BACKEND = (process.env.APIARY_BACKEND ?? 'http://127.0.0.1:8081').replace(/\/$/, '') +const READ_METHODS = new Set(['GET', 'HEAD', 'OPTIONS']) + +const MIME = { + '.html': 'text/html; charset=utf-8', + '.css': 'text/css; charset=utf-8', + '.js': 'text/javascript; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.svg': 'image/svg+xml', + '.png': 'image/png', + '.woff2': 'font/woff2', +} + +const children = [] + +function log(scope, message) { + process.stdout.write(`[lab:${scope}] ${message}\n`) +} + +/** + * The read-only door. Anything that is not a read is refused here rather + * than reaching Elasticsearch, and the refusal is printed -- a variant that + * quietly stopped working because the lab blocked a write is worse than one + * that says so. + */ +function startGate() { + const server = createServer(async (req, res) => { + if (!READ_METHODS.has(req.method ?? '')) { + log('gate', `REFUSED ${req.method} ${req.url} -- the lab is read-only`) + res.writeHead(405, { 'content-type': 'application/json', allow: 'GET, HEAD' }) + res.end( + JSON.stringify({ + error: 'design lab is read-only', + detail: `${req.method} ${req.url} was not forwarded. Real captured data is not a scratch pad.`, + }), + ) + return + } + try { + const upstream = await fetch(`${REAL_BACKEND}${req.url}`, { + method: req.method, + headers: { ...req.headers, host: new URL(REAL_BACKEND).host }, + }) + res.writeHead(upstream.status, Object.fromEntries(upstream.headers)) + if (upstream.body) { + const reader = upstream.body.getReader() + for (;;) { + const { done, value } = await reader.read() + if (done) break + res.write(value) + } + } + res.end() + } catch (error) { + log('gate', `upstream ${REAL_BACKEND} unreachable: ${error.message}`) + res.writeHead(502, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ error: 'backend unreachable', detail: error.message })) + } + }) + server.listen(GATE_PORT, '127.0.0.1', () => log('gate', `read-only -> ${REAL_BACKEND} on :${GATE_PORT}`)) + return server +} + +/** + * The write-capable tier, absent. Returning 503 rather than leaving the port + * closed is deliberate: a connection refused reads as "the lab is broken", + * a 503 with this body reads as "this is not something the lab has". + */ +function startAbsentBackend() { + const server = createServer((req, res) => { + log('absent', `${req.method} ${req.url} -- no write-capable backend in the lab`) + res.writeHead(503, { 'content-type': 'application/json' }) + res.end( + JSON.stringify({ + error: 'write-capable backend absent', + detail: 'The design lab runs without backend-service-mounted by design. Sandbox, Ghidra and analysis submits do not exist here.', + }), + ) + }) + server.listen(ABSENT_PORT, '127.0.0.1', () => log('absent', `write tier absent on :${ABSENT_PORT}`)) + return server +} + +function startStatic(root, port, scope) { + const server = createServer((req, res) => { + const requested = decodeURIComponent((req.url ?? '/').split('?')[0]) + const path = join(root, requested === '/' ? 'index.html' : requested) + if (!path.startsWith(root) || !existsSync(path)) { + res.writeHead(404, { 'content-type': 'text/plain' }) + res.end('not found') + return + } + res.writeHead(200, { 'content-type': MIME[extname(path)] ?? 'application/octet-stream' }) + createReadStream(path).pipe(res) + }) + server.listen(port, '127.0.0.1', () => log(scope, `http://127.0.0.1:${port}/`)) + return server +} + +function startVariant(cssFile, port) { + const source = resolve(LAB, cssFile) + if (!existsSync(source)) { + log('variant', `no such stylesheet: ${source}`) + return null + } + const linked = join(LINK_DIR, cssFile.replace(/\//g, '_')) + symlinkSync(source, linked) + const href = `/static/lab/${cssFile.replace(/\//g, '_')}` + + const child = spawn('npx', ['vite', 'dev', '--port', String(port), '--strictPort'], { + cwd: FRONTEND, + stdio: 'inherit', + env: { + ...process.env, + VARIANT_CSS: href, + // No login round-trip per variant per page -- the reviewer is looking + // at type and colour, not at Keycloak. + OIDC_DISABLED: '1', + SERVE_MODE: 'all', + BACKEND_URL: `http://127.0.0.1:${GATE_PORT}`, + BACKEND_MOUNTED_URL: `http://127.0.0.1:${ABSENT_PORT}`, + }, + }) + children.push(child) + log('variant', `${cssFile} -> http://127.0.0.1:${port}/ (css at ${href})`) + return child +} + +function main() { + const requested = process.argv.slice(2) + const variants = requested.length ? requested : ['v5-picks-override.css'] + if (variants.length > MAX_VARIANTS) { + log('lab', `at most ${MAX_VARIANTS} variants (ports ${FIRST_VARIANT_PORT}-${FIRST_VARIANT_PORT + MAX_VARIANTS - 1})`) + process.exit(2) + } + + rmSync(LINK_DIR, { recursive: true, force: true }) + mkdirSync(LINK_DIR, { recursive: true }) + + const servers = [startGate(), startAbsentBackend(), startStatic(PLAYGROUND, PLAYGROUND_PORT, 'playground')] + variants.forEach((css, index) => startVariant(css, FIRST_VARIANT_PORT + index)) + + const shutdown = () => { + for (const child of children) child.kill('SIGTERM') + for (const server of servers) server.close() + rmSync(LINK_DIR, { recursive: true, force: true }) + process.exit(0) + } + process.on('SIGINT', shutdown) + process.on('SIGTERM', shutdown) +} + +// Exported for the harness's own test; main() only runs when invoked directly. +export { READ_METHODS, GATE_PORT, ABSENT_PORT, FIRST_VARIANT_PORT, MAX_VARIANTS, PLAYGROUND_PORT } + +if (process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url))) { + main() +} diff --git a/branding/design-lab/lab.test.mjs b/branding/design-lab/lab.test.mjs new file mode 100644 index 000000000..e97a67a83 --- /dev/null +++ b/branding/design-lab/lab.test.mjs @@ -0,0 +1,79 @@ +// The design lab's read-only guarantee, asserted against the real harness +// process rather than by reading its source (#1828). +// +// This is the requirement the whole tool exists to satisfy: the lab points a +// dashboard at real captured Elasticsearch data, so "we won't call the write +// paths" is not good enough. A stand-in backend records everything that +// reaches it, and the test fails if a write ever does. +// +// node --test branding/design-lab/lab.test.mjs +import { spawn } from 'node:child_process' +import { createServer } from 'node:http' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import test from 'node:test' +import assert from 'node:assert/strict' + +const LAB = join(dirname(fileURLToPath(import.meta.url)), 'lab.mjs') + +// Stand in for the real Rust backend so the test needs no infrastructure -- +// and so a write reaching it is observable rather than merely unlikely. +const reachedUpstream = [] +const upstream = createServer((req, res) => { + reachedUpstream.push(`${req.method} ${req.url}`) + res.writeHead(200, { 'content-type': 'application/json' }) + res.end('{"ok":true}') +}) +await new Promise((r) => upstream.listen(19099, '127.0.0.1', r)) + +// The real harness, not a re-creation of it -- named with a stylesheet that +// does not exist so it starts its servers without spawning vite. +const child = spawn('node', [LAB, 'definitely-not-a-real-variant.css'], { + env: { ...process.env, APIARY_BACKEND: 'http://127.0.0.1:19099' }, + stdio: ['ignore', 'pipe', 'pipe'], +}) +let out = '' +child.stdout.on('data', (d) => (out += d)) +child.stderr.on('data', (d) => (out += d)) + +await new Promise((r) => setTimeout(r, 1500)) + +// 1. A read is forwarded. +const read = await fetch('http://127.0.0.1:19199/api/v1/source-health') +test('a read is forwarded to the backend', () => { + assert.equal(read.status, 200) + assert.deepEqual(reachedUpstream, ['GET /api/v1/source-health']) +}) + +// 2. A delete is refused *and never reaches upstream* -- the whole point. +const before = reachedUpstream.length +const del = await fetch('http://127.0.0.1:19199/api/v1/stores/events/abc123', { method: 'DELETE' }) +test('a delete is refused', () => assert.equal(del.status, 405)) +test('the delete never reached the backend', () => assert.equal(reachedUpstream.length, before)) + +// 3. A preferences PUT, likewise. +const put = await fetch('http://127.0.0.1:19199/api/v1/preferences', { method: 'PUT', body: '{}' }) +test('a preferences write is refused', () => assert.equal(put.status, 405)) +test('the preferences write never reached the backend', () => assert.equal(reachedUpstream.length, before)) + +// 4. The write-capable tier is absent, not merely unused. +const mounted = await fetch('http://127.0.0.1:19198/api/v1/sandbox/submit', { method: 'POST' }) +const mountedBody = await mounted.json() +test('the write-capable tier is absent, not merely unused', () => { + assert.equal(mounted.status, 503) + assert.equal(mountedBody.error, 'write-capable backend absent') +}) + +// 5. Even a GET to the mounted tier finds nothing -- absence is total. +const mountedGet = await fetch('http://127.0.0.1:19198/api/v1/sandbox/status') +test('the write-capable tier is absent for reads too', () => assert.equal(mountedGet.status, 503)) + +// 6. The playground is served on its documented port. +const playground = await fetch('http://127.0.0.1:19300/elements.html') +test('the elements playground is on its documented port', () => assert.equal(playground.status, 200)) + +test.after(() => { + child.kill('SIGTERM') + upstream.close() + if (process.env.LAB_TEST_VERBOSE) console.log(out.trim()) +}) From c4c36914874ffbeae9490d5f4004d24d43b90b40 Mon Sep 17 00:00:00 2001 From: Xore Date: Tue, 25 Aug 2026 14:59:07 +0200 Subject: [PATCH 2/3] fix(stores): three lists were sorted by a field their documents do not have Chasing #1566's "near-duplicate rows" on /ml-anomalies turned up why they looked the way they did: the list was not in any order at all. Store pages sort with `unmapped_type` set, so a store whose index does not exist yet returns an empty list instead of an error. The same setting means a field the documents do not have sorts every hit as null rather than failing -- and nothing anywhere says so. Three stores shipped that way: ml-anomalies asked for `timestamp` where the worker writes `@timestamp`, auth-events asked for `last_seen` where Keycloak writes `@timestamp`, and static-analysis asked for `Analysis.GeneratedUTC`, which its documents do not carry in any spelling -- they hold only Analysis and Fingerprint, and no time field whatsoever. Measured on the live index, the first page of /ml-anomalies read 20:58, 20:58, 20:59, 20:59, 20:59, 20:57, 20:57, 20:58. The newest document was third by luck. Worse than the disorder: an all-null sort is one enormous tie, and `from`/`size` over a tie leaves the order inside it undefined. Paging could hand back the same document twice and never show another. So every store sort now carries `_doc` as a tiebreak -- which the numeric sorts needed anyway, since campaigns-by-score and attackers-by-events tie constantly -- and each declares its field's real type, because an `unmapped_type` that disagrees with the mapping is the same silent no-op for keyword sorts. CI cannot see Elasticsearch, so scripts/check-store-sorts.py is the guard: it checks all 23 declared sort fields against a live mapping dump. Run against the homeserver it passes, and re-introducing either bug class fails it with the reason. With the order fixed, the duplicates #1566 actually reported are adjacent for the first time, so they can be folded. 66% of the 7,305 live anomalies sit in an (address, same second) bucket with siblings; the worst single burst is 48 rows for one IP inside one second, every one scoring exactly 0.8000. Consecutive rows from one address in one second scoring within 0.01 now collapse to one row badged with what it stands for. Acknowledging a folded row acknowledges all of it -- otherwise it would return as open on the next refresh and the button would look broken while behaving exactly as written -- and the status badge reports "3 of 48 open" rather than speaking only for whichever row represents the run. Folding is deliberately page-local and says so in the badge: doing it in the Rust tier would change the shape of the shared /api/v1/store endpoint for every other consumer, and a burst longer than one page still splits at the boundary. Also verified while here, and already correct: page-title casing is uniformly sentence case across all 27 headers, and /campaigns renders 7 columns rather than 15 -- the rest are behind `detail: true`. Closes #1566 --- .../backend-service/src/stores.rs | 127 ++++++++++++++---- .../frontend-next/src/lib/mlGrouping.test.ts | 95 +++++++++++++ .../frontend-next/src/lib/mlGrouping.ts | 96 +++++++++++++ .../frontend-next/src/routes/ml-anomalies.tsx | 69 ++++++++-- scripts/check-store-sorts.py | 120 +++++++++++++++++ 5 files changed, 467 insertions(+), 40 deletions(-) create mode 100644 arcane/home/honeypot-dashboard/frontend-next/src/lib/mlGrouping.test.ts create mode 100644 arcane/home/honeypot-dashboard/frontend-next/src/lib/mlGrouping.ts create mode 100644 scripts/check-store-sorts.py diff --git a/arcane/home/honeypot-dashboard/backend-service/src/stores.rs b/arcane/home/honeypot-dashboard/backend-service/src/stores.rs index 581e73c54..9478dc5cc 100644 --- a/arcane/home/honeypot-dashboard/backend-service/src/stores.rs +++ b/arcane/home/honeypot-dashboard/backend-service/src/stores.rs @@ -51,22 +51,50 @@ fn bad_gateway(error: anyhow::Error) -> (StatusCode, String) { (StatusCode::BAD_GATEWAY, error.to_string()) } +/// The sort every store list is paged by: the store's own field, then a +/// deterministic tiebreak. +/// +/// Both keys are load-bearing. +/// +/// `unmapped_type` keeps a store whose index does not exist yet from +/// erroring -- but it also means a *misspelled* field sorts every document +/// as null instead of failing. That is how three lists shipped in no order +/// at all: ml-anomalies asked for "timestamp" where the worker writes +/// "@timestamp", auth-events asked for "last_seen" where Keycloak writes +/// "@timestamp", and static-analysis asked for "Analysis.GeneratedUTC", +/// which its documents do not carry in any spelling. It has to match the +/// field's real type, or a keyword sort reintroduces the same silent no-op. +/// +/// The tiebreak is what makes paging correct rather than merely tidy. +/// `from`/`size` over a sort with ties leaves the order within a tie +/// undefined, so the same document can come back on two pages while another +/// is never shown at all -- and an all-null sort is one giant tie. `_doc` is +/// the cheapest total order Elasticsearch offers. +fn sort_spec(sort_field: &str, unmapped_type: &str) -> Value { + json!([ + {sort_field: {"order": "desc", "unmapped_type": unmapped_type}}, + {"_doc": {"order": "asc"}} + ]) +} + /// Generic hits page: {"total": N, "rows": [ _source... ]}. The BFF/routes /// know each store's shape; this tier guarantees ordering + paging. async fn store_page( state: &AppState, indices: &[&str], sort_field: &str, + unmapped_type: &str, q: &StoreQuery, extra_filter: Option, ) -> anyhow::Result { - store_page_excluding(state, indices, sort_field, q, extra_filter, &[]).await + store_page_excluding(state, indices, sort_field, unmapped_type, q, extra_filter, &[]).await } async fn store_page_excluding( state: &AppState, indices: &[&str], sort_field: &str, + unmapped_type: &str, q: &StoreQuery, extra_filter: Option, excludes: &[&str], @@ -83,7 +111,7 @@ async fn store_page_excluding( "from": q.offset, "size": size, "track_total_hits": true, - "sort": [{sort_field: {"order": "desc", "unmapped_type": "date"}}], + "sort": sort_spec(sort_field, unmapped_type), "query": query }); if !excludes.is_empty() { @@ -112,7 +140,7 @@ pub async fn campaigns( State(state): State, Query(q): Query, ) -> Result, (StatusCode, String)> { - store_page(&state, &["campaigns-v1"], "score", &q, None) + store_page(&state, &["campaigns-v1"], "score", "long", &q, None) .await .map(Json) .map_err(bad_gateway) @@ -122,7 +150,7 @@ pub async fn clusters( State(state): State, Query(q): Query, ) -> Result, (StatusCode, String)> { - store_page(&state, &["attacker-clusters-v1"], "events", &q, None) + store_page(&state, &["attacker-clusters-v1"], "events", "long", &q, None) .await .map(Json) .map_err(bad_gateway) @@ -132,7 +160,7 @@ pub async fn attackers( State(state): State, Query(q): Query, ) -> Result, (StatusCode, String)> { - store_page(&state, &["attackers-v1"], "events", &q, None) + store_page(&state, &["attackers-v1"], "events", "long", &q, None) .await .map(Json) .map_err(bad_gateway) @@ -224,7 +252,7 @@ pub async fn alerts( State(state): State, Query(q): Query, ) -> Result, (StatusCode, String)> { - store_page(&state, &["dashboard-alert-state-v1"], "LastSeen", &q, None) + store_page(&state, &["dashboard-alert-state-v1"], "LastSeen", "date", &q, None) .await .map(Json) .map_err(bad_gateway) @@ -234,7 +262,7 @@ pub async fn payloads( State(state): State, Query(q): Query, ) -> Result, (StatusCode, String)> { - store_page(&state, &["dashboard-payload-inventory-v1"], "MtimeUTC", &q, None) + store_page(&state, &["dashboard-payload-inventory-v1"], "MtimeUTC", "date", &q, None) .await .map(Json) .map_err(bad_gateway) @@ -270,40 +298,53 @@ pub async fn generic( Query(q): Query, ) -> Result, (StatusCode, String)> { // (index, sort field, heavy fields excluded from list responses). - let (index, sort, excludes): (&str, &str, &[&str]) = match name.as_str() { + // (index, sort field, that field's type, heavy fields excluded from + // list responses). The type is not decoration -- see the sort spec in + // store_page_excluding for what a wrong one costs. + let (index, sort, sort_type, excludes): (&str, &str, &str, &[&str]) = match name.as_str() { // #1611 workstream E.9: `error`, `details.username`, and // `details.redirect_uri` are already present here (no excludes) — // the workstream's ask is first-class *columns* for them on the // auth-events.tsx table, a frontend-only change; this passthrough // already carries every field they'd need. - "auth-events" => ("auth-failure-events", "last_seen", &[]), + // "last_seen" is not a field these documents have -- Keycloak's + // event stream writes @timestamp -- so this list came back in no + // order at all until #1566. + "auth-events" => ("auth-failure-events", "@timestamp", "date", &[]), // llm-worker output; index may not exist yet (ignore_unavailable). - "llm-analysis" => ("llm-analysis", "@timestamp", &[]), - "ml-anomalies" => ("ml-anomalies", "timestamp", &[]), + "llm-analysis" => ("llm-analysis", "@timestamp", "date", &[]), + // Likewise "timestamp": the ml-worker writes @timestamp. Measured + // on the live index, the first page mixed 20:58, 20:59 and 20:57 + // rows in that order (#1566). + "ml-anomalies" => ("ml-anomalies", "@timestamp", "date", &[]), // Matches dashboard/agent_campaigns.go's own refreshAgentCampaigns // sort (`sort=@timestamp:asc`) — the campaign-verdict documents // this index holds have no last_seen field at all (see the // agent-intrusion-worker port's build_campaign_verdict, #1610). - "agent-campaigns" => ("agent-intrusion-campaigns", "@timestamp", &[]), - "canarytokens" => ("dashboard-canarytokens-v1", "created_at", &[]), - "problem-reports" => ("dashboard-problem-reports-v1", "submitted_at", &["dom_snapshot"]), - "dead-letters" => ("dead-letter-honeypot", "@timestamp", &[]), - "yara" => ("yara-analysis-v1", "@timestamp", &[]), - "sandbox-runs" => ("sandbox-analysis-v1", "@timestamp", &[]), - "ghidra-runs" => ("ghidra-analysis-v1", "@timestamp", &[]), - "static-analysis" => ("dashboard-static-analysis-v1", "Analysis.GeneratedUTC", &[]), + "agent-campaigns" => ("agent-intrusion-campaigns", "@timestamp", "date", &[]), + "canarytokens" => ("dashboard-canarytokens-v1", "created_at", "date", &[]), + "problem-reports" => ("dashboard-problem-reports-v1", "submitted_at", "date", &["dom_snapshot"]), + "dead-letters" => ("dead-letter-honeypot", "@timestamp", "date", &[]), + "yara" => ("yara-analysis-v1", "@timestamp", "date", &[]), + "sandbox-runs" => ("sandbox-analysis-v1", "@timestamp", "date", &[]), + "ghidra-runs" => ("ghidra-analysis-v1", "@timestamp", "date", &[]), + // These documents carry only Analysis and Fingerprint -- there is + // no GeneratedUTC, and no time field at all, so there is nothing to + // order by chronologically. Fingerprint is the one mapped field, so + // it is what makes paging deterministic instead of arbitrary. + "static-analysis" => ("dashboard-static-analysis-v1", "Fingerprint", "keyword", &[]), // Result families that may not exist yet on a given deployment // (ignore_unavailable keeps them safe): revdeck, CAPE, GitHub. - "revdeck" => ("revdeck-analysis-v1", "@timestamp", &[]), - "cape" => ("cape-analysis-v1", "@timestamp", &[]), - "github-analysis" => ("github-analysis-v1", "@timestamp", &[]), - "workbench-runs" => ("dashboard-workbench-runs-v1", "created_at", &[]), - "generated-reports" => ("dashboard-generated-reports-v1", "created_at", &["pdf_base64"]), - "report-definitions" => ("dashboard-reports-definitions-v1", "updated", &[]), - "intelligence" => ("dashboard-intelligence-archive-v1", "generated", &[]), + "revdeck" => ("revdeck-analysis-v1", "@timestamp", "date", &[]), + "cape" => ("cape-analysis-v1", "@timestamp", "date", &[]), + "github-analysis" => ("github-analysis-v1", "@timestamp", "date", &[]), + "workbench-runs" => ("dashboard-workbench-runs-v1", "created_at", "date", &[]), + "generated-reports" => ("dashboard-generated-reports-v1", "created_at", "date", &["pdf_base64"]), + "report-definitions" => ("dashboard-reports-definitions-v1", "updated", "date", &[]), + "intelligence" => ("dashboard-intelligence-archive-v1", "generated", "date", &[]), _ => return Err((StatusCode::NOT_FOUND, format!("unknown store {name}"))), }; - store_page_excluding(&state, &[index], sort, &q, None, excludes) + store_page_excluding(&state, &[index], sort, sort_type, &q, None, excludes) .await .map(Json) .map_err(bad_gateway) @@ -342,3 +383,35 @@ pub async fn generic_delete( let deleted = state.es.delete_by_query("dead-letter-honeypot", query).await.map_err(bad_gateway)?; Ok(Json(json!({"deleted": deleted}))) } + + +#[cfg(test)] +mod sort_tests { + use super::sort_spec; + + #[test] + fn the_store_field_leads_and_carries_its_own_type() { + // A date sort and a keyword sort must not both claim "date": an + // unmapped_type that disagrees with the field is the same silent + // no-op as a misspelled field name. + let dated = sort_spec("@timestamp", "date"); + assert_eq!(dated[0]["@timestamp"]["order"], "desc"); + assert_eq!(dated[0]["@timestamp"]["unmapped_type"], "date"); + + let keyed = sort_spec("Fingerprint", "keyword"); + assert_eq!(keyed[0]["Fingerprint"]["unmapped_type"], "keyword"); + } + + #[test] + fn every_sort_has_a_tiebreak_so_paging_cannot_repeat_or_skip() { + // The bug this guards. from/size over a sort with ties leaves the + // order inside a tie undefined, so a document can appear on two + // pages while another never appears -- and the measured live data + // ties hard (48 ml-anomaly rows on one IP in a single second). + for (field, kind) in [("@timestamp", "date"), ("score", "long"), ("Fingerprint", "keyword")] { + let spec = sort_spec(field, kind); + assert_eq!(spec.as_array().map(Vec::len), Some(2), "{field} lost its tiebreak"); + assert_eq!(spec[1]["_doc"]["order"], "asc", "{field}'s tiebreak is not a total order"); + } + } +} diff --git a/arcane/home/honeypot-dashboard/frontend-next/src/lib/mlGrouping.test.ts b/arcane/home/honeypot-dashboard/frontend-next/src/lib/mlGrouping.test.ts new file mode 100644 index 000000000..19ee7dd50 --- /dev/null +++ b/arcane/home/honeypot-dashboard/frontend-next/src/lib/mlGrouping.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from 'vitest' +import { collapseRuns, foldedCount, idsFor } from './mlGrouping' +import type { StoreRow } from '../components/StoreList' + +function row(id: string, ip: string, ts: string, score: number): StoreRow { + return { _doc_id: id, src_ip: ip, '@timestamp': ts, composite_score: score } +} + +describe('collapseRuns', () => { + it('folds a burst from one address in one second into a single row', () => { + // The measured shape of the bug: 48 rows for one IP inside one second, + // every one scoring exactly 0.8000. + const burst = Array.from({ length: 48 }, (_, i) => + row(`d${i}`, '153.75.87.176', `2026-08-24T20:59:23.${String(i).padStart(3, '0')}Z`, 0.8), + ) + const out = collapseRuns(burst) + expect(out).toHaveLength(1) + expect(foldedCount(out[0])).toBe(48) + expect(idsFor(out[0])).toHaveLength(48) + }) + + it('keeps different addresses apart even in the same second', () => { + const out = collapseRuns([ + row('a', '1.1.1.1', '2026-08-24T20:59:23.100Z', 0.8), + row('b', '2.2.2.2', '2026-08-24T20:59:23.200Z', 0.8), + ]) + expect(out).toHaveLength(2) + }) + + it('keeps different seconds apart even from the same address', () => { + const out = collapseRuns([ + row('a', '1.1.1.1', '2026-08-24T20:59:23.900Z', 0.8), + row('b', '1.1.1.1', '2026-08-24T20:59:24.000Z', 0.8), + ]) + expect(out).toHaveLength(2) + }) + + it('keeps materially different scores apart', () => { + const out = collapseRuns([ + row('a', '1.1.1.1', '2026-08-24T20:59:23.100Z', 0.8), + row('b', '1.1.1.1', '2026-08-24T20:59:23.200Z', 0.95), + ]) + expect(out).toHaveLength(2) + }) + + it('does not chain a drift into one arbitrarily wide group', () => { + // Each row is within the epsilon of the one before it, but the run spans + // 0.05 end to end. Comparing against the representative rather than the + // predecessor is what stops that becoming a single "group". + const drift = Array.from({ length: 6 }, (_, i) => + row(`d${i}`, '1.1.1.1', '2026-08-24T20:59:23.100Z', 0.8 + i * 0.009), + ) + const out = collapseRuns(drift) + expect(out.length).toBeGreaterThan(1) + }) + + it('only folds rows that are actually adjacent', () => { + // A lookalike separated by an unrelated row is a different moment, and + // is left alone rather than reached across for. + const out = collapseRuns([ + row('a', '1.1.1.1', '2026-08-24T20:59:23.100Z', 0.8), + row('x', '9.9.9.9', '2026-08-24T20:59:23.150Z', 0.4), + row('b', '1.1.1.1', '2026-08-24T20:59:23.200Z', 0.8), + ]) + expect(out).toHaveLength(3) + }) + + it('never folds a row that has no score', () => { + const out = collapseRuns([ + { _doc_id: 'a', src_ip: '1.1.1.1', '@timestamp': '2026-08-24T20:59:23.100Z' }, + { _doc_id: 'b', src_ip: '1.1.1.1', '@timestamp': '2026-08-24T20:59:23.200Z' }, + ]) + expect(out).toHaveLength(2) + }) + + it('leaves an ungrouped row reporting itself, so callers need no special case', () => { + const out = collapseRuns([row('a', '1.1.1.1', '2026-08-24T20:59:23.100Z', 0.8)]) + expect(foldedCount(out[0])).toBe(1) + expect(idsFor(out[0])).toEqual(['a']) + }) + + it('does not mutate the rows it was given', () => { + const input = [ + row('a', '1.1.1.1', '2026-08-24T20:59:23.100Z', 0.8), + row('b', '1.1.1.1', '2026-08-24T20:59:23.200Z', 0.8), + ] + const snapshot = JSON.parse(JSON.stringify(input)) + collapseRuns(input) + expect(input).toEqual(snapshot) + }) + + it('handles an empty page', () => { + expect(collapseRuns([])).toEqual([]) + }) +}) diff --git a/arcane/home/honeypot-dashboard/frontend-next/src/lib/mlGrouping.ts b/arcane/home/honeypot-dashboard/frontend-next/src/lib/mlGrouping.ts new file mode 100644 index 000000000..4a27dff9e --- /dev/null +++ b/arcane/home/honeypot-dashboard/frontend-next/src/lib/mlGrouping.ts @@ -0,0 +1,96 @@ +// Collapsing near-duplicate ML anomaly rows (#1566). +// +// The ml-worker scores each event independently, so one burst from one +// address produces one row per event. Measured on the live index: 66% of all +// 7,305 anomalies sit in an (address, same second) bucket with siblings, and +// the worst single burst is 48 rows for one IP inside one second, every one +// of them scoring exactly 0.8000. Paging through that is reading the same +// sentence forty-eight times. +// +// This only became possible once the store's sort was fixed in the same +// change: the list was previously ordered by a field the documents do not +// have, so a burst arrived scattered through the page rather than contiguous, +// and there were no runs to collapse. +// +// Scope, deliberately: runs are collapsed *within the fetched page*. The +// alternative is an aggregation in the Rust tier, which would change the +// shape of the shared /api/v1/store endpoint for every other consumer. A +// burst longer than the page still splits across the boundary, so the badge +// says how many rows were folded together here rather than claiming to be +// the size of the whole burst. + +import type { StoreRow } from '../components/StoreList' + +/** How far two composite scores may differ and still count as the same finding. */ +export const SCORE_EPSILON = 0.01 + +/** Rows folded into this one, including itself. Absent means an ungrouped row. */ +export const DUPES = '_dupes' +/** Every `_doc_id` folded into this row, so an acknowledgement covers them all. */ +export const DUPE_IDS = '_dupe_ids' + +function second(row: StoreRow): string { + const raw = typeof row['@timestamp'] === 'string' ? (row['@timestamp'] as string) : '' + // "2026-08-24T20:59:23.138Z" -> "2026-08-24T20:59:23"; a value with no + // sub-second part is already truncated and slicing it is harmless. + return raw.slice(0, 19) +} + +function score(row: StoreRow): number { + const raw = row['composite_score'] + return typeof raw === 'number' ? raw : Number.NaN +} + +function docId(row: StoreRow): string { + return typeof row['_doc_id'] === 'string' ? (row['_doc_id'] as string) : '' +} + +/** + * Fold consecutive near-identical rows into one, carrying the count and the + * ids that went into it. + * + * "Consecutive" is what makes this correct without a global view: the rows + * arrive newest-first by `@timestamp`, so every member of a burst is adjacent + * to the rest of it. A row that merely resembles one three pages away is left + * alone, which is the honest outcome — it is not part of the same moment. + * + * Comparison is against the run's representative rather than the previous + * row, so a slow drift of 0.009 per row cannot chain an arbitrarily wide + * range of scores into a single group. + */ +export function collapseRuns(rows: StoreRow[]): StoreRow[] { + const out: StoreRow[] = [] + for (const row of rows) { + const head = out[out.length - 1] + const sameRun = + head !== undefined && + head['src_ip'] === row['src_ip'] && + second(head) === second(row) && + Math.abs(score(head) - score(row)) <= SCORE_EPSILON && + // NaN scores never compare equal, so a row with no score is never + // folded into another — it is genuinely a different kind of finding. + !Number.isNaN(score(head)) && + !Number.isNaN(score(row)) + + if (sameRun) { + head[DUPES] = ((head[DUPES] as number) ?? 1) + 1 + ;(head[DUPE_IDS] as string[]).push(docId(row)) + continue + } + out.push({ ...row, [DUPES]: 1, [DUPE_IDS]: [docId(row)] }) + } + return out +} + +/** Ids an action on this row should apply to — the group, or just the row. */ +export function idsFor(row: StoreRow): string[] { + const ids = row[DUPE_IDS] + if (Array.isArray(ids) && ids.length) return ids as string[] + return [docId(row)].filter(Boolean) +} + +/** How many rows this one stands for. 1 when it stands only for itself. */ +export function foldedCount(row: StoreRow): number { + const count = row[DUPES] + return typeof count === 'number' && count > 0 ? count : 1 +} diff --git a/arcane/home/honeypot-dashboard/frontend-next/src/routes/ml-anomalies.tsx b/arcane/home/honeypot-dashboard/frontend-next/src/routes/ml-anomalies.tsx index 1caff698b..e6c4d44b4 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/src/routes/ml-anomalies.tsx +++ b/arcane/home/honeypot-dashboard/frontend-next/src/routes/ml-anomalies.tsx @@ -12,6 +12,7 @@ import { confirmAction } from '../components/ConfirmDialog' import { EChart } from '../components/EChart' import { FiltersButton, FiltersModal } from '../components/FiltersModal' import { formatTimestamp } from '../lib/time' +import { collapseRuns, foldedCount, idsFor } from '../lib/mlGrouping' import { countryName } from '../lib/country' type AckRecord = { Acknowledged: boolean; AckedBy?: string; AckedAt?: string } @@ -130,10 +131,17 @@ const ackAll = createServerFn({ method: 'POST' }).handler(async (): Promise; onChanged: () => void }) { +// #1566: `docIds` rather than one id, because a row may stand for a whole +// folded run. Acknowledging only the representative would leave its siblings +// open, and the row would come straight back as unacknowledged on the next +// refresh -- the button would look broken while working exactly as written. +function AckControl({ docIds, acks, onChanged }: { docIds: string[]; acks: Record; onChanged: () => void }) { const [busy, setBusy] = useState(false) - if (!docId) return null - const acked = acks[docId]?.Acknowledged ?? false + if (!docIds.length) return null + // A folded run is "acknowledged" only when all of it is; a partially + // acknowledged run still has open findings in it. + const acked = docIds.every((id) => acks[id]?.Acknowledged ?? false) + const many = docIds.length > 1 return ( ) } @@ -235,7 +244,28 @@ function sourceEventLink(row: StoreRow) { function buildColumns(acks: Record): Column[] { return [ - { header: 'time', render: (row) => when(str(row, '@timestamp')) }, + { + header: 'time', + render: (row) => { + const folded = foldedCount(row) + return ( + <> + {when(str(row, '@timestamp'))} + {folded > 1 ? ( + <> + {' '} + + ×{folded} + + + ) : null} + + ) + }, + }, { header: 'severity', render: (row) => severityBadge(str(row, 'severity')) }, { header: 'score', className: 'n', render: (row) => num(row, 'composite_score').toFixed(2) }, { @@ -267,11 +297,19 @@ function buildColumns(acks: Record): Column[] { { header: 'status', render: (row) => { - const record = acks[str(row, '_doc_id')] - return record?.Acknowledged ? ( - acknowledged{record.AckedBy ? ` by ${record.AckedBy}` : ''} - ) : ( - open + // #1566: a folded row speaks for every anomaly in it. Reading only + // the representative's ack would show "acknowledged" over a run + // that still has open findings inside it. + const ids = idsFor(row) + const open = ids.filter((id) => !(acks[id]?.Acknowledged ?? false)) + if (!open.length) { + const by = acks[ids[0]]?.AckedBy + return acknowledged{by ? ` by ${by}` : ''} + } + return ( + + {open.length === ids.length ? 'open' : `${open.length} of ${ids.length} open`} + ) }, }, @@ -353,6 +391,11 @@ function Page() { }) }, [rows, severity, eventType, status, acks]) + // #1566: fold last, over what the filters actually left on screen. Folding + // first would group rows the filters then tear apart, leaving a badge + // counting siblings that are no longer shown. + const grouped = useMemo(() => (filtered ? collapseRuns(filtered) : null), [filtered]) + return ( <> `${str(row, 'source_event_id')}-${index}`} emptyState={{ @@ -500,7 +543,7 @@ function Page() { {acks[str(row, '_doc_id')]?.AckedAt ? ` at ${formatTimestamp((acks[str(row, '_doc_id')]!.AckedAt as string))}` : ''}

) : null} - + )} /> diff --git a/scripts/check-store-sorts.py b/scripts/check-store-sorts.py new file mode 100644 index 000000000..68b962598 --- /dev/null +++ b/scripts/check-store-sorts.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Check every store list's sort field against the live Elasticsearch mapping. + +A store list is paged with `sort: [{field: {..., unmapped_type: ...}}]`. That +`unmapped_type` exists so a deployment where the index has not been created +yet gets an empty list instead of an error -- but it also means a field the +documents do not actually have sorts every hit as null rather than failing. +The list then comes back in no order at all, and nothing anywhere says so. + +Three shipped that way (#1566): ml-anomalies asked for `timestamp` where the +worker writes `@timestamp`, auth-events asked for `last_seen` where Keycloak +writes `@timestamp`, and static-analysis asked for `Analysis.GeneratedUTC`, +which its documents do not carry in any spelling. + +CI cannot see Elasticsearch, so this is not a CI check -- run it against a +real cluster after a deploy, or when adding a store: + + ssh homeserver "docker exec hp-elasticsearch curl -s \\ + 'http://localhost:9200/_mapping'" > mappings.json + python scripts/check-store-sorts.py mappings.json + +Exits non-zero if any store's sort field is missing from an index that +exists. An index that does not exist on this deployment is reported and +skipped -- that is the case `unmapped_type` is legitimately for. +""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +STORES_RS = ROOT / "arcane/home/honeypot-dashboard/backend-service/src/stores.rs" + +# ("index", "sort field", "type", ...) in the generic store table, and the +# direct store_page(&state, &["index"], "field", "type", ...) callers. +TABLE_ENTRY = re.compile( + r'=>\s*\(\s*"(?P[^"]+)"\s*,\s*"(?P[^"]+)"\s*,\s*"(?P[^"]+)"', +) +DIRECT_CALL = re.compile( + r'store_page(?:_excluding)?\(\s*&state,\s*&\[\s*"(?P[^"]+)"\s*\]\s*,' + r'\s*"(?P[^"]+)"\s*,\s*"(?P[^"]+)"', +) + + +def declared_sorts(source: str) -> list[tuple[str, str, str]]: + seen: dict[tuple[str, str], str] = {} + for pattern in (TABLE_ENTRY, DIRECT_CALL): + for match in pattern.finditer(source): + seen[(match["index"], match["field"])] = match["kind"] + return sorted((index, field, kind) for (index, field), kind in seen.items()) + + +def field_type(properties: dict, dotted: str) -> str | None: + """Resolve a possibly-dotted field path through a mapping's properties.""" + cursor = properties + parts = dotted.split(".") + for depth, part in enumerate(parts): + if part not in cursor: + return None + node = cursor[part] + if depth == len(parts) - 1: + return node.get("type", "object") + cursor = node.get("properties", {}) + return None + + +def indices_matching(mappings: dict, name: str) -> list[tuple[str, dict]]: + """Concrete indices behind a name, which may be an alias or a datastream.""" + exact = [(key, value) for key, value in mappings.items() if key == name] + if exact: + return exact + # A datastream-backed name appears as .ds---. + return [(key, value) for key, value in mappings.items() if key.startswith(f".ds-{name}-")] + + +def main() -> int: + if len(sys.argv) != 2: + print(__doc__, file=sys.stderr) + return 2 + mappings = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) + + problems: list[str] = [] + skipped: list[str] = [] + checked = 0 + + for index, field, kind in declared_sorts(STORES_RS.read_text(encoding="utf-8")): + concrete = indices_matching(mappings, index) + if not concrete: + skipped.append(f"{index}: not present on this deployment (sort={field})") + continue + for name, body in concrete: + actual = field_type(body.get("mappings", {}).get("properties", {}), field) + checked += 1 + if actual is None: + problems.append( + f"{name}: sort field {field!r} is not in the mapping -- " + "every hit sorts as null and the list is unordered" + ) + elif actual != kind and not (actual, kind) in {("float", "double"), ("integer", "long"), ("half_float", "double")}: + problems.append( + f"{name}: sort field {field!r} is mapped as {actual!r} but " + f"declared as {kind!r} -- unmapped_type must match the real type" + ) + + for line in skipped: + print(f" skipped: {line}") + if problems: + print(f"\nStore sort check failed ({checked} checked):", file=sys.stderr) + for problem in sorted(set(problems)): + print(f" - {problem}", file=sys.stderr) + return 1 + print(f"\nStore sort check passed ({checked} sort fields verified against live mappings).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 4bca0acddc3da0bd63e5e4971600f754d988be90 Mon Sep 17 00:00:00 2001 From: Xore Date: Tue, 25 Aug 2026 15:08:42 +0200 Subject: [PATCH 3/3] fix(overview): the attack map never re-measured the card it was drawn into Leaflet sizes its tile grid once, against whatever the container measured at construction time, and the map is built before the card has settled at its final width. Measured at an 1854px viewport: six 256px tile columns laid out from the wrong origin for a 1442px container, leaving a 92px strip of bare card down the right-hand edge. It reads as a rendering fault rather than as ocean, which is how #1565 reported it. invalidateSize() re-measures and recomputes the grid. A ResizeObserver repeats that whenever the card changes width, because a fixed zoom covers a fixed pixel width and the card's width is not fixed -- a window resize puts the strip straight back otherwise. Verified on a rendered page against live data, at two widths. Before, at 1442px: tiles ended 92px short on the right. After: every edge overflows the container (-75 left, -19 right, -179 top, -69 bottom) and the whole world is still in frame, Australia and southern Africa included. Narrowed to a 488px card the observer holds it (-552/-496/-249/-139). Fitting the world to *cover* the card was the other candidate and is worse: it zooms to 3 and crops the southern hemisphere, trading data for coverage. This keeps zoom 2 and simply stops lying to leaflet about how big the card is. Closes #1565 --- .../src/components/OverviewPanels.tsx | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/arcane/home/honeypot-dashboard/frontend-next/src/components/OverviewPanels.tsx b/arcane/home/honeypot-dashboard/frontend-next/src/components/OverviewPanels.tsx index e2bde49c4..caab7e1ba 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/src/components/OverviewPanels.tsx +++ b/arcane/home/honeypot-dashboard/frontend-next/src/components/OverviewPanels.tsx @@ -208,6 +208,9 @@ const MARKER_RADIUS_PX = 6 export function AttackMap({ points }: { points: MapPoint[] | null }) { const containerRef = useRef(null) const mapRef = useRef(null) + // Torn down alongside the map itself; the ResizeObserver outlives the + // async import that creates it, so it needs its own handle. + const cleanupRef = useRef<(() => void) | null>(null) useEffect(() => { const container = containerRef.current @@ -217,8 +220,32 @@ export function AttackMap({ points }: { points: MapPoint[] | null }) { const L = (await import('leaflet')).default await import('leaflet/dist/leaflet.css') if (disposed || mapRef.current) return - const map = L.map(container, { worldCopyJump: true, minZoom: 1 }).setView([25, 10], 2) + const map = L.map(container, { worldCopyJump: true, minZoom: 1 }) mapRef.current = map + + // #1565: the map is built before the card has settled at its final + // width, so leaflet sizes its tile grid against whatever the container + // measured at construction time and never revisits it. Measured at an + // 1854px viewport, that left six 256px tile columns covering 1536px of + // a 1442px container from the wrong origin -- a 92px strip of bare + // card down the right edge, which reads as a rendering fault rather + // than as ocean. + // + // invalidateSize() re-measures and recomputes the grid, which is the + // whole fix; the setView after it re-centres on the world at the same + // zoom the card has always used, so the full world stays visible + // rather than being cropped to fill the width. + const fitWorld = () => { + map.invalidateSize({ animate: false }) + map.setView([25, 10], 2, { animate: false }) + } + fitWorld() + + // A card that changes width -- a sidebar opening, a window resize, + // the print stylesheet -- puts the strip straight back otherwise. + const observer = new ResizeObserver(fitWorld) + observer.observe(container) + cleanupRef.current = () => observer.disconnect() L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', { attribution: '© OpenStreetMap contributors', }).addTo(map) @@ -259,6 +286,8 @@ export function AttackMap({ points }: { points: MapPoint[] | null }) { })() return () => { disposed = true + cleanupRef.current?.() + cleanupRef.current = null mapRef.current?.remove() mapRef.current = null }