diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 0f92350b8..c108255c4 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -49,6 +49,20 @@ jobs: - run: python scripts/check-public-leaks.py - run: python scripts/validate-oidc-redirects.py + design-lab-readonly: + # #1828: the design lab serves variants against the real captured-data + # Elasticsearch, so its read-only guarantee is a safety property, not a + # convenience. The test drives the actual harness against a recording + # stand-in backend and fails if a write ever reaches it. + name: Design lab is read-only + runs-on: ${{ (github.event_name == 'workflow_dispatch' && inputs.use_self_hosted_runner) && 'self-hosted' || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: "22" + - run: node --test branding/design-lab/lab.test.mjs + go-fmt: name: Go formatting runs-on: ${{ (github.event_name == 'workflow_dispatch' && inputs.use_self_hosted_runner) && 'self-hosted' || 'ubuntu-latest' }} diff --git a/arcane/home/honeypot-dashboard/backend-service/src/stores.rs b/arcane/home/honeypot-dashboard/backend-service/src/stores.rs index 581e73c54..9478dc5cc 100644 --- a/arcane/home/honeypot-dashboard/backend-service/src/stores.rs +++ b/arcane/home/honeypot-dashboard/backend-service/src/stores.rs @@ -51,22 +51,50 @@ fn bad_gateway(error: anyhow::Error) -> (StatusCode, String) { (StatusCode::BAD_GATEWAY, error.to_string()) } +/// The sort every store list is paged by: the store's own field, then a +/// deterministic tiebreak. +/// +/// Both keys are load-bearing. +/// +/// `unmapped_type` keeps a store whose index does not exist yet from +/// erroring -- but it also means a *misspelled* field sorts every document +/// as null instead of failing. That is how three lists shipped in no order +/// at all: ml-anomalies asked for "timestamp" where the worker writes +/// "@timestamp", auth-events asked for "last_seen" where Keycloak writes +/// "@timestamp", and static-analysis asked for "Analysis.GeneratedUTC", +/// which its documents do not carry in any spelling. It has to match the +/// field's real type, or a keyword sort reintroduces the same silent no-op. +/// +/// The tiebreak is what makes paging correct rather than merely tidy. +/// `from`/`size` over a sort with ties leaves the order within a tie +/// undefined, so the same document can come back on two pages while another +/// is never shown at all -- and an all-null sort is one giant tie. `_doc` is +/// the cheapest total order Elasticsearch offers. +fn sort_spec(sort_field: &str, unmapped_type: &str) -> Value { + json!([ + {sort_field: {"order": "desc", "unmapped_type": unmapped_type}}, + {"_doc": {"order": "asc"}} + ]) +} + /// Generic hits page: {"total": N, "rows": [ _source... ]}. The BFF/routes /// know each store's shape; this tier guarantees ordering + paging. async fn store_page( state: &AppState, indices: &[&str], sort_field: &str, + unmapped_type: &str, q: &StoreQuery, extra_filter: Option, ) -> anyhow::Result { - store_page_excluding(state, indices, sort_field, q, extra_filter, &[]).await + store_page_excluding(state, indices, sort_field, unmapped_type, q, extra_filter, &[]).await } async fn store_page_excluding( state: &AppState, indices: &[&str], sort_field: &str, + unmapped_type: &str, q: &StoreQuery, extra_filter: Option, excludes: &[&str], @@ -83,7 +111,7 @@ async fn store_page_excluding( "from": q.offset, "size": size, "track_total_hits": true, - "sort": [{sort_field: {"order": "desc", "unmapped_type": "date"}}], + "sort": sort_spec(sort_field, unmapped_type), "query": query }); if !excludes.is_empty() { @@ -112,7 +140,7 @@ pub async fn campaigns( State(state): State, Query(q): Query, ) -> Result, (StatusCode, String)> { - store_page(&state, &["campaigns-v1"], "score", &q, None) + store_page(&state, &["campaigns-v1"], "score", "long", &q, None) .await .map(Json) .map_err(bad_gateway) @@ -122,7 +150,7 @@ pub async fn clusters( State(state): State, Query(q): Query, ) -> Result, (StatusCode, String)> { - store_page(&state, &["attacker-clusters-v1"], "events", &q, None) + store_page(&state, &["attacker-clusters-v1"], "events", "long", &q, None) .await .map(Json) .map_err(bad_gateway) @@ -132,7 +160,7 @@ pub async fn attackers( State(state): State, Query(q): Query, ) -> Result, (StatusCode, String)> { - store_page(&state, &["attackers-v1"], "events", &q, None) + store_page(&state, &["attackers-v1"], "events", "long", &q, None) .await .map(Json) .map_err(bad_gateway) @@ -224,7 +252,7 @@ pub async fn alerts( State(state): State, Query(q): Query, ) -> Result, (StatusCode, String)> { - store_page(&state, &["dashboard-alert-state-v1"], "LastSeen", &q, None) + store_page(&state, &["dashboard-alert-state-v1"], "LastSeen", "date", &q, None) .await .map(Json) .map_err(bad_gateway) @@ -234,7 +262,7 @@ pub async fn payloads( State(state): State, Query(q): Query, ) -> Result, (StatusCode, String)> { - store_page(&state, &["dashboard-payload-inventory-v1"], "MtimeUTC", &q, None) + store_page(&state, &["dashboard-payload-inventory-v1"], "MtimeUTC", "date", &q, None) .await .map(Json) .map_err(bad_gateway) @@ -270,40 +298,53 @@ pub async fn generic( Query(q): Query, ) -> Result, (StatusCode, String)> { // (index, sort field, heavy fields excluded from list responses). - let (index, sort, excludes): (&str, &str, &[&str]) = match name.as_str() { + // (index, sort field, that field's type, heavy fields excluded from + // list responses). The type is not decoration -- see the sort spec in + // store_page_excluding for what a wrong one costs. + let (index, sort, sort_type, excludes): (&str, &str, &str, &[&str]) = match name.as_str() { // #1611 workstream E.9: `error`, `details.username`, and // `details.redirect_uri` are already present here (no excludes) — // the workstream's ask is first-class *columns* for them on the // auth-events.tsx table, a frontend-only change; this passthrough // already carries every field they'd need. - "auth-events" => ("auth-failure-events", "last_seen", &[]), + // "last_seen" is not a field these documents have -- Keycloak's + // event stream writes @timestamp -- so this list came back in no + // order at all until #1566. + "auth-events" => ("auth-failure-events", "@timestamp", "date", &[]), // llm-worker output; index may not exist yet (ignore_unavailable). - "llm-analysis" => ("llm-analysis", "@timestamp", &[]), - "ml-anomalies" => ("ml-anomalies", "timestamp", &[]), + "llm-analysis" => ("llm-analysis", "@timestamp", "date", &[]), + // Likewise "timestamp": the ml-worker writes @timestamp. Measured + // on the live index, the first page mixed 20:58, 20:59 and 20:57 + // rows in that order (#1566). + "ml-anomalies" => ("ml-anomalies", "@timestamp", "date", &[]), // Matches dashboard/agent_campaigns.go's own refreshAgentCampaigns // sort (`sort=@timestamp:asc`) — the campaign-verdict documents // this index holds have no last_seen field at all (see the // agent-intrusion-worker port's build_campaign_verdict, #1610). - "agent-campaigns" => ("agent-intrusion-campaigns", "@timestamp", &[]), - "canarytokens" => ("dashboard-canarytokens-v1", "created_at", &[]), - "problem-reports" => ("dashboard-problem-reports-v1", "submitted_at", &["dom_snapshot"]), - "dead-letters" => ("dead-letter-honeypot", "@timestamp", &[]), - "yara" => ("yara-analysis-v1", "@timestamp", &[]), - "sandbox-runs" => ("sandbox-analysis-v1", "@timestamp", &[]), - "ghidra-runs" => ("ghidra-analysis-v1", "@timestamp", &[]), - "static-analysis" => ("dashboard-static-analysis-v1", "Analysis.GeneratedUTC", &[]), + "agent-campaigns" => ("agent-intrusion-campaigns", "@timestamp", "date", &[]), + "canarytokens" => ("dashboard-canarytokens-v1", "created_at", "date", &[]), + "problem-reports" => ("dashboard-problem-reports-v1", "submitted_at", "date", &["dom_snapshot"]), + "dead-letters" => ("dead-letter-honeypot", "@timestamp", "date", &[]), + "yara" => ("yara-analysis-v1", "@timestamp", "date", &[]), + "sandbox-runs" => ("sandbox-analysis-v1", "@timestamp", "date", &[]), + "ghidra-runs" => ("ghidra-analysis-v1", "@timestamp", "date", &[]), + // These documents carry only Analysis and Fingerprint -- there is + // no GeneratedUTC, and no time field at all, so there is nothing to + // order by chronologically. Fingerprint is the one mapped field, so + // it is what makes paging deterministic instead of arbitrary. + "static-analysis" => ("dashboard-static-analysis-v1", "Fingerprint", "keyword", &[]), // Result families that may not exist yet on a given deployment // (ignore_unavailable keeps them safe): revdeck, CAPE, GitHub. - "revdeck" => ("revdeck-analysis-v1", "@timestamp", &[]), - "cape" => ("cape-analysis-v1", "@timestamp", &[]), - "github-analysis" => ("github-analysis-v1", "@timestamp", &[]), - "workbench-runs" => ("dashboard-workbench-runs-v1", "created_at", &[]), - "generated-reports" => ("dashboard-generated-reports-v1", "created_at", &["pdf_base64"]), - "report-definitions" => ("dashboard-reports-definitions-v1", "updated", &[]), - "intelligence" => ("dashboard-intelligence-archive-v1", "generated", &[]), + "revdeck" => ("revdeck-analysis-v1", "@timestamp", "date", &[]), + "cape" => ("cape-analysis-v1", "@timestamp", "date", &[]), + "github-analysis" => ("github-analysis-v1", "@timestamp", "date", &[]), + "workbench-runs" => ("dashboard-workbench-runs-v1", "created_at", "date", &[]), + "generated-reports" => ("dashboard-generated-reports-v1", "created_at", "date", &["pdf_base64"]), + "report-definitions" => ("dashboard-reports-definitions-v1", "updated", "date", &[]), + "intelligence" => ("dashboard-intelligence-archive-v1", "generated", "date", &[]), _ => return Err((StatusCode::NOT_FOUND, format!("unknown store {name}"))), }; - store_page_excluding(&state, &[index], sort, &q, None, excludes) + store_page_excluding(&state, &[index], sort, sort_type, &q, None, excludes) .await .map(Json) .map_err(bad_gateway) @@ -342,3 +383,35 @@ pub async fn generic_delete( let deleted = state.es.delete_by_query("dead-letter-honeypot", query).await.map_err(bad_gateway)?; Ok(Json(json!({"deleted": deleted}))) } + + +#[cfg(test)] +mod sort_tests { + use super::sort_spec; + + #[test] + fn the_store_field_leads_and_carries_its_own_type() { + // A date sort and a keyword sort must not both claim "date": an + // unmapped_type that disagrees with the field is the same silent + // no-op as a misspelled field name. + let dated = sort_spec("@timestamp", "date"); + assert_eq!(dated[0]["@timestamp"]["order"], "desc"); + assert_eq!(dated[0]["@timestamp"]["unmapped_type"], "date"); + + let keyed = sort_spec("Fingerprint", "keyword"); + assert_eq!(keyed[0]["Fingerprint"]["unmapped_type"], "keyword"); + } + + #[test] + fn every_sort_has_a_tiebreak_so_paging_cannot_repeat_or_skip() { + // The bug this guards. from/size over a sort with ties leaves the + // order inside a tie undefined, so a document can appear on two + // pages while another never appears -- and the measured live data + // ties hard (48 ml-anomaly rows on one IP in a single second). + for (field, kind) in [("@timestamp", "date"), ("score", "long"), ("Fingerprint", "keyword")] { + let spec = sort_spec(field, kind); + assert_eq!(spec.as_array().map(Vec::len), Some(2), "{field} lost its tiebreak"); + assert_eq!(spec[1]["_doc"]["order"], "asc", "{field}'s tiebreak is not a total order"); + } + } +} diff --git a/arcane/home/honeypot-dashboard/frontend-next/.gitignore b/arcane/home/honeypot-dashboard/frontend-next/.gitignore index 713db202c..549515269 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/.gitignore +++ b/arcane/home/honeypot-dashboard/frontend-next/.gitignore @@ -4,3 +4,7 @@ node_modules/ .tanstack/ dist/ .env + +# #1828: the design lab symlinks variant stylesheets in here at start-up +# and clears it on exit. Never content, always disposable. +public/static/lab/ diff --git a/arcane/home/honeypot-dashboard/frontend-next/src/components/OverviewPanels.tsx b/arcane/home/honeypot-dashboard/frontend-next/src/components/OverviewPanels.tsx index e2bde49c4..caab7e1ba 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/src/components/OverviewPanels.tsx +++ b/arcane/home/honeypot-dashboard/frontend-next/src/components/OverviewPanels.tsx @@ -208,6 +208,9 @@ const MARKER_RADIUS_PX = 6 export function AttackMap({ points }: { points: MapPoint[] | null }) { const containerRef = useRef(null) const mapRef = useRef(null) + // Torn down alongside the map itself; the ResizeObserver outlives the + // async import that creates it, so it needs its own handle. + const cleanupRef = useRef<(() => void) | null>(null) useEffect(() => { const container = containerRef.current @@ -217,8 +220,32 @@ export function AttackMap({ points }: { points: MapPoint[] | null }) { const L = (await import('leaflet')).default await import('leaflet/dist/leaflet.css') if (disposed || mapRef.current) return - const map = L.map(container, { worldCopyJump: true, minZoom: 1 }).setView([25, 10], 2) + const map = L.map(container, { worldCopyJump: true, minZoom: 1 }) mapRef.current = map + + // #1565: the map is built before the card has settled at its final + // width, so leaflet sizes its tile grid against whatever the container + // measured at construction time and never revisits it. Measured at an + // 1854px viewport, that left six 256px tile columns covering 1536px of + // a 1442px container from the wrong origin -- a 92px strip of bare + // card down the right edge, which reads as a rendering fault rather + // than as ocean. + // + // invalidateSize() re-measures and recomputes the grid, which is the + // whole fix; the setView after it re-centres on the world at the same + // zoom the card has always used, so the full world stays visible + // rather than being cropped to fill the width. + const fitWorld = () => { + map.invalidateSize({ animate: false }) + map.setView([25, 10], 2, { animate: false }) + } + fitWorld() + + // A card that changes width -- a sidebar opening, a window resize, + // the print stylesheet -- puts the strip straight back otherwise. + const observer = new ResizeObserver(fitWorld) + observer.observe(container) + cleanupRef.current = () => observer.disconnect() L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', { attribution: '© OpenStreetMap contributors', }).addTo(map) @@ -259,6 +286,8 @@ export function AttackMap({ points }: { points: MapPoint[] | null }) { })() return () => { disposed = true + cleanupRef.current?.() + cleanupRef.current = null mapRef.current?.remove() mapRef.current = null } diff --git a/arcane/home/honeypot-dashboard/frontend-next/src/lib/mlGrouping.test.ts b/arcane/home/honeypot-dashboard/frontend-next/src/lib/mlGrouping.test.ts new file mode 100644 index 000000000..19ee7dd50 --- /dev/null +++ b/arcane/home/honeypot-dashboard/frontend-next/src/lib/mlGrouping.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from 'vitest' +import { collapseRuns, foldedCount, idsFor } from './mlGrouping' +import type { StoreRow } from '../components/StoreList' + +function row(id: string, ip: string, ts: string, score: number): StoreRow { + return { _doc_id: id, src_ip: ip, '@timestamp': ts, composite_score: score } +} + +describe('collapseRuns', () => { + it('folds a burst from one address in one second into a single row', () => { + // The measured shape of the bug: 48 rows for one IP inside one second, + // every one scoring exactly 0.8000. + const burst = Array.from({ length: 48 }, (_, i) => + row(`d${i}`, '153.75.87.176', `2026-08-24T20:59:23.${String(i).padStart(3, '0')}Z`, 0.8), + ) + const out = collapseRuns(burst) + expect(out).toHaveLength(1) + expect(foldedCount(out[0])).toBe(48) + expect(idsFor(out[0])).toHaveLength(48) + }) + + it('keeps different addresses apart even in the same second', () => { + const out = collapseRuns([ + row('a', '1.1.1.1', '2026-08-24T20:59:23.100Z', 0.8), + row('b', '2.2.2.2', '2026-08-24T20:59:23.200Z', 0.8), + ]) + expect(out).toHaveLength(2) + }) + + it('keeps different seconds apart even from the same address', () => { + const out = collapseRuns([ + row('a', '1.1.1.1', '2026-08-24T20:59:23.900Z', 0.8), + row('b', '1.1.1.1', '2026-08-24T20:59:24.000Z', 0.8), + ]) + expect(out).toHaveLength(2) + }) + + it('keeps materially different scores apart', () => { + const out = collapseRuns([ + row('a', '1.1.1.1', '2026-08-24T20:59:23.100Z', 0.8), + row('b', '1.1.1.1', '2026-08-24T20:59:23.200Z', 0.95), + ]) + expect(out).toHaveLength(2) + }) + + it('does not chain a drift into one arbitrarily wide group', () => { + // Each row is within the epsilon of the one before it, but the run spans + // 0.05 end to end. Comparing against the representative rather than the + // predecessor is what stops that becoming a single "group". + const drift = Array.from({ length: 6 }, (_, i) => + row(`d${i}`, '1.1.1.1', '2026-08-24T20:59:23.100Z', 0.8 + i * 0.009), + ) + const out = collapseRuns(drift) + expect(out.length).toBeGreaterThan(1) + }) + + it('only folds rows that are actually adjacent', () => { + // A lookalike separated by an unrelated row is a different moment, and + // is left alone rather than reached across for. + const out = collapseRuns([ + row('a', '1.1.1.1', '2026-08-24T20:59:23.100Z', 0.8), + row('x', '9.9.9.9', '2026-08-24T20:59:23.150Z', 0.4), + row('b', '1.1.1.1', '2026-08-24T20:59:23.200Z', 0.8), + ]) + expect(out).toHaveLength(3) + }) + + it('never folds a row that has no score', () => { + const out = collapseRuns([ + { _doc_id: 'a', src_ip: '1.1.1.1', '@timestamp': '2026-08-24T20:59:23.100Z' }, + { _doc_id: 'b', src_ip: '1.1.1.1', '@timestamp': '2026-08-24T20:59:23.200Z' }, + ]) + expect(out).toHaveLength(2) + }) + + it('leaves an ungrouped row reporting itself, so callers need no special case', () => { + const out = collapseRuns([row('a', '1.1.1.1', '2026-08-24T20:59:23.100Z', 0.8)]) + expect(foldedCount(out[0])).toBe(1) + expect(idsFor(out[0])).toEqual(['a']) + }) + + it('does not mutate the rows it was given', () => { + const input = [ + row('a', '1.1.1.1', '2026-08-24T20:59:23.100Z', 0.8), + row('b', '1.1.1.1', '2026-08-24T20:59:23.200Z', 0.8), + ] + const snapshot = JSON.parse(JSON.stringify(input)) + collapseRuns(input) + expect(input).toEqual(snapshot) + }) + + it('handles an empty page', () => { + expect(collapseRuns([])).toEqual([]) + }) +}) diff --git a/arcane/home/honeypot-dashboard/frontend-next/src/lib/mlGrouping.ts b/arcane/home/honeypot-dashboard/frontend-next/src/lib/mlGrouping.ts new file mode 100644 index 000000000..4a27dff9e --- /dev/null +++ b/arcane/home/honeypot-dashboard/frontend-next/src/lib/mlGrouping.ts @@ -0,0 +1,96 @@ +// Collapsing near-duplicate ML anomaly rows (#1566). +// +// The ml-worker scores each event independently, so one burst from one +// address produces one row per event. Measured on the live index: 66% of all +// 7,305 anomalies sit in an (address, same second) bucket with siblings, and +// the worst single burst is 48 rows for one IP inside one second, every one +// of them scoring exactly 0.8000. Paging through that is reading the same +// sentence forty-eight times. +// +// This only became possible once the store's sort was fixed in the same +// change: the list was previously ordered by a field the documents do not +// have, so a burst arrived scattered through the page rather than contiguous, +// and there were no runs to collapse. +// +// Scope, deliberately: runs are collapsed *within the fetched page*. The +// alternative is an aggregation in the Rust tier, which would change the +// shape of the shared /api/v1/store endpoint for every other consumer. A +// burst longer than the page still splits across the boundary, so the badge +// says how many rows were folded together here rather than claiming to be +// the size of the whole burst. + +import type { StoreRow } from '../components/StoreList' + +/** How far two composite scores may differ and still count as the same finding. */ +export const SCORE_EPSILON = 0.01 + +/** Rows folded into this one, including itself. Absent means an ungrouped row. */ +export const DUPES = '_dupes' +/** Every `_doc_id` folded into this row, so an acknowledgement covers them all. */ +export const DUPE_IDS = '_dupe_ids' + +function second(row: StoreRow): string { + const raw = typeof row['@timestamp'] === 'string' ? (row['@timestamp'] as string) : '' + // "2026-08-24T20:59:23.138Z" -> "2026-08-24T20:59:23"; a value with no + // sub-second part is already truncated and slicing it is harmless. + return raw.slice(0, 19) +} + +function score(row: StoreRow): number { + const raw = row['composite_score'] + return typeof raw === 'number' ? raw : Number.NaN +} + +function docId(row: StoreRow): string { + return typeof row['_doc_id'] === 'string' ? (row['_doc_id'] as string) : '' +} + +/** + * Fold consecutive near-identical rows into one, carrying the count and the + * ids that went into it. + * + * "Consecutive" is what makes this correct without a global view: the rows + * arrive newest-first by `@timestamp`, so every member of a burst is adjacent + * to the rest of it. A row that merely resembles one three pages away is left + * alone, which is the honest outcome — it is not part of the same moment. + * + * Comparison is against the run's representative rather than the previous + * row, so a slow drift of 0.009 per row cannot chain an arbitrarily wide + * range of scores into a single group. + */ +export function collapseRuns(rows: StoreRow[]): StoreRow[] { + const out: StoreRow[] = [] + for (const row of rows) { + const head = out[out.length - 1] + const sameRun = + head !== undefined && + head['src_ip'] === row['src_ip'] && + second(head) === second(row) && + Math.abs(score(head) - score(row)) <= SCORE_EPSILON && + // NaN scores never compare equal, so a row with no score is never + // folded into another — it is genuinely a different kind of finding. + !Number.isNaN(score(head)) && + !Number.isNaN(score(row)) + + if (sameRun) { + head[DUPES] = ((head[DUPES] as number) ?? 1) + 1 + ;(head[DUPE_IDS] as string[]).push(docId(row)) + continue + } + out.push({ ...row, [DUPES]: 1, [DUPE_IDS]: [docId(row)] }) + } + return out +} + +/** Ids an action on this row should apply to — the group, or just the row. */ +export function idsFor(row: StoreRow): string[] { + const ids = row[DUPE_IDS] + if (Array.isArray(ids) && ids.length) return ids as string[] + return [docId(row)].filter(Boolean) +} + +/** How many rows this one stands for. 1 when it stands only for itself. */ +export function foldedCount(row: StoreRow): number { + const count = row[DUPES] + return typeof count === 'number' && count > 0 ? count : 1 +} diff --git a/arcane/home/honeypot-dashboard/frontend-next/src/routes/routeShape.test.ts b/arcane/home/honeypot-dashboard/frontend-next/src/routes/-routeShape.test.ts similarity index 91% rename from arcane/home/honeypot-dashboard/frontend-next/src/routes/routeShape.test.ts rename to arcane/home/honeypot-dashboard/frontend-next/src/routes/-routeShape.test.ts index ebb038118..2fc4f988a 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/src/routes/routeShape.test.ts +++ b/arcane/home/honeypot-dashboard/frontend-next/src/routes/-routeShape.test.ts @@ -1,3 +1,7 @@ +// Named with a leading "-" so TanStack's route generator skips it: without +// the prefix it scans this file for a Route export, finds none, and warns on +// every dev-server start (routeFileIgnorePrefix, see the generator's own +// message). vitest still collects it -- its include is src/**/*.test.ts. // A route file that has children is a layout, and a layout that redirects // is an infinite loop. // diff --git a/arcane/home/honeypot-dashboard/frontend-next/src/routes/__root.tsx b/arcane/home/honeypot-dashboard/frontend-next/src/routes/__root.tsx index 9f212f123..5a5b50fe2 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/src/routes/__root.tsx +++ b/arcane/home/honeypot-dashboard/frontend-next/src/routes/__root.tsx @@ -60,6 +60,11 @@ const getAppearance = createServerFn({ method: 'GET' }).handler(async (): Promis } }) +// #1828: an optional stylesheet layered after theme.css, for the design +// lab's variants. Read once at module scope -- it is a property of the +// process, not of a request, and production never sets it. +const variantCSS = process.env.VARIANT_CSS ?? '' + export const Route = createRootRoute({ // BFF-owned auth: every navigation resolves the redis session on the // server; unauthenticated requests bounce to the Keycloak flow. The @@ -95,7 +100,17 @@ export const Route = createRootRoute({ { name: 'robots', content: 'noindex, nofollow' }, { title: 'APIARY' }, ], - links: [{ rel: 'stylesheet', href: '/static/theme.css' }], + links: [ + { rel: 'stylesheet', href: '/static/theme.css' }, + // #1828: a design-lab variant layers its token overrides on the + // vendored stylesheet instead of replacing it, so a variant is a diff + // against what ships rather than a fork of it — the authoring pattern + // branding/design-lab/v5-picks-override.css already documents. + // + // Absent unless VARIANT_CSS names one, so production renders exactly + // one stylesheet and this costs nothing. + ...(variantCSS ? [{ rel: 'stylesheet' as const, href: variantCSS }] : []), + ], scripts: [ { // Pre-paint theme + palette boot, byte-compatible with the Go diff --git a/arcane/home/honeypot-dashboard/frontend-next/src/routes/ml-anomalies.tsx b/arcane/home/honeypot-dashboard/frontend-next/src/routes/ml-anomalies.tsx index 1caff698b..e6c4d44b4 100644 --- a/arcane/home/honeypot-dashboard/frontend-next/src/routes/ml-anomalies.tsx +++ b/arcane/home/honeypot-dashboard/frontend-next/src/routes/ml-anomalies.tsx @@ -12,6 +12,7 @@ import { confirmAction } from '../components/ConfirmDialog' import { EChart } from '../components/EChart' import { FiltersButton, FiltersModal } from '../components/FiltersModal' import { formatTimestamp } from '../lib/time' +import { collapseRuns, foldedCount, idsFor } from '../lib/mlGrouping' import { countryName } from '../lib/country' type AckRecord = { Acknowledged: boolean; AckedBy?: string; AckedAt?: string } @@ -130,10 +131,17 @@ const ackAll = createServerFn({ method: 'POST' }).handler(async (): Promise; onChanged: () => void }) { +// #1566: `docIds` rather than one id, because a row may stand for a whole +// folded run. Acknowledging only the representative would leave its siblings +// open, and the row would come straight back as unacknowledged on the next +// refresh -- the button would look broken while working exactly as written. +function AckControl({ docIds, acks, onChanged }: { docIds: string[]; acks: Record; onChanged: () => void }) { const [busy, setBusy] = useState(false) - if (!docId) return null - const acked = acks[docId]?.Acknowledged ?? false + if (!docIds.length) return null + // A folded run is "acknowledged" only when all of it is; a partially + // acknowledged run still has open findings in it. + const acked = docIds.every((id) => acks[id]?.Acknowledged ?? false) + const many = docIds.length > 1 return ( ) } @@ -235,7 +244,28 @@ function sourceEventLink(row: StoreRow) { function buildColumns(acks: Record): Column[] { return [ - { header: 'time', render: (row) => when(str(row, '@timestamp')) }, + { + header: 'time', + render: (row) => { + const folded = foldedCount(row) + return ( + <> + {when(str(row, '@timestamp'))} + {folded > 1 ? ( + <> + {' '} + + ×{folded} + + + ) : null} + + ) + }, + }, { header: 'severity', render: (row) => severityBadge(str(row, 'severity')) }, { header: 'score', className: 'n', render: (row) => num(row, 'composite_score').toFixed(2) }, { @@ -267,11 +297,19 @@ function buildColumns(acks: Record): Column[] { { header: 'status', render: (row) => { - const record = acks[str(row, '_doc_id')] - return record?.Acknowledged ? ( - acknowledged{record.AckedBy ? ` by ${record.AckedBy}` : ''} - ) : ( - open + // #1566: a folded row speaks for every anomaly in it. Reading only + // the representative's ack would show "acknowledged" over a run + // that still has open findings inside it. + const ids = idsFor(row) + const open = ids.filter((id) => !(acks[id]?.Acknowledged ?? false)) + if (!open.length) { + const by = acks[ids[0]]?.AckedBy + return acknowledged{by ? ` by ${by}` : ''} + } + return ( + + {open.length === ids.length ? 'open' : `${open.length} of ${ids.length} open`} + ) }, }, @@ -353,6 +391,11 @@ function Page() { }) }, [rows, severity, eventType, status, acks]) + // #1566: fold last, over what the filters actually left on screen. Folding + // first would group rows the filters then tear apart, leaving a badge + // counting siblings that are no longer shown. + const grouped = useMemo(() => (filtered ? collapseRuns(filtered) : null), [filtered]) + return ( <> `${str(row, 'source_event_id')}-${index}`} emptyState={{ @@ -500,7 +543,7 @@ function Page() { {acks[str(row, '_doc_id')]?.AckedAt ? ` at ${formatTimestamp((acks[str(row, '_doc_id')]!.AckedAt as string))}` : ''}

) : null} - + )} /> diff --git a/branding/design-lab/README.md b/branding/design-lab/README.md index 03d6a9c46..a5a6a81fa 100644 --- a/branding/design-lab/README.md +++ b/branding/design-lab/README.md @@ -81,13 +81,42 @@ session, a `STATIC_DIR` override and nil write-services so real Elasticsearch stayed read-only. It went with the Go dashboard in `cb77cdf8` and is not recoverable from here. -The `frontend-next` equivalent — a dev server with the same read-only -guarantees — still needs building. That is the remaining part of #1763 and is -worth its own issue once #1753's approach is settled; `gen_palettes.py` has -meanwhile been superseded upstream by `scripts/theme-tokens.mjs` and -`check-contrast.mjs` in Xore/theme, which do the same job in CI over 422 -pairs. It is kept here as the record of how the palettes were derived, not as -something to run. +`lab.mjs` is the `frontend-next` equivalent (#1828). Run it from the repo +root: + +``` +node branding/design-lab/lab.mjs # v5-picks-override.css on 19201 +node branding/design-lab/lab.mjs a.css b.css # two variants, side by side +APIARY_BACKEND=http://10.8.0.2:8081 node branding/design-lab/lab.mjs +``` + +Variants take 19201-19205 and the elements playground is on 19300, the ports +this lab has always used. A variant stylesheet is symlinked into the dev +server's `public/static/lab/`, so saving the file and reloading the page is +enough — no rebuild, and no change to the shipped vite config. + +The read-only guarantee is made at the one seam `frontend-next` has, since +there is no service handle to pass nil to: + +| | in the lab | +|---|---| +| `BACKEND_URL` | a gate that forwards `GET`/`HEAD` and refuses everything else with 405 | +| `BACKEND_MOUNTED_URL` | absent — every request 503s | + +That is not decoration. The Rust tier really exposes `generic_delete`, +`preferences::put`, the honeyfs implant writer and the canarytoken minter, so +a reviewer clicking around a variant would otherwise delete real documents and +mint real tokens against live infrastructure. `lab.test.mjs` drives the actual +harness against a recording stand-in backend and fails if a write reaches it; +it runs in CI as `Design lab is read-only`. + +`OIDC_DISABLED=1` supplies the stubbed session, so there is no login +round-trip per variant per page. + +`gen_palettes.py` has meanwhile been superseded upstream by +`scripts/theme-tokens.mjs` and `check-contrast.mjs` in Xore/theme, which do the +same job in CI over 422 pairs. It is kept here as the record of how the +palettes were derived, not as something to run. ## Related diff --git a/branding/design-lab/lab.mjs b/branding/design-lab/lab.mjs new file mode 100644 index 000000000..f98cbd906 --- /dev/null +++ b/branding/design-lab/lab.mjs @@ -0,0 +1,216 @@ +#!/usr/bin/env node +// The design lab's variant harness (#1828). +// +// Reviewing a theme against fixtures tells you the theme looks fine against +// fixtures. Both previous design refreshes were run against real captured +// data and that is what shipped, so #1753 wants the nine themes drafted the +// same way -- which needs a dashboard pointed at the real Elasticsearch that +// structurally cannot write to it. +// +// The Go harness this replaces (dev_serve_test.go, lost with cb77cdf8) got +// that by constructing the server with nil write-services. frontend-next has +// no such handle: it reaches data over HTTP through two bases, and the +// mounted one is write-capable by definition (see backend.server.ts). So the +// same guarantee is made at that seam instead -- +// +// BACKEND_URL -> a gate that forwards GET/HEAD and refuses the +// rest with 405, so a delete is a visible failure +// BACKEND_MOUNTED_URL -> nothing at all; every request 503s +// +// -- which is stronger than the original rather than weaker: it is enforced +// per request at runtime, not by remembering to pass nil. +// +// This matters concretely. The Rust tier really does expose stores.rs's +// generic_delete, preferences.rs's put, the honeyfs implant writer and the +// canarytoken minter. A reviewer clicking around a variant would otherwise +// delete real documents and mint real tokens against live infrastructure. +// +// Usage: +// node branding/design-lab/lab.mjs # v5-picks-override.css +// node branding/design-lab/lab.mjs a.css b.css # two variants, side by side +// APIARY_BACKEND=http://10.8.0.2:8081 node .../lab.mjs # against the homeserver +// +// Variants land on 19201+, the elements playground on 19300 -- the ports the +// lab has always used, so the review log's links keep resolving. + +import { spawn } from 'node:child_process' +import { createServer } from 'node:http' +import { createReadStream, existsSync, mkdirSync, rmSync, symlinkSync } from 'node:fs' +import { dirname, extname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const LAB = dirname(fileURLToPath(import.meta.url)) +const FRONTEND = resolve(LAB, '../../arcane/home/honeypot-dashboard/frontend-next') +const PLAYGROUND = join(LAB, 'playground') + +// Where variant stylesheets are exposed to the browser. vite serves public/ +// at the root, so a symlink here is reachable at /static/lab/.css with +// no rebuild and no vite config change -- and because it is a symlink, saving +// the source file and reloading is enough to see the change. The directory is +// disposable and gitignored; the lab clears it on every start. +const LINK_DIR = join(FRONTEND, 'public/static/lab') + +const GATE_PORT = 19199 // read-only door to the real backend +const ABSENT_PORT = 19198 // the write-capable backend, deliberately not here +const FIRST_VARIANT_PORT = 19201 +const MAX_VARIANTS = 5 // 19201-19205, the documented range +const PLAYGROUND_PORT = 19300 + +const REAL_BACKEND = (process.env.APIARY_BACKEND ?? 'http://127.0.0.1:8081').replace(/\/$/, '') +const READ_METHODS = new Set(['GET', 'HEAD', 'OPTIONS']) + +const MIME = { + '.html': 'text/html; charset=utf-8', + '.css': 'text/css; charset=utf-8', + '.js': 'text/javascript; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.svg': 'image/svg+xml', + '.png': 'image/png', + '.woff2': 'font/woff2', +} + +const children = [] + +function log(scope, message) { + process.stdout.write(`[lab:${scope}] ${message}\n`) +} + +/** + * The read-only door. Anything that is not a read is refused here rather + * than reaching Elasticsearch, and the refusal is printed -- a variant that + * quietly stopped working because the lab blocked a write is worse than one + * that says so. + */ +function startGate() { + const server = createServer(async (req, res) => { + if (!READ_METHODS.has(req.method ?? '')) { + log('gate', `REFUSED ${req.method} ${req.url} -- the lab is read-only`) + res.writeHead(405, { 'content-type': 'application/json', allow: 'GET, HEAD' }) + res.end( + JSON.stringify({ + error: 'design lab is read-only', + detail: `${req.method} ${req.url} was not forwarded. Real captured data is not a scratch pad.`, + }), + ) + return + } + try { + const upstream = await fetch(`${REAL_BACKEND}${req.url}`, { + method: req.method, + headers: { ...req.headers, host: new URL(REAL_BACKEND).host }, + }) + res.writeHead(upstream.status, Object.fromEntries(upstream.headers)) + if (upstream.body) { + const reader = upstream.body.getReader() + for (;;) { + const { done, value } = await reader.read() + if (done) break + res.write(value) + } + } + res.end() + } catch (error) { + log('gate', `upstream ${REAL_BACKEND} unreachable: ${error.message}`) + res.writeHead(502, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ error: 'backend unreachable', detail: error.message })) + } + }) + server.listen(GATE_PORT, '127.0.0.1', () => log('gate', `read-only -> ${REAL_BACKEND} on :${GATE_PORT}`)) + return server +} + +/** + * The write-capable tier, absent. Returning 503 rather than leaving the port + * closed is deliberate: a connection refused reads as "the lab is broken", + * a 503 with this body reads as "this is not something the lab has". + */ +function startAbsentBackend() { + const server = createServer((req, res) => { + log('absent', `${req.method} ${req.url} -- no write-capable backend in the lab`) + res.writeHead(503, { 'content-type': 'application/json' }) + res.end( + JSON.stringify({ + error: 'write-capable backend absent', + detail: 'The design lab runs without backend-service-mounted by design. Sandbox, Ghidra and analysis submits do not exist here.', + }), + ) + }) + server.listen(ABSENT_PORT, '127.0.0.1', () => log('absent', `write tier absent on :${ABSENT_PORT}`)) + return server +} + +function startStatic(root, port, scope) { + const server = createServer((req, res) => { + const requested = decodeURIComponent((req.url ?? '/').split('?')[0]) + const path = join(root, requested === '/' ? 'index.html' : requested) + if (!path.startsWith(root) || !existsSync(path)) { + res.writeHead(404, { 'content-type': 'text/plain' }) + res.end('not found') + return + } + res.writeHead(200, { 'content-type': MIME[extname(path)] ?? 'application/octet-stream' }) + createReadStream(path).pipe(res) + }) + server.listen(port, '127.0.0.1', () => log(scope, `http://127.0.0.1:${port}/`)) + return server +} + +function startVariant(cssFile, port) { + const source = resolve(LAB, cssFile) + if (!existsSync(source)) { + log('variant', `no such stylesheet: ${source}`) + return null + } + const linked = join(LINK_DIR, cssFile.replace(/\//g, '_')) + symlinkSync(source, linked) + const href = `/static/lab/${cssFile.replace(/\//g, '_')}` + + const child = spawn('npx', ['vite', 'dev', '--port', String(port), '--strictPort'], { + cwd: FRONTEND, + stdio: 'inherit', + env: { + ...process.env, + VARIANT_CSS: href, + // No login round-trip per variant per page -- the reviewer is looking + // at type and colour, not at Keycloak. + OIDC_DISABLED: '1', + SERVE_MODE: 'all', + BACKEND_URL: `http://127.0.0.1:${GATE_PORT}`, + BACKEND_MOUNTED_URL: `http://127.0.0.1:${ABSENT_PORT}`, + }, + }) + children.push(child) + log('variant', `${cssFile} -> http://127.0.0.1:${port}/ (css at ${href})`) + return child +} + +function main() { + const requested = process.argv.slice(2) + const variants = requested.length ? requested : ['v5-picks-override.css'] + if (variants.length > MAX_VARIANTS) { + log('lab', `at most ${MAX_VARIANTS} variants (ports ${FIRST_VARIANT_PORT}-${FIRST_VARIANT_PORT + MAX_VARIANTS - 1})`) + process.exit(2) + } + + rmSync(LINK_DIR, { recursive: true, force: true }) + mkdirSync(LINK_DIR, { recursive: true }) + + const servers = [startGate(), startAbsentBackend(), startStatic(PLAYGROUND, PLAYGROUND_PORT, 'playground')] + variants.forEach((css, index) => startVariant(css, FIRST_VARIANT_PORT + index)) + + const shutdown = () => { + for (const child of children) child.kill('SIGTERM') + for (const server of servers) server.close() + rmSync(LINK_DIR, { recursive: true, force: true }) + process.exit(0) + } + process.on('SIGINT', shutdown) + process.on('SIGTERM', shutdown) +} + +// Exported for the harness's own test; main() only runs when invoked directly. +export { READ_METHODS, GATE_PORT, ABSENT_PORT, FIRST_VARIANT_PORT, MAX_VARIANTS, PLAYGROUND_PORT } + +if (process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url))) { + main() +} diff --git a/branding/design-lab/lab.test.mjs b/branding/design-lab/lab.test.mjs new file mode 100644 index 000000000..e97a67a83 --- /dev/null +++ b/branding/design-lab/lab.test.mjs @@ -0,0 +1,79 @@ +// The design lab's read-only guarantee, asserted against the real harness +// process rather than by reading its source (#1828). +// +// This is the requirement the whole tool exists to satisfy: the lab points a +// dashboard at real captured Elasticsearch data, so "we won't call the write +// paths" is not good enough. A stand-in backend records everything that +// reaches it, and the test fails if a write ever does. +// +// node --test branding/design-lab/lab.test.mjs +import { spawn } from 'node:child_process' +import { createServer } from 'node:http' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import test from 'node:test' +import assert from 'node:assert/strict' + +const LAB = join(dirname(fileURLToPath(import.meta.url)), 'lab.mjs') + +// Stand in for the real Rust backend so the test needs no infrastructure -- +// and so a write reaching it is observable rather than merely unlikely. +const reachedUpstream = [] +const upstream = createServer((req, res) => { + reachedUpstream.push(`${req.method} ${req.url}`) + res.writeHead(200, { 'content-type': 'application/json' }) + res.end('{"ok":true}') +}) +await new Promise((r) => upstream.listen(19099, '127.0.0.1', r)) + +// The real harness, not a re-creation of it -- named with a stylesheet that +// does not exist so it starts its servers without spawning vite. +const child = spawn('node', [LAB, 'definitely-not-a-real-variant.css'], { + env: { ...process.env, APIARY_BACKEND: 'http://127.0.0.1:19099' }, + stdio: ['ignore', 'pipe', 'pipe'], +}) +let out = '' +child.stdout.on('data', (d) => (out += d)) +child.stderr.on('data', (d) => (out += d)) + +await new Promise((r) => setTimeout(r, 1500)) + +// 1. A read is forwarded. +const read = await fetch('http://127.0.0.1:19199/api/v1/source-health') +test('a read is forwarded to the backend', () => { + assert.equal(read.status, 200) + assert.deepEqual(reachedUpstream, ['GET /api/v1/source-health']) +}) + +// 2. A delete is refused *and never reaches upstream* -- the whole point. +const before = reachedUpstream.length +const del = await fetch('http://127.0.0.1:19199/api/v1/stores/events/abc123', { method: 'DELETE' }) +test('a delete is refused', () => assert.equal(del.status, 405)) +test('the delete never reached the backend', () => assert.equal(reachedUpstream.length, before)) + +// 3. A preferences PUT, likewise. +const put = await fetch('http://127.0.0.1:19199/api/v1/preferences', { method: 'PUT', body: '{}' }) +test('a preferences write is refused', () => assert.equal(put.status, 405)) +test('the preferences write never reached the backend', () => assert.equal(reachedUpstream.length, before)) + +// 4. The write-capable tier is absent, not merely unused. +const mounted = await fetch('http://127.0.0.1:19198/api/v1/sandbox/submit', { method: 'POST' }) +const mountedBody = await mounted.json() +test('the write-capable tier is absent, not merely unused', () => { + assert.equal(mounted.status, 503) + assert.equal(mountedBody.error, 'write-capable backend absent') +}) + +// 5. Even a GET to the mounted tier finds nothing -- absence is total. +const mountedGet = await fetch('http://127.0.0.1:19198/api/v1/sandbox/status') +test('the write-capable tier is absent for reads too', () => assert.equal(mountedGet.status, 503)) + +// 6. The playground is served on its documented port. +const playground = await fetch('http://127.0.0.1:19300/elements.html') +test('the elements playground is on its documented port', () => assert.equal(playground.status, 200)) + +test.after(() => { + child.kill('SIGTERM') + upstream.close() + if (process.env.LAB_TEST_VERBOSE) console.log(out.trim()) +}) diff --git a/scripts/check-store-sorts.py b/scripts/check-store-sorts.py new file mode 100644 index 000000000..68b962598 --- /dev/null +++ b/scripts/check-store-sorts.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Check every store list's sort field against the live Elasticsearch mapping. + +A store list is paged with `sort: [{field: {..., unmapped_type: ...}}]`. That +`unmapped_type` exists so a deployment where the index has not been created +yet gets an empty list instead of an error -- but it also means a field the +documents do not actually have sorts every hit as null rather than failing. +The list then comes back in no order at all, and nothing anywhere says so. + +Three shipped that way (#1566): ml-anomalies asked for `timestamp` where the +worker writes `@timestamp`, auth-events asked for `last_seen` where Keycloak +writes `@timestamp`, and static-analysis asked for `Analysis.GeneratedUTC`, +which its documents do not carry in any spelling. + +CI cannot see Elasticsearch, so this is not a CI check -- run it against a +real cluster after a deploy, or when adding a store: + + ssh homeserver "docker exec hp-elasticsearch curl -s \\ + 'http://localhost:9200/_mapping'" > mappings.json + python scripts/check-store-sorts.py mappings.json + +Exits non-zero if any store's sort field is missing from an index that +exists. An index that does not exist on this deployment is reported and +skipped -- that is the case `unmapped_type` is legitimately for. +""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +STORES_RS = ROOT / "arcane/home/honeypot-dashboard/backend-service/src/stores.rs" + +# ("index", "sort field", "type", ...) in the generic store table, and the +# direct store_page(&state, &["index"], "field", "type", ...) callers. +TABLE_ENTRY = re.compile( + r'=>\s*\(\s*"(?P[^"]+)"\s*,\s*"(?P[^"]+)"\s*,\s*"(?P[^"]+)"', +) +DIRECT_CALL = re.compile( + r'store_page(?:_excluding)?\(\s*&state,\s*&\[\s*"(?P[^"]+)"\s*\]\s*,' + r'\s*"(?P[^"]+)"\s*,\s*"(?P[^"]+)"', +) + + +def declared_sorts(source: str) -> list[tuple[str, str, str]]: + seen: dict[tuple[str, str], str] = {} + for pattern in (TABLE_ENTRY, DIRECT_CALL): + for match in pattern.finditer(source): + seen[(match["index"], match["field"])] = match["kind"] + return sorted((index, field, kind) for (index, field), kind in seen.items()) + + +def field_type(properties: dict, dotted: str) -> str | None: + """Resolve a possibly-dotted field path through a mapping's properties.""" + cursor = properties + parts = dotted.split(".") + for depth, part in enumerate(parts): + if part not in cursor: + return None + node = cursor[part] + if depth == len(parts) - 1: + return node.get("type", "object") + cursor = node.get("properties", {}) + return None + + +def indices_matching(mappings: dict, name: str) -> list[tuple[str, dict]]: + """Concrete indices behind a name, which may be an alias or a datastream.""" + exact = [(key, value) for key, value in mappings.items() if key == name] + if exact: + return exact + # A datastream-backed name appears as .ds---. + return [(key, value) for key, value in mappings.items() if key.startswith(f".ds-{name}-")] + + +def main() -> int: + if len(sys.argv) != 2: + print(__doc__, file=sys.stderr) + return 2 + mappings = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) + + problems: list[str] = [] + skipped: list[str] = [] + checked = 0 + + for index, field, kind in declared_sorts(STORES_RS.read_text(encoding="utf-8")): + concrete = indices_matching(mappings, index) + if not concrete: + skipped.append(f"{index}: not present on this deployment (sort={field})") + continue + for name, body in concrete: + actual = field_type(body.get("mappings", {}).get("properties", {}), field) + checked += 1 + if actual is None: + problems.append( + f"{name}: sort field {field!r} is not in the mapping -- " + "every hit sorts as null and the list is unordered" + ) + elif actual != kind and not (actual, kind) in {("float", "double"), ("integer", "long"), ("half_float", "double")}: + problems.append( + f"{name}: sort field {field!r} is mapped as {actual!r} but " + f"declared as {kind!r} -- unmapped_type must match the real type" + ) + + for line in skipped: + print(f" skipped: {line}") + if problems: + print(f"\nStore sort check failed ({checked} checked):", file=sys.stderr) + for problem in sorted(set(problems)): + print(f" - {problem}", file=sys.stderr) + return 1 + print(f"\nStore sort check passed ({checked} sort fields verified against live mappings).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())