diff --git a/backend/database/migrations/089_add_memory_layout_tiles.ts b/backend/database/migrations/089_add_memory_layout_tiles.ts new file mode 100644 index 00000000..835c8691 --- /dev/null +++ b/backend/database/migrations/089_add_memory_layout_tiles.ts @@ -0,0 +1,58 @@ +import type { DatabaseConnection } from '$shared/types/database/connection'; +import { debug } from '$shared/utils/logger'; + +export const description = + 'Persist memory-graph node degree and derived edges so the view can be served as tiles'; + +/** + * What a tiled Memory view needs from the arrangement and could not get. + * + * The view no longer asks for "the graph"; it asks for the rectangles of it that + * are on screen, at the resolution the camera is at. Two things a tile has to + * carry were only ever computed for a whole fetch: + * + * - DEGREE sizes a node. It was counted over whatever was drawn, which for a + * tile is a meaningless slice — the same memory would change size as the + * tiles around it loaded. The layout pass already builds the full graph, so + * it writes each node's degree alongside its position. + * + * - DERIVED EDGES (two memories naming the same subject or file) are most of + * the structure the episodic half has. They were derived at read time from + * the set being drawn, which is unbounded for the whole store and wrong for a + * tile: an edge whose other end sits in the next tile was never derived at + * all. The layout pass derives them over every live node anyway — it lays out + * on them — so it now keeps them. Stored edges stay in `graph_edges` and are + * read from there, so a connection a model records shows up without waiting + * for a pass. + * + * Both are derived and disposable like the rest of `graph_layout`: deleting them + * costs one background pass and no memory. + */ +export const up = (db: DatabaseConnection): void => { + debug.log('migration', 'Adding memory layout degree and derived edges...'); + + const columns = db.prepare(`PRAGMA table_info(graph_layout)`).all() as { name: string }[]; + if (!columns.some(column => column.name === 'degree')) { + db.exec(`ALTER TABLE graph_layout ADD COLUMN degree INTEGER NOT NULL DEFAULT 0`); + } + + // Stored once per pair, smaller id first, so "edges touching this node" is the + // union of a lookup on each column — the same shape `graph_edges` is read in. + db.exec(` + CREATE TABLE IF NOT EXISTS graph_layout_edges ( + src_id TEXT NOT NULL REFERENCES graph_nodes(id) ON DELETE CASCADE, + dst_id TEXT NOT NULL REFERENCES graph_nodes(id) ON DELETE CASCADE, + weight REAL NOT NULL DEFAULT 1, + PRIMARY KEY (src_id, dst_id) + ) WITHOUT ROWID + `); + db.exec(`CREATE INDEX IF NOT EXISTS idx_graph_layout_edges_dst ON graph_layout_edges (dst_id)`); + + debug.log('migration', 'Memory layout tiles ready'); +}; + +export const down = (db: DatabaseConnection): void => { + db.exec(`DROP INDEX IF EXISTS idx_graph_layout_edges_dst`); + db.exec(`DROP TABLE IF EXISTS graph_layout_edges`); + db.exec(`ALTER TABLE graph_layout DROP COLUMN degree`); +}; diff --git a/backend/database/migrations/index.ts b/backend/database/migrations/index.ts index 0384fd7d..3e02c9a5 100644 --- a/backend/database/migrations/index.ts +++ b/backend/database/migrations/index.ts @@ -87,6 +87,7 @@ import * as migration085 from './085_add_home_projects'; import * as migration086 from './086_single_owner'; import * as migration087 from './087_create_blob_line_stats'; import * as migration088 from './088_create_messages_fts_map'; +import * as migration089 from './089_add_memory_layout_tiles'; // Export all migrations in order export const migrations = [ @@ -617,6 +618,12 @@ export const migrations = [ description: migration088.description, up: migration088.up, down: migration088.down + }, + { + id: '089', + description: migration089.description, + up: migration089.up, + down: migration089.down } ]; diff --git a/backend/database/queries/graph-queries.ts b/backend/database/queries/graph-queries.ts index fd499e2b..d670b8f7 100644 --- a/backend/database/queries/graph-queries.ts +++ b/backend/database/queries/graph-queries.ts @@ -1544,36 +1544,6 @@ export const graphQueries = { return rows.map(toNode); }, - /** - * The same listing, narrowed to a rectangle of the persisted layout. - * - * This is what opening a bin reads, and it is why the view can be complete - * without being unbounded: zooming into part of the map costs that part. The - * bounds come from the layout's own coordinate space, which the client already - * holds — every mark it drew carries the cell it stands for. - */ - listInRegion( - filter: GraphListFilter & { limit?: number }, - region: { minX: number; maxX: number; minY: number; maxY: number } - ): GraphNode[] { - const { where, params } = buildNodeFilter(filter); - where.push( - `id IN (SELECT node_id FROM graph_layout - WHERE x >= ? AND x <= ? AND y >= ? AND y <= ?)` - ); - params.push(region.minX, region.maxX, region.minY, region.maxY); - params.push(filter.limit ?? 2000); - - const rows = getDatabase() - .prepare( - `SELECT * FROM graph_nodes WHERE ${where.join(' AND ')} - ORDER BY pinned DESC, weight DESC, updated_at DESC - LIMIT ?` - ) - .all(...params) as GraphNodeRow[]; - return rows.map(toNode); - }, - /** * Whether the graph holds any live memory at all. * @@ -1906,11 +1876,14 @@ export interface GraphLayoutRow { x: number; y: number; /** - * 1 when the force simulation computed this position, 0 when it is a guess - * from the community's neighbourhood. See migration 067 for why the two must - * stay distinguishable. + * 1 when the position is a real one — computed by the simulation or by the + * arrival solver — and 0 when it is not usable. See migration 067 for why the + * two must stay distinguishable; the pass no longer writes 0, but rows from + * before it may still hold it, and those are treated as unplaced. */ placed: number; + /** Connections in the graph the pass laid out, stored and derived. Sizes the node. */ + degree: number; } /** How many nodes match a filter, and the rectangle their layout occupies. */ @@ -1924,51 +1897,91 @@ export interface GraphLayoutExtent { maxY: number; } -/** The grid a binned view is rolled up to, in layout coordinates. */ -export interface GraphBinGrid { +/** A rectangle of layout space, half-open: `[minX, maxX) × [minY, maxY)`. */ +export interface GraphLayoutRect { + minX: number; + maxX: number; + minY: number; + maxY: number; +} + +/** A square grid laid over layout space — tiles at one zoom level, or cells inside them. */ +export interface GraphLayoutGrid { originX: number; originY: number; - cellWidth: number; - cellHeight: number; + size: number; } -/** One occupied cell: how many it holds, where they average, and which speaks for it. */ -export interface GraphBinRow { - cellX: number; - cellY: number; - members: number; - x: number; - y: number; - /** The most reinforced member — the mark itself when `members` is 1. */ +/** A memory as a tile carries it: enough to draw and hover, nothing to inspect. */ +export interface GraphTileNodeRow { id: string; label: string; community: number; + degree: number; + x: number; + y: number; +} + +/** One occupied cell of a crowded tile, with the member that speaks for it. */ +export interface GraphTileCellRow extends GraphTileNodeRow { + cellX: number; + cellY: number; + members: number; +} + +/** An edge leaving a tile's memory, with where its far end sits. */ +export interface GraphTileEdgeRow { + source: string; + target: string; + weight: number; + targetX: number; + targetY: number; } -export interface GraphBinEdgeRow { - srcX: number; - srcY: number; - dstX: number; - dstY: number; +/** Edges between the cells of a crowded tile and the cells they reach. */ +export interface GraphTileCellEdgeRow { + sourceCellX: number; + sourceCellY: number; + targetCellX: number; + targetCellY: number; weight: number; } +/** Longest caption a tile carries; the full text belongs to the inspector. */ +const TILE_LABEL_MAX = 80; + +/** + * Every edge touching a set of nodes, from both of the places edges live, and + * oriented so `s` is always the member of the set. + * + * Stored edges come from `graph_edges` directly rather than from the layout + * pass, so a connection a model records is drawn on the next fetch instead of + * after the next pass. Derived ones come from `graph_layout_edges`, which the + * pass keeps (see migration 089). Both are read through their own indexes, one + * lookup per id per column. + */ +const PAIRS_OF = (set: string): string => ` + SELECT e.src_id AS s, e.dst_id AS t, e.weight AS w FROM ${set} ids JOIN graph_edges e ON e.src_id = ids.id + UNION ALL SELECT e.dst_id, e.src_id, e.weight FROM ${set} ids JOIN graph_edges e ON e.dst_id = ids.id + UNION ALL SELECT d.src_id, d.dst_id, d.weight FROM ${set} ids JOIN graph_layout_edges d ON d.src_id = ids.id + UNION ALL SELECT d.dst_id, d.src_id, d.weight FROM ${set} ids JOIN graph_layout_edges d ON d.dst_id = ids.id`; + /** - * Reading and writing the persisted arrangement (see migration 067). + * Reading and writing the persisted arrangement (see migrations 067 and 089). * * Kept apart from `graphQueries` because it answers a different question: * `graphQueries` is about what is remembered, this is about how it is drawn. - * Every row here is derived and disposable — deleting the table costs one + * Every row here is derived and disposable — deleting the tables costs one * background pass and no memory. */ export const graphLayoutQueries = { - /** Positions and communities for a set of nodes. */ + /** Positions, communities and degrees for a set of nodes. */ read(nodeIds: string[]): Map { const result = new Map(); if (nodeIds.length === 0) return result; const rows = getDatabase() .prepare( - `SELECT node_id AS nodeId, community, x, y, placed FROM graph_layout + `SELECT node_id AS nodeId, community, x, y, placed, degree FROM graph_layout WHERE node_id IN ${ID_SET}` ) .all(idSet(nodeIds)) as GraphLayoutRow[]; @@ -1976,24 +1989,69 @@ export const graphLayoutQueries = { return result; }, + /** + * Memories that are no longer live — archived or superseded — for the pass + * to place. + * + * The pass arranges live memories, but "show archived" draws these too, and a + * memory with no position cannot be drawn. `all` asks for every one of them, + * which a cold pass needs because it draws a new map; otherwise only those + * with no usable position yet. Capped, most recent first, because the + * forgotten pile is allowed to grow far beyond anything worth drawing. + */ + retiredIds(all: boolean, limit = 20_000): string[] { + const rows = getDatabase() + .prepare( + `SELECT n.id AS id FROM graph_nodes n + LEFT JOIN graph_layout l ON l.node_id = n.id + WHERE (n.archived_at IS NOT NULL OR n.superseded_by IS NOT NULL) + AND (? = 1 OR l.node_id IS NULL OR l.placed = 0) + ORDER BY n.updated_at DESC, n.id ASC + LIMIT ?` + ) + .all(all ? 1 : 0, limit) as { id: string }[]; + return rows.map(row => row.id).sort(); + }, + /** Replace the arrangement for the nodes given, in one transaction. */ writeMany(rows: GraphLayoutRow[]): number { if (rows.length === 0) return 0; return graphQueries.transaction(() => { const statement = getDatabase().prepare( - `INSERT INTO graph_layout (node_id, community, x, y, placed, updated_at) - VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + `INSERT INTO graph_layout (node_id, community, x, y, placed, degree, updated_at) + VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) ON CONFLICT(node_id) DO UPDATE SET community = excluded.community, x = excluded.x, y = excluded.y, - placed = excluded.placed, updated_at = CURRENT_TIMESTAMP` + placed = excluded.placed, degree = excluded.degree, updated_at = CURRENT_TIMESTAMP` ); for (const row of rows) { - statement.run(row.nodeId, row.community, row.x, row.y, row.placed); + statement.run(row.nodeId, row.community, row.x, row.y, row.placed, row.degree); } return rows.length; }); }, + /** + * Replace every derived edge with the set the pass just laid out on. + * + * Wholesale rather than diffed: the set is a function of the whole graph — one + * memory joining a subject re-chains that subject's members — so there is no + * cheaper correct answer than the new set itself. + */ + replaceDerivedEdges(edges: { srcId: string; dstId: string; weight: number }[]): void { + graphQueries.transaction(() => { + const db = getDatabase(); + db.prepare(`DELETE FROM graph_layout_edges`).run(); + const statement = db.prepare( + `INSERT OR REPLACE INTO graph_layout_edges (src_id, dst_id, weight) VALUES (?, ?, ?)` + ); + for (const edge of edges) { + const [a, b] = edge.srcId < edge.dstId ? [edge.srcId, edge.dstId] : [edge.dstId, edge.srcId]; + statement.run(a, b, edge.weight); + } + }); + }, + /** * Live nodes in the order the view itself ranks them, for the layout pass. * @@ -2019,39 +2077,102 @@ export const graphLayoutQueries = { return rows.map(row => row.id); }, + /** + * A cheap fingerprint of everything the edges of the layout graph come from. + * + * Reinforcement rings the change doorbell on every recall without moving a + * node or an edge, so the pass compares this before doing any work. Counts and + * the highest id are enough: rows are inserted and deleted, never rewritten in + * place in a way that changes what they connect. + */ + sourceFingerprint(): string { + const row = getDatabase() + .prepare( + `SELECT (SELECT COUNT(*) FROM graph_edges) AS edges, + (SELECT COALESCE(MAX(id), 0) FROM graph_edges) AS lastEdge, + (SELECT COUNT(*) FROM graph_node_entities) AS entities, + (SELECT COUNT(*) FROM graph_node_paths) AS paths` + ) + .get() as { edges: number; lastEdge: number; entities: number; paths: number }; + return `${row.edges}:${row.lastEdge}:${row.entities}:${row.paths}`; + }, + /** Whether any live node is still waiting to be placed. */ hasUnplaced(): boolean { const row = getDatabase() .prepare( `SELECT 1 AS present FROM graph_nodes n LEFT JOIN graph_layout l ON l.node_id = n.id - WHERE n.archived_at IS NULL AND n.superseded_by IS NULL AND l.node_id IS NULL + WHERE n.archived_at IS NULL AND n.superseded_by IS NULL + AND (l.node_id IS NULL OR l.placed = 0) LIMIT 1` ) .get() as { present: number } | null; return row !== null && row !== undefined; }, + /** + * The rectangle every placed row occupies — live or not — and the largest + * coordinate magnitude in it. + * + * The magnitude is what catches a corrupted arrangement: a healthy one spans + * thousands of units, and anything near the float range means a pass diverged. + */ + bounds(): (GraphLayoutRect & { count: number; magnitude: number }) | null { + const row = getDatabase() + .prepare( + `SELECT COUNT(*) AS count, MIN(x) AS minX, MAX(x) AS maxX, MIN(y) AS minY, MAX(y) AS maxY, + MAX(MAX(ABS(x)), MAX(ABS(y))) AS magnitude + FROM graph_layout WHERE placed = 1` + ) + .get() as { + count: number; + minX: number | null; + maxX: number | null; + minY: number | null; + maxY: number | null; + magnitude: number | null; + }; + if (!row || row.count === 0 || row.minX === null || row.maxX === null || row.minY === null || row.maxY === null) { + return null; + } + return { + count: row.count, + minX: row.minX, + maxX: row.maxX, + minY: row.minY, + maxY: row.maxY, + magnitude: row.magnitude ?? 0 + }; + }, + /** Rows whose node no longer exists. Foreign keys are not always enforced. */ pruneOrphans(): number { - const result = getDatabase() + const db = getDatabase(); + db.prepare( + `DELETE FROM graph_layout_edges + WHERE src_id NOT IN (SELECT id FROM graph_nodes) OR dst_id NOT IN (SELECT id FROM graph_nodes)` + ).run(); + const result = db .prepare(`DELETE FROM graph_layout WHERE node_id NOT IN (SELECT id FROM graph_nodes)`) .run() as { changes?: number }; return Number(result.changes ?? 0); }, - /** Drop the whole arrangement — used when the graph is purged. */ + /** Drop the whole arrangement — used when the graph is purged or the pass changes. */ clear(): void { - getDatabase().prepare(`DELETE FROM graph_layout`).run(); + const db = getDatabase(); + db.prepare(`DELETE FROM graph_layout_edges`).run(); + db.prepare(`DELETE FROM graph_layout`).run(); }, /** * How many nodes match, and the rectangle their arrangement occupies. * - * One pass answering both, because the two are always wanted together: the - * count decides whether the view has to bin at all, and the extent is what the - * grid is laid over. `MIN`/`MAX` skip NULLs, so a node still waiting to be - * placed is counted without dragging the bounds toward the origin. + * The count is what the header reports and the rectangle is what "Fit" frames + * — the filtered part of the map rather than the whole of it. `MIN`/`MAX` skip + * NULLs, so a node still waiting to be placed is counted without dragging the + * bounds toward the origin. */ extent(filter: GraphListFilter): GraphLayoutExtent { const { where, params } = buildNodeFilter(filter, 'n.'); @@ -2060,9 +2181,11 @@ export const graphLayoutQueries = { const row = getDatabase() .prepare( `SELECT COUNT(*) AS total, - SUM(CASE WHEN l.node_id IS NULL THEN 1 ELSE 0 END) AS unplaced, - MIN(l.x) AS minX, MAX(l.x) AS maxX, - MIN(l.y) AS minY, MAX(l.y) AS maxY + SUM(CASE WHEN l.node_id IS NULL OR l.placed = 0 THEN 1 ELSE 0 END) AS unplaced, + MIN(CASE WHEN l.placed = 1 THEN l.x END) AS minX, + MAX(CASE WHEN l.placed = 1 THEN l.x END) AS maxX, + MIN(CASE WHEN l.placed = 1 THEN l.y END) AS minY, + MAX(CASE WHEN l.placed = 1 THEN l.y END) AS maxY FROM graph_nodes n LEFT JOIN graph_layout l ON l.node_id = n.id ${clause}` @@ -2087,107 +2210,241 @@ export const graphLayoutQueries = { }, /** - * One row per occupied cell of the grid: its size, its centroid, and its most - * reinforced member. + * How many matching memories fall in each tile of `grid` inside `rect`. * - * The count of rows is bounded by the grid, not by the store — that is the - * whole point. A cell holding one memory comes back with `members = 1` and - * that memory's id, so the caller can return the memory ITSELF rather than a - * bin standing for it; below the render budget that is every cell, and the - * view is bit-for-bit the flat one. - * - * `CAST(... AS INTEGER)` truncates toward zero, which would fold the cells on - * either side of the origin into one — the grid is therefore anchored at the - * arrangement's minimum, so every offset it sees is positive. + * Asked once per request over the union of the tiles wanted, so the caller can + * decide per tile whether it fits its budget before reading a single label. + * `CAST` truncates toward zero, which is only a floor for non-negative input — + * the world box is anchored below every placed row, so it always is. */ - binnedNodes(filter: GraphListFilter, grid: GraphBinGrid): GraphBinRow[] { + tileCounts(filter: GraphListFilter, grid: GraphLayoutGrid, rect: GraphLayoutRect): { tileX: number; tileY: number; count: number }[] { const { where, params } = buildNodeFilter(filter, 'n.'); - const clause = where.length ? `WHERE ${where.join(' AND ')}` : ''; + const clause = where.length ? `AND ${where.join(' AND ')}` : ''; + return getDatabase() + .prepare( + `SELECT CAST((l.x - ?) / ? AS INTEGER) AS tileX, + CAST((l.y - ?) / ? AS INTEGER) AS tileY, + COUNT(*) AS count + FROM graph_layout l + INNER JOIN graph_nodes n ON n.id = l.node_id + WHERE l.placed = 1 AND l.x >= ? AND l.x < ? AND l.y >= ? AND l.y < ? ${clause} + GROUP BY 1, 2` + ) + .all( + grid.originX, + grid.size, + grid.originY, + grid.size, + rect.minX, + rect.maxX, + rect.minY, + rect.maxY, + ...params + ) as { tileX: number; tileY: number; count: number }[]; + }, + /** Every matching memory inside a rectangle, drawn individually. */ + tileNodes(filter: GraphListFilter, rect: GraphLayoutRect, limit: number): GraphTileNodeRow[] { + const { where, params } = buildNodeFilter(filter, 'n.'); + const clause = where.length ? `AND ${where.join(' AND ')}` : ''; return getDatabase() .prepare( - `SELECT cellX, cellY, members, cx AS x, cy AS y, id, label, community + `SELECT n.id AS id, substr(n.label, 1, ?) AS label, l.community AS community, + l.degree AS degree, l.x AS x, l.y AS y + FROM graph_layout l + INNER JOIN graph_nodes n ON n.id = l.node_id + WHERE l.placed = 1 AND l.x >= ? AND l.x < ? AND l.y >= ? AND l.y < ? ${clause} + ORDER BY n.pinned DESC, n.weight DESC, n.updated_at DESC, n.id ASC + LIMIT ?` + ) + .all(TILE_LABEL_MAX, rect.minX, rect.maxX, rect.minY, rect.maxY, ...params, limit) as GraphTileNodeRow[]; + }, + + /** + * A crowded tile folded into the cells of `cells`: one row per occupied cell, + * with its size, its centroid and its most reinforced member. + * + * Cell coordinates are GLOBAL — counted from the world origin, not the tile — + * so an edge from any tile can name the cell it lands in without knowing which + * tile that cell belongs to. + */ + tileCells(filter: GraphListFilter, rect: GraphLayoutRect, cells: GraphLayoutGrid): GraphTileCellRow[] { + const { where, params } = buildNodeFilter(filter, 'n.'); + const clause = where.length ? `AND ${where.join(' AND ')}` : ''; + return getDatabase() + .prepare( + `SELECT cellX, cellY, members, cx AS x, cy AS y, id, label, community, degree FROM ( - SELECT cellX, cellY, id, label, community, - COUNT(*) OVER (PARTITION BY cellX, cellY) AS members, - AVG(x) OVER (PARTITION BY cellX, cellY) AS cx, - AVG(y) OVER (PARTITION BY cellX, cellY) AS cy, + SELECT cellX, cellY, id, label, community, degree, + COUNT(*) OVER cell AS members, + AVG(x) OVER cell AS cx, + AVG(y) OVER cell AS cy, ROW_NUMBER() OVER ( PARTITION BY cellX, cellY - ORDER BY pinned DESC, weight DESC, updated_at DESC + ORDER BY pinned DESC, weight DESC, updated_at DESC, id ASC ) AS rn FROM ( SELECT CAST((l.x - ?) / ? AS INTEGER) AS cellX, CAST((l.y - ?) / ? AS INTEGER) AS cellY, - l.x AS x, l.y AS y, l.community AS community, - n.id AS id, n.label AS label, + l.x AS x, l.y AS y, l.community AS community, l.degree AS degree, + n.id AS id, substr(n.label, 1, ?) AS label, n.pinned AS pinned, n.weight AS weight, n.updated_at AS updated_at - FROM graph_nodes n - INNER JOIN graph_layout l ON l.node_id = n.id - ${clause} + FROM graph_layout l + INNER JOIN graph_nodes n ON n.id = l.node_id + WHERE l.placed = 1 AND l.x >= ? AND l.x < ? AND l.y >= ? AND l.y < ? ${clause} ) + WINDOW cell AS (PARTITION BY cellX, cellY) ) WHERE rn = 1` ) .all( - grid.originX, - grid.cellWidth, - grid.originY, - grid.cellHeight, + cells.originX, + cells.size, + cells.originY, + cells.size, + TILE_LABEL_MAX, + rect.minX, + rect.maxX, + rect.minY, + rect.maxY, ...params - ) as GraphBinRow[]; + ) as GraphTileCellRow[]; }, /** - * Edges rolled up to the same grid, so the binned picture keeps its structure. + * Edges touching a set of memories, each with where its far end sits. * - * Stored edges only. The derived ones (see `derivedEdges`) exist to hold a - * subject's memories together, and the layout pass already consumed them — - * their effect is in the POSITIONS, which is where it belongs at this - * resolution. Recomputing them here would mean handing this query the id of - * every matching node, which is the unbounded thing the grid exists to avoid. - * - * Edges inside a single cell are dropped: both ends are the same mark. + * The far end is filtered exactly like the near one, so an edge never leads + * into a memory the current filter hides — and it must have a position, or + * there is nowhere to draw it to. */ - binnedEdges(filter: GraphListFilter, grid: GraphBinGrid, limit: number): GraphBinEdgeRow[] { - const src = buildNodeFilter(filter, 'na.'); - const dst = buildNodeFilter(filter, 'nb.'); - const conditions = [...src.where, ...dst.where]; - const clause = conditions.length ? `WHERE ${conditions.join(' AND ')}` : ''; + edgesTouching(filter: GraphListFilter, nodeIds: string[], limit: number): GraphTileEdgeRow[] { + if (nodeIds.length === 0) return []; + const { where, params } = buildNodeFilter(filter, 'nb.'); + const clause = where.length ? `AND ${where.join(' AND ')}` : ''; + return getDatabase() + .prepare( + `WITH wanted(id) AS (SELECT value FROM json_each(?)), + pairs(s, t, w) AS (${PAIRS_OF('wanted')}) + SELECT p.s AS source, p.t AS target, MAX(p.w) AS weight, l.x AS targetX, l.y AS targetY + FROM pairs p + INNER JOIN graph_layout l ON l.node_id = p.t + INNER JOIN graph_nodes nb ON nb.id = p.t + WHERE p.s != p.t AND l.placed = 1 ${clause} + GROUP BY p.s, p.t + ORDER BY weight DESC + LIMIT ?` + ) + .all(idSet(nodeIds), ...params, limit) as GraphTileEdgeRow[]; + }, + /** + * Edges leaving a crowded tile, rolled up to the cells at both ends. + * + * Same sources and the same filtering as `edgesTouching`, aggregated so a tile + * of ten thousand memories costs a few hundred lines rather than ten thousand. + * Edges that stay inside one cell are dropped: both ends are the same mark. + */ + cellEdges(filter: GraphListFilter, rect: GraphLayoutRect, cells: GraphLayoutGrid, limit: number): GraphTileCellEdgeRow[] { + const near = buildNodeFilter(filter, 'n.'); + const far = buildNodeFilter(filter, 'nb.'); + const nearClause = near.where.length ? `AND ${near.where.join(' AND ')}` : ''; + const farClause = far.where.length ? `AND ${far.where.join(' AND ')}` : ''; return getDatabase() .prepare( - `SELECT srcX, srcY, dstX, dstY, SUM(weight) AS weight + `WITH inside(id) AS ( + SELECT n.id FROM graph_layout l + INNER JOIN graph_nodes n ON n.id = l.node_id + WHERE l.placed = 1 AND l.x >= ? AND l.x < ? AND l.y >= ? AND l.y < ? ${nearClause} + ), + pairs(s, t, w) AS (${PAIRS_OF('inside')}) + SELECT sourceCellX, sourceCellY, targetCellX, targetCellY, SUM(w) AS weight FROM ( - SELECT CAST((la.x - ?) / ? AS INTEGER) AS srcX, - CAST((la.y - ?) / ? AS INTEGER) AS srcY, - CAST((lb.x - ?) / ? AS INTEGER) AS dstX, - CAST((lb.y - ?) / ? AS INTEGER) AS dstY, - e.weight AS weight - FROM graph_edges e - INNER JOIN graph_layout la ON la.node_id = e.src_id - INNER JOIN graph_layout lb ON lb.node_id = e.dst_id - INNER JOIN graph_nodes na ON na.id = e.src_id - INNER JOIN graph_nodes nb ON nb.id = e.dst_id - ${clause} + SELECT CAST((la.x - ?) / ? AS INTEGER) AS sourceCellX, + CAST((la.y - ?) / ? AS INTEGER) AS sourceCellY, + CAST((lb.x - ?) / ? AS INTEGER) AS targetCellX, + CAST((lb.y - ?) / ? AS INTEGER) AS targetCellY, + p.w AS w + FROM pairs p + INNER JOIN graph_layout la ON la.node_id = p.s + INNER JOIN graph_layout lb ON lb.node_id = p.t + INNER JOIN graph_nodes nb ON nb.id = p.t + WHERE p.s != p.t AND lb.placed = 1 ${farClause} ) - WHERE NOT (srcX = dstX AND srcY = dstY) + WHERE NOT (sourceCellX = targetCellX AND sourceCellY = targetCellY) GROUP BY 1, 2, 3, 4 ORDER BY weight DESC LIMIT ?` ) .all( - grid.originX, - grid.cellWidth, - grid.originY, - grid.cellHeight, - grid.originX, - grid.cellWidth, - grid.originY, - grid.cellHeight, - ...src.params, - ...dst.params, + rect.minX, + rect.maxX, + rect.minY, + rect.maxY, + ...near.params, + cells.originX, + cells.size, + cells.originY, + cells.size, + cells.originX, + cells.size, + cells.originY, + cells.size, + ...far.params, limit - ) as GraphBinEdgeRow[]; + ) as GraphTileCellEdgeRow[]; + }, + + /** + * Where particular memories are, whatever is loaded around them. + * + * What lets the view go TO something it has not drawn — a search result, a + * connection in the inspector, a step back through history. Unfiltered on + * purpose: the caller already decided these ids are worth showing. + */ + locate(nodeIds: string[]): GraphTileNodeRow[] { + if (nodeIds.length === 0) return []; + return getDatabase() + .prepare( + `SELECT n.id AS id, substr(n.label, 1, ?) AS label, l.community AS community, + l.degree AS degree, l.x AS x, l.y AS y + FROM json_each(?) ids + INNER JOIN graph_nodes n ON n.id = ids.value + INNER JOIN graph_layout l ON l.node_id = n.id + WHERE l.placed = 1` + ) + .all(TILE_LABEL_MAX, idSet(nodeIds)) as GraphTileNodeRow[]; + }, + + /** + * The degree that counts as "fully connected" for sizing, over what matches. + * + * The 95th percentile rather than the maximum — one hub with a hundred edges + * would otherwise flatten every ordinary node to the floor. Computed over the + * whole filtered set, never over a tile, or a memory would change size as the + * tiles around it loaded. + */ + degreeScale(filter: GraphListFilter): number { + const { where, params } = buildNodeFilter(filter, 'n.'); + const clause = where.length ? `AND ${where.join(' AND ')}` : ''; + const db = getDatabase(); + const count = ( + db + .prepare( + `SELECT COUNT(*) AS c FROM graph_layout l INNER JOIN graph_nodes n ON n.id = l.node_id + WHERE l.placed = 1 ${clause}` + ) + .get(...params) as { c: number } + ).c; + if (count === 0) return 1; + const row = db + .prepare( + `SELECT l.degree AS degree FROM graph_layout l INNER JOIN graph_nodes n ON n.id = l.node_id + WHERE l.placed = 1 ${clause} + ORDER BY l.degree ASC + LIMIT 1 OFFSET ?` + ) + .get(...params, Math.min(count - 1, Math.floor(count * 0.95))) as { degree: number } | null; + return Math.max(1, row?.degree ?? 1); } }; diff --git a/backend/memory/README.md b/backend/memory/README.md index d15a8528..c4f830ba 100644 --- a/backend/memory/README.md +++ b/backend/memory/README.md @@ -308,7 +308,8 @@ graph_node_entities subjects a memory is about (an ATTRIBUTE, not nodes) graph_node_paths files a memory is about (likewise) graph_vectors one int8 vector per node, 260 bytes at 256 dims graph_nodes_fts FTS5 mirror for BM25 -graph_layout persisted positions and communities (migration 067) +graph_layout persisted positions, communities and degree (067, 089) +graph_layout_edges derived edges the layout pass laid out on (089) memory_extraction_queue durable queue of turns awaiting summarisation ``` @@ -349,8 +350,9 @@ of sentences all shaped "X is a project that does Y", every pair is similar in shape rather than in subject, so 144 of one graph's 371 edges were pairings like *"Vantum monorepo phase 1.8"* ↔ *"ChatKit runs on Node.js 18+"*. The fault was never that they connected memories — it was that **cosine chose which**. Edges are -now derived at read time from the subjects extraction already recorded, and never -stored. +now derived from the subjects extraction already recorded. They are never stored +as relations; the layout pass keeps the set it laid out on in +`graph_layout_edges`, which is disposable like the rest of the arrangement. --- @@ -364,7 +366,8 @@ bootstrap.ts startup; idempotent, also re-run after Clear All Data retrieval.ts BM25 + vector, RRF fusion, spreading activation context.ts builds the injected block; standing instructions -view.ts graph serialization for the UI +layout.ts background arrangement: cold passes, warm arrival placement, the world +view.ts the graph view as map tiles, plus locate and node detail extract/ scheduler.ts durable queue, retries, concurrency diff --git a/backend/memory/layout.test.ts b/backend/memory/layout.test.ts new file mode 100644 index 00000000..c4dbba9a --- /dev/null +++ b/backend/memory/layout.test.ts @@ -0,0 +1,377 @@ +/** + * Memory graph layout + tile tests. + * + * Run against a REAL in-memory SQLite database with the graph migrations + * applied, because what is under test is the arrangement the pass leaves in the + * table and what the tile queries read back out of it. + * + * The regression these exist for: warm passes used to re-settle the whole map + * from its own previous result, which inflated it by ~1.4× per pass until the + * coordinates overflowed to NaN and every pass failed on the NOT NULL + * constraint. Stability across passes is therefore asserted EXACTLY, not + * approximately. + */ + +import { beforeEach, describe, expect, it, mock } from 'bun:test'; +import { Database } from 'bun:sqlite'; +import type { DatabaseConnection } from '$shared/types/database/connection'; +import * as migration005 from '$backend/database/migrations/005_create_settings_table'; +import * as migration066 from '$backend/database/migrations/066_create_memory_graph'; +import * as migration067 from '$backend/database/migrations/067_add_memory_graph_layout'; +import * as migration076 from '$backend/database/migrations/076_remove_memory_code_graph'; +import * as migration089 from '$backend/database/migrations/089_add_memory_layout_tiles'; + +let db: Database; + +// The whole module surface, not just `getDatabase` — see retrieval.test.ts. +mock.module('$backend/database', () => ({ + getDatabase: () => db, + initializeDatabase: async () => db, + closeDatabase: () => {}, + resetDatabase: async () => {}, + getDatabaseInfo: async () => ({}), + vacuumDatabase: async () => {} +})); + +// The pass announces its result over the WebSocket, which has no server here. +mock.module('$backend/memory/notify', () => ({ + notifyGraphChanged: () => {}, + broadcastGraphChanged: () => {}, + notifyMemoryStatus: () => {}, + notifyMemoryReadiness: () => {} +})); + +const { graphQueries, graphLayoutQueries } = await import('$backend/database/queries/graph-queries'); +const { runGraphLayout, resetGraphLayoutState, getLayoutState, alignTo } = await import('./layout'); +const { buildGraphMeta, buildGraphTiles, locateNodes } = await import('./view'); + +const PROJECT = 'project-a'; + +function resetDatabase(): void { + db = new Database(':memory:'); + db.exec('PRAGMA foreign_keys = ON'); + const connection = db as unknown as DatabaseConnection; + migration005.up(connection); + migration066.up(connection); + migration067.up(connection); + migration076.up(connection); + migration089.up(connection); + resetGraphLayoutState(); +} + +/** A store with structure: subjects that group memories, a few stored edges, and loners. */ +function seed(subjects = 6, perSubject = 12, loners = 8): string[] { + const ids: string[] = []; + for (let s = 0; s < subjects; s++) { + const members: string[] = []; + for (let m = 0; m < perSubject; m++) { + const node = graphQueries.upsert({ + subkind: 'decision', + projectId: PROJECT, + label: `Subject ${s} decision ${m}` + }); + graphQueries.setEntities(node.id, [{ key: `subject-${s}`, name: `Subject ${s}` }]); + members.push(node.id); + } + // A stored edge between neighbouring subjects, so the graph is one mass + // with lobes rather than six islands. + if (s > 0) graphQueries.link({ srcId: ids[ids.length - 1], dstId: members[0], rel: 'relates_to' }); + ids.push(...members); + } + for (let l = 0; l < loners; l++) { + ids.push(graphQueries.upsert({ subkind: 'observation', projectId: PROJECT, label: `Loner ${l}` }).id); + } + return ids; +} + +function positions(): Map { + const rows = db.prepare('SELECT node_id AS id, x, y, community FROM graph_layout').all() as { + id: string; + x: number; + y: number; + community: number; + }[]; + return new Map(rows.map(row => [row.id, { x: row.x, y: row.y, community: row.community }])); +} + +/** Force the next pass to do real work rather than skip on an unchanged fingerprint. */ +async function pass(): Promise { + resetGraphLayoutState(); + await runGraphLayout(); +} + +beforeEach(() => { + resetDatabase(); +}); + +describe('memory layout', () => { + it('lays out every live memory with finite positions inside the world', async () => { + const ids = seed(); + await pass(); + + const placed = positions(); + const state = getLayoutState(); + expect(placed.size).toBe(ids.length); + expect(state.world).not.toBeNull(); + expect(state.coldNodes).toBe(ids.length); + // What the view sizes nodes by: the median room a memory has. + expect(state.spacing).toBeGreaterThan(0); + expect(Number.isFinite(state.spacing)).toBe(true); + + const world = state.world!; + for (const point of placed.values()) { + expect(Number.isFinite(point.x) && Number.isFinite(point.y)).toBe(true); + expect(point.x).toBeGreaterThanOrEqual(world.minX); + expect(point.y).toBeGreaterThanOrEqual(world.minY); + expect(point.x).toBeLessThan(world.minX + world.size); + expect(point.y).toBeLessThan(world.minY + world.size); + } + // Derived edges are kept for the view to draw. + const derived = (db.prepare('SELECT COUNT(*) AS c FROM graph_layout_edges').get() as { c: number }).c; + expect(derived).toBeGreaterThan(0); + }); + + it('leaves the map exactly where it was across passes over unchanged data', async () => { + seed(); + await pass(); + const first = positions(); + const epoch = getLayoutState().epoch; + + for (let i = 0; i < 4; i++) await pass(); + + const later = positions(); + for (const [id, point] of first) expect(later.get(id)).toEqual(point); + expect(getLayoutState().epoch).toBe(epoch); + }); + + it('places arrivals next to what they connect to without moving anything else', async () => { + seed(6, 12, 8); + await pass(); + const before = positions(); + const stamp = getLayoutState().stamp; + + // Two arrivals about an existing subject — under the re-arrangement threshold. + const arrivals = [0, 1].map(i => { + const node = graphQueries.upsert({ subkind: 'decision', projectId: PROJECT, label: `Arrival ${i}` }); + graphQueries.setEntities(node.id, [{ key: 'subject-0', name: 'Subject 0' }]); + return node.id; + }); + await pass(); + + const after = positions(); + for (const [id, point] of before) expect(after.get(id)).toEqual(point); + for (const id of arrivals) { + const point = after.get(id); + expect(point).toBeDefined(); + expect(Number.isFinite(point!.x) && Number.isFinite(point!.y)).toBe(true); + // Next to what it connects to, not flung across the map: at the centre + // of the memories its edges lead to, within a neighbour's distance or + // two. Measured against its EDGES rather than its subject's members, + // because Louvain is free to split one subject across several lobes. + const partners = ( + db + .prepare( + `SELECT CASE WHEN src_id = ? THEN dst_id ELSE src_id END AS id FROM graph_layout_edges + WHERE src_id = ? OR dst_id = ?` + ) + .all(id, id, id) as { id: string }[] + ) + .map(row => row.id) + .filter(partner => !arrivals.includes(partner)) + .map(partner => before.get(partner)!); + expect(partners.length).toBeGreaterThan(0); + const cx = partners.reduce((sum, partner) => sum + partner.x, 0) / partners.length; + const cy = partners.reduce((sum, partner) => sum + partner.y, 0) / partners.length; + expect(Math.hypot(point!.x - cx, point!.y - cy)).toBeLessThan(getLayoutState().spacing * 2); + + // And it takes the colour most of them carry. + const votes = new Map(); + for (const partner of partners) votes.set(partner.community, (votes.get(partner.community) ?? 0) + 1); + const top = Math.max(...votes.values()); + expect(votes.get(point!.community)).toBe(top); + } + expect(getLayoutState().arrivals).toBe(2); + // The stamp is what tells a client its tiles are out of date. + expect(getLayoutState().stamp).toBeGreaterThan(stamp); + }); + + it('recovers from a diverged arrangement by laying out from scratch', async () => { + seed(); + await pass(); + const epoch = getLayoutState().epoch; + + // What the old warm passes compounded to: coordinates near the float range. + db.exec('UPDATE graph_layout SET x = x * 1e34, y = y * 1e34'); + await pass(); + + const bounds = graphLayoutQueries.bounds(); + expect(bounds).not.toBeNull(); + expect(bounds!.magnitude).toBeLessThan(1e6); + expect(getLayoutState().epoch).toBeGreaterThan(epoch); + }); + + it('re-arranges once the store outgrows its arrangement, keeping its colours', async () => { + const ids = seed(6, 12, 8); + await pass(); + const before = positions(); + const epoch = getLayoutState().epoch; + + // Well past a fifth of the store, all joining existing subjects. + for (let i = 0; i < 24; i++) { + const node = graphQueries.upsert({ subkind: 'pattern', projectId: PROJECT, label: `Growth ${i}` }); + graphQueries.setEntities(node.id, [{ key: `subject-${i % 6}`, name: `Subject ${i % 6}` }]); + } + await pass(); + + const state = getLayoutState(); + expect(state.epoch).toBeGreaterThan(epoch); + expect(state.arrivals).toBe(0); + expect(state.coldNodes).toBe(ids.length + 24); + + // Most memories keep the community NUMBER they had, so their colour holds + // even though Louvain numbered the new communities however it liked. + const after = positions(); + let sameColour = 0; + for (const [id, point] of before) if (after.get(id)?.community === point.community) sameColour++; + expect(sameColour / before.size).toBeGreaterThan(0.6); + + const bounds = graphLayoutQueries.bounds(); + expect(bounds!.magnitude).toBeLessThan(1e6); + }); + + it('turns a fresh arrangement to face the way the previous one did, without scaling it', () => { + const old = new Map(); + const fresh: { nodeId: string; community: number; x: number; y: number; placed: number; degree: number }[] = []; + const angle = 1.27; + for (let i = 0; i < 40; i++) { + const x = Math.cos(i * 0.7) * (100 + i * 13); + const y = Math.sin(i * 1.3) * (60 + i * 7); + old.set(`n${i}`, { nodeId: `n${i}`, community: 0, x, y, placed: 1, degree: 0 }); + // The same arrangement mirrored, rotated and moved — what a fresh + // simulation of the same graph is free to come back as. + const mx = x; + const my = -y; + fresh.push({ + nodeId: `n${i}`, + community: 0, + x: Math.cos(angle) * mx - Math.sin(angle) * my + 5_000, + y: Math.sin(angle) * mx + Math.cos(angle) * my - 3_000, + placed: 1, + degree: 0 + }); + } + + alignTo(fresh, old); + + for (const row of fresh) { + const target = old.get(row.nodeId)!; + expect(Math.abs(row.x - target.x)).toBeLessThan(1e-6); + expect(Math.abs(row.y - target.y)).toBeLessThan(1e-6); + } + }); + + it('gives archived memories a place so "show archived" can draw them', async () => { + const ids = seed(); + graphQueries.archive(ids[3]); + await pass(); + + expect(positions().has(ids[3])).toBe(true); + const meta = buildGraphMeta({ includeArchived: true }); + expect(meta.total).toBe(ids.length); + expect(meta.unplaced).toBe(0); + }); +}); + +describe('graph tiles', () => { + const allTiles = (z: number) => { + const tiles: { x: number; y: number }[] = []; + for (let x = 0; x < 2 ** z; x++) for (let y = 0; y < 2 ** z; y++) tiles.push({ x, y }); + return tiles; + }; + + it('serves every memory individually when the screen can afford them', async () => { + const ids = seed(); + await pass(); + const meta = buildGraphMeta({}); + + for (const z of [0, 1, 2]) { + const response = buildGraphTiles({ epoch: meta.epoch, z, tiles: allTiles(z), budget: 12_000 }); + expect(response.stale).toBe(false); + const nodes = response.tiles.flatMap(tile => tile.nodes.map(node => node.id)); + expect(new Set(nodes).size).toBe(ids.length); + expect(response.tiles.every(tile => tile.clusters.length === 0)).toBe(true); + } + }); + + it('folds crowded tiles when the budget is small, without losing a memory', async () => { + const ids = seed(8, 40, 10); + await pass(); + const meta = buildGraphMeta({}); + + const response = buildGraphTiles({ epoch: meta.epoch, z: 0, tiles: [{ x: 0, y: 0 }], budget: 256 }); + const [tile] = response.tiles; + expect(tile.cells).toBeGreaterThan(0); + expect(tile.clusters.length).toBeGreaterThan(0); + expect(tile.nodes.length + tile.clusters.length).toBeLessThanOrEqual(256); + + const covered = tile.nodes.length + tile.clusters.reduce((sum, cluster) => sum + cluster.members, 0); + expect(covered).toBe(ids.length); + }); + + it('answers stale instead of serving tiles from another map', async () => { + seed(); + await pass(); + const meta = buildGraphMeta({}); + + const response = buildGraphTiles({ epoch: meta.epoch + 1, z: 0, tiles: [{ x: 0, y: 0 }], budget: 12_000 }); + expect(response.stale).toBe(true); + expect(response.tiles).toHaveLength(0); + }); + + it('sends each edge once per request, with where its far end is', async () => { + seed(); + await pass(); + const meta = buildGraphMeta({}); + + const response = buildGraphTiles({ epoch: meta.epoch, z: 2, tiles: allTiles(2), budget: 12_000 }); + const pairs = response.tiles.flatMap(tile => + tile.edges.map(edge => (edge.source < edge.target ? `${edge.source}|${edge.target}` : `${edge.target}|${edge.source}`)) + ); + expect(pairs.length).toBeGreaterThan(0); + expect(new Set(pairs).size).toBe(pairs.length); + + const placed = positions(); + for (const tile of response.tiles) { + for (const edge of tile.edges) { + const far = placed.get(edge.target)!; + expect(Math.abs(edge.targetX - far.x)).toBeLessThan(0.1); + expect(Math.abs(edge.targetY - far.y)).toBeLessThan(0.1); + } + } + }); + + it('narrows tiles and counts by the filter', async () => { + seed(); + const other = graphQueries.upsert({ subkind: 'decision', projectId: 'project-b', label: 'Elsewhere' }); + await pass(); + + const meta = buildGraphMeta({ projectIds: ['project-b'] }); + expect(meta.total).toBe(1); + const response = buildGraphTiles({ + epoch: meta.epoch, + z: 0, + tiles: [{ x: 0, y: 0 }], + budget: 12_000, + projectIds: ['project-b'] + }); + expect(response.tiles[0].nodes.map(node => node.id)).toEqual([other.id]); + }); + + it('locates memories by id for going to one that is not drawn', async () => { + const ids = seed(); + await pass(); + const located = locateNodes([ids[5], 'missing-id']); + expect(located.map(node => node.id)).toEqual([ids[5]]); + expect(Number.isFinite(located[0].x)).toBe(true); + }); +}); diff --git a/backend/memory/layout.ts b/backend/memory/layout.ts index d92ac80e..76d0f5d2 100644 --- a/backend/memory/layout.ts +++ b/backend/memory/layout.ts @@ -4,32 +4,58 @@ * Community detection and force layout both used to run on the read path — one * on the server inside `buildGraphView`, the other in the browser on every open — * and both produce the SAME answer for the same dataset. They were being - * recomputed, not computed: opening the modal twice paid for the identical - * arrangement twice, and every conversation that recorded a memory made every - * connected client pay for it again. - * - * That is what made the modal's opening animation stutter once the graph got - * large. The animation is two hundred milliseconds of one thread, and a few - * hundred milliseconds of layout was landing inside it. - * - * So it happens HERE instead: once, in the background, written to `graph_layout`, - * and read back as two numbers per node. Reading a laid-out graph is then a - * table lookup whatever the graph costs to lay out, and the browser never runs a - * force simulation for the whole store again. - * - * ── Warm, not cold ──────────────────────────────────────────────────────────── - * A pass starts from where the last one finished. That is not only cheaper, it is - * the thing that makes the view stable to look at: a memory arriving must not - * rearrange the map somebody has learned. New nodes are seeded at the centroid of - * whatever they connect to and a short pass settles them into the gaps, exactly - * as the client's incremental path used to do for a single session. + * recomputed, not computed. So they happen HERE instead: in the background, + * written to `graph_layout`, and read back as a position per node. Reading a + * laid-out graph is then a table lookup whatever the graph costs to lay out. + * + * ── Two kinds of pass ───────────────────────────────────────────────────────── + * A COLD pass arranges every live memory from nothing: communities, a force + * simulation per component, lobes pushed apart, components packed. It is the + * only thing that decides the map's shape, and it is the expensive half — + * seconds for a few thousand memories. + * + * A WARM pass places what arrived since, and moves NOTHING that was already + * placed. New memories are dropped next to what they connect to and relaxed + * into the gaps around them by a small solver whose cost is the arrivals times + * their neighbourhood, not the store. That is what keeps a map somebody has + * learned in place while a conversation writes into it. + * + * ── Why a warm pass may not re-settle the map ──────────────────────────────── + * It used to, and the map grew without limit. A warm pass re-ran the simulation + * from the stored positions and then pushed the lobes apart again — but the + * stored positions were ALREADY pushed apart, and the simulation only pulls a + * part of that back. Measured on one unchanged store, every pass inflated the + * map by about 1.4×: 9.8k units, then 13.8k, 18.9k, 28.9k, 39.3k, 52.4k. Over + * weeks of passes that compounded to 10³⁸, where squaring a coordinate + * overflows to Infinity, Infinity / Infinity is NaN, and every pass after that + * failed on the NOT NULL constraint — so nothing written since was ever placed, + * and the overview binned five thousand memories into thirty-five marks because + * the extent it divided into a grid was astronomically wide. + * + * Re-settling has no fixed point to converge to, so it is not done at all. The + * map changes shape only on a cold pass, and a cold pass runs when the store has + * grown enough since the last one that the shape is worth revisiting (see + * `RECOLD_RATIO`) — or when the stored arrangement is unusable. + * + * ── A cold pass must not look like a new map ───────────────────────────────── + * Two things would make it one, and both are undone after the fact. Louvain + * numbers its communities arbitrarily, so colours would shuffle; the new + * communities are matched to the old ones by overlap and take their numbers. + * And a fresh simulation has no reason to face the same way as the last one; the + * result is rotated (or mirrored) and translated onto the previous arrangement — + * rigidly, never scaled, so the map cannot creep in size either way. + * + * ── The world ──────────────────────────────────────────────────────────────── + * The view is served as tiles of a fixed square, the WORLD, which covers every + * placed row with a margin. It is part of the stored state together with an + * EPOCH that changes whenever the world does — a cold pass, or an arrival that + * lands outside it — because a client's cached tiles and camera are expressed in + * it and have to know when they no longer mean the same thing. * * ── Bounded on purpose ──────────────────────────────────────────────────────── - * `MAX_LAYOUT_NODES` caps the simulation. A force layout is superlinear and the - * store grows without limit, so an uncapped pass would eventually take longer - * than the interval between the writes that trigger it. What survives the cap is - * what the view ranks highest anyway; anything below it is reachable by search - * and by expanding a lobe, neither of which needs a position. + * `MAX_LAYOUT_NODES` caps the simulation. What survives the cap is what the view + * ranks highest; everything below it is placed by the same solver that places + * arrivals, next to what it connects to. */ import Graph from 'graphology'; @@ -44,50 +70,92 @@ import { debug } from '$shared/utils/logger'; * What produced the stored arrangement. BUMP IT whenever that changes — either * the edges this pass lays out, or how it places them. * - * A pass is warm by default: every node is seeded at the position it already had - * and only lightly re-settled, which is what stops one new memory rearranging a - * map somebody has learned. That is exactly wrong after the pass itself changes, - * because the arrangement it would preserve was computed by different rules — - * when removing the code half took the store from 1,351 components to one, the - * surviving memories stayed in the shell those nine thousand nodes had shaped. - * - * So the marker is stored alongside the arrangement, and a mismatch drops the - * arrangement once. It costs one cold pass, which the store pays for anyway the - * first time it is laid out. + * A mismatch drops the arrangement once and lays the store out cold. Version 7 + * is the one that stopped warm passes re-settling the map; every arrangement + * written before it was inflated by them, and some of them to the point of being + * unreadable, so none of them is worth preserving. */ -const LAYOUT_VERSION = '6'; +const LAYOUT_VERSION = '8'; const LAYOUT_VERSION_KEY = 'memory_layout_version'; /** The key this used before it covered placement as well as derivation. */ const LEGACY_LAYOUT_KEY = 'memory_layout_derivation'; +/** Where the world, the epoch and the cold-pass bookkeeping live. */ +const LAYOUT_STATE_KEY = 'memory_layout_state'; /** * Two caps, because the two halves of a pass scale differently. * - * COMMUNITY DETECTION is roughly linear in the edges, and it is what the overview - * groups by — a node left out of it has no lobe to belong to and would vanish - * from the picture entirely, which is exactly the failure a cap is supposed to - * prevent. So it runs over everything live, up to a ceiling far beyond any real - * store. + * COMMUNITY DETECTION is roughly linear in the edges and colours every node, so + * it runs over everything live, up to a ceiling far beyond any real store. * * THE FORCE SIMULATION is superlinear and measurably so — around 15 ms per * iteration at 1,200 nodes and 90 ms at 5,000 — so it runs only over the - * highest-ranked nodes. The rest keep their real community and get a position - * near it, marked `placed = 0` so nothing downstream mistakes the guess for a - * computed arrangement. + * highest-ranked nodes. The rest are placed by the arrival solver next to what + * they connect to. */ const MAX_COMMUNITY_NODES = 50_000; const MAX_LAYOUT_NODES = 6_000; +/** + * How much the store may grow by warm placement before the shape is revisited. + * + * Arrivals are placed well, but locally: a subject that grows from three + * memories to three hundred still sits in the room its first three were given. + * Past a fifth of the store arriving since the last cold pass, the arrangement + * is describing a graph that no longer exists, and a cold pass is cheaper than + * the confusion. Low enough that small stores — where a cold pass is a few + * milliseconds — are re-arranged often. + */ +const RECOLD_RATIO = 0.2; + +/** + * Past this magnitude a coordinate is not an arrangement, it is a divergence. + * + * A healthy store spans thousands of units and a very large one tens of + * thousands; a stored row beyond this means some earlier pass went wrong, and + * the only sound response is to lay out from nothing. + */ +const MAX_SANE_COORDINATE = 1e7; + +/** Margin around the placed rows, as a fraction of the world's side. */ +const WORLD_MARGIN = 0.08; + /** * How long to wait after a write before laying out again. * * `notifyGraphChanged` already folds a turn's ingestion burst into one event at - * 600 ms; this sits on top because a layout pass is far more expensive than a - * broadcast and a user watching the graph does not need the new node to have - * found its final place within the second — it appears immediately either way, - * seeded next to what it connects to. + * 600 ms; this sits on top so a burst becomes one pass. Short, because a warm + * pass is cheap now — and until it runs a new memory has no position, so it is + * not on the map at all. */ -const DEBOUNCE_MS = 4_000; +const DEBOUNCE_MS = 1_500; + +/** The arrangement's bookkeeping, persisted next to it. */ +export interface LayoutState { + /** Changes whenever the world does, so clients know their tiles are stale. */ + epoch: number; + /** + * Changes whenever a pass writes anything — a new memory placed, an edge laid + * out. What a client compares to decide whether the tiles it holds still say + * the same thing; reinforcement rings the change doorbell constantly and moves + * none of this. + */ + stamp: number; + /** The square every tile is cut from, or null before anything is placed. */ + world: { minX: number; minY: number; size: number } | null; + /** + * The median distance from a live memory to its nearest neighbour — how much + * room a memory has on this map. The view sizes nodes from it, so a dot never + * outgrows the space around it at any zoom or store size. + */ + spacing: number; + /** Live memories the last cold pass arranged. */ + coldNodes: number; + /** Memories the arrival solver has placed since then. */ + arrivals: number; +} + +const EMPTY_STATE: LayoutState = { epoch: 0, stamp: 0, world: null, spacing: 0, coldNodes: 0, arrivals: 0 }; let timer: ReturnType | null = null; let running = false; @@ -95,24 +163,58 @@ let running = false; let rerunRequested = false; /** - * What the last pass laid out. + * What the last pass was computed from. * * Reinforcement bumps `weight` and `updated_at` on every recall, so the change - * notification fires constantly without the SET of nodes moving at all. Laying - * out an unchanged set produces an unchanged answer, so this skips it. + * notification fires constantly without the set of nodes or edges moving at + * all. Laying out an unchanged graph produces an unchanged answer, so this + * skips it. */ -let lastMembership = ''; +let lastFingerprint = ''; + +/** The state as last read or written, so the read path does not parse JSON per fetch. */ +let cachedState: LayoutState | null = null; /** Cheap order-sensitive fingerprint of an id list. */ function membershipOf(ids: string[]): string { - let hash = 2166136261; - for (const id of ids) { - for (let i = 0; i < id.length; i++) { - hash ^= id.charCodeAt(i); - hash = Math.imul(hash, 16777619); + return `${ids.length}:${hashOf(ids.join('\u0000')).toString(36)}`; +} + +/** + * The stored world and epoch — what the view tiles against. + * + * Read through a cache because every tile request asks; written only by a pass. + */ +export function getLayoutState(): LayoutState { + if (cachedState) return cachedState; + const raw = settingsQueries.get(LAYOUT_STATE_KEY)?.value; + let state = EMPTY_STATE; + if (raw) { + try { + const parsed = JSON.parse(raw) as Partial; + const world = parsed.world; + state = { + epoch: Number.isFinite(parsed.epoch) ? (parsed.epoch as number) : 0, + stamp: Number.isFinite(parsed.stamp) ? (parsed.stamp as number) : 0, + world: + world && [world.minX, world.minY, world.size].every(Number.isFinite) && world.size > 0 + ? { minX: world.minX, minY: world.minY, size: world.size } + : null, + spacing: Number.isFinite(parsed.spacing) ? (parsed.spacing as number) : 0, + coldNodes: Number.isFinite(parsed.coldNodes) ? (parsed.coldNodes as number) : 0, + arrivals: Number.isFinite(parsed.arrivals) ? (parsed.arrivals as number) : 0 + }; + } catch { + state = EMPTY_STATE; } } - return `${ids.length}:${(hash >>> 0).toString(36)}`; + cachedState = state; + return state; +} + +function saveLayoutState(state: LayoutState): void { + cachedState = state; + settingsQueries.set(LAYOUT_STATE_KEY, JSON.stringify(state)); } /** @@ -131,17 +233,17 @@ export function scheduleGraphLayout(): void { timer.unref?.(); } -/** Forget what was laid out, so the next pass rebuilds from nothing. */ +/** Forget what was laid out, so the next pass rebuilds from what the database holds. */ export function resetGraphLayoutState(): void { - lastMembership = ''; + lastFingerprint = ''; + cachedState = null; } /** - * Recompute communities and positions for the live graph. + * Bring the arrangement up to date with the live graph. * - * Never throws: a missing arrangement costs the client a seeded layout of its - * own, which is what it used to do for everything, so a failure here degrades to - * the previous behaviour instead of breaking the view. + * Never throws: a failed pass leaves the previous arrangement in place, and the + * memories it would have placed simply wait for the next one. */ export async function runGraphLayout(): Promise { if (running) { @@ -151,105 +253,100 @@ export async function runGraphLayout(): Promise { running = true; try { - // Rank-ordered, so "the first N" is also "the N the view would have shown". - const ids = graphLayoutQueries.liveNodeIds(MAX_COMMUNITY_NODES); - if (ids.length === 0) { - graphLayoutQueries.clear(); - lastMembership = membershipOf(ids); - return; - } - // Before anything reads the stored arrangement: if a different version of // this pass produced it, it is not an arrangement of this graph. if (settingsQueries.get(LAYOUT_VERSION_KEY)?.value !== LAYOUT_VERSION) { graphLayoutQueries.clear(); settingsQueries.set(LAYOUT_VERSION_KEY, LAYOUT_VERSION); settingsQueries.delete(LEGACY_LAYOUT_KEY); - lastMembership = ''; + const previous = getLayoutState(); + saveLayoutState({ ...EMPTY_STATE, epoch: previous.epoch + 1, stamp: previous.stamp + 1 }); + lastFingerprint = ''; debug.log('memory', 'Memory layout: the pass changed, arranging from scratch'); } - const membership = membershipOf(ids); - // Same nodes as last time and nothing waiting to be placed: the answer is - // the one already in the table. - if (membership === lastMembership && !graphLayoutQueries.hasUnplaced()) return; + // Rank-ordered, so "the first N" is also "the N the view would have shown". + const ids = graphLayoutQueries.liveNodeIds(MAX_COMMUNITY_NODES); + const state = getLayoutState(); - const started = Date.now(); - const existing = graphLayoutQueries.read(ids); - const graph = buildLayoutGraph(ids); - - // Louvain needs at least one edge; an edgeless graph leaves every node in - // its own community, which is also the honest answer. - let communities: Record = {}; - if (graph.size > 0) { - try { - // `rng` is NOT optional here, whatever the signature suggests. - // graphology-communities-louvain defaults to `Math.random` and walks the - // graph in random order, so the same store produced different lobes on - // every pass — and since everything downstream is seeded from the - // community structure, that alone made the whole map move. A fixed seed - // keeps the randomised traversal (which is what the algorithm wants) - // while making the answer reproducible. - communities = louvain(graph, { getEdgeWeight: 'weight', rng: seededRandom() }); - } catch (error) { - debug.warn('memory', 'Community detection failed; laying out without lobes', error); + if (ids.length === 0) { + graphLayoutQueries.clear(); + if (state.world) { + saveLayoutState({ ...EMPTY_STATE, epoch: state.epoch + 1, stamp: state.stamp + 1 }); + broadcastGraphChanged('layout'); } + lastFingerprint = ''; + return; } - const simulatedIds = ids.slice(0, MAX_LAYOUT_NODES); - const simulated = simulatedIds.length === ids.length ? graph : restrictTo(graph, simulatedIds); - - const cold = existing.size === 0; - const rows = await arrange(simulated, communities, existing, cold); - - // Everything below the simulation cap. It keeps its real community — so the - // view counts it and groups it correctly — and sits near the members of that - // community that WERE placed, spread by its own id so a lobe of ten thousand - // does not become one dot. `placed: 0` tells the view this is a - // neighbourhood rather than an arrangement. - if (simulatedIds.length < ids.length) { - const centroids = new Map(); - for (const row of rows) { - const centroid = centroids.get(row.community) ?? { x: 0, y: 0, count: 0 }; - centroid.x += row.x; - centroid.y += row.y; - centroid.count++; - centroids.set(row.community, centroid); - } + // Same nodes, same edges, and nothing waiting to be placed: the answer is the + // one already in the table. + const fingerprint = `${membershipOf(ids)}|${graphLayoutQueries.sourceFingerprint()}`; + if (fingerprint === lastFingerprint && !graphLayoutQueries.hasUnplaced()) return; - for (let i = simulatedIds.length; i < ids.length; i++) { - const id = ids[i]; - const community = communities[id] ?? 0; - const centroid = centroids.get(community); - const anchor = centroid - ? { x: centroid.x / centroid.count, y: centroid.y / centroid.count } - : { x: 0, y: 0 }; - const radius = centroid ? 40 + 8 * Math.sqrt(centroid.count) : 200; - const jitter = jitterOf(id); + const started = Date.now(); + const existing = graphLayoutQueries.read(ids); + const { graph, derived } = buildLayoutGraph(ids); - rows.push({ - nodeId: id, - community, - x: anchor.x + jitter.x * radius, - y: anchor.y + jitter.y * radius, - placed: 0 - }); - } + const placed = new Map(); + for (const [id, row] of existing) { + if (row.placed === 1 && Number.isFinite(row.x) && Number.isFinite(row.y)) placed.set(id, row); } - graphLayoutQueries.writeMany(rows); + const bounds = graphLayoutQueries.bounds(); + const corrupt = bounds !== null && !(bounds.magnitude <= MAX_SANE_COORDINATE); + const arriving = ids.length - placed.size; + const reason = corrupt + ? 'the stored arrangement diverged' + : placed.size === 0 || state.world === null + ? 'nothing is placed yet' + : state.arrivals + arriving > RECOLD_RATIO * Math.max(1, state.coldNodes) + ? 'the store outgrew its arrangement' + : null; + + const rows = reason + ? await coldPass(ids, graph, corrupt ? new Map() : placed) + : warmPass(ids, graph, placed); + + // Retired memories are drawn by the "show archived" view, so they need a + // place too. A cold pass draws a new map and re-places every one of them; a + // warm pass only places those that have none yet, and leaves the rest where + // they were, which is still the same map. + const retired = placeRetired(rows, reason !== null); + + const written = sanitise([...rows, ...retired]); + const liveIds = new Set(ids); + const world = worldFor(written, reason ? null : state.world); + const worldChanged = + !state.world || + world.minX !== state.world.minX || + world.minY !== state.world.minY || + world.size !== state.world.size; + + graphLayoutQueries.writeMany(written); + graphLayoutQueries.replaceDerivedEdges(derived); graphLayoutQueries.pruneOrphans(); - lastMembership = membership; - // Tell open views, or they never learn. The arrangement changes several - // seconds AFTER the write that triggered it — long after the doorbell for - // that write has been answered — so without this a modal that was open the - // whole time keeps drawing positions it computed for itself, and only a - // close-and-reopen picks up the real ones. + saveLayoutState({ + epoch: reason || worldChanged ? state.epoch + 1 : state.epoch, + stamp: state.stamp + 1, + world, + spacing: medianSpacing(written.filter(row => liveIds.has(row.nodeId))), + coldNodes: reason ? ids.length : state.coldNodes, + arrivals: reason ? 0 : state.arrivals + arriving + }); + lastFingerprint = fingerprint; + + // Tell open views, or they never learn. The arrangement changes a moment + // AFTER the write that triggered it — after the doorbell for that write has + // been answered — so without this a new memory would stay off the map until + // the modal was reopened. broadcastGraphChanged('layout'); debug.log( 'memory', - `Laid out ${simulated.order} of ${rows.length} memory node(s) in ${Date.now() - started}ms` + reason + ? `Laid out ${ids.length} memory node(s) from scratch (${reason}) in ${Date.now() - started}ms` + : `Placed ${arriving} arriving memory node(s) in ${Date.now() - started}ms` ); } catch (error) { debug.warn('memory', 'Memory graph layout failed (non-fatal)', error); @@ -262,14 +359,640 @@ export async function runGraphLayout(): Promise { } } +/** + * Arrange every live memory from nothing, then make it look like the map it + * replaces. + * + * `previous` is the last arrangement's placed rows, empty when there was none or + * it was unusable — in which case there is nothing to stay faithful to and the + * fresh arrangement stands as computed. + */ +async function coldPass( + ids: string[], + graph: Graph, + previous: Map +): Promise { + // Louvain needs at least one edge; an edgeless graph leaves every node in its + // own community, which is also the honest answer. + let communities: Record = {}; + if (graph.size > 0) { + try { + // `rng` is NOT optional here, whatever the signature suggests. + // graphology-communities-louvain defaults to `Math.random` and walks the + // graph in random order, so the same store produced different lobes on + // every pass. A fixed seed keeps the randomised traversal (which is what + // the algorithm wants) while making the answer reproducible. + communities = louvain(graph, { getEdgeWeight: 'weight', rng: seededRandom() }); + } catch (error) { + debug.warn('memory', 'Community detection failed; laying out without lobes', error); + } + } + for (const id of ids) communities[id] ??= 0; + communities = keepCommunityNumbers(ids, communities, previous); + + const simulatedIds = ids.slice(0, MAX_LAYOUT_NODES); + const simulated = simulatedIds.length === ids.length ? graph : restrictTo(graph, simulatedIds); + const arranged = await arrange(simulated, communities); + + alignTo(arranged, previous); + + // Everything below the simulation cap is placed the way an arrival is: next to + // what it connects to, among what the simulation already placed. + const positions = new Map(); + for (const row of arranged) positions.set(row.nodeId, { x: row.x, y: row.y }); + const remaining = ids.slice(simulatedIds.length); + const solved = placeArrivals(remaining, graph, positions, communities); + + return [ + ...arranged.map(row => ({ ...row, degree: graph.degree(row.nodeId) })), + ...remaining.map(id => { + const at = solved.positions.get(id) as { x: number; y: number }; + return { nodeId: id, community: communities[id], x: at.x, y: at.y, placed: 1, degree: graph.degree(id) }; + }) + ]; +} + +/** + * Place what arrived since the last pass, leaving everything else untouched. + * + * Every existing row is rewritten with its own position and community — only + * its degree moves, because a new memory changes how connected its neighbours + * are. + */ +function warmPass(ids: string[], graph: Graph, placed: Map): GraphLayoutRow[] { + const positions = new Map(); + const communities: Record = {}; + for (const [id, row] of placed) { + positions.set(id, { x: row.x, y: row.y }); + communities[id] = row.community; + } + + const arriving = ids.filter(id => !placed.has(id)); + const solved = placeArrivals(arriving, graph, positions, communities); + + return ids.map(id => { + const row = placed.get(id); + if (row) return { ...row, degree: graph.degree(id) }; + const at = solved.positions.get(id) as { x: number; y: number }; + return { + nodeId: id, + community: solved.communities[id] ?? 0, + x: at.x, + y: at.y, + placed: 1, + degree: graph.degree(id) + }; + }); +} + +/** + * Place archived and superseded memories among the live ones. + * + * They are not part of the layout graph — the pass arranges what is live — so + * their neighbours are read from the stored edges, and one with no live + * neighbour is placed in the field like any lone memory. They never move a live + * memory: the live rows are the fixed half of the solve. + */ +function placeRetired(live: GraphLayoutRow[], all: boolean): GraphLayoutRow[] { + const retired = graphLayoutQueries.retiredIds(all); + if (retired.length === 0) return []; + + const positions = new Map(); + const communities: Record = {}; + for (const row of live) { + positions.set(row.nodeId, { x: row.x, y: row.y }); + communities[row.nodeId] = row.community; + } + + const graph = new Graph({ type: 'undirected', multi: false }); + for (const row of live) graph.addNode(row.nodeId); + for (const id of retired) if (!graph.hasNode(id)) graph.addNode(id); + for (const edge of graphQueries.edgesWithin([...retired, ...live.map(row => row.nodeId)])) { + if (edge.srcId === edge.dstId || graph.hasEdge(edge.srcId, edge.dstId)) continue; + graph.addUndirectedEdge(edge.srcId, edge.dstId, { weight: edge.weight }); + } + + const solved = placeArrivals(retired, graph, positions, communities); + return retired.map(id => { + const at = solved.positions.get(id) as { x: number; y: number }; + return { + nodeId: id, + community: solved.communities[id] ?? 0, + x: at.x, + y: at.y, + placed: 1, + degree: graph.degree(id) + }; + }); +} + +/** + * Only finite rows are written — the last line of defence for the table. + * + * Nothing upstream should produce anything else, and that is exactly why this + * exists: the failure it replaces was a single NaN that made every subsequent + * pass throw on the NOT NULL constraint, permanently, with the store growing + * unplaced behind it. A row dropped here stays unplaced and is retried. + */ +function sanitise(rows: GraphLayoutRow[]): GraphLayoutRow[] { + const kept = rows.filter( + row => + Number.isFinite(row.x) && + Number.isFinite(row.y) && + Math.abs(row.x) <= MAX_SANE_COORDINATE && + Math.abs(row.y) <= MAX_SANE_COORDINATE && + Number.isFinite(row.community) + ); + if (kept.length < rows.length) { + debug.warn('memory', `Memory layout: dropped ${rows.length - kept.length} unusable position(s)`); + } + return kept; +} + +/** + * The median distance from a memory to its nearest neighbour. + * + * Found through a spatial hash whose cell is grown until most memories have a + * neighbour within reach, so the cost stays linear whatever the scale. + */ +function medianSpacing(rows: { x: number; y: number }[]): number { + if (rows.length < 2) return 0; + let minX = Infinity; + let maxX = -Infinity; + let minY = Infinity; + let maxY = -Infinity; + for (const row of rows) { + minX = Math.min(minX, row.x); + maxX = Math.max(maxX, row.x); + minY = Math.min(minY, row.y); + maxY = Math.max(maxY, row.y); + } + // The spacing a uniform scatter would have: a starting cell, not the answer. + const cell = Math.max(1e-6, Math.sqrt(((maxX - minX) * (maxY - minY)) / rows.length) || 1); + const grid = new Map(); + for (const row of rows) { + const key = `${Math.floor(row.x / cell)}:${Math.floor(row.y / cell)}`; + const bucket = grid.get(key); + if (bucket) bucket.push(row); + else grid.set(key, [row]); + } + + const nearest: number[] = []; + for (const row of rows) { + const cx = Math.floor(row.x / cell); + const cy = Math.floor(row.y / cell); + let best = Infinity; + // Ring by ring until something is found and no closer ring can beat it. + for (let ring = 1; ring <= 64; ring++) { + for (let gx = cx - ring; gx <= cx + ring; gx++) { + for (let gy = cy - ring; gy <= cy + ring; gy++) { + if (Math.max(Math.abs(gx - cx), Math.abs(gy - cy)) !== ring && ring > 1) continue; + for (const other of grid.get(`${gx}:${gy}`) ?? []) { + if (other === row) continue; + best = Math.min(best, Math.hypot(other.x - row.x, other.y - row.y)); + } + } + } + if (best <= (ring - 1) * cell) break; + } + if (Number.isFinite(best)) nearest.push(best); + } + if (nearest.length === 0) return 0; + nearest.sort((a, b) => a - b); + return nearest[nearest.length >> 1]; +} + +/** + * The square the view tiles, covering every row with a margin. + * + * `current` is kept whenever everything still fits inside it, so a warm pass + * does not invalidate every client's tiles for a memory that landed in the + * middle of the map. When something lands outside, the world is recomputed with + * the margin restored — growing in steps rather than per arrival. + */ +function worldFor( + rows: GraphLayoutRow[], + current: { minX: number; minY: number; size: number } | null +): { minX: number; minY: number; size: number } { + let minX = Infinity; + let maxX = -Infinity; + let minY = Infinity; + let maxY = -Infinity; + for (const row of rows) { + minX = Math.min(minX, row.x); + maxX = Math.max(maxX, row.x); + minY = Math.min(minY, row.y); + maxY = Math.max(maxY, row.y); + } + if (!Number.isFinite(minX)) return current ?? { minX: -500, minY: -500, size: 1000 }; + + if ( + current && + minX >= current.minX && + minY >= current.minY && + maxX < current.minX + current.size && + maxY < current.minY + current.size + ) { + return current; + } + + const side = Math.max(maxX - minX, maxY - minY, 200); + const size = side * (1 + 2 * WORLD_MARGIN); + return { + minX: (minX + maxX) / 2 - size / 2, + minY: (minY + maxY) / 2 - size / 2, + size + }; +} + +/** + * Give the new communities the numbers the old ones had, wherever they overlap. + * + * Colour is derived from the community number, and Louvain's numbering is + * arbitrary: without this every cold pass would recolour the whole map even + * when the lobes were the same lobes. Largest communities choose first, each + * taking the old number most of its members carried; anything with no + * predecessor gets a number nothing has used. + */ +function keepCommunityNumbers( + ids: string[], + communities: Record, + previous: Map +): Record { + if (previous.size === 0) return communities; + + const members = new Map(); + for (const id of ids) { + const community = communities[id]; + const list = members.get(community); + if (list) list.push(id); + else members.set(community, [id]); + } + + let next = 0; + for (const row of previous.values()) next = Math.max(next, row.community + 1); + for (const community of members.keys()) next = Math.max(next, community + 1); + + const taken = new Set(); + const renumber = new Map(); + const ordered = [...members.entries()].sort((a, b) => b[1].length - a[1].length || a[0] - b[0]); + + for (const [community, list] of ordered) { + const votes = new Map(); + for (const id of list) { + const old = previous.get(id); + if (old) votes.set(old.community, (votes.get(old.community) ?? 0) + 1); + } + let best: number | null = null; + let bestVotes = 0; + for (const [old, count] of [...votes.entries()].sort((a, b) => a[0] - b[0])) { + if (taken.has(old) || count <= bestVotes) continue; + best = old; + bestVotes = count; + } + const assigned = best ?? next++; + taken.add(assigned); + renumber.set(community, assigned); + } + + const result: Record = {}; + for (const id of ids) result[id] = renumber.get(communities[id]) ?? communities[id]; + return result; +} + +/** + * Turn a fresh arrangement to face the way the previous one did. + * + * Orthogonal Procrustes over the memories both arrangements placed: the + * rotation — or rotation with a mirror, whichever fits better — and translation + * that bring the new positions closest to the old. RIGID: no scaling, so the + * map's size is always the simulation's own and can never ratchet the way the + * old warm passes did. + * + * @internal Exported for tests. + */ +export function alignTo(rows: GraphLayoutRow[], previous: Map): void { + const pairs: { nx: number; ny: number; ox: number; oy: number }[] = []; + for (const row of rows) { + const old = previous.get(row.nodeId); + if (old) pairs.push({ nx: row.x, ny: row.y, ox: old.x, oy: old.y }); + } + if (pairs.length < 3) return; + + let ncx = 0; + let ncy = 0; + let ocx = 0; + let ocy = 0; + for (const pair of pairs) { + ncx += pair.nx; + ncy += pair.ny; + ocx += pair.ox; + ocy += pair.oy; + } + ncx /= pairs.length; + ncy /= pairs.length; + ocx /= pairs.length; + ocy /= pairs.length; + + let sxx = 0; + let sxy = 0; + let syx = 0; + let syy = 0; + for (const pair of pairs) { + const ax = pair.nx - ncx; + const ay = pair.ny - ncy; + const bx = pair.ox - ocx; + const by = pair.oy - ocy; + sxx += ax * bx; + sxy += ax * by; + syx += ay * bx; + syy += ay * by; + } + + // The best proper rotation maximises cos θ (Sxx + Syy) + sin θ (Sxy − Syx); + // mirroring the new arrangement in y first flips the signs of Syx and Syy. + const rotation = Math.hypot(sxx + syy, sxy - syx); + const mirrored = Math.hypot(sxx - syy, sxy + syx); + const mirror = mirrored > rotation; + const angle = mirror ? Math.atan2(sxy + syx, sxx - syy) : Math.atan2(sxy - syx, sxx + syy); + const cos = Math.cos(angle); + const sin = Math.sin(angle); + + for (const row of rows) { + const ax = row.x - ncx; + const ay = mirror ? -(row.y - ncy) : row.y - ncy; + row.x = ocx + cos * ax - sin * ay; + row.y = ocy + sin * ax + cos * ay; + } +} + +/** + * Place memories among an arrangement that does not move. + * + * Each is dropped at the centroid of whatever it connects to that already has a + * place — so a chain of arrivals resolves outward from whichever end touches the + * map — and one connected to nothing placed is dropped into the field the map + * already covers. Then a short relaxation moves ONLY the arrivals: pulled to an + * edge's natural length from their neighbours, pushed off anything closer than + * that. The natural length is measured from the arrangement itself, so the + * solver works at whatever scale the map is drawn at. + * + * Cost is the arrivals times their neighbourhood. Repulsion reads a spatial + * hash, never the whole store, which is what makes a warm pass milliseconds + * where re-running the simulation was seconds. + * + * Deterministic: ids are visited in sorted order and every scatter comes from + * `jitterOf`. + */ +function placeArrivals( + arrivals: string[], + graph: Graph, + fixed: Map, + knownCommunities: Record +): { positions: Map; communities: Record } { + const positions = new Map(fixed); + const communities: Record = { ...knownCommunities }; + if (arrivals.length === 0) return { positions, communities }; + + const neighboursOf = (id: string): string[] => (graph.hasNode(id) ? graph.neighbors(id) : []); + // Two scales, because a declumped map has two. Edges are long — they span a + // lobe — but neighbours sit only `spacing` apart, and a solver that kept an + // edge's length of clear space around each arrival found no room left inside + // a lobe and pushed newcomers out of it, sometimes nearer a different subject + // than their own. So arrivals are pulled to within a few spacings of what + // they connect to and keep only a neighbour's distance clear. + const spacing = medianSpacing([...fixed.values()]); + const length = spacing > 0 ? Math.min(naturalLength(graph, fixed), spacing * 1.5) : naturalLength(graph, fixed); + const personal = spacing > 0 ? spacing * 0.6 : length * 0.75; + + let nextCommunity = 0; + for (const value of Object.values(communities)) nextCommunity = Math.max(nextCommunity, value + 1); + + // The field: where the map already is, so a lone memory lands inside it rather + // than deciding how far it reaches. + let centreX = 0; + let centreY = 0; + for (const point of fixed.values()) { + centreX += point.x; + centreY += point.y; + } + if (fixed.size > 0) { + centreX /= fixed.size; + centreY /= fixed.size; + } + let reach = 0; + for (const point of fixed.values()) reach = Math.max(reach, Math.hypot(point.x - centreX, point.y - centreY)); + if (reach === 0) reach = length * Math.sqrt(arrivals.length + 1) * 2; + + const pending = [...arrivals].sort(); + const waiting = new Set(pending); + + while (waiting.size > 0) { + // Everything that touches the map, repeatedly, until nothing new does. + let progressed = true; + while (progressed) { + progressed = false; + for (const id of pending) { + if (!waiting.has(id)) continue; + let sumX = 0; + let sumY = 0; + let count = 0; + const votes = new Map(); + for (const neighbour of neighboursOf(id)) { + const at = positions.get(neighbour); + if (!at) continue; + sumX += at.x; + sumY += at.y; + count++; + const community = communities[neighbour]; + if (community !== undefined) votes.set(community, (votes.get(community) ?? 0) + 1); + } + if (count === 0) continue; + + const jitter = jitterOf(id); + positions.set(id, { x: sumX / count + jitter.x * length * 0.6, y: sumY / count + jitter.y * length * 0.6 }); + if (communities[id] === undefined) { + let best = -1; + let bestVotes = 0; + for (const [community, votesFor] of [...votes.entries()].sort((a, b) => a[0] - b[0])) { + if (votesFor > bestVotes) { + best = community; + bestVotes = votesFor; + } + } + communities[id] = best >= 0 ? best : nextCommunity++; + } + waiting.delete(id); + progressed = true; + } + } + + // One that connects to nothing placed seeds its own little group in the + // field; the loop above then grows the rest of that group around it. + const seed = pending.find(id => waiting.has(id)); + if (seed === undefined) break; + const draw = jitterOf(seed); + const radius = reach * Math.sqrt((draw.x + 1) / 2); + positions.set(seed, { + x: centreX + Math.cos(draw.angle) * radius, + y: centreY + Math.sin(draw.angle) * radius + }); + communities[seed] ??= nextCommunity++; + waiting.delete(seed); + } + + relaxArrivals(pending, graph, fixed, positions, length, personal); + return { positions, communities }; +} + +/** + * The typical edge length of an arrangement — the scale the solver works at. + * + * The median over edges both of whose ends are placed, so a handful of long + * bridges between lobes does not inflate it. Falls back to the spacing a + * uniform scatter of the placed nodes would have, and to a constant when there + * is nothing placed at all. + */ +function naturalLength(graph: Graph, fixed: Map): number { + const lengths: number[] = []; + graph.forEachEdge((_edge, _attributes, source, target) => { + if (lengths.length >= 20_000) return; + const a = fixed.get(source); + const b = fixed.get(target); + if (!a || !b) return; + const distance = Math.hypot(a.x - b.x, a.y - b.y); + if (distance > 0 && Number.isFinite(distance)) lengths.push(distance); + }); + if (lengths.length >= 8) { + lengths.sort((a, b) => a - b); + return lengths[lengths.length >> 1]; + } + + if (fixed.size >= 2) { + let minX = Infinity; + let maxX = -Infinity; + let minY = Infinity; + let maxY = -Infinity; + for (const point of fixed.values()) { + minX = Math.min(minX, point.x); + maxX = Math.max(maxX, point.x); + minY = Math.min(minY, point.y); + maxY = Math.max(maxY, point.y); + } + const area = Math.max(1, (maxX - minX) * (maxY - minY)); + return Math.max(10, Math.sqrt(area / fixed.size)); + } + return 40; +} + +/** + * Settle the arrivals into the gaps, moving nothing else. + * + * Spring toward each neighbour to `length`, push off anything nearer than + * `personal`, and never move further than half of `length` in one step — the + * bound is what makes it impossible for this to diverge the way an unbounded + * force step can. + */ +function relaxArrivals( + arrivals: string[], + graph: Graph, + fixed: Map, + positions: Map, + length: number, + personal: number +): void { + const ITERATIONS = 40; + const cell = Math.max(1e-6, personal); + const key = (x: number, y: number): string => `${Math.floor(x / cell)}:${Math.floor(y / cell)}`; + + const fixedGrid = new Map(); + for (const point of fixed.values()) { + const k = key(point.x, point.y); + const list = fixedGrid.get(k); + if (list) list.push(point); + else fixedGrid.set(k, [point]); + } + + const moving = arrivals.filter(id => positions.has(id) && !fixed.has(id)); + if (moving.length === 0) return; + + for (let iteration = 0; iteration < ITERATIONS; iteration++) { + const cooling = 1 - iteration / ITERATIONS; + + const arrivalGrid = new Map(); + for (const id of moving) { + const at = positions.get(id) as { x: number; y: number }; + const k = key(at.x, at.y); + const list = arrivalGrid.get(k); + if (list) list.push(id); + else arrivalGrid.set(k, [id]); + } + + for (const id of moving) { + const at = positions.get(id) as { x: number; y: number }; + let dx = 0; + let dy = 0; + + for (const neighbour of graph.hasNode(id) ? graph.neighbors(id) : []) { + const other = positions.get(neighbour); + if (!other) continue; + const ox = other.x - at.x; + const oy = other.y - at.y; + const distance = Math.hypot(ox, oy); + if (distance < 1e-9) continue; + const pull = ((distance - length) / distance) * 0.3; + dx += ox * pull; + dy += oy * pull; + } + + const cx = Math.floor(at.x / cell); + const cy = Math.floor(at.y / cell); + for (let gx = cx - 1; gx <= cx + 1; gx++) { + for (let gy = cy - 1; gy <= cy + 1; gy++) { + const k = `${gx}:${gy}`; + const push = (other: { x: number; y: number }, salt: string): void => { + let ox = at.x - other.x; + let oy = at.y - other.y; + let distance = Math.hypot(ox, oy); + if (distance >= personal) return; + if (distance < 1e-9) { + const angle = jitterOf(id + salt).angle; + ox = Math.cos(angle); + oy = Math.sin(angle); + distance = 1; + } + const strength = ((personal - distance) / distance) * 0.5; + dx += ox * strength; + dy += oy * strength; + }; + for (const point of fixedGrid.get(k) ?? []) push(point, `${point.x},${point.y}`); + for (const other of arrivalGrid.get(k) ?? []) { + if (other !== id) push(positions.get(other) as { x: number; y: number }, other); + } + } + } + + const step = Math.hypot(dx, dy); + const limit = length * 0.5 * cooling; + if (!Number.isFinite(step) || step === 0) continue; + const scale = step > limit ? limit / step : 1; + positions.set(id, { x: at.x + dx * scale, y: at.y + dy * scale }); + } + } +} + /** * The graph the simulation runs on: stored edges plus the derived ones. * * Both are included because the derived edges are most of the structure the * episodic half has (see `graphQueries.derivedEdges`) — laying out without them - * scatters memories that plainly belong together. + * scatters memories that plainly belong together. The derived set is returned as + * well, because the view draws exactly the edges the map was laid out on. */ -function buildLayoutGraph(ids: string[]): Graph { +function buildLayoutGraph(ids: string[]): { + graph: Graph; + derived: { srcId: string; dstId: string; weight: number }[]; +} { const graph = new Graph({ type: 'undirected', multi: false }); for (const id of ids) graph.addNode(id, { x: 0, y: 0, size: 1 }); @@ -281,14 +1004,15 @@ function buildLayoutGraph(ids: string[]): Graph { }; for (const edge of graphQueries.edgesWithin(ids)) link(edge.srcId, edge.dstId, edge.weight); - for (const edge of graphQueries.derivedEdges(ids)) link(edge.srcId, edge.dstId, edge.weight); + const derived = graphQueries.derivedEdges(ids); + for (const edge of derived) link(edge.srcId, edge.dstId, edge.weight); // ForceAtlas2 reads mass from `size`; a hub that weighs more holds its // neighbours' fan open instead of being dragged into the middle of it. graph.forEachNode(id => { graph.setNodeAttribute(id, 'size', 1 + Math.sqrt(graph.degree(id))); }); - return graph; + return { graph, derived }; } /** The induced subgraph over `ids` — the part the simulation will actually move. */ @@ -345,12 +1069,7 @@ function restrictTo(graph: Graph, ids: string[]): Graph { * (Σnᵢ)². On the store measured above that is 1.0M against 3.2M — the shape is * better AND the pass is a third of the work. */ -async function arrange( - simulated: Graph, - communities: Record, - existing: Map, - cold: boolean -): Promise { +async function arrange(simulated: Graph, communities: Record): Promise { const components = componentsOf(simulated); const singletons: string[] = []; const placed: PlacedComponent[] = []; @@ -362,15 +1081,12 @@ async function arrange( } const part = restrictTo(simulated, member); - // A component that was already on the map keeps its place. Only its interior - // is re-settled, so a memory arriving in one lobe cannot rearrange the - // others — the map somebody has learned survives the pass. - const anchor = existingCentroid(member, existing); - seedComponent(part, communities, existing, anchor); - await settle(part, cold || !anchor ? iterationsFor(part.order) : warmIterationsFor(part.order)); + seedComponent(part, communities); + await settle(part, iterationsFor(part.order)); + declump(part, communities); const lobes = spreadCommunities(part, communities); - placed.push(measure(part, anchor, lobes)); + placed.push(measure(part, lobes)); } packComponents(placed); @@ -383,7 +1099,9 @@ async function arrange( community: communities[component.ids[i]] ?? 0, x: component.xs[i] + component.offsetX, y: component.ys[i] + component.offsetY, - placed: 1 + placed: 1, + // Filled in by the caller, which holds the whole graph. + degree: 0 }); } } @@ -398,10 +1116,106 @@ async function arrange( } } - for (const row of placeLoneNodes(singletons, field, communities, existing)) rows.push(row); + for (const row of placeLoneNodes(singletons, field, communities)) rows.push(row); return rows; } +/** + * Give every memory room of its own, at the scale the lobe was drawn at. + * + * ForceAtlas2 with mass-distributed attraction pulls a lobe's members onto its + * hubs, so a settled lobe is a dense core with a thin rim. Measured on a real + * store, the median distance to the nearest neighbour was 46 units against a + * median edge of 416 — at the zoom that shows the whole map that is under two + * pixels, so the cores drew as solid blobs and nothing inside them could be + * told apart, hovered or clicked. + * + * So members closer than `MIN_SPACING` of the lobe's own typical edge are pushed + * apart, pairwise, until none are — which grows the dense cores and leaves the + * already-spacious rims where they were. Measured against the component's + * INTRA-lobe edges, so the spacing follows the scale this layout happened to + * settle at rather than a constant that is right for one store. + * + * Neighbours are found through a spatial hash, so a pass is linear in the + * members; deterministic, because every tie is broken by id. + */ +function declump(part: Graph, communities: Record): void { + /** Clear space between two members, as a fraction of the lobe's median edge. */ + const MIN_SPACING = 0.8; + /** + * A ceiling, not a count: the loop stops the moment nothing overlaps. Dense + * cores need many — each pass only resolves the overlaps it sees, and pushing + * one pair apart presses both into their other neighbours. + */ + const PASSES = 400; + + const lengths: number[] = []; + part.forEachUndirectedEdge((_edge, _attributes, source, target) => { + if ((communities[source] ?? 0) !== (communities[target] ?? 0)) return; + const a = part.getNodeAttributes(source) as { x: number; y: number }; + const b = part.getNodeAttributes(target) as { x: number; y: number }; + const length = Math.hypot(a.x - b.x, a.y - b.y); + if (length > 0 && Number.isFinite(length)) lengths.push(length); + }); + if (lengths.length === 0) return; + lengths.sort((a, b) => a - b); + const spacing = lengths[lengths.length >> 1] * MIN_SPACING; + if (!(spacing > 0)) return; + + const ids = part.nodes().sort(); + const xs = new Float64Array(ids.length); + const ys = new Float64Array(ids.length); + for (let i = 0; i < ids.length; i++) { + xs[i] = part.getNodeAttribute(ids[i], 'x') as number; + ys[i] = part.getNodeAttribute(ids[i], 'y') as number; + } + + for (let pass = 0; pass < PASSES; pass++) { + const grid = new Map(); + for (let i = 0; i < ids.length; i++) { + const key = `${Math.floor(xs[i] / spacing)}:${Math.floor(ys[i] / spacing)}`; + const bucket = grid.get(key); + if (bucket) bucket.push(i); + else grid.set(key, [i]); + } + + let moved = false; + for (let i = 0; i < ids.length; i++) { + const cx = Math.floor(xs[i] / spacing); + const cy = Math.floor(ys[i] / spacing); + for (let gx = cx - 1; gx <= cx + 1; gx++) { + for (let gy = cy - 1; gy <= cy + 1; gy++) { + for (const j of grid.get(`${gx}:${gy}`) ?? []) { + if (j <= i) continue; + let dx = xs[j] - xs[i]; + let dy = ys[j] - ys[i]; + let distance = Math.hypot(dx, dy); + if (distance >= spacing) continue; + if (distance < 1e-9) { + const angle = jitterOf(ids[i] + ids[j]).angle; + dx = Math.cos(angle); + dy = Math.sin(angle); + distance = 1; + } + const push = (spacing - distance) / distance / 2; + xs[i] -= dx * push; + ys[i] -= dy * push; + xs[j] += dx * push; + ys[j] += dy * push; + moved = true; + } + } + } + } + if (!moved) break; + } + + for (let i = 0; i < ids.length; i++) { + part.setNodeAttribute(ids[i], 'x', xs[i]); + part.setNodeAttribute(ids[i], 'y', ys[i]); + } +} + /** * Push the lobes apart from each other, leaving each one's interior alone. * @@ -444,8 +1258,10 @@ function spreadCommunities(part: Graph, communities: Record): Di * uses 0.15 between whole components; lobes want far more, because unlike * components they are joined by edges that read as bridges. */ - const GAP = 0.9; + const GAP = 0.35; const PASSES = 240; + /** Share of the distance to the centre a lobe moves per compaction pass. */ + const COMPACT_PULL = 0.06; /** * The percentile of member distances taken as a lobe's radius. * @@ -493,11 +1309,12 @@ function spreadCommunities(part: Graph, communities: Record): Di // One lobe has nothing to be spread away from, but it is still the field. if (lobes.length < 2) return lobes.map(lobe => ({ x: lobe.x, y: lobe.y, radius: lobe.radius })); - for (let pass = 0; pass < PASSES; pass++) { + const separate = (limit: number): void => { + for (let pass = 0; pass < limit; pass++) { // Over-relaxed early and settling toward an exact correction, for the same // reason `packComponents` does it: separating one pair nudges both into // their other neighbours. - const relax = 1.4 - 0.4 * (pass / PASSES); + const relax = 1.4 - 0.4 * (pass / limit); let collided = false; for (let i = 0; i < lobes.length; i++) { @@ -531,6 +1348,38 @@ function spreadCommunities(part: Graph, communities: Record): Di if (!collided) break; } + }; + + separate(PASSES); + + // Then close the gaps. Separation only ever pushes apart, so lobes stayed as + // far apart as the simulation left them — and once `declump` had grown their + // interiors, that was most of the map: measured, the lobes covered a sixth of + // the world, the rest was empty, and every edge between two lobes crossed it + // as a long line. Drawing the lobes toward their common centre while the + // separation keeps them from overlapping packs them like circles: the same + // neighbours, a fraction of the empty space, and shorter bridges. + const COMPACT_PASSES = 160; + for (let pass = 0; pass < COMPACT_PASSES; pass++) { + let weight = 0; + let centreX = 0; + let centreY = 0; + for (const lobe of lobes) { + const area = lobe.radius * lobe.radius; + centreX += lobe.x * area; + centreY += lobe.y * area; + weight += area; + } + centreX /= weight; + centreY /= weight; + for (const lobe of lobes) { + lobe.x += (centreX - lobe.x) * COMPACT_PULL; + lobe.y += (centreY - lobe.y) * COMPACT_PULL; + } + separate(8); + } + // Whatever the last pull left touching is pulled apart for good. + separate(PASSES); for (const lobe of lobes) { const shiftX = lobe.x - lobe.originX; @@ -561,7 +1410,7 @@ interface PlacedComponent { radius: number; offsetX: number; offsetY: number; - /** Where it already sat, when it was on the map before this pass. */ + /** Held still by the packing — the largest component, which holds the origin. */ anchored: boolean; /** Its lobes, in the same local space as `xs`/`ys`. See `placeLoneNodes`. */ lobes: Disc[]; @@ -601,24 +1450,6 @@ function componentsOf(graph: Graph): string[][] { return components; } -/** The centroid this component occupied before the pass, if it was on the map. */ -function existingCentroid( - member: string[], - existing: Map -): { x: number; y: number } | null { - let sumX = 0; - let sumY = 0; - let count = 0; - for (const id of member) { - const previous = existing.get(id); - if (!previous || previous.placed !== 1) continue; - sumX += previous.x; - sumY += previous.y; - count++; - } - return count > 0 ? { x: sumX / count, y: sumY / count } : null; -} - /** * Where a component's nodes start, in LOCAL space. * @@ -627,49 +1458,14 @@ function existingCentroid( * passes every component would have crept into one pile — the disc again, arrived * at slowly instead of immediately. * - * Nodes that already had a position keep it, translated into local space, which - * is what makes successive passes refinements of one arrangement rather than a - * new arrangement each time. + * Every node starts inside its community's region, scattered by its own id, so + * the same store always starts — and therefore settles — the same way. */ -function seedComponent( - part: Graph, - communities: Record, - existing: Map, - anchor: { x: number; y: number } | null -): void { +function seedComponent(part: Graph, communities: Record): void { const meta = communityMetaSeed(part, communities); - const placedIds = new Set(); part.forEachNode(id => { - const previous = existing.get(id); - if (!previous || previous.placed !== 1 || !anchor) return; - part.setNodeAttribute(id, 'x', previous.x - anchor.x); - part.setNodeAttribute(id, 'y', previous.y - anchor.y); - placedIds.add(id); - }); - - part.forEachNode(id => { - if (placedIds.has(id)) return; - - // Near whatever it connects to that is already placed — an arriving memory - // belongs next to its subject, not wherever its community happens to sit. - let sumX = 0; - let sumY = 0; - let count = 0; - part.forEachNeighbor(id, neighbour => { - if (!placedIds.has(neighbour)) return; - sumX += part.getNodeAttribute(neighbour, 'x') as number; - sumY += part.getNodeAttribute(neighbour, 'y') as number; - count++; - }); - const jitter = jitterOf(id); - if (count > 0) { - part.setNodeAttribute(id, 'x', sumX / count + jitter.x * 12); - part.setNodeAttribute(id, 'y', sumY / count + jitter.y * 12); - return; - } - const region = meta.get(communities[id] ?? 0); part.setNodeAttribute(id, 'x', (region?.x ?? 0) + jitter.x * (region?.radius ?? 120)); part.setNodeAttribute(id, 'y', (region?.y ?? 0) + jitter.y * (region?.radius ?? 120)); @@ -761,7 +1557,7 @@ function communityMetaSeed( } /** Centre a settled component on its own centroid and measure what it needs. */ -function measure(part: Graph, anchor: { x: number; y: number } | null, lobes: Disc[]): PlacedComponent { +function measure(part: Graph, lobes: Disc[]): PlacedComponent { const ids = part.nodes(); const xs = new Float64Array(ids.length); const ys = new Float64Array(ids.length); @@ -792,16 +1588,15 @@ function measure(part: Graph, anchor: { x: number; y: number } | null, lobes: Di // actually are once the component is given a place. lobes: lobes.map(lobe => ({ x: lobe.x - centreX, y: lobe.y - centreY, radius: lobe.radius })), radius: Math.max(radius, 20), - // An anchored component keeps its old centroid; the drift the settle - // introduced is absorbed by re-centring rather than accumulating. - offsetX: anchor ? anchor.x + centreX : 0, - offsetY: anchor ? anchor.y + centreY : 0, - anchored: anchor !== null + // Given a place by `packComponents`. + offsetX: 0, + offsetY: 0, + anchored: false }; } /** - * Give every unanchored component a place, and separate any that overlap. + * Give every component a place, and separate any that overlap. * * A store this feature has been running in for any length of time is ONE mass * and a handful of stragglers, so the placement is built around that rather than @@ -810,7 +1605,8 @@ function measure(part: Graph, anchor: { x: number; y: number } | null, lobes: Di * whatever still collides. The silhouette is then decided by how big the mass is * and how many stragglers there are, both of which are properties of the data. * - * Anchored components do not move: they are where the user last saw them. + * The largest component is held still once it is given the origin; everything + * else moves around it. */ function packComponents(placed: PlacedComponent[]): void { if (placed.length === 0) return; @@ -931,38 +1727,12 @@ function packComponents(placed: PlacedComponent[]): void { function placeLoneNodes( singletons: string[], field: Disc[], - communities: Record, - existing: Map + communities: Record ): GraphLayoutRow[] { if (singletons.length === 0) return []; const rows: GraphLayoutRow[] = []; - - /** - * One that was already on the map stays exactly where it was. - * - * Anchoring these matters more than anchoring components, and measurement is - * what showed it: the old free-cell grid was derived from where the components - * ended up, so a single memory arriving anywhere reassigned EVERY lone node. A - * warm pass moved a hundred of them clear across the map — the one thing a - * background pass must never do to a view somebody is reading. - */ - const ordered: string[] = []; - for (const id of [...singletons].sort()) { - const previous = existing.get(id); - if (previous && previous.placed === 1) { - rows.push({ - nodeId: id, - community: communities[id] ?? 0, - x: previous.x, - y: previous.y, - placed: 1 - }); - continue; - } - ordered.push(id); - } - if (ordered.length === 0) return rows; + const ordered = [...singletons].sort(); // Nothing else on the map — a field of their own, which is the only case where // they set the scale, because there is nothing else to set it. @@ -976,7 +1746,8 @@ function placeLoneNodes( community: communities[id] ?? 0, x: Math.cos(draw.angle) * radius, y: Math.sin(draw.angle) * radius, - placed: 1 + placed: 1, + degree: 0 }); } return rows; @@ -1030,7 +1801,7 @@ function placeLoneNodes( y = hit.y + (dy / distance) * target; } - rows.push({ nodeId: id, community: communities[id] ?? 0, x, y, placed: 1 }); + rows.push({ nodeId: id, community: communities[id] ?? 0, x, y, placed: 1, degree: 0 }); } return rows; @@ -1080,13 +1851,6 @@ function iterationsFor(order: number): number { return 600; } -/** A warm pass only has to absorb what arrived since the last one. */ -function warmIterationsFor(order: number): number { - if (order > 3_000) return 40; - if (order > 1_500) return 60; - return 90; -} - /** * Run the simulation, yielding between chunks. * diff --git a/backend/memory/view.ts b/backend/memory/view.ts index e5bc2217..0c99cf22 100644 --- a/backend/memory/view.ts +++ b/backend/memory/view.ts @@ -1,71 +1,73 @@ /** - * Graph view assembly for the visualization. + * The Memory graph, served the way a map is. * - * ── What this file is NOT allowed to do any more ───────────────────────────── - * It used to run Louvain on every call. Community detection is stable per - * dataset, so opening the modal, changing a filter, or any conversation - * recording a memory made every connected client pay for the same clustering - * again — and the cost grew with the store rather than with what was on screen. - * It now lives in `graph_layout`, written by the background pass in `layout.ts`, - * and this file only reads it. + * ── Why tiles ──────────────────────────────────────────────────────────────── + * The view used to answer "show me the graph" with the graph: every memory up + * to a render budget, and past it a single grid laid over the whole extent that + * merged whatever shared a cell into one mark — which then had to be clicked to + * see inside. The cost of a look grew with the store, and past the budget the + * picture was mostly marks standing in for memories. * - * ── Showing everything, at bounded cost ────────────────────────────────────── - * The old view answered scale with a cap: return the three thousand most - * reinforced memories and set `truncated`. That is not a scale strategy, it is a - * silent ceiling — past it the rest of the store simply was not in the picture, - * with no way to reach it but narrowing a filter. + * Now the client asks for the squares of the map that are on screen, at the + * resolution the camera is at, exactly as a map viewer does. The world (see + * `layout.ts`) is cut into 2^z × 2^z tiles at zoom level z; the client picks the + * level from its zoom, so a tile is always a few hundred pixels across whatever + * the zoom. What is on screen therefore costs about the same at any zoom and + * any store size, and zooming in is how you see more — never clicking. * - * The answer here is to aggregate by POSITION rather than to trim by rank, and - * the difference is what the user sees. Grouping by community — the obvious - * alternative, and the first thing tried — replaces the picture: you get lobes - * instead of memories, and reaching any memory means opening one first. Grouping - * by position keeps the picture. The layout plane is divided into a grid fine - * enough that the number of OCCUPIED cells is bounded, and each occupied cell - * becomes one mark. Marks merge only where their dots would have overlapped - * anyway, so the silhouette, the lobes and the colours are the ones the full - * graph would have drawn, and nothing is missing from it. + * ── What a tile holds ──────────────────────────────────────────────────────── + * The budget is the SCREEN's, not a tile's: how many marks the client can draw + * at once. When everything in the requested tiles fits it, every memory arrives + * individually — so a store of a few thousand memories is drawn in full at every + * zoom, and zooming in can never fold something the overview showed. Only when + * the screen holds more than the budget is it shared out: sparse tiles keep + * every memory, and what is left is split between the crowded ones, which are + * folded into a grid of cells (a cell holding one memory still arrives as that + * memory). Zooming in shows less of the map, which fits, so every cluster + * resolves by zooming alone. * - * Below `RENDER_BUDGET` no cell holds more than one memory, so every mark IS a - * memory and the view is exactly the flat one it always was. Above it, a cell - * holding one memory still returns that memory — fidelity is lost only where - * the display had none to give. - * - * Zooming into a mark asks for its cell as a rectangle, which costs that - * rectangle rather than the store. + * Edges travel with the tile they leave from, carrying where their far end is, + * so the client can attach them to whichever mark stands for that end at the + * moment — the memory itself, or the cluster it is folded into. */ -import { graphQueries, graphLayoutQueries, type GraphBinGrid } from '$backend/database/queries/graph-queries'; +import { graphLayoutQueries, graphQueries, type GraphListFilter } from '$backend/database/queries/graph-queries'; +import { getLayoutState, scheduleGraphLayout } from './layout'; import type { - GraphNode, - GraphRegion, GraphScope, GraphSource, - GraphView, - GraphViewBin, - GraphViewBinEdge, - GraphViewEdge, - GraphViewNode + GraphTile, + GraphTileCellEdge, + GraphTileCluster, + GraphTileNode, + GraphTilesResponse, + GraphViewMeta } from '$shared/types/memory'; +/** The deepest zoom level a client may ask for — far past one memory per tile. */ +export const MAX_TILE_ZOOM = 20; + +/** Tiles served per request. A screen needs a few dozen at most. */ +const MAX_TILES_PER_REQUEST = 64; + +/** The range a client may ask the screen's budget to be in. */ +const MIN_SCREEN_BUDGET = 256; +const MAX_SCREEN_BUDGET = 40_000; + /** - * Marks the client is asked to draw at once. + * Cells per side when a crowded tile is folded. * - * A ceiling on the WORK, not on the content: past it marks merge rather than - * disappear. Set where a WebGL scene stays comfortable on a low-powered machine - * — the graph build, sigma's indexation and the per-frame reducers are all - * linear in this number, and it is the only number they are linear in. + * A tile is drawn a few hundred pixels across, so 32 makes a cell 8–16 pixels — + * about the size a cluster is drawn, which is what stops neighbouring clusters + * overlapping. Smaller when the budget is smaller, so a folded tile never holds + * more marks than its budget. */ -const RENDER_BUDGET = 3_000; - -/** Bin edges drawn at once, heaviest first. Lines are cheaper than marks, not free. */ -const MAX_BIN_EDGES = 6_000; +const MAX_CELLS_PER_SIDE = 32; -/** Longest caption a mark carries; the full text belongs to the inspector. */ -const BIN_LABEL_MAX = 60; +/** Edges served per tile, as a multiple of the marks it holds, heaviest first. */ +const EDGES_PER_MARK = 5; export interface GraphViewFilter { - /** `undefined` = every project (cross-project view); `null` = global only. */ - projectId?: string | null; /** The multi-select's answer: absent = every project, empty = global only. */ projectIds?: string[]; /** Narrow to particular subkinds — decisions only, failures only, and so on. */ @@ -76,385 +78,254 @@ export interface GraphViewFilter { includeArchived?: boolean; /** Show memories that have been replaced by a newer belief. Off by default. */ includeSuperseded?: boolean; - /** - * A rectangle of the layout to restrict to — how a mark is opened. - * - * Navigation, not filtering: it narrows what is DRAWN without changing what - * the graph is, which is why the totals it reports are the region's. - */ - region?: GraphRegion; - limit?: number; } -export function buildGraphView(filter: GraphViewFilter): GraphView { - const limit = Math.min(filter.limit ?? RENDER_BUDGET, RENDER_BUDGET); - const region = filter.region ?? null; - - // Zoomed in: the rectangle is already a bounded slice of the arrangement, so - // it is drawn at full fidelity and counted within itself. - if (region) { - const nodes = graphQueries.listInRegion({ ...filter, limit }, region); - return flat(nodes, region, nodes.length, false); - } - - const extent = graphLayoutQueries.extent(filter); - - // Either it fits, or nothing has been laid out yet — a fresh install, or the - // first fetch after the upgrade added the table. Both draw the flat view: the - // second because binning needs positions, and falling back to what the feature - // always did means the graph is never empty while the background pass catches - // up. - const laidOut = extent.total - extent.unplaced; - if (extent.total <= RENDER_BUDGET || laidOut === 0) { - const nodes = graphQueries.list({ ...filter, limit }); - return flat(nodes, null, extent.total, extent.total > nodes.length); - } +export interface GraphTileRequest extends GraphViewFilter { + /** The epoch the client's tile addresses were computed against. */ + epoch: number; + z: number; + tiles: { x: number; y: number }[]; + /** Marks the client can draw at once, shared out across `tiles`. */ + budget: number; +} - return binned(filter, extent, limit); +function listFilter(filter: GraphViewFilter): GraphListFilter { + return { + projectIds: filter.projectIds, + subkinds: filter.subkinds, + scopes: filter.scopes, + sources: filter.sources, + includeArchived: filter.includeArchived, + includeSuperseded: filter.includeSuperseded + }; } /** - * Every matching memory, drawn individually. + * What the client needs before it can ask for tiles: the world, its epoch, and + * the filter's counts and extent. * - * The ordinary case and the one the view is tuned for — below the render budget - * this is the whole story, and it is unchanged from before any of this existed. + * Also the nudge for anything still waiting to be placed. A memory with no + * position is counted but not drawn, and the pass that places it is normally + * already scheduled by the write — this covers a store that was written while + * the server was not running a pass at all (an upgrade, a clear-data). */ -function flat( - nodes: GraphNode[], - region: GraphRegion | null, - totalNodes: number, - truncated: boolean -): GraphView { - if (nodes.length === 0) { - return { - level: 'flat', - nodes: [], - edges: [], - bins: [], - binEdges: [], - region, - totalNodes, - truncated: false - }; - } - - const ids = nodes.map(node => node.id); - const stored = graphQueries.edgesWithin(ids); - - // Two memories about the same subject ARE connected, and that connection is - // stated by extraction rather than inferred from how they are worded. It is - // derived here rather than stored, so it can never outlive the claim it came - // from — see `derivedEdges`. - // - // Without it the episodic half has almost no structure at all. Similarity - // linking used to supply it, badly: on a real graph 144 of 371 edges were - // fabricated by cosine over a corpus with no variance. Removing that was right - // and removing it ALONE was not — the view became ninety-three disconnected - // dots, which is a less honest picture than the wrong one, because these - // memories genuinely do belong together. - const derived: GraphViewEdge[] = graphQueries.derivedEdges(ids).map((edge, index) => ({ - // Negative ids, so nothing can mistake a derived edge for a stored row and - // try to delete or re-weight it. - id: -(index + 1), - source: edge.srcId, - target: edge.dstId, - rel: 'relates_to' as const, - weight: edge.weight - })); - - // Degree is counted over what is being DRAWN, so a node's size answers "how - // connected is this within what I am looking at" — which is the question the - // eye is actually asking. - const degree = new Map(); - const bump = (srcId: string, dstId: string): void => { - degree.set(srcId, (degree.get(srcId) ?? 0) + 1); - degree.set(dstId, (degree.get(dstId) ?? 0) + 1); - }; - for (const edge of stored) bump(edge.srcId, edge.dstId); - for (const edge of derived) bump(edge.source, edge.target); - - const layout = graphLayoutQueries.read(ids); - const viewNodes = nodes.map(node => toViewNode(node, degree.get(node.id) ?? 0, layout.get(node.id))); - placeArrivals(viewNodes, [...stored.map(edge => ({ source: edge.srcId, target: edge.dstId })), ...derived]); +export function buildGraphMeta(filter: GraphViewFilter): GraphViewMeta { + const narrowing = listFilter(filter); + const state = getLayoutState(); + const extent = graphLayoutQueries.extent(narrowing); + if (extent.unplaced > 0) scheduleGraphLayout(); + const placed = extent.total - extent.unplaced; return { - level: 'flat', - nodes: viewNodes, - edges: [ - ...stored.map(edge => ({ - id: edge.id, - source: edge.srcId, - target: edge.dstId, - rel: edge.rel, - weight: edge.weight - })), - ...derived - ], - bins: [], - binEdges: [], - region, - totalNodes, - truncated + epoch: state.epoch, + stamp: state.stamp, + world: state.world, + spacing: state.spacing, + total: extent.total, + unplaced: extent.unplaced, + extent: + placed > 0 ? { minX: extent.minX, maxX: extent.maxX, minY: extent.minY, maxY: extent.maxY } : null, + degreeScale: placed > 0 ? graphLayoutQueries.degreeScale(narrowing) : 1 }; } /** - * The whole graph, with crowded neighbourhoods folded into single marks. + * The requested tiles of the map at one zoom level. * - * Cells holding exactly one memory come back as that memory, so this is a flat - * view everywhere the display could have drawn one — which, in a force layout, - * is most of the periphery. Only the dense middle merges, and only as far as it - * was going to overlap. + * Answers `stale` instead of tiles when the client's epoch is not the current + * one: its addresses were computed against a world that no longer exists, and + * serving them would draw the right memories in the wrong places. */ -function binned( - filter: GraphViewFilter, - extent: { total: number; minX: number; maxX: number; minY: number; maxY: number }, - limit: number -): GraphView { - const grid = gridFor(extent); - const cells = graphLayoutQueries.binnedNodes(filter, grid); - - // A cell of one is not a bin, it is a memory. Splitting them here is what - // keeps the picture inspectable: everything the grid did not have to merge - // stays clickable, hoverable and highlightable exactly as before. - const singles = cells.filter(cell => cell.members === 1); - const groups = cells.filter(cell => cell.members > 1); - - const nodes = graphQueries.getByIds(singles.map(cell => cell.id)); - const byId = new Map(nodes.map(node => [node.id, node])); - const layout = graphLayoutQueries.read(nodes.map(node => node.id)); - - // Which mark each cell is, so the rolled-up edges can name their ends. A - // single's mark is the memory's own id; a group's is the cell. - const markOf = new Map(); - for (const cell of cells) { - markOf.set(cellKey(cell.cellX, cell.cellY), cell.members === 1 ? cell.id : binId(cell.cellX, cell.cellY)); +export function buildGraphTiles(request: GraphTileRequest): GraphTilesResponse { + const state = getLayoutState(); + const world = state.world; + if (!world || request.epoch !== state.epoch) return { epoch: state.epoch, stale: true, tiles: [] }; + + const z = Math.max(0, Math.min(MAX_TILE_ZOOM, Math.floor(request.z))); + const span = 2 ** z; + const tileSize = world.size / span; + const budget = Math.max(MIN_SCREEN_BUDGET, Math.min(MAX_SCREEN_BUDGET, Math.floor(request.budget))); + const filter = listFilter(request); + + const wanted: { x: number; y: number }[] = []; + const seen = new Set(); + for (const tile of request.tiles) { + const x = Math.floor(tile.x); + const y = Math.floor(tile.y); + if (x < 0 || y < 0 || x >= span || y >= span) continue; + const key = `${x}:${y}`; + if (seen.has(key)) continue; + seen.add(key); + wanted.push({ x, y }); + if (wanted.length >= MAX_TILES_PER_REQUEST) break; + } + if (wanted.length === 0) return { epoch: state.epoch, stale: false, tiles: [] }; + + const rectOf = (x: number, y: number) => ({ + minX: world.minX + x * tileSize, + maxX: world.minX + (x + 1) * tileSize, + minY: world.minY + y * tileSize, + maxY: world.minY + (y + 1) * tileSize + }); + + // One counting query over the union of what was asked, so each tile's + // resolution is decided before a single label is read. + let minTileX = Infinity; + let maxTileX = -Infinity; + let minTileY = Infinity; + let maxTileY = -Infinity; + for (const tile of wanted) { + minTileX = Math.min(minTileX, tile.x); + maxTileX = Math.max(maxTileX, tile.x); + minTileY = Math.min(minTileY, tile.y); + maxTileY = Math.max(maxTileY, tile.y); + } + const union = { + minX: world.minX + minTileX * tileSize, + maxX: world.minX + (maxTileX + 1) * tileSize, + minY: world.minY + minTileY * tileSize, + maxY: world.minY + (maxTileY + 1) * tileSize + }; + const counts = new Map(); + for (const row of graphLayoutQueries.tileCounts(filter, { originX: world.minX, originY: world.minY, size: tileSize }, union)) { + counts.set(`${row.tileX}:${row.tileY}`, row.count); } - const binEdges: GraphViewBinEdge[] = []; - const nodeEdges: GraphViewEdge[] = []; - let syntheticId = -1; + const allowance = shareBudget( + wanted.map(({ x, y }) => counts.get(`${x}:${y}`) ?? 0), + budget + ); + + // An edge between two tiles of the same request would otherwise travel twice, + // once from each end — measured, that was most of the payload at overview. + const sentPairs = new Set(); + const firstTime = (a: string, b: string): boolean => { + const key = a < b ? `${a}|${b}` : `${b}|${a}`; + if (sentPairs.has(key)) return false; + sentPairs.add(key); + return true; + }; - for (const edge of graphLayoutQueries.binnedEdges(filter, grid, MAX_BIN_EDGES)) { - const source = markOf.get(cellKey(edge.srcX, edge.srcY)); - const target = markOf.get(cellKey(edge.dstX, edge.dstY)); - if (!source || !target || source === target) continue; + const tiles: GraphTile[] = wanted.map(({ x, y }, index) => { + const count = counts.get(`${x}:${y}`) ?? 0; + const allowed = allowance[index]; + const rect = rectOf(x, y); + const empty: GraphTile = { z, x, y, cells: 0, nodes: [], clusters: [], edges: [], cellEdges: [] }; + if (count === 0) return empty; + const edgeLimit = Math.max(1, Math.min(count, allowed)) * EDGES_PER_MARK; + + if (count <= allowed) { + const nodes = graphLayoutQueries.tileNodes(filter, rect, count).map(round); + const edges = graphLayoutQueries + .edgesTouching( + filter, + nodes.map(node => node.id), + edgeLimit + ) + .filter(edge => firstTime(edge.source, edge.target)) + .map(edge => ({ ...edge, targetX: tidy(edge.targetX), targetY: tidy(edge.targetY) })); + return { ...empty, nodes, edges }; + } - // An edge between two singles is a real edge between two real memories, and - // the canvas dims and highlights those against the node it is anchored on. - // Demoting it to a bin edge would break hover on exactly the marks that did - // not need binning in the first place. - if (byId.has(source) && byId.has(target)) { - nodeEdges.push({ - id: syntheticId--, + // Folded, into no more cells than its share of the budget. Cell coordinates + // are global at this zoom, so an edge from any other tile can name the cell + // it lands in without knowing this tile. + const cellsPerSide = Math.max(4, Math.min(MAX_CELLS_PER_SIDE, Math.floor(Math.sqrt(allowed)))); + const cellGrid = { originX: world.minX, originY: world.minY, size: tileSize / cellsPerSide }; + const nodes: GraphTileNode[] = []; + const clusters: GraphTileCluster[] = []; + const markOfCell = new Map(); + + for (const cell of graphLayoutQueries.tileCells(filter, rect, cellGrid)) { + const key = `${cell.cellX}:${cell.cellY}`; + if (cell.members === 1) { + nodes.push(round(cell)); + markOfCell.set(key, cell.id); + continue; + } + const id = `cluster:${z}:${cell.cellX}:${cell.cellY}`; + clusters.push({ + id, + members: cell.members, + label: cell.label, + community: cell.community, + x: tidy(cell.x), + y: tidy(cell.y), + cellX: cell.cellX, + cellY: cell.cellY + }); + markOfCell.set(key, id); + } + + const cellEdges: GraphTileCellEdge[] = []; + for (const edge of graphLayoutQueries.cellEdges(filter, rect, cellGrid, edgeLimit)) { + const source = markOfCell.get(`${edge.sourceCellX}:${edge.sourceCellY}`); + if (!source) continue; + if (!firstTime(`${z}:${edge.sourceCellX}:${edge.sourceCellY}`, `${z}:${edge.targetCellX}:${edge.targetCellY}`)) { + continue; + } + cellEdges.push({ source, - target, - rel: 'relates_to', + targetCellX: edge.targetCellX, + targetCellY: edge.targetCellY, weight: edge.weight }); - } else { - binEdges.push({ source, target, weight: edge.weight }); } - } - // Degree over what is drawn, same as the flat path — counting both kinds of - // edge, because a memory next to a dense clump is well connected whether or - // not the thing it connects to had to be merged. - const degree = new Map(); - const bump = (id: string): void => { - degree.set(id, (degree.get(id) ?? 0) + 1); - }; - for (const edge of nodeEdges) { - bump(edge.source); - bump(edge.target); - } - for (const edge of binEdges) { - bump(edge.source); - bump(edge.target); - } + return { ...empty, cells: cellsPerSide, nodes, clusters, cellEdges }; + }); - const bins: GraphViewBin[] = groups.map(cell => ({ - id: binId(cell.cellX, cell.cellY), - members: cell.members, - label: truncate(cell.label ?? ''), - community: cell.community, - x: cell.x, - y: cell.y, - region: { - minX: grid.originX + cell.cellX * grid.cellWidth, - maxX: grid.originX + (cell.cellX + 1) * grid.cellWidth, - minY: grid.originY + cell.cellY * grid.cellHeight, - maxY: grid.originY + (cell.cellY + 1) * grid.cellHeight - } - })); - - return { - level: 'binned', - nodes: singles - .map(cell => byId.get(cell.id)) - .filter((node): node is GraphNode => node !== undefined) - .map(node => toViewNode(node, degree.get(node.id) ?? 0, layout.get(node.id))), - edges: nodeEdges, - bins, - binEdges, - region: null, - totalNodes: extent.total, - // Nothing was left out — every memory is inside one of these marks. The - // only thing a binned view withholds is which memory is which inside a - // crowd, and opening the mark answers that. - truncated: nodes.length + bins.length > limit - }; + return { epoch: state.epoch, stale: false, tiles }; } /** - * The grid, sized so the number of cells cannot exceed the render budget. + * Share the screen's budget between tiles, sparse ones first. * - * Cells are square in layout space rather than fitted to the extent's aspect - * ratio: a non-square cell merges more readily along one axis than the other, - * which shows up as marks that smear horizontally or vertically — the eye reads - * that as structure, and there is none. + * Water-filling: walking the tiles from least to most crowded, each takes what + * it holds if that is within an even split of what is left, and the crowded + * remainder share the rest equally. So a tile is only ever folded when the + * screen as a whole could not afford it, and then only the crowded ones are. */ -function gridFor(extent: { minX: number; maxX: number; minY: number; maxY: number }): GraphBinGrid { - const width = Math.max(1e-6, extent.maxX - extent.minX); - const height = Math.max(1e-6, extent.maxY - extent.minY); - - // One axis of the grid. Squared, this is the cell count, so it is the budget's - // square root — and the longer side sets the cell size for both. - const perAxis = Math.max(1, Math.floor(Math.sqrt(RENDER_BUDGET))); - const cell = Math.max(width, height) / perAxis; - - return { - originX: extent.minX, - originY: extent.minY, - cellWidth: cell, - cellHeight: cell - }; -} - -function toViewNode( - node: GraphNode, - degree: number, - placed?: { community: number; x: number; y: number; placed: number } -): GraphViewNode { - return { - id: node.id, - subkind: node.subkind, - scope: node.scope, - label: node.label, - projectId: node.projectId, - degree, - weight: node.weight, - pinned: node.pinned, - community: placed?.community ?? 0, - createdAt: node.createdAt, - // Sent ONLY when the simulation computed it. A memory written since the - // last pass has no row at all, and one below the simulation cap has a row - // holding a guess — in both cases the client seeds it from its neighbours, - // which is a far better answer than drawing a whole lobe on one point. - ...(placed?.placed === 1 && { x: placed.x, y: placed.y }) - }; +function shareBudget(counts: number[], budget: number): number[] { + const allowance = new Array(counts.length).fill(0); + const order = counts.map((count, index) => ({ count, index })).sort((a, b) => a.count - b.count); + let remaining = budget; + for (let i = 0; i < order.length; i++) { + const share = Math.floor(remaining / (order.length - i)); + const take = Math.min(order[i].count, share); + allowance[order[i].index] = order[i].count <= share ? order[i].count : Math.max(16, share); + remaining -= take; + } + return allowance; } /** - * Give a position to anything the arrangement has not reached yet. - * - * A memory recorded while the modal is open has no layout row: the background - * pass is debounced behind the notification that announces the memory, so the - * first fetch after it lands is always a few seconds early. Sending it without - * coordinates left the client to invent some, and what it invented was the - * middle of the map — a new memory appearing nowhere near the subject it is - * about. + * Where particular memories are on the map, whatever the client has loaded. * - * Derived HERE instead, from the same rule the layout pass itself uses: the - * centroid of whatever it connects to. The answer is therefore the one the pass - * will confirm a few seconds later rather than one it will have to correct, and - * the client stops having an opinion about placement at all. - * - * Read-time only — nothing is written. A position that has not been through the - * simulation is a guess, and guesses do not belong in the table the simulation - * owns. - * - * A node with no placed neighbour is deliberately LEFT without a position. That - * is the honest signal for "the arrangement does not cover this yet", and it is - * what the client's cold fallback keys on — inventing a centroid for a store that - * has never been laid out would replace a considered scatter with a meaningless - * one. + * What lets the view go TO a memory it has not drawn: a search result, a + * connection listed in the inspector, a step back through history. A memory the + * layout has not placed yet is simply absent from the answer. */ -function placeArrivals(nodes: GraphViewNode[], edges: { source: string; target: string }[]): void { - const arrivals = nodes.filter(node => node.x === undefined); - if (arrivals.length === 0) return; - - const byId = new Map(nodes.map(node => [node.id, node])); - const adjacency = new Map(); - const link = (from: string, to: string): void => { - const neighbours = adjacency.get(from); - if (neighbours) neighbours.push(to); - else adjacency.set(from, [to]); - }; - for (const edge of edges) { - link(edge.source, edge.target); - link(edge.target, edge.source); - } - - // Three passes, so a small cluster of arrivals resolves outward from whichever - // end touches the existing map rather than all of them giving up at once. - for (let pass = 0; pass < 3; pass++) { - let settled = 0; - - for (const node of arrivals) { - if (node.x !== undefined) continue; - - let sumX = 0; - let sumY = 0; - let count = 0; - for (const neighbour of adjacency.get(node.id) ?? []) { - const other = byId.get(neighbour); - if (!other || other.x === undefined || other.y === undefined) continue; - sumX += other.x; - sumY += other.y; - count++; - } - if (count === 0) continue; - - // Nudged off the exact centroid, or several arrivals about one subject - // land on the same point and read as a single node. - const jitter = jitterOf(node.id); - node.x = sumX / count + jitter.x * 24; - node.y = sumY / count + jitter.y * 24; - settled++; - } +export function locateNodes(nodeIds: string[]): GraphTileNode[] { + return graphLayoutQueries.locate(nodeIds.slice(0, 500)).map(round); +} - if (settled === 0) break; - } +/** Positions are drawn on a screen; a tenth of a unit is far below a pixel. */ +function tidy(value: number): number { + return Math.round(value * 10) / 10; } -/** A deterministic offset in [-1, 1]², derived from an id. FNV-1a. */ -function jitterOf(id: string): { x: number; y: number } { - let hash = 2166136261; - for (let i = 0; i < id.length; i++) { - hash ^= id.charCodeAt(i); - hash = Math.imul(hash, 16777619); - } - hash = hash >>> 0; +function round(node: GraphTileNode): GraphTileNode { return { - x: ((hash % 65536) / 65536) * 2 - 1, - y: (((hash >>> 16) % 65536) / 65536) * 2 - 1 + id: node.id, + label: node.label, + community: node.community, + degree: node.degree, + x: tidy(node.x), + y: tidy(node.y) }; } -/** Namespaced so a bin's id can never collide with a node's. */ -function binId(cellX: number, cellY: number): string { - return `bin:${cellX}:${cellY}`; -} - -function cellKey(cellX: number, cellY: number): string { - return `${cellX}:${cellY}`; -} - -function truncate(label: string): string { - return label.length > BIN_LABEL_MAX ? `${label.slice(0, BIN_LABEL_MAX - 1)}…` : label; -} - /** * A node plus the nodes it reaches — what the inspector renders. * diff --git a/backend/ws/memory/crud.ts b/backend/ws/memory/crud.ts index bd70a112..d8c6cdc2 100644 --- a/backend/ws/memory/crud.ts +++ b/backend/ws/memory/crud.ts @@ -21,7 +21,7 @@ import { createRouter } from '$shared/utils/ws-server'; import { projectContextService } from '$backend/mcp/internal/project-context'; import { debug } from '$shared/utils/logger'; import { graphQueries, graphLayoutQueries } from '$backend/database/queries/graph-queries'; -import { buildGraphView, buildNodeDetail } from '$backend/memory/view'; +import { buildGraphMeta, buildGraphTiles, buildNodeDetail, locateNodes, type GraphViewFilter } from '$backend/memory/view'; import { resetGraphLayoutState } from '$backend/memory/layout'; import { markConsulted, retrieve } from '$backend/memory/retrieval'; import { scheduleVectorIndexing } from '$backend/memory/indexer'; @@ -78,6 +78,56 @@ const NODE = t.Object({ entityKey: t.Union([t.String(), t.Null()]) }); +/** + * The graph view's narrowing, shared by the meta and tile requests so the two + * can never disagree about what is being looked at. + */ +const FILTER = t.Object({ + // Tri-state, and each state means something different: ABSENT is every + // project, an EMPTY array is global only, and a non-empty array is those + // projects plus the globals that apply inside every one of them. + projectIds: t.Optional(t.Array(t.String())), + subkinds: t.Optional(t.Array(t.String())), + scopes: t.Optional(t.Array(SCOPE)), + sources: t.Optional(t.Array(SOURCE)), + includeArchived: t.Optional(t.Boolean()), + includeSuperseded: t.Optional(t.Boolean()) +}); + +const REGION = t.Object({ + minX: t.Number(), + maxX: t.Number(), + minY: t.Number(), + maxY: t.Number() +}); + +const TILE_NODE = t.Object({ + id: t.String(), + label: t.String(), + community: t.Number(), + degree: t.Number(), + x: t.Number(), + y: t.Number() +}); + +function filterOf(data: { + projectIds?: string[]; + subkinds?: string[]; + scopes?: string[]; + sources?: string[]; + includeArchived?: boolean; + includeSuperseded?: boolean; +}): GraphViewFilter { + return { + projectIds: data.projectIds, + subkinds: data.subkinds, + scopes: data.scopes as GraphScope[] | undefined, + sources: data.sources as GraphSource[] | undefined, + includeArchived: data.includeArchived, + includeSuperseded: data.includeSuperseded + }; +} + const MEMORY_MODEL = t.Object({ engine: t.String(), providerSlug: t.Optional(t.String()), @@ -131,95 +181,82 @@ const READINESS = t.Object({ export const memoryCrudHandler = createRouter() .http('memory:graph', { + data: FILTER, + response: t.Object({ + epoch: t.Number(), + stamp: t.Number(), + world: t.Union([t.Object({ minX: t.Number(), minY: t.Number(), size: t.Number() }), t.Null()]), + spacing: t.Number(), + total: t.Number(), + unplaced: t.Number(), + extent: t.Union([REGION, t.Null()]), + degreeScale: t.Number() + }) + }, async ({ data }) => { + debug.log('path', 'memory:graph'); + return buildGraphMeta(filterOf(data)); + }) + .http('memory:graph-tiles', { data: t.Object({ - // Tri-state, and each state means something different: ABSENT is every - // project, an EMPTY array is global only, and a non-empty array is those - // projects plus the globals that apply inside every one of them. - projectIds: t.Optional(t.Array(t.String())), - subkinds: t.Optional(t.Array(t.String())), - scopes: t.Optional(t.Array(SCOPE)), - sources: t.Optional(t.Array(SOURCE)), - includeArchived: t.Optional(t.Boolean()), - includeSuperseded: t.Optional(t.Boolean()), - // The rectangle of the layout to zoom into, when a mark was opened. - // Absent asks for the whole graph at whatever resolution fits. - region: t.Optional( - t.Object({ - minX: t.Number(), - maxX: t.Number(), - minY: t.Number(), - maxY: t.Number() - }) - ), - limit: t.Optional(t.Number()) + ...FILTER.properties, + // The epoch the tile addresses were computed against; a mismatch is + // answered with `stale` rather than tiles from a different map. + epoch: t.Number(), + z: t.Number(), + tiles: t.Array(t.Object({ x: t.Number(), y: t.Number() })), + budget: t.Number() }), response: t.Object({ - level: t.String(), - nodes: t.Array(t.Object({ - id: t.String(), - subkind: t.String(), - scope: t.String(), - label: t.String(), - projectId: t.Union([t.String(), t.Null()]), - degree: t.Number(), - weight: t.Number(), - pinned: t.Boolean(), - community: t.Number(), - createdAt: t.String(), - x: t.Optional(t.Number()), - y: t.Optional(t.Number()) - })), - edges: t.Array(t.Object({ - id: t.Number(), - source: t.String(), - target: t.String(), - rel: t.String(), - weight: t.Number() - })), - bins: t.Array(t.Object({ - id: t.String(), - members: t.Number(), - label: t.String(), - community: t.Number(), + epoch: t.Number(), + stale: t.Boolean(), + tiles: t.Array(t.Object({ + z: t.Number(), x: t.Number(), y: t.Number(), - region: t.Object({ - minX: t.Number(), - maxX: t.Number(), - minY: t.Number(), - maxY: t.Number() - }) - })), - binEdges: t.Array(t.Object({ - source: t.String(), - target: t.String(), - weight: t.Number() - })), - region: t.Union([ - t.Object({ - minX: t.Number(), - maxX: t.Number(), - minY: t.Number(), - maxY: t.Number() - }), - t.Null() - ]), - totalNodes: t.Number(), - truncated: t.Boolean() + cells: t.Number(), + nodes: t.Array(TILE_NODE), + clusters: t.Array(t.Object({ + id: t.String(), + members: t.Number(), + label: t.String(), + community: t.Number(), + x: t.Number(), + y: t.Number(), + cellX: t.Number(), + cellY: t.Number() + })), + edges: t.Array(t.Object({ + source: t.String(), + target: t.String(), + weight: t.Number(), + targetX: t.Number(), + targetY: t.Number() + })), + cellEdges: t.Array(t.Object({ + source: t.String(), + targetCellX: t.Number(), + targetCellY: t.Number(), + weight: t.Number() + })) + })) }) }, async ({ data }) => { - debug.log('path', 'memory:graph'); - return buildGraphView({ - projectIds: data.projectIds, - subkinds: data.subkinds, - scopes: data.scopes as GraphScope[] | undefined, - sources: data.sources as GraphSource[] | undefined, - includeArchived: data.includeArchived, - includeSuperseded: data.includeSuperseded, - region: data.region, - limit: data.limit + debug.log('path', `memory:graph-tiles z=${data.z} ×${data.tiles.length}`); + return buildGraphTiles({ + ...filterOf(data), + epoch: data.epoch, + z: data.z, + tiles: data.tiles, + budget: data.budget }); }) + .http('memory:locate', { + data: t.Object({ nodeIds: t.Array(t.String()) }), + response: t.Object({ nodes: t.Array(TILE_NODE) }) + }, async ({ data }) => { + debug.log('path', `memory:locate ×${data.nodeIds.length}`); + return { nodes: locateNodes(data.nodeIds) }; + }) .http('memory:node', { data: t.Object({ nodeId: t.String(), hops: t.Optional(t.Number()) }), response: t.Object({ diff --git a/bun.lock b/bun.lock index 12e1049c..3bf751b6 100644 --- a/bun.lock +++ b/bun.lock @@ -13,8 +13,8 @@ "@modelcontextprotocol/sdk": "1.31.0", "@monaco-editor/loader": "1.7.0", "@myrialabs/chatkit": "0.0.3", - "@myrialabs/ptykit": "0.1.0", - "@myrialabs/tunnelkit": "0.0.15", + "@myrialabs/ptykit": "0.1.1", + "@myrialabs/tunnelkit": "0.1.0", "@myrialabs/zipkit": "0.0.11", "@sinclair/typebox": "0.34.48", "@xyflow/svelte": "1.6.2", @@ -525,9 +525,9 @@ "@myrialabs/chatkit": ["@myrialabs/chatkit@0.0.3", "", { "dependencies": { "@slack/socket-mode": "^2.0.0", "@slack/web-api": "^7.0.0", "@whiskeysockets/baileys": "^6.7.0", "discord.js": "^14.0.0", "zod": "^3.23.0" }, "optionalDependencies": { "qrcode-terminal": "^0.12.0" }, "bin": { "chatkit": "dist/cli.js" } }, "sha512-QzRhVKyDtyAzeQQ8Y6qbyX9Up0zEvUSEYmDnPl3oKcz2WFISXKiff7Lw8lWKx/DqXlqWVGlsJPTHkm6LAfdoPw=="], - "@myrialabs/ptykit": ["@myrialabs/ptykit@0.1.0", "", { "dependencies": { "@xterm/addon-clipboard": "^0.2.0", "@xterm/addon-fit": "^0.11.0", "@xterm/addon-ligatures": "^0.10.0", "@xterm/addon-serialize": "^0.14.0", "@xterm/addon-unicode11": "^0.9.0", "@xterm/addon-web-links": "^0.12.0", "@xterm/headless": "^6.0.0", "@xterm/xterm": "^6.0.0" }, "optionalDependencies": { "bun-pty": "^0.4.10", "node-pty": "^1.1.0", "ws": "^8.21.0" }, "peerDependencies": { "svelte": "^5.0.0" }, "optionalPeers": ["svelte"] }, "sha512-h3wxmnM7Vzia2Cy+ZDcqDIiu0gqazBzmbPmMybMM3cAn/4uXER0NaQGuAcr+LpxHX9efXNPrWCML2eqrSbennA=="], + "@myrialabs/ptykit": ["@myrialabs/ptykit@0.1.1", "", { "dependencies": { "@xterm/addon-clipboard": "^0.2.0", "@xterm/addon-fit": "^0.11.0", "@xterm/addon-ligatures": "^0.10.0", "@xterm/addon-serialize": "^0.14.0", "@xterm/addon-unicode11": "^0.9.0", "@xterm/addon-web-links": "^0.12.0", "@xterm/headless": "^6.0.0", "@xterm/xterm": "^6.0.0" }, "optionalDependencies": { "bun-pty": "^0.4.10", "node-pty": "^1.1.0", "ws": "^8.21.0" }, "peerDependencies": { "svelte": "^5.0.0" }, "optionalPeers": ["svelte"] }, "sha512-49qOjycA63QOgBkTLm+tknBWTJ+neRm/tMpoEl1jy5iX6mqlfVwArt4SN0oinvUSES3KaR0VuWpOrA06YURG4g=="], - "@myrialabs/tunnelkit": ["@myrialabs/tunnelkit@0.0.15", "", { "bin": { "tunnelkit": "dist/cli.js" } }, "sha512-IxOED4DtZGTnpNy/s0VmFZyaWNFyyVJpN7symy7Dyj6Qu2/7TK8hUsB6gbyrbFrfix/+2ZgwnlIoXkGmYvfAMA=="], + "@myrialabs/tunnelkit": ["@myrialabs/tunnelkit@0.1.0", "", { "bin": { "tunnelkit": "dist/cli.js" } }, "sha512-4Asfo6MKK9E6nNQiPbtlD3P6FnOuoPVgVILf8Iejlad47pRmYFBfaVoD5Yp4X/c2J0X6wO0sPo1Q0htN4oPAAA=="], "@myrialabs/zipkit": ["@myrialabs/zipkit@0.0.11", "", { "bin": { "zipkit": "dist/cli.js" } }, "sha512-7IHU/b+zuxQvS3u6+0EwpyHIQexCoYUnqCX9/6TuI8ETOrani4F/UiL0fVzv6AdCEyuoJxy2u1ZTq3AtdNnZaw=="], diff --git a/frontend/components/memory/MemoryGraphCanvas.svelte b/frontend/components/memory/MemoryGraphCanvas.svelte index f4438cae..0617fb77 100644 --- a/frontend/components/memory/MemoryGraphCanvas.svelte +++ b/frontend/components/memory/MemoryGraphCanvas.svelte @@ -1,45 +1,45 @@ diff --git a/frontend/components/memory/MemoryModal.svelte b/frontend/components/memory/MemoryModal.svelte index 347b470d..0dfa8b88 100644 --- a/frontend/components/memory/MemoryModal.svelte +++ b/frontend/components/memory/MemoryModal.svelte @@ -31,7 +31,7 @@ import Dialog from '$frontend/components/common/overlay/Dialog.svelte'; import { showAlert, showConfirm } from '$frontend/stores/ui/dialog.svelte'; import { EPISODIC_SUBKINDS } from '$frontend/stores/features/memory-graph.svelte'; - import type { GraphRegion, GraphSource } from '$shared/types/memory'; + import type { GraphSource } from '$shared/types/memory'; interface Props { isOpen: boolean; @@ -95,29 +95,33 @@ let purgeScope = $state<'selection' | 'all'>('all'); let purgeTyped = $state(''); + /** + * Clusters on screen right now, as the canvas reports them. Only ever non-zero + * when the screen holds more memories than it can draw, and zooming in is what + * resolves them — so the hint below says so only while it is true. + */ + let clustersOnScreen = $state(0); + const isMobile = $derived(windowWidth < 768); - const view = $derived(memoryGraphStore.view); + const meta = $derived(memoryGraphStore.meta); const stats = $derived(memoryGraphStore.stats); const selected = $derived(memoryGraphStore.selected); const hits = $derived(memoryGraphStore.searchHits); /** - * Hits always highlight, at either resolution. A memory that survived the grid - * as its own mark lights up exactly as it always did; one that was merged into - * a crowd simply has no mark of its own to light, and the results list is where - * it is reached from. That degrades quietly, which the community grouping did - * not — there every hit was invisible. + * Hits always highlight. The canvas draws every result on top of whatever the + * tiles around it say, so a hit inside a folded or unloaded part of the map + * still has a mark of its own to light. */ const highlightIds = $derived(hits.map(hit => hit.node.id)); - const region = $derived(memoryGraphStore.region); - /** Marks on the canvas: memories drawn individually, plus the merged ones. */ - const drawnCount = $derived(view.nodes.length + view.bins.length); - /** How many memories the merged marks stand for, for the header's tooltip. */ - const mergedCount = $derived(view.bins.reduce((sum, bin) => sum + bin.members, 0)); + /** Memories matching the filter — what the header counts. */ + const totalCount = $derived(meta?.total ?? 0); const filter = $derived(memoryGraphStore.filter); const forgotten = $derived(memoryGraphStore.forgotten); const forgottenSelection = $derived(memoryGraphStore.forgottenSelection); /** Any narrowing beyond the default, so the filter button can say so. */ const activeFilterCount = $derived(filter.subkinds.length + filter.sources.length); + /** Whether an empty graph is the store being empty or the filter excluding it. */ + const narrowed = $derived(activeFilterCount > 0 || filter.projectIds !== null); /** Projects offered by the multi-select, with their path to tell namesakes apart. */ const projectOptions = $derived( projectState.projects.map(project => ({ @@ -228,12 +232,6 @@ void memoryGraphStore.refreshForgotten(); }); - /** `bin:12:34` — a crowd of memories rather than one of them. */ - function binRegionOf(nodeId: string): GraphRegion | null { - if (!nodeId.startsWith('bin:')) return null; - return view.bins.find(bin => bin.id === nodeId)?.region ?? null; - } - function visit(nodeId: string | null): void { if (!nodeId) { history = []; @@ -242,21 +240,6 @@ return; } - // A merged mark has nothing to inspect — it stands for several memories - // rather than being one — so choosing it zooms into the patch of the map it - // covers. - const binRegion = binRegionOf(nodeId); - if (binRegion) { - history = []; - historyIndex = -1; - // The inspector is showing a memory from wherever the user was before. - // Leaving it open next to a patch that does not contain it is the kind of - // stale panel that makes a view feel like it lost track of itself. - void memoryGraphStore.select(null); - memoryGraphStore.focusRegion(binRegion); - return; - } - if (nodeId === selectedId) return; // Truncate the forward entries: navigating somewhere new from the middle of @@ -271,7 +254,7 @@ if (next < 0 || next >= history.length) return; historyIndex = next; void memoryGraphStore.select(history[next]); - canvas?.focusNode(history[next]); + void canvas?.focusNode(history[next]); } function handleSearchInput(): void { @@ -297,7 +280,7 @@ */ function visitAndFocus(nodeId: string): void { visit(nodeId); - canvas?.focusNode(nodeId); + void canvas?.focusNode(nodeId); } /** @@ -607,28 +590,20 @@ - {#if drawnCount > 0} + {#if totalCount > 0} {/if} {#if stats && !stats.embedding.ready} @@ -671,17 +646,34 @@
- {#if !canvasReady || (memoryGraphStore.loading && drawnCount === 0)} + {#if !canvasReady || meta === null}
- {:else if drawnCount === 0} + {:else if totalCount === 0}
- No memories yet - - This fills itself as you work — what gets decided, what breaks and the files involved - are recorded after each conversation. - + {#if narrowed} + Nothing matches these filters + + Widen the project or filter selection to see more of what is remembered. + + {:else} + No memories yet + + This fills itself as you work — what gets decided, what breaks and the files involved + are recorded after each conversation. + + {/if} +
+ {:else if meta.world === null} + +
+ + Arranging {totalCount} memories…
{:else} (clustersOnScreen = scene.clusters)} /> {/if} - {#if canvasReady && region} - - {:else if canvasReady && view.level === 'binned'} + {#if canvasReady && meta?.world && clustersOnScreen > 0}
- {view.bins.length} crowded spots · click one to zoom in + Dense areas are grouped · zoom in to see every memory
{/if} diff --git a/frontend/services/memory/graph-layout.worker.ts b/frontend/services/memory/graph-layout.worker.ts deleted file mode 100644 index 7b63e8da..00000000 --- a/frontend/services/memory/graph-layout.worker.ts +++ /dev/null @@ -1,168 +0,0 @@ -/** - * Memory Graph Layout Worker - * - * ForceAtlas2 runs here instead of on the page's main thread. - * - * The reason is the same one behind the preview decode worker: a force - * simulation is a per-iteration O(n log n) grind over every node, and on a - * low-powered machine it competes directly with Svelte reactivity, the chat view - * and the terminal for one thread. Off-thread, the main thread's only remaining - * job is handing sigma a fresh set of coordinates. - * - * Positions are streamed back in batches as a flat Float32Array (transferred, - * zero copy) rather than as an object per node, because at a few thousand nodes - * the structured-clone cost of the object form is itself enough to stutter. - * - * ── One phase, deliberately ─────────────────────────────────────────────────── - * This used to run three: spread with plain ForceAtlas2, condense with LinLog, - * then a pass that pushed community lobes apart as rigid circles. All three - * survived a long time on the theory that the last two made the grouping legible. - * They did the opposite, and the reason is that sigma renders with `autoRescale`: - * positions are normalised to their bounding box before being drawn, so the - * layout's absolute size is invisible and only the RELATIVE distribution counts. - * - * Under that transform, LinLog contracts each group into a ball and the - * separation pass then inflates the bounding box by shoving the balls apart — - * which, after normalisation, shrinks every lobe on screen and packs it tighter - * still. Two passes intended to reveal structure were jointly manufacturing the - * blobs, and the node sizes on the other side had to fight them. - * - * What is left is ForceAtlas2 with hub dissuasion and high repulsion, which is - * the setup that produces the open, fanned-out look this view wants: leaves - * radiate off their hub instead of collapsing onto it, and communities end up - * adjacent because their edges say so rather than because a post-process placed - * them. Colour carries which lobe is which; the arrangement carries the shape. - */ - -import Graph from 'graphology'; -import forceAtlas2 from 'graphology-layout-forceatlas2'; - -interface StartMessage { - type: 'start'; - nodes: { id: string; x: number; y: number; degree: number }[]; - edges: { source: string; target: string; weight: number }[]; - /** Total iterations to run. Higher settles more but costs more. */ - iterations: number; -} - -interface StopMessage { - type: 'stop'; -} - -type Incoming = StartMessage | StopMessage; - -/** Iterations per chunk between position emissions — the layout visibly settles. */ -const CHUNK = 12; - -let cancelled = false; - -self.onmessage = (event: MessageEvent) => { - const message = event.data; - - if (message.type === 'stop') { - cancelled = true; - return; - } - - if (message.type === 'start') { - cancelled = false; - run(message); - } -}; - -function run(message: StartMessage): void { - const graph = new Graph({ type: 'undirected', multi: false }); - - for (const node of message.nodes) { - graph.addNode(node.id, { x: node.x, y: node.y, size: 1 + Math.sqrt(node.degree) }); - } - for (const edge of message.edges) { - if (!graph.hasNode(edge.source) || !graph.hasNode(edge.target)) continue; - if (edge.source === edge.target) continue; - if (graph.hasEdge(edge.source, edge.target)) continue; - graph.addUndirectedEdge(edge.source, edge.target, { weight: edge.weight }); - } - - const order = graph.order; - if (order === 0) { - self.postMessage({ type: 'done' }); - return; - } - - const inferred = forceAtlas2.inferSettings(graph); - - /** - * ONE set of settings, tuned for openness. - * - * `outboundAttractionDistribution` is the setting that shapes this view more - * than any other: it divides attraction by a node's mass, so a hub does not - * drag its neighbours onto itself. That is what turns a hub and its forty leaves - * into a fan rather than a dot, and fans are most of what makes a large graph - * readable at a glance. - * - * `scalingRatio` is repulsion. It matters less than it looks under sigma's - * `autoRescale` — doubling it mostly scales the whole layout up, which the - * rescale then undoes — but it does change the BALANCE against attraction, so - * higher means longer edges relative to how tightly a group holds together. - * 24 was chosen back when a LinLog phase followed and re-contracted everything; - * with that gone it left the graph tighter than intended. - * - * Anti-collision (`adjustSizes`) stays off throughout. It is a local correction - * that enforces a minimum spacing from the node sizes, which flattens exactly - * the density differences between a busy lobe and a sparse one that the layout - * exists to show. - */ - const settings = { - ...inferred, - outboundAttractionDistribution: true, - // Gravity holds disconnected components in frame; without it they drift off - // and the user is left panning across empty space. - // - // Kept LOW on purpose. Gravity is an isotropic pull toward a single point, so - // the stronger it is the more every graph converges on the same disc - // regardless of what its edges say. Together with the ring-shaped seed this - // used to have (see `seedPosition` in MemoryGraphCanvas) that is why the view - // always rendered as a circle whatever the data looked like. - gravity: 0.12, - scalingRatio: 50, - adjustSizes: false, - // Similarity edges carry their cosine as a weight, so letting weight - // influence attraction means a 0.9 pair sits visibly closer than a 0.4 one — - // the strength of a relationship becomes a distance the user can read. - edgeWeightInfluence: 1, - barnesHutOptimize: order > 500 - }; - - // Node order is stable for the lifetime of a run, so positions can be sent as - // a flat array and re-associated by index on the other side. - const ids = graph.nodes(); - self.postMessage({ type: 'order', ids }); - - let done = 0; - const step = (): void => { - if (cancelled) return; - - const iterations = Math.min(CHUNK, message.iterations - done); - forceAtlas2.assign(graph, { iterations, settings }); - done += iterations; - - const coords = new Float32Array(ids.length * 2); - for (let i = 0; i < ids.length; i++) { - const attributes = graph.getNodeAttributes(ids[i]) as { x: number; y: number }; - coords[i * 2] = attributes.x; - coords[i * 2 + 1] = attributes.y; - } - self.postMessage({ type: 'positions', coords }, [coords.buffer]); - - if (done >= message.iterations) { - self.postMessage({ type: 'done' }); - return; - } - - // Yield between chunks so a 'stop' message can be delivered — a tight loop - // would ignore cancellation until the whole layout finished. - setTimeout(step, 0); - }; - - step(); -} diff --git a/frontend/stores/features/memory-graph.svelte.ts b/frontend/stores/features/memory-graph.svelte.ts index ec1e2e27..056b388b 100644 --- a/frontend/stores/features/memory-graph.svelte.ts +++ b/frontend/stores/features/memory-graph.svelte.ts @@ -1,16 +1,17 @@ /** * Memory Graph Store * - * Backs the Memory modal: the graph view, the selected node's neighbourhood, and - * hybrid search. It is a view over server state — the graph is - * instance-global and every engine writes to it — so nothing is cached + * Backs the Memory modal: what the map covers, the selected node's + * neighbourhood, and hybrid search. It is a view over server state — the graph + * is instance-global and every engine writes to it — so nothing is cached * optimistically: a mutation refetches, because another session may have * changed the same node between renders. * - * Community assignment and node degree arrive precomputed from the server - * (`backend/memory/view.ts`); clustering on the client would fight the force - * simulation for the main thread on exactly the low-powered devices this has to - * stay usable on. + * The map itself is served as tiles (see `backend/memory/view.ts`), and the + * canvas asks for the ones it is looking at. This store holds what the tiles + * are relative to — the meta: world, epoch, counts — plus a `revision` that + * moves whenever what the tiles would say has changed, so the canvas knows to + * fetch the ones on screen again. */ import ws, { onWsReconnect } from '$frontend/utils/ws'; @@ -19,10 +20,11 @@ import type { MemoryQueueStatus, MemoryDraft, GraphNode, - GraphRegion, GraphScope, GraphSource, - GraphView, + GraphTileNode, + GraphTilesResponse, + GraphViewMeta, RetrievalHit } from '$shared/types/memory'; @@ -94,130 +96,37 @@ export const EPISODIC_SUBKINDS = [ 'entity' ] as const; -const EMPTY_VIEW: GraphView = { - level: 'flat', - nodes: [], - edges: [], - bins: [], - binEdges: [], - region: null, - totalNodes: 0, - truncated: false -}; - -/** - * A fingerprint of everything the canvas actually DRAWS. - * - * It lives here rather than in the canvas because it belongs to a FETCH, not to - * a render: computed once when a response lands, it used to be recomputed inside - * a reactive effect — hashing every label of every node again on any store write - * that happened to invalidate the effect. - * - * Every field that feeds a colour, a size or a caption is folded in, and nothing - * else: `weight` and `rel` are absent because no drawn property is derived from - * them, and including them would buy repaints that change no pixels. Positions - * are absent too, but for a different reason — they move a node without changing - * what it IS, so they are watched separately by `layoutSignatureOf` and applied - * without a rebuild. - * - * Hashed rather than concatenated: the ids alone are a string joined from up to - * three thousand entries, and adding labels would build a few hundred kilobytes - * on every refetch to then throw it away. - */ -function signatureOf(view: GraphView): string { - let hash = 2166136261; - const feed = (value: string | number): void => { - const text = typeof value === 'number' ? value.toString(36) : value; - for (let i = 0; i < text.length; i++) { - hash ^= text.charCodeAt(i); - hash = Math.imul(hash, 16777619); - } - // A field separator, so ("ab","c") and ("a","bc") cannot collide. - hash ^= 0x1f; - hash = Math.imul(hash, 16777619); - }; +/** Forgotten memories fetched per page. The list is a review queue, not a feed. */ +const FORGOTTEN_PAGE = 200; - for (const node of view.nodes) { - feed(node.id); - feed(node.label); - feed(node.community); - feed(node.degree); - } - for (const edge of view.edges) { - feed(edge.source); - feed(edge.target); - } - for (const bin of view.bins) { - feed(bin.id); - feed(bin.label); - feed(bin.members); - feed(bin.community); - } - for (const edge of view.binEdges) { - feed(edge.source); - feed(edge.target); - } - return `${view.level}:${view.nodes.length}:${view.edges.length}:${view.bins.length}:${(hash >>> 0).toString(36)}`; -} +/** What the map covers under the current filter; null until first fetched. */ +let meta = $state(null); +let loading = $state(false); /** - * A fingerprint of WHERE things are, kept apart from what they are. + * Moves whenever the tiles on screen may say something different — a write, a + * layout pass, a filter change, a mutation from this client. * - * The two change on different schedules and want different responses. A changed - * node set has to rebuild the graph; a changed arrangement must not, because a - * rebuild resets the camera and replays the entrance — and the arrangement - * changes on its own, seconds after any memory is recorded, when the background - * layout pass lands. - * - * Without this the canvas never saw those positions at all: the node-set - * signature was identical, so a graph the client had laid out for itself stayed - * on screen until the modal was closed and reopened. + * A counter rather than a signature of the data, because the data is not here: + * it is in whichever tiles the canvas happens to be showing, and only the + * canvas can refetch those. */ -function layoutSignatureOf(view: GraphView): string { - let hash = 2166136261; - const feed = (value: number): void => { - // Rounded, so a position that moved a fraction of a pixel does not read as - // news. The layout pass re-settles slightly on every run. - const text = Math.round(value).toString(36); - for (let i = 0; i < text.length; i++) { - hash ^= text.charCodeAt(i); - hash = Math.imul(hash, 16777619); - } - hash ^= 0x1f; - hash = Math.imul(hash, 16777619); - }; - - let placed = 0; - for (const node of view.nodes) { - if (node.x === undefined || node.y === undefined) continue; - placed++; - feed(node.x); - feed(node.y); - } - for (const bin of view.bins) { - feed(bin.x); - feed(bin.y); - } - return `${placed}:${(hash >>> 0).toString(36)}`; -} - -/** Forgotten memories fetched per page. The list is a review queue, not a feed. */ -const FORGOTTEN_PAGE = 200; - -let view = $state(EMPTY_VIEW); -let viewSignature = $state(signatureOf(EMPTY_VIEW)); -let layoutSignature = $state(layoutSignatureOf(EMPTY_VIEW)); -let loading = $state(false); +let revision = $state(0); /** - * The part of the layout being looked at closely, when a crowded mark was opened. + * Memories that must be on the map whatever the tiles around them say: the + * selected one, its connections, and the search results. * - * NAVIGATION, not filtering, which is why it does not live in `filter`: a filter - * narrows what exists, this chooses what to look at closely. Changing a filter - * clears it, because the rectangle was read off an arrangement the new filter - * may not produce. + * They are the things the user is looking FOR, and the tile they sit in may be + * folded into a cluster or not loaded at all. Drawing them on top keeps + * selection, hover and highlight working on exactly what was asked about. */ -let region = $state(null); +let pinned = $state([]); +/** The edges from the selected memory to its listed connections. */ +let pinnedEdges = $state<{ source: string; target: string }[]>([]); +let searchPinned: GraphTileNode[] = []; +let selectionPinned: GraphTileNode[] = []; + let stats = $state(null); let selected = $state(null); let selectedLoading = $state(false); @@ -237,7 +146,7 @@ let filter = $state({ let status = $state(null); /** - * Which graph fetch is the current one. + * Which meta fetch is the current one. * * Refreshes overlap routinely — a filter change while a `memory:changed` refetch * is still in flight — and nothing about a WebSocket request guarantees the @@ -246,6 +155,57 @@ let status = $state(null); * and stays there until something triggers yet another refresh. */ let refreshSeq = 0; +let selectSeq = 0; +let searchSeq = 0; + +/** The filter as the graph requests send it. */ +function filterParams(): { + projectIds?: string[]; + subkinds?: string[]; + sources?: GraphSource[]; + includeArchived: boolean; +} { + return { + ...(filter.projectIds !== null && { projectIds: filter.projectIds }), + ...(filter.subkinds.length > 0 && { subkinds: filter.subkinds }), + ...(filter.sources.length > 0 && { sources: filter.sources }), + includeArchived: filter.includeArchived + }; +} + +/** + * Everything in the meta whose change means the tiles on screen may be out of + * date. The PLACED count rather than the total: a memory that has just been + * written has no position yet, so it is in no tile until the pass that places + * it moves the stamp — refetching for it any earlier fetches the same tiles. + */ +function metaSignature(value: GraphViewMeta): string { + return JSON.stringify([ + value.epoch, + value.stamp, + value.total - value.unplaced, + value.degreeScale, + value.world, + value.extent + ]); +} + +function republishPinned(): void { + const byId = new Map(); + for (const node of [...selectionPinned, ...searchPinned]) byId.set(node.id, node); + pinned = [...byId.values()]; +} + +async function locate(nodeIds: string[]): Promise { + if (nodeIds.length === 0) return []; + try { + const result = (await ws.http('memory:locate', { nodeIds })) as { nodes: GraphTileNode[] }; + return result.nodes; + } catch (error) { + debug.error('settings', 'Failed to locate memory nodes:', error); + return []; + } +} /** The forgotten list: archived and superseded memories, and what is selected. */ let forgotten = $state([]); @@ -261,12 +221,13 @@ let forgottenSelection = $state>(new Set()); let hoveredId = $state(null); export const memoryGraphStore = { - get view() { return view; }, - /** Changes exactly when the drawn graph changes; see `signatureOf`. */ - get viewSignature() { return viewSignature; }, - /** Changes when the persisted arrangement moves; see `layoutSignatureOf`. */ - get layoutSignature() { return layoutSignature; }, - get region() { return region; }, + get meta() { return meta; }, + /** Moves whenever the tiles on screen should be fetched again. */ + get revision() { return revision; }, + /** A key for the current filter — tiles fetched under another are not these. */ + get filterKey() { return JSON.stringify(filterParams()); }, + get pinned() { return pinned; }, + get pinnedEdges() { return pinnedEdges; }, get loading() { return loading; }, get stats() { return stats; }, get selected() { return selected; }, @@ -307,60 +268,41 @@ export const memoryGraphStore = { setFilter(patch: Partial): void { filter = { ...filter, ...patch }; - // The rectangle was read off an arrangement this filter may not produce, so - // keeping it would zoom the new graph to a patch of the old one's plane. - region = null; - void this.refresh(); + void this.refresh({ force: true }); + // The results were retrieved under the old filter, so they may name + // memories this one hides. + if (searchQuery.trim()) void this.search(searchQuery); }, /** - * Zoom into one crowded mark, drawing the memories it stands for. + * Re-read what the map covers, and tell the canvas to refetch its tiles if + * what they would say has changed. * - * REPLACES rather than accumulates. Holding several unrelated rectangles at - * once has no coherent picture to draw and would make "back" undo them one at - * a time — a history nobody asked to build. One region, and the whole graph is - * always one step away. + * "Changed" is decided by the meta — epoch, layout stamp, counts — because the + * change doorbell rings for every write, including reinforcement on every + * recall, which moves nothing on the map. Refetching every tile on screen for + * those was a multi-megabyte stall every few seconds while a conversation ran. + * `force` is for this client's own mutations: an edited label changes a tile + * without changing any count. + * + * The revision moves when the answer lands rather than when it is asked for, + * so the canvas refetches against the epoch and world it is about to be told + * about rather than the ones it already had. */ - focusRegion(next: GraphRegion): void { - region = next; - void this.refresh(); - }, - - /** Back to the whole graph. */ - clearRegion(): void { - if (!region) return; - region = null; - selected = null; - void this.refresh(); - }, - - async refresh(): Promise { + async refresh(options: { force?: boolean } = {}): Promise { const seq = ++refreshSeq; loading = true; try { - const next = (await ws.http('memory:graph', { - ...(filter.projectIds !== null && { projectIds: filter.projectIds }), - ...(filter.subkinds.length > 0 && { subkinds: filter.subkinds }), - ...(filter.sources.length > 0 && { sources: filter.sources }), - ...(region !== null && { region }), - includeArchived: filter.includeArchived - })) as GraphView; + const next = (await ws.http('memory:graph', filterParams())) as GraphViewMeta; // A newer fetch has already been issued, so this answer describes a filter // or a moment the user has moved on from. Dropping it is the whole point. if (seq !== refreshSeq) return; - view = next; - viewSignature = signatureOf(next); - layoutSignature = layoutSignatureOf(next); - // The server decides the resolution — a graph can fall back to the flat - // view while its arrangement is still being computed — so what it - // answered with is the truth about what is drawn, not what was asked. - region = next.region; + const changed = options.force || meta === null || metaSignature(meta) !== metaSignature(next); + meta = next; + if (changed) revision++; } catch (error) { if (seq !== refreshSeq) return; debug.error('settings', 'Failed to load memory graph:', error); - view = EMPTY_VIEW; - viewSignature = signatureOf(EMPTY_VIEW); - layoutSignature = layoutSignatureOf(EMPTY_VIEW); } finally { // Still loading, as far as the user is concerned — the request that // overtook this one has not answered yet. @@ -368,6 +310,35 @@ export const memoryGraphStore = { } }, + /** + * Some tiles of the map, at one zoom level, under the current filter. + * + * Called by the canvas, which is the only thing that knows what is on screen. + * A `stale` answer means the map changed since the meta was read; the meta is + * refetched, and the revision it bumps makes the canvas ask again. + */ + async fetchTiles(request: { + epoch: number; + z: number; + tiles: { x: number; y: number }[]; + budget: number; + }): Promise { + try { + const response = (await ws.http('memory:graph-tiles', { + ...filterParams(), + ...request + })) as GraphTilesResponse; + if (response.stale) void this.refresh({ force: true }); + return response; + } catch (error) { + debug.error('settings', 'Failed to load memory graph tiles:', error); + return null; + } + }, + + /** Where memories are on the map, for going to one the canvas has not drawn. */ + locate, + /** * Poll-free: this is refetched when the server says the queue moved, and once * when a view opens. Recording is background work, so without it the UI has no @@ -399,8 +370,12 @@ export const memoryGraphStore = { }, async select(nodeId: string | null, hops = 1): Promise { + const seq = ++selectSeq; if (!nodeId) { selected = null; + selectionPinned = []; + pinnedEdges = []; + republishPinned(); return; } selectedLoading = true; @@ -409,22 +384,43 @@ export const memoryGraphStore = { node: GraphNode | null; neighbours: { node: GraphNode; hops: number }[]; }; + if (seq !== selectSeq) return; // A node can vanish between the click and the fetch (another session // forgot it), which is a cleared selection rather than an error. selected = detail.node ? { node: detail.node, neighbours: detail.neighbours } : null; + + // The selection and what it connects to are drawn whatever the tiles say, + // so its neighbourhood lights up even where the map is folded or not + // loaded yet. + const direct = detail.neighbours.filter(entry => entry.hops === 1).map(entry => entry.node.id); + const located = detail.node ? await locate([detail.node.id, ...direct]) : []; + if (seq !== selectSeq) return; + selectionPinned = located; + const present = new Set(located.map(node => node.id)); + pinnedEdges = present.has(nodeId) + ? direct.filter(id => present.has(id)).map(target => ({ source: nodeId, target })) + : []; + republishPinned(); } catch (error) { + if (seq !== selectSeq) return; debug.error('settings', 'Failed to load memory node:', error); selected = null; + selectionPinned = []; + pinnedEdges = []; + republishPinned(); } finally { - selectedLoading = false; + if (seq === selectSeq) selectedLoading = false; } }, async search(query: string): Promise { + const seq = ++searchSeq; searchQuery = query; const trimmed = query.trim(); if (!trimmed) { searchHits = []; + searchPinned = []; + republishPinned(); return; } @@ -432,25 +428,36 @@ export const memoryGraphStore = { try { const result = (await ws.http('memory:search', { query: trimmed, - ...(filter.projectIds !== null && { projectIds: filter.projectIds }), - ...(filter.subkinds.length > 0 && { subkinds: filter.subkinds }), - ...(filter.sources.length > 0 && { sources: filter.sources }), - includeArchived: filter.includeArchived, + ...filterParams(), limit: 30 })) as { hits: RetrievalHit[]; vectorUsed: boolean }; + if (seq !== searchSeq) return; searchHits = result.hits; vectorUsed = result.vectorUsed; + // Results are drawn on the map whatever the tiles say, so every one of + // them can be hovered, highlighted and gone to. + const located = await locate(result.hits.map(hit => hit.node.id)); + if (seq !== searchSeq) return; + searchPinned = located; + republishPinned(); } catch (error) { + if (seq !== searchSeq) return; debug.error('settings', 'Memory search failed:', error); searchHits = []; + searchPinned = []; + republishPinned(); } finally { - searching = false; + if (seq === searchSeq) searching = false; } }, clearSearch(): void { + searchSeq++; searchQuery = ''; searchHits = []; + searchPinned = []; + searching = false; + republishPinned(); }, async saveNode( @@ -459,13 +466,13 @@ export const memoryGraphStore = { ): Promise { await ws.http('memory:save-node', { nodeId, ...patch }); // The label may have changed, so the graph view's node caption is stale too. - await Promise.all([this.select(nodeId), this.refresh()]); + await Promise.all([this.select(nodeId), this.refresh({ force: true })]); }, async archiveNode(nodeId: string): Promise { await ws.http('memory:archive-node', { nodeId }); - selected = null; - await Promise.all([this.refresh(), this.refreshStats()]); + await this.select(null); + await Promise.all([this.refresh({ force: true }), this.refreshStats()]); }, /** @@ -479,7 +486,7 @@ export const memoryGraphStore = { if (nodeIds.length === 0) return 0; const result = (await ws.http('memory:restore-nodes', { nodeIds })) as { restored: number }; forgottenSelection = new Set(); - await Promise.all([this.refresh(), this.refreshStats(), this.refreshForgotten()]); + await Promise.all([this.refresh({ force: true }), this.refreshStats(), this.refreshForgotten()]); if (nodeIds.length === 1) await this.select(nodeIds[0]); return result.restored; }, @@ -522,7 +529,7 @@ export const memoryGraphStore = { pinned: input.pinned ?? false, ...(input.reinforceId && { reinforceId: input.reinforceId }) })) as { node: GraphNode | null }; - await Promise.all([this.refresh(), this.refreshStats()]); + await Promise.all([this.refresh({ force: true }), this.refreshStats()]); return result.node; }, @@ -570,7 +577,7 @@ export const memoryGraphStore = { const result = (await ws.http('memory:delete-nodes', { nodeIds })) as { deleted: number }; forgottenSelection = new Set(); - await Promise.all([this.refreshForgotten(), this.refresh(), this.refreshStats()]); + await Promise.all([this.refreshForgotten(), this.refresh({ force: true }), this.refreshStats()]); return result.deleted; }, @@ -583,9 +590,9 @@ export const memoryGraphStore = { ...(projectIds !== undefined && { projectIds }) })) as { nodes: number; edges: number }; - selected = null; - searchHits = []; - await Promise.all([this.refresh(), this.refreshStats(), this.refreshForgotten()]); + await this.select(null); + this.clearSearch(); + await Promise.all([this.refresh({ force: true }), this.refreshStats(), this.refreshForgotten()]); return result; }, @@ -634,7 +641,7 @@ export const memoryGraphStore = { // without a refetch on the way back the graph is silently stale with no // event left to correct it. const offReconnect = onWsReconnect(() => { - void this.refresh(); + void this.refresh({ force: true }); void this.refreshStats(); void this.refreshStatus(); }); diff --git a/shared/types/memory.ts b/shared/types/memory.ts index 373a01d7..18ea76e9 100644 --- a/shared/types/memory.ts +++ b/shared/types/memory.ts @@ -351,40 +351,19 @@ export interface RetrievalResult { // Visualization // ───────────────────────────────────────────────────────────────────────────── -/** A node as sent to the graph view: display fields plus its community. */ -export interface GraphViewNode { - id: string; - subkind: GraphNodeSubkind; - scope: GraphScope; - label: string; - projectId: string | null; - /** Degree, used to size the node. */ - degree: number; - weight: number; - pinned: boolean; - /** Louvain community index — the "lobe" a node belongs to. */ - community: number; - createdAt: string; - /** - * Where the persisted layout put this node, when it has been laid out. - * - * Absent means "no position yet" — a memory written since the last layout - * pass — and the client seeds those from their neighbours rather than - * re-solving the whole graph. See `backend/memory/layout.ts`. - */ - x?: number; - y?: number; -} - -export interface GraphViewEdge { - id: number; - source: string; - target: string; - rel: GraphRelation; - weight: number; +/** + * The square the graph view is cut into tiles from, in layout coordinates. + * + * Fixed between layout passes that change the map, so a tile's address means + * the same rectangle for as long as the client's `epoch` matches the server's. + */ +export interface GraphWorld { + minX: number; + minY: number; + size: number; } -/** A rectangle in layout space — what a zoomed-in view is scoped to. */ +/** A rectangle in layout space. */ export interface GraphRegion { minX: number; maxX: number; @@ -393,65 +372,109 @@ export interface GraphRegion { } /** - * Several memories that fell in one cell of the layout grid, drawn as one mark. + * Everything the view needs before it can ask for a single tile. * - * This is what keeps the view's cost bounded without taking anything out of the - * picture, and the distinction from the alternative matters. Grouping by - * COMMUNITY replaces the picture — you see lobes instead of memories, and - * reaching a memory means opening one. Grouping by POSITION keeps the picture: - * marks merge only where their dots would have overlapped anyway, so the - * silhouette, the lobes and the colours are the ones the full graph would have - * drawn. Below the render budget nothing merges at all and every memory is its - * own mark, exactly as before. + * The counts are the FILTER's, not the screen's: how many memories match, how + * many of those have a place on the map, and where they are — which is what + * "Fit" frames. + */ +export interface GraphViewMeta { + /** Changes whenever the world does; a tile from another epoch is from another map. */ + epoch: number; + /** Changes whenever a layout pass writes; tiles from an older stamp may be missing memories. */ + stamp: number; + /** Null until the first layout pass has placed anything. */ + world: GraphWorld | null; + /** Median distance between a memory and its nearest neighbour, in layout units. */ + spacing: number; + /** Memories matching the filter. */ + total: number; + /** Of those, how many have no position yet — they appear once a pass places them. */ + unplaced: number; + /** Where the placed matches are; null when none are. */ + extent: GraphRegion | null; + /** The degree that counts as "fully connected" when sizing nodes; see `degreeScale`. */ + degreeScale: number; +} + +/** A memory as a tile carries it: enough to draw and hover, nothing to inspect. */ +export interface GraphTileNode { + id: string; + label: string; + /** Louvain community index — the "lobe" a node belongs to, and its colour. */ + community: number; + /** Connections in the whole graph, stored and derived. Sizes the node. */ + degree: number; + x: number; + y: number; +} + +/** + * Several memories sharing one cell of a crowded tile, drawn as one mark. * - * A bin is not a memory — it has no body, no scope and nothing to inspect — - * which is why it travels in its own array rather than as a node wearing a fake - * subkind. + * Only a tile with more memories than its budget is folded like this, and only + * where a cell holds more than one; zooming in asks for smaller tiles, which + * fit, so a cluster always resolves into its memories by zooming alone. */ -export interface GraphViewBin { - /** `bin::` — namespaced so it cannot collide with a node id. */ +export interface GraphTileCluster { + /** `cluster:::` — namespaced so it cannot collide with a node id. */ id: string; - /** How many memories it stands for; drives its size. */ members: number; /** The most reinforced member's summary, for the hover pill. */ label: string; /** Community of that same member, so the mark keeps its lobe's colour. */ community: number; - /** Centroid of its members — not the cell's centre, which would grid the map. */ + /** Centroid of the members. */ x: number; y: number; - /** The cell itself, so opening the mark asks for exactly this rectangle. */ - region: GraphRegion; + /** The cell, in global cell coordinates at this tile's zoom. */ + cellX: number; + cellY: number; } -/** A connection between two bins, weighted by the edges that cross between them. */ -export interface GraphViewBinEdge { +/** + * An edge from one of a tile's memories, with where its far end sits. + * + * The far end may be in a tile the client has not loaded, or folded into a + * cluster there — its position is what lets the client attach the edge to + * whichever mark currently stands for it. + */ +export interface GraphTileEdge { source: string; target: string; weight: number; + targetX: number; + targetY: number; } -export interface GraphView { - /** - * `flat` when every matching memory is drawn individually — the ordinary case, - * and the only one below the render budget. `binned` when some marks stand for - * more than one memory. - * - * Both populate `nodes`: a cell holding a single memory returns that memory, - * not a bin of one. So the two are not alternatives to each other — a binned - * view is a full-fidelity view everywhere it can afford to be. - */ - level: 'flat' | 'binned'; - nodes: GraphViewNode[]; - edges: GraphViewEdge[]; - bins: GraphViewBin[]; - binEdges: GraphViewBinEdge[]; - /** The rectangle this view was scoped to, when the caller zoomed into one. */ - region: GraphRegion | null; - /** Total nodes matching the filter before any display cap was applied. */ - totalNodes: number; - /** True when `totalNodes` exceeded the cap and the view was trimmed. */ - truncated: boolean; +/** An edge out of a crowded tile, rolled up to the cell it lands in. */ +export interface GraphTileCellEdge { + /** The mark in this tile it leaves from — a cluster id or a memory id. */ + source: string; + targetCellX: number; + targetCellY: number; + weight: number; +} + +/** One square of the map at one zoom level. */ +export interface GraphTile { + z: number; + x: number; + y: number; + /** Cells per tile side when this tile was folded; 0 when it was not. */ + cells: number; + nodes: GraphTileNode[]; + clusters: GraphTileCluster[]; + edges: GraphTileEdge[]; + cellEdges: GraphTileCellEdge[]; +} + +export interface GraphTilesResponse { + /** The server's epoch. When it is not the one asked about, `stale` is set. */ + epoch: number; + /** The map changed since the client last read the meta; nothing was served. */ + stale: boolean; + tiles: GraphTile[]; } /**