refactor(erd): decode canonical column handles during export

.jules/bolt.md

@@ -77,3 +77,6 @@ Optimized metric route processing to O(N) by creating a mapping of routes direct
 ## 2024-07-13 - [Optimize Export Dictionary FK lookups]
 **Learning:** Found O(N * C * E) performance bottleneck in ERD export dictionaries due to repeated array searching with `edges.some()` inside a nested loop over nodes and columns.
 **Action:** Replace repeated linear array scans for edges by precomputing O(1) Set lookups of foreign key column handles per node before looping.
+## 2025-03-02 - Optimize column parsing in ERD export
+**Learning:** Identifying and reducing repeated iterations over large arrays of node columns (like checking `column.column_name` by encoding it as a handle ID iteratively in `export.ts`) is crucial. A simple helper `parseColumnNameFromHandle` effectively achieves O(1) string decodes on the edge handles themselves without O(N) generation of handles for each element of columns array, saving allocation and iteration regressions when resolving edge handles. However, directly performing a `.some()` search for membership inside edge processing still leads to an $O(E \times C)$ iteration bottleneck, as warned by reviewer constraints.
+**Action:** Add a column handle parsing utility directly, to pre-parse handles during export functionality, avoiding array scanning and multiple DOM ID generation. Pre-compute node columns into Map/Sets to guarantee $O(1)$ amortized membership verification, making the final edge processing complexity genuinely $O(L)$ where $L$ is handle string length.

frontend/src/erd/__tests__/exportDictionary.benchmark.test.ts

@@ -0,0 +1,57 @@
+import { describe, it, expect } from 'vitest';
+import { exportDictionaryCsv } from '../exportDataDictionary';
+import { sourceColumnHandleId, targetColumnHandleId } from '../handleUtils';
+import type { Node, Edge } from '@xyflow/react';
+import type { TableNodeData } from '../convert';
+
+describe('Export Dictionary Benchmark', () => {
+  it('should efficiently export dictionaries for large graphs without N^2 scaling', () => {
+    const nodes: Node<TableNodeData>[] = [];
+    const edges: Edge[] = [];
+
+    // Generate 500 tables, each with 20 columns
+    const numTables = 500;
+    const numCols = 20;
+
+    for (let i = 0; i < numTables; i++) {
+      nodes.push({
+        id: `t${i}`,
+        data: {
+          title: `table_${i}`,
+          columns: Array.from({ length: numCols }, (_, c) => ({
+            column_name: `col_${c}`,
+            data_type: 'text',
+            is_not_null: false,
+            is_pk: c === 0,
+          })),
+          badges: { pk: true, fk: i > 0 }
+        },
+        position: { x: 0, y: 0 }
+      });
+
+      // Connect each table to the previous one
+      if (i > 0) {
+        edges.push({
+          id: `e${i}`,
+          source: `t${i}`,
+          target: `t${i-1}`,
+          sourceHandle: sourceColumnHandleId('col_1'), // FK column
+          targetHandle: targetColumnHandleId('col_0'), // PK column
+          data: {}
+        });
+      }
+    }
+
+    const start = performance.now();
+    const csv = exportDictionaryCsv(nodes, edges);
+    const elapsed = performance.now() - start;
+
+    expect(csv).toContain('table_0');
+    expect(csv).toContain('table_499');
+
+    // This previously took hundreds of milliseconds due to O(N * C * E)
+    // and is now expected to be well under 50ms.
+    console.log(`Large CSV export took: ${elapsed.toFixed(2)}ms`);
+    expect(elapsed).toBeLessThan(100);
+  });
+});

frontend/src/erd/__tests__/handleUtils.property.test.ts

@@ -0,0 +1,43 @@
+import { describe, it, expect } from 'vitest';
+import fc from 'fast-check';
+import { sanitizeHandleId, sourceColumnHandleId, targetColumnHandleId, parseColumnNameFromHandle } from '../handleUtils';
+
+describe('Handle encoding/decoding properties', () => {
+  it('should round-trip correctly for arbitrary valid column names (including ASCII, CJK, emoji, punctuation)', () => {
+    fc.assert(
+      fc.property(fc.string({ minLength: 0 }), (str) => {
+        const sourceHandle = sourceColumnHandleId(str);
+        const parsedSource = parseColumnNameFromHandle(sourceHandle, 'src');
+        expect(parsedSource).toBe(str);
+
+        const targetHandle = targetColumnHandleId(str);
+        const parsedTarget = parseColumnNameFromHandle(targetHandle, 'tgt');
+        expect(parsedTarget).toBe(str);
+      })
+    );
+  });
+
+  it('should reject malformed handles, noncanonical padded hex, missing digits, and direction mismatches', () => {
+    // Malformed/empty
+    expect(parseColumnNameFromHandle(null)).toBeNull();
+    expect(parseColumnNameFromHandle(undefined)).toBeNull();
+    expect(parseColumnNameFromHandle('')).toBeNull();
+    expect(parseColumnNameFromHandle('invalid-format')).toBeNull();
+
+    // Direction swaps
+    expect(parseColumnNameFromHandle('tgt-c-0069-0064', 'src')).toBeNull();
+    expect(parseColumnNameFromHandle('src-c-0069-0064', 'tgt')).toBeNull();
+
+    // Invalid hex / bad scalars
+    expect(parseColumnNameFromHandle('src-c-nothex')).toBeNull();
+    expect(parseColumnNameFromHandle('src-c-g000')).toBeNull();
+    expect(parseColumnNameFromHandle('src-c-1000000')).toBeNull(); // Out of range (> 0x10FFFF)
+
+    // Non-canonical padding (e.g. 00069 instead of 0069)
+    // The letter 'i' (0069 in hex)
+    expect(parseColumnNameFromHandle('src-c-00069')).toBeNull();
+    // Uppercase letters in hex (should be lowercase according to toString(16))
+    // '006A' will round trip back to '006a'. Thus '006A' fails the round trip check.
+    expect(parseColumnNameFromHandle('src-c-006A')).toBeNull();
+  });
+});

frontend/src/erd/export.ts

@@ -2,7 +2,7 @@ import type { Node, Edge } from '@xyflow/react';
 import { normalizeBusinessGroupColor } from './businessGroups';
 import type { IndexRecommendation } from './cardinality';
 import type { ForeignKeyEdgeData, TableNodeData } from './convert';
-import { sourceColumnHandleId, targetColumnHandleId } from './handleUtils';
+import { sourceColumnHandleId, targetColumnHandleId, parseColumnNameFromHandle } from './handleUtils';
 
 export * from './exportDataDictionary';
 
@@ -59,6 +59,7 @@ function fkColumnsForEdge(
   edge: Edge,
   sourceNode: Node<TableNodeData>,
   targetNode: Node<TableNodeData>,
+  columnsByNode: Map<string, Set<string>>
 ): { sourceColumns: string[]; targetColumns: string[] } | null {
   const data = edge.data as ForeignKeyEdgeData | undefined;
   const sourceColumns = data?.sourceColumns?.filter(Boolean) || [];
@@ -67,12 +68,15 @@ function fkColumnsForEdge(
     return { sourceColumns, targetColumns };
   }
 
-  const sourceHandleColumn = (sourceNode.data.columns || [])
-    .find((column) => sourceColumnHandleId(column.column_name) === edge.sourceHandle)
-    ?.column_name;
-  const targetHandleColumn = (targetNode.data.columns || [])
-    .find((column) => targetColumnHandleId(column.column_name) === edge.targetHandle)
-    ?.column_name;
+  const parsedSource = parseColumnNameFromHandle(edge.sourceHandle, 'src');
+  const parsedTarget = parseColumnNameFromHandle(edge.targetHandle, 'tgt');
+
+  const sourceCols = columnsByNode.get(sourceNode.id);
+  const targetCols = columnsByNode.get(targetNode.id);
+
+  const sourceHandleColumn = parsedSource && sourceCols?.has(parsedSource) ? parsedSource : undefined;
+  const targetHandleColumn = parsedTarget && targetCols?.has(parsedTarget) ? parsedTarget : undefined;
+
   if (sourceHandleColumn && targetHandleColumn) {
     return { sourceColumns: [sourceHandleColumn], targetColumns: [targetHandleColumn] };
   }
@@ -127,13 +131,22 @@ export function exportDDL(nodes: Node<TableNodeData>[], edges: Edge[]): string {
     ddl += '\n);\n\n';
   }
 
+  const columnsByNode = new Map<string, Set<string>>();
+  for (const node of nodes) {
+    const set = new Set<string>();
+    for (const c of node.data.columns || []) {
+      if (c && c.column_name) set.add(c.column_name);
+    }
+    columnsByNode.set(node.id, set);
+  }
+
   // Export foreign keys
   for (const edge of edges) {
     const sourceNode = nodesById.get(edge.source);
     const targetNode = nodesById.get(edge.target);
 
     if (sourceNode && targetNode) {
-      const fkColumns = fkColumnsForEdge(edge, sourceNode, targetNode);
+      const fkColumns = fkColumnsForEdge(edge, sourceNode, targetNode, columnsByNode);
       const constraintName = edge.label ? edge.label : `fk_${edge.source}_${edge.target}`;
       const sourceTable = quoteSqlIdentifier(sourceNode.data.title || sourceNode.id);
       const targetTable = quoteSqlIdentifier(targetNode.data.title || targetNode.id);

frontend/src/erd/exportDataDictionary.ts

@@ -1,7 +1,7 @@
 import type { Edge, Node } from '@xyflow/react';
 
 import type { ForeignKeyEdgeData, TableNodeData } from './convert';
-import { sourceColumnHandleId } from './handleUtils';
+import { sourceColumnHandleId, parseColumnNameFromHandle } from './handleUtils';
 
 const CONTROL_TEXT_RE = /[\u0000-\u001f\u007f]+/g;
 const CSV_FORMULA_RE = /^[=+\-@]/;
@@ -59,7 +59,12 @@ function foreignKeyColumnsByNode(edges: Edge[]): Map<string, ForeignKeyNodeInfo>
     }
 
     if (edge.sourceHandle) {
-      info.handles.add(edge.sourceHandle);
+      const parsedColumn = parseColumnNameFromHandle(edge.sourceHandle, 'src');
+      if (parsedColumn) {
+        info.columns.add(parsedColumn);
+      } else {
+        info.handles.add(edge.sourceHandle);
+      }
     }
   }
 

frontend/src/erd/handleUtils.ts

@@ -14,3 +14,26 @@ export function sourceColumnHandleId(columnName: string): string {
 export function targetColumnHandleId(columnName: string): string {
   return `tgt-${sanitizeHandleId(columnName)}`
 }
+
+export function parseColumnNameFromHandle(handleId: string | null | undefined, direction?: 'src' | 'tgt'): string | null {
+  if (!handleId) return null;
+  const match = handleId.match(/^(src|tgt)-c-([0-9a-f-]+|empty)$/);
+  if (!match) return null;
+  const parsedDirection = match[1];
+  if (direction && parsedDirection !== direction) return null;
+  const encoded = match[2];
+  if (encoded === 'empty') return '';
+  try {
+    const decoded = encoded.split('-').map((hex) => {
+      if (!/^[0-9a-f]{4,6}$/.test(hex)) throw new Error('Invalid hex format');
+      const codePoint = parseInt(hex, 16);
+      if (codePoint > 0x10FFFF) throw new Error('Invalid code point');
+      return String.fromCodePoint(codePoint);
+    }).join('');
+    // Verify canonical re-encoding to reject padded or noncanonical hex casing.
+    if (sanitizeHandleId(decoded) !== `c-${encoded}`) return null;
+    return decoded;
+  } catch (e) {
+    return null;
+  }
+}