Skip to content

Commit 75dd28e

Browse files
authored
feat(databases): CSV-backed databases — Table + Board, record pages, vim grid (#81)
* feat(databases): CSV-backed databases with editable Table + Board views Add a general "databases" primitive: a .csv file in the vault is a database (rows = records, typed columns = fields) shown through multiple views — an editable spreadsheet Table and a Board grouped by a select field. Separate from, and leaving untouched, the markdown-task kanban. Backend (shared-domain + main): - databases.ts types + virtual tab-path helpers; hand-rolled RFC-4180 database-csv.ts (parse/serialize, schema inference); type-aware database-transforms.ts (filter/sort/group). Zero new dependencies. - main/databases.ts: read/write/create/list over a .csv + co-located .csv.base.json sidecar (types + views); extracted writeFileAtomic; adopt-a-plain-CSV by inferring schema + materializing stable row ids. - IPC channels + 'database' watch scope; ZenBridge/preload; watcher detection for .csv/.csv.base.json; web-bridge stubs (desktop-only for now). Renderer (app-core): - a database tab is a virtual tab (never hits the markdown pipeline); routed in EditorPane. - store slice: load/open/create + optimistic edits with debounced persistence and watcher-echo reconciliation. - DatabaseView host + DatabaseTableView (inline per-type cell editing, field menu rename/retype/sort/delete, add row/field) + DatabaseBoardView (group by a select field, HTML5 drag-to-move, add column). "New Database" command; Table/Database icons. Tests: 20 new (CSV round-trip incl. commas/quotes/multiSelect, inference, transforms, IO round-trip/adopt/create). typecheck 7/7, all tests green, desktop + web build clean. Note: the UI has not been runtime-verified yet; the Board uses native HTML5 drag-and-drop (lighter than the TasksKanban pointer engine). Sidebar .csv surfacing is a follow-up. * feat(databases): record pages, vim grid, and Notion-style polish Builds on the CSV-backed databases (bc7682f) with the follow-ups from live testing: - Records as pages: each row links to a real markdown note in a per-db folder; row properties mirror into the note's frontmatter, the body is the freeform page. The title cell shows a page icon that reflects whether the note has content, plus a hover "Open" button; opening a page moves the cursor into its editor. - Vim grid: the Table view is a focusable grid with h/j/k/l, gg/G, 0/$, i/Enter to edit, Space/x to select, o to open, a to add, dd to delete. VimNav yields to it via data-zen-db-grid so motions aren't stolen by sidebar/note-list navigation. - Sidebar: right-click a folder to create a database in that location. - Views: rename/delete views; raw-CSV toggle; row-selection checkboxes with bulk delete; right-click a row to Open or Delete it. - Frontmatter rendered compact and muted in the editor (cm-frontmatter). - Drop .bak clutter; route asset-csv tabs to the database view; portal the select dropdown so it isn't clipped by the table's overflow.
1 parent b6ce1b6 commit 75dd28e

27 files changed

Lines changed: 3147 additions & 11 deletions
Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
import { describe, expect, it } from 'vitest'
2+
import {
3+
parseCsv,
4+
serializeCsv,
5+
parseRows,
6+
serializeRows,
7+
inferFields,
8+
buildDefaultViews,
9+
type GenId
10+
} from '@shared/database-csv'
11+
import {
12+
filterRows,
13+
sortRows,
14+
boardColumns,
15+
splitMultiSelect,
16+
joinMultiSelect
17+
} from '@shared/database-transforms'
18+
import { EMPTY_GROUP, type DbField, type DbRow } from '@shared/databases'
19+
20+
/** Deterministic id factory for tests. */
21+
function counterGenId(): GenId {
22+
let n = 0
23+
return () => `id-${++n}`
24+
}
25+
26+
const fieldsById = (fields: DbField[]): Map<string, DbField> =>
27+
new Map(fields.map((f) => [f.id, f]))
28+
29+
describe('parseCsv / serializeCsv round-trip', () => {
30+
it('round-trips embedded commas, quotes, and newlines', () => {
31+
const grid = [
32+
['id', 'name', 'note'],
33+
['1', 'a,b', 'he said "hi"'],
34+
['2', 'line1\nline2', 'plain'],
35+
['3', '', 'trailing space ']
36+
]
37+
expect(parseCsv(serializeCsv(grid))).toEqual(grid)
38+
})
39+
40+
it('parses CRLF and a leading BOM, drops blank lines', () => {
41+
const text = 'id,name\r\n1,Alpha\r\n\r\n2,Beta\r\n'
42+
expect(parseCsv(text)).toEqual([
43+
['id', 'name'],
44+
['1', 'Alpha'],
45+
['2', 'Beta']
46+
])
47+
})
48+
49+
it('preserves empty trailing cells and a no-trailing-newline last row', () => {
50+
expect(parseCsv('a,')).toEqual([['a', '']])
51+
expect(parseCsv('a,b\nc,d')).toEqual([
52+
['a', 'b'],
53+
['c', 'd']
54+
])
55+
})
56+
57+
it('round-trips a cell that is a single double-quote', () => {
58+
const grid = [['x'], ['"']]
59+
expect(parseCsv(serializeCsv(grid))).toEqual(grid)
60+
})
61+
})
62+
63+
describe('parseRows / serializeRows', () => {
64+
const fields: DbField[] = [
65+
{ id: 'fid', name: 'id', type: 'text', hidden: true },
66+
{ id: 'fname', name: 'Name', type: 'text' },
67+
{ id: 'fdone', name: 'Done', type: 'checkbox' }
68+
]
69+
70+
it('keys cells by field id and round-trips', () => {
71+
const csv = serializeCsv([
72+
['id', 'Name', 'Done'],
73+
['u1', 'Task A', 'true'],
74+
['u2', 'Task, B', 'false']
75+
])
76+
const rows = parseRows(csv, fields, 'fid')
77+
expect(rows).toEqual([
78+
{ id: 'u1', cells: { fid: 'u1', fname: 'Task A', fdone: 'true' } },
79+
{ id: 'u2', cells: { fid: 'u2', fname: 'Task, B', fdone: 'false' } }
80+
])
81+
expect(serializeRows(rows, fields)).toEqual(csv)
82+
})
83+
84+
it('synthesizes ids for rows missing one', () => {
85+
const csv = 'id,Name,Done\n,No Id,true\n'
86+
const rows = parseRows(csv, fields, 'fid', counterGenId())
87+
expect(rows[0].id).toBe('id-1')
88+
expect(rows[0].cells.fid).toBe('id-1')
89+
})
90+
91+
it('matches columns by header name regardless of order', () => {
92+
const csv = 'Name,Done,id\nReordered,true,u9\n'
93+
const rows = parseRows(csv, fields, 'fid')
94+
expect(rows[0]).toEqual({ id: 'u9', cells: { fid: 'u9', fname: 'Reordered', fdone: 'true' } })
95+
})
96+
})
97+
98+
describe('inferFields', () => {
99+
it('infers number, checkbox, date, and text columns', () => {
100+
const { fields, idFieldId } = inferFields(
101+
['id', 'Count', 'Done', 'Due', 'Title'],
102+
[
103+
['1', '10', 'true', '2026-01-01', 'hello'],
104+
['2', '20', 'false', '2026-02-15', 'world']
105+
],
106+
counterGenId()
107+
)
108+
const byName = new Map(fields.map((f) => [f.name, f]))
109+
expect(byName.get('Count')!.type).toBe('number')
110+
expect(byName.get('Done')!.type).toBe('checkbox')
111+
expect(byName.get('Due')!.type).toBe('date')
112+
expect(byName.get('Title')!.type).toBe('text')
113+
// existing unique id column is used + hidden
114+
expect(byName.get('id')!.id).toBe(idFieldId)
115+
expect(byName.get('id')!.hidden).toBe(true)
116+
})
117+
118+
it('synthesizes a leading id field when no id column exists', () => {
119+
const { fields, idFieldId } = inferFields(['Name'], [['a'], ['b']], counterGenId())
120+
expect(fields[0].name).toBe('id')
121+
expect(fields[0].id).toBe(idFieldId)
122+
expect(fields[0].hidden).toBe(true)
123+
expect(fields).toHaveLength(2)
124+
})
125+
126+
it('dedupes blank and duplicate headers', () => {
127+
const { fields } = inferFields(['', 'Name', 'Name'], [], counterGenId())
128+
const names = fields.map((f) => f.name)
129+
// includes synthesized id + Column 1 + Name + Name (2)
130+
expect(names).toContain('Column 1')
131+
expect(names.filter((n) => n.startsWith('Name'))).toEqual(['Name', 'Name (2)'])
132+
})
133+
134+
it('buildDefaultViews creates one table view with the id field hidden', () => {
135+
const { fields, idFieldId } = inferFields(['id', 'Name'], [['1', 'a']], counterGenId())
136+
const { views, activeViewId } = buildDefaultViews(fields, counterGenId())
137+
expect(views).toHaveLength(1)
138+
expect(views[0].type).toBe('table')
139+
expect(views[0].id).toBe(activeViewId)
140+
expect(views[0].hiddenFieldIds).toContain(idFieldId)
141+
expect(views[0].columnOrder).toEqual(fields.map((f) => f.id))
142+
})
143+
})
144+
145+
describe('transforms', () => {
146+
const fields: DbField[] = [
147+
{ id: 'name', name: 'Name', type: 'text' },
148+
{ id: 'count', name: 'Count', type: 'number' },
149+
{ id: 'due', name: 'Due', type: 'date' },
150+
{ id: 'status', name: 'Status', type: 'select' }
151+
]
152+
const map = fieldsById(fields)
153+
const rows: DbRow[] = [
154+
{ id: '1', cells: { name: 'Beta', count: '2', due: '2026-03-01', status: 'todo' } },
155+
{ id: '2', cells: { name: 'Alpha', count: '10', due: '2026-01-15', status: 'done' } },
156+
{ id: '3', cells: { name: 'Gamma', count: '1', due: '', status: '' } }
157+
]
158+
159+
it('sorts numerically, not lexically', () => {
160+
const sorted = sortRows(rows, [{ fieldId: 'count', direction: 'asc' }], map)
161+
expect(sorted.map((r) => r.cells.count)).toEqual(['1', '2', '10'])
162+
})
163+
164+
it('sorts dates chronologically and pushes empty values last', () => {
165+
const sorted = sortRows(rows, [{ fieldId: 'due', direction: 'asc' }], map)
166+
expect(sorted.map((r) => r.id)).toEqual(['2', '1', '3'])
167+
})
168+
169+
it('filters by contains and is', () => {
170+
expect(filterRows(rows, [{ fieldId: 'name', op: 'contains', value: 'a' }], map).map((r) => r.id)).toEqual([
171+
'1',
172+
'2',
173+
'3'
174+
])
175+
expect(filterRows(rows, [{ fieldId: 'status', op: 'is', value: 'done' }], map).map((r) => r.id)).toEqual([
176+
'2'
177+
])
178+
expect(filterRows(rows, [{ fieldId: 'due', op: 'isEmpty' }], map).map((r) => r.id)).toEqual(['3'])
179+
})
180+
181+
it('groups into board columns with an EMPTY_GROUP bucket', () => {
182+
const cols = boardColumns(rows, fields[3], ['todo', 'done'])
183+
expect(cols.map((c) => c.key)).toEqual(['todo', 'done', EMPTY_GROUP])
184+
expect(cols[0].rows.map((r) => r.id)).toEqual(['1'])
185+
expect(cols[1].rows.map((r) => r.id)).toEqual(['2'])
186+
expect(cols[2].rows.map((r) => r.id)).toEqual(['3'])
187+
})
188+
189+
it('splits and joins multiSelect cells', () => {
190+
expect(splitMultiSelect('a, b ,c')).toEqual(['a', 'b', 'c'])
191+
expect(joinMultiSelect(['a', 'b'])).toBe('a, b')
192+
expect(splitMultiSelect('')).toEqual([])
193+
})
194+
})
Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
import { mkdtemp, readFile, rm, writeFile, mkdir } from 'node:fs/promises'
2+
import os from 'node:os'
3+
import path from 'node:path'
4+
import { afterEach, describe, expect, it } from 'vitest'
5+
import { createDatabase, createRecordPage, readDatabase, writeDatabaseRows } from './databases'
6+
7+
const tmpDirs: string[] = []
8+
async function makeVault(): Promise<string> {
9+
const dir = await mkdtemp(path.join(os.tmpdir(), 'zennotes-db-'))
10+
tmpDirs.push(dir)
11+
await mkdir(path.join(dir, 'inbox'), { recursive: true })
12+
return dir
13+
}
14+
15+
afterEach(async () => {
16+
await Promise.all(tmpDirs.splice(0).map((d) => rm(d, { recursive: true, force: true })))
17+
})
18+
19+
describe('createDatabase + readDatabase', () => {
20+
it('creates a .csv + sidecar and reads it back', async () => {
21+
const root = await makeVault()
22+
const doc = await createDatabase(root, 'inbox', '', 'Projects')
23+
expect(doc.path.endsWith('Projects.csv')).toBe(true)
24+
expect(doc.title).toBe('Projects')
25+
expect(doc.fields.map((f) => f.name)).toEqual(['id', 'Name'])
26+
expect(doc.rows).toEqual([])
27+
expect(doc.views).toHaveLength(1)
28+
29+
// both files exist on disk
30+
await expect(readFile(path.join(root, doc.path), 'utf8')).resolves.toContain('id,Name')
31+
const sidecar = JSON.parse(
32+
await readFile(path.join(root, `${doc.path}.base.json`), 'utf8')
33+
)
34+
expect(sidecar.version).toBe(1)
35+
expect(sidecar.fields).toHaveLength(2)
36+
37+
// re-open yields the same shape
38+
const reopened = await readDatabase(root, doc.path)
39+
expect(reopened.idFieldId).toBe(doc.idFieldId)
40+
})
41+
42+
it('avoids filename collisions', async () => {
43+
const root = await makeVault()
44+
const a = await createDatabase(root, 'inbox', '', 'Notes')
45+
const b = await createDatabase(root, 'inbox', '', 'Notes')
46+
expect(a.path).not.toBe(b.path)
47+
})
48+
})
49+
50+
describe('writeDatabaseRows round-trip', () => {
51+
it('persists rows (incl. embedded commas) and reads them back', async () => {
52+
const root = await makeVault()
53+
const doc = await createDatabase(root, 'inbox', '', 'Tasks')
54+
const idField = doc.fields.find((f) => f.id === doc.idFieldId)!
55+
const nameField = doc.fields.find((f) => f.name === 'Name')!
56+
57+
const written = await writeDatabaseRows(root, doc.path, [
58+
{ id: 'r1', cells: { [idField.id]: 'r1', [nameField.id]: 'Alpha, with comma' } },
59+
{ id: 'r2', cells: { [idField.id]: 'r2', [nameField.id]: 'Beta' } }
60+
])
61+
expect(written.rows).toHaveLength(2)
62+
63+
const reread = await readDatabase(root, doc.path)
64+
expect(reread.rows.map((r) => r.cells[nameField.id])).toEqual(['Alpha, with comma', 'Beta'])
65+
expect(reread.rows.map((r) => r.id)).toEqual(['r1', 'r2'])
66+
})
67+
})
68+
69+
describe('createRecordPage', () => {
70+
it('creates a page note in a per-database folder', async () => {
71+
const root = await makeVault()
72+
const doc = await createDatabase(root, 'inbox', '', 'Projects')
73+
const noteRel = await createRecordPage(
74+
root,
75+
doc.path,
76+
'My Task',
77+
'---\nName: My Task\n---\n# My Task\n'
78+
)
79+
expect(noteRel.endsWith('.md')).toBe(true)
80+
expect(noteRel).toContain('Projects/') // pages folder named after the db
81+
await expect(readFile(path.join(root, noteRel), 'utf8')).resolves.toContain('# My Task')
82+
})
83+
})
84+
85+
describe('adopting a plain CSV (no sidecar)', () => {
86+
it('infers schema, materializes the sidecar + stable ids, and is stable on re-read', async () => {
87+
const root = await makeVault()
88+
await writeFile(
89+
path.join(root, 'inbox/people.csv'),
90+
'Name,Age,Active\nAda,36,true\nGrace,40,false\n',
91+
'utf8'
92+
)
93+
94+
const doc = await readDatabase(root, 'inbox/people.csv')
95+
const byName = new Map(doc.fields.map((f) => [f.name, f]))
96+
expect(byName.get('Age')!.type).toBe('number')
97+
expect(byName.get('Active')!.type).toBe('checkbox')
98+
expect(doc.rows).toHaveLength(2)
99+
100+
// sidecar was materialized
101+
await expect(
102+
readFile(path.join(root, 'inbox/people.csv.base.json'), 'utf8')
103+
).resolves.toContain('"version": 1')
104+
105+
// ids are stable across re-read (the CSV gained an id column)
106+
const firstIds = doc.rows.map((r) => r.id)
107+
const reread = await readDatabase(root, 'inbox/people.csv')
108+
expect(reread.rows.map((r) => r.id)).toEqual(firstIds)
109+
expect(firstIds.every((id) => id.length > 0)).toBe(true)
110+
})
111+
})

0 commit comments

Comments
 (0)