diff --git a/.changeset/grumpy-grapheme-fix.md b/.changeset/grumpy-grapheme-fix.md new file mode 100644 index 0000000..de6cb69 --- /dev/null +++ b/.changeset/grumpy-grapheme-fix.md @@ -0,0 +1,5 @@ +--- +"@bomb.sh/tty": patch +--- + +Fixes combining marks (accents, ZWJ, variation selectors, kitty-graphics placeholder diacritics) being silently dropped from rendered output instead of attaching to their base character's cell. Cells that overflow the 8-mark-per-cell limit now also surface a `COMBINING_MARKS_EXCEEDED` render error instead of truncating silently. diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index d204b4e..e07cee7 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -43,6 +43,9 @@ jobs: - name: build wasm run: make + - name: test + run: deno task test + - name: upload wasm artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/specs/renderer-spec.md b/specs/renderer-spec.md index f6b7e64..ffa780e 100644 --- a/specs/renderer-spec.md +++ b/specs/renderer-spec.md @@ -603,6 +603,15 @@ NOT override the background already present in each glyph cell; element backgrounds established by `open({ bg })` remain in effect, and the terminal default remains in effect where no element background applies. +**Grapheme cluster preservation.** When `content` contains grapheme clusters — a +base codepoint followed by one or more combining marks (Unicode codepoints with +`wcwidth` ≤ 0) — the renderer MUST preserve and emit the full cluster. Combining +marks MUST NOT be silently dropped. Combining marks do not advance the cursor +position; they are attached to the preceding base codepoint's cell and emitted +immediately after the base codepoint's bytes in the output stream. A cell +occupied by a grapheme cluster with combining marks MUST be treated as changed +(and thus emitted) when any mark in the cluster changes between frames. + The set of styling properties accepted by `props` is part of the current implementation surface and may be extended. @@ -1062,8 +1071,8 @@ The `errors` field contains any errors reported by the Clay layout engine during the most recent `render()` call. Each error is a `ClayError` object with: - `type`: a string identifying the error category. The following types are - defined. Most mirror Clay's error taxonomy; `"CLIP_DEPTH_EXCEEDED"` is - Clayterm-specific. + defined. Most mirror Clay's error taxonomy; `"CLIP_DEPTH_EXCEEDED"` and + `"COMBINING_MARKS_EXCEEDED"` are Clayterm-specific. - `"TEXT_MEASUREMENT_FUNCTION_NOT_PROVIDED"` - `"ARENA_CAPACITY_EXCEEDED"` - `"ELEMENTS_CAPACITY_EXCEEDED"` @@ -1076,6 +1085,10 @@ the most recent `render()` call. Each error is a `ClayError` object with: - `"CLIP_DEPTH_EXCEEDED"` — A frame nested clip regions more deeply than the renderer could track. See §7.5 for the guarantees that still hold in this case. The `message` SHOULD identify the renderer's tracking limit. + - `"COMBINING_MARKS_EXCEEDED"` — A frame attached more combining marks to a + single cell than the cell can store. The excess marks are truncated (see + §13, Cell representation). Reported at most once per frame, on the first + truncation. The `message` SHOULD identify the per-cell limit. - `message`: a human-readable string describing the error in detail. Errors are collected per-render; each call to `render()` returns only the errors @@ -1138,11 +1151,25 @@ elements; clip regions; and scroll containers. **Text measurement.** Text width measurement uses `wcwidth`-based character width computation, supporting ASCII, CJK wide characters, and other Unicode -codepoints. - -**Cell representation.** Each cell in the buffer stores a Unicode codepoint, a -foreground color (packed ARGB with attribute flags in the high byte), and a -background color. +codepoints. Combining marks (codepoints with `wcwidth` ≤ 0) contribute zero to +measured width; they attach to the preceding base codepoint's cell and are not +counted as separate cells in layout. Measurement and rendering MUST agree: if +the measurer ignores a combining mark for width purposes, the renderer MUST +still attach and emit it. + +**Cell representation.** Each cell in the buffer stores a grapheme cluster — a +base Unicode codepoint plus up to 8 combining-mark codepoints — together with a +foreground color (packed ARGB with attribute flags in the high byte) and a +background color. The combining-mark slots are zero-terminated; a cell with no +combining marks stores zero in every slot. When a text string produces more than +8 combining marks for a single base codepoint, the excess marks are truncated +from the end (marks 1–8 are kept, marks 9+ are discarded), ensuring that the +first and most semantically significant marks always survive. The first +truncation in a frame is reported as a `"COMBINING_MARKS_EXCEEDED"` error +(§12.3); later truncations in the same frame are not reported again. Cell +comparison for diffing considers combining marks: a cell is considered changed +when any combining mark differs from the front buffer, not only when the base +codepoint or color attributes differ. **Border junction resolution.** When bordered elements share edges, the renderer accumulates per-cell direction bitmasks and resolves them to correct box-drawing diff --git a/src/cell.c b/src/cell.c index 476a33b..4a5e14f 100644 --- a/src/cell.c +++ b/src/cell.c @@ -4,13 +4,18 @@ void cells_fill(Cell *buf, int w, int h, uint32_t ch, uint32_t fg, uint32_t bg) { - for (int i = 0; i < w * h; i++) { - buf[i].ch = ch; - buf[i].fg = fg; - buf[i].bg = bg; - } + /* Designated init zeros unspecified fields (including combining[]). */ + Cell tmpl = {.ch = ch, .fg = fg, .bg = bg}; + for (int i = 0; i < w * h; i++) + buf[i] = tmpl; } int cell_cmp(Cell *a, Cell *b) { - return a->ch != b->ch || a->fg != b->fg || a->bg != b->bg; + if (a->ch != b->ch || a->fg != b->fg || a->bg != b->bg) + return 1; + for (int i = 0; i < CELL_MAX_COMBINING; i++) { + if (a->combining[i] != b->combining[i]) + return 1; + } + return 0; } diff --git a/src/cell.h b/src/cell.h index 5a1db90..1755b2c 100644 --- a/src/cell.h +++ b/src/cell.h @@ -5,10 +5,16 @@ #include +/* Maximum combining marks stored per cell. Marks beyond this limit are + * silently truncated from the end; the first CELL_MAX_COMBINING are kept. */ +#define CELL_MAX_COMBINING 8 + typedef struct { uint32_t ch; uint32_t fg; /* 0xAARRGGBB — upper byte: attribute flags */ uint32_t bg; /* 0xAARRGGBB — upper byte: attribute flags */ + uint32_t combining[CELL_MAX_COMBINING]; /* zero-terminated combining-mark + codepoints */ } Cell; /* Attribute flags (packed into high byte of fg) */ diff --git a/src/clayterm.c b/src/clayterm.c index c1edb83..7e4e62e 100644 --- a/src/clayterm.c +++ b/src/clayterm.c @@ -51,9 +51,10 @@ struct Clayterm *ct_active_context = NULL; /* clip stack depth: nesting beyond this clamps to the deepest rect */ #define CLIP_STACK_MAX 16 -/* Clayterm-specific error code, numbered past Clay's error enum (0..8). +/* Clayterm-specific error codes, numbered past Clay's error enum (0..8). * Mirrored by ERROR_TYPES in term.ts. */ #define CLAYTERM_ERR_CLIP_DEPTH_EXCEEDED 9 +#define CLAYTERM_ERR_COMBINING_MARKS_EXCEEDED 10 #define CLAYTERM_STR_(x) #x #define CLAYTERM_STR(x) CLAYTERM_STR_(x) @@ -80,6 +81,8 @@ struct Clayterm { int clipoverflow; /* set once per frame when nesting first exceeds the tracked depth */ int clip_depth_exceeded; + /* set once per frame when a cell first runs out of combining-mark slots */ + int combining_exceeded; /* error collection */ Clay_ErrorData errors[MAX_ERRORS]; int error_count; @@ -109,7 +112,9 @@ struct Clayterm { * Output buffer is sized at 64 bytes per cell — enough for worst-case * full-screen redraws with truecolor SGR sequences on every cell. */ -#define OUT_BYTES_PER_CELL 64 +/* 128 bytes per cell: ~84 bytes worst-case for CUP + SGR sequences, plus up + * to 4 (base SMP char) + 8×4 (combining marks) = 36 bytes of cluster text. */ +#define OUT_BYTES_PER_CELL 128 /* ── Cell buffer ops ──────────────────────────────────────────────── */ @@ -117,15 +122,15 @@ static Cell *cell_at(struct Clayterm *ct, Cell *buf, int x, int y) { return &buf[y * ct->w + x]; } -static void setcell(struct Clayterm *ct, int x, int y, uint32_t ch, uint32_t fg, - uint32_t bg) { +static int setcell(struct Clayterm *ct, int x, int y, uint32_t ch, uint32_t fg, + uint32_t bg) { if (x < 0 || x >= ct->w || y < 0 || y >= ct->h) - return; + return 0; if (ct->clipping) { if (x < ct->clipx || x >= ct->clipx + ct->clipw) - return; + return 0; if (y < ct->clipy || y >= ct->clipy + ct->cliph) - return; + return 0; } Cell *c = cell_at(ct, ct->back, x, y); c->ch = ch; @@ -133,6 +138,45 @@ static void setcell(struct Clayterm *ct, int x, int y, uint32_t ch, uint32_t fg, if (!(bg & ATTR_DEFAULT)) { c->bg = bg; } + for (int i = 0; i < CELL_MAX_COMBINING; i++) + c->combining[i] = 0; + return 1; +} + +/* Surface a COMBINING_MARKS_EXCEEDED error once per frame. */ +static void report_combining_exceeded(struct Clayterm *ct) { + if (ct->combining_exceeded) + return; + ct->combining_exceeded = 1; + if (ct->error_count >= MAX_ERRORS) + return; + static const char msg[] = + "cell exceeds combining-mark limit of " CLAYTERM_STR( + CELL_MAX_COMBINING) "; excess marks truncated"; + ct->errors[ct->error_count++] = (Clay_ErrorData){ + .errorType = (Clay_ErrorType)CLAYTERM_ERR_COMBINING_MARKS_EXCEEDED, + .errorText = {.isStaticallyAllocated = true, + .length = (int32_t)(sizeof(msg) - 1), + .chars = msg}, + .userData = ct, + }; +} + +/* Append a combining-mark codepoint to the cell at (x, y) in the back buffer. + * Marks beyond CELL_MAX_COMBINING are dropped (truncation from end) and + * reported once per frame. + */ +static void append_combining(struct Clayterm *ct, int x, int y, uint32_t cp) { + if (x < 0 || x >= ct->w || y < 0 || y >= ct->h) + return; + Cell *c = cell_at(ct, ct->back, x, y); + for (int i = 0; i < CELL_MAX_COMBINING; i++) { + if (c->combining[i] == 0) { + c->combining[i] = cp; + return; + } + } + report_combining_exceeded(ct); } /* ── Escape sequence generation ───────────────────────────────────── */ @@ -238,6 +282,8 @@ static void present_cups(struct Clayterm *ct, int row) { emit_ch(ct, i, y, row, ' '); } else { emit_ch(ct, x, y, row, back->ch); + for (int ci = 0; ci < CELL_MAX_COMBINING && back->combining[ci]; ci++) + buf_char(&ct->out, back->combining[ci]); /* mark trailing cells of wide char as invalid in front * so they'll diff when overwritten by narrow chars */ for (int i = 1; i < w; i++) { @@ -298,6 +344,8 @@ static void present_lines(struct Clayterm *ct) { if (!iswprint(ch)) ch = 0xfffd; buf_char(&ct->out, ch); + for (int ci = 0; ci < CELL_MAX_COMBINING && back->combining[ci]; ci++) + buf_char(&ct->out, back->combining[ci]); for (int i = 1; i < w; i++) { Cell *fw = cell_at(ct, ct->front, x + i, y); fw->ch = 0xffffffff; @@ -391,6 +439,7 @@ static void render_text(struct Clayterm *ct, int x0, int y0, const char *p = slice; int rem = slice_len; int x = x0; + int last_x = -1; /* column of the most-recently written base cell */ while (rem > 0) { /* Check at the top of each iteration: if the pointer we are about to @@ -414,8 +463,11 @@ static void render_text(struct Clayterm *ct, int x0, int y0, if (cw < 0) cw = 1; if (cw > 0) { - setcell(ct, x, y0, cp, fg, bg); + last_x = setcell(ct, x, y0, cp, fg, bg) ? x : -1; x += cw; + } else if (last_x >= 0) { + /* combining mark: attach to the preceding base cell */ + append_combining(ct, last_x, y0, cp); } p += n; rem -= n; @@ -870,6 +922,7 @@ void reduce(struct Clayterm *ct, uint32_t *buf, int len, int mode, int row, ct->clipdepth = 0; ct->clipoverflow = 0; ct->clip_depth_exceeded = 0; + ct->combining_exceeded = 0; ct->clipping = 0; cells_fill(ct->back, ct->w, ct->h, ' ', ATTR_DEFAULT, ATTR_DEFAULT); diff --git a/term.ts b/term.ts index b1305ff..174f311 100644 --- a/term.ts +++ b/term.ts @@ -69,6 +69,7 @@ const ERROR_TYPES = [ "INTERNAL_ERROR", "UNBALANCED_OPEN_CLOSE", "CLIP_DEPTH_EXCEEDED", + "COMBINING_MARKS_EXCEEDED", ] as const; export interface ClayError { diff --git a/test/print.ts b/test/print.ts index c4ba54a..db0e516 100644 --- a/test/print.ts +++ b/test/print.ts @@ -1,3 +1,5 @@ +const ZERO_WIDTH = /^[\p{Mn}\p{Me}\u200B-\u200F\uFE00-\uFE0F]$/u; + /** * Interpret ANSI escape sequences into a plain text grid. * Handles CSI cursor positioning (row;colH), DECTCEM show/hide @@ -7,6 +9,9 @@ * cell is marked with U+0332 COMBINING LOW LINE appended to the base * character. The base char is preserved and the underline spans its * rendered width — including the full width of CJK/wide chars. + * + * Zero-width codepoints (combining marks, ZWJ, variation selectors) + * attach to the most recently written cell instead of advancing. */ export function print(ansi: string, w: number, h: number): string { let grid: string[][] = []; @@ -20,6 +25,7 @@ export function print(ansi: string, w: number, h: number): string { let x = 0; let y = 0; let i = 0; + let last: [number, number] | undefined; let cursorVisible = false; while (i < ansi.length) { @@ -36,6 +42,7 @@ export function print(ansi: string, w: number, h: number): string { let cmd = ansi[i++]; if (cmd === "H") { + last = undefined; // cursor position: row;col (1-indexed) let parts = params.split(";"); y = (parseInt(parts[0]) || 1) - 1; @@ -50,15 +57,23 @@ export function print(ansi: string, w: number, h: number): string { y++; x = 0; i++; + last = undefined; } else { // regular character — could be multi-byte UTF-8 let cp = ansi.codePointAt(i)!; let ch = String.fromCodePoint(cp); + i += ch.length; + if (ZERO_WIDTH.test(ch)) { + if (last) grid[last[1]][last[0]] += ch; + continue; + } if (x >= 0 && x < w && y >= 0 && y < h) { grid[y][x] = ch; + last = [x, y]; + } else { + last = undefined; } x++; - i += ch.length; } } diff --git a/test/term.test.ts b/test/term.test.ts index b12bede..fe72c6c 100644 --- a/test/term.test.ts +++ b/test/term.test.ts @@ -335,6 +335,102 @@ describe("term", () => { }); }); + describe("grapheme clusters", () => { + let border = { + color: rgba(255, 255, 255), + left: 1, + right: 1, + top: 1, + bottom: 1, + }; + + async function render(content: string): Promise { + let t = await createTerm({ width: 12, height: 3 }); + let ansi = decode( + t.render([ + open("root", { + layout: { width: grow(), height: grow(), direction: "ttb" }, + border, + }), + text(content), + close(), + ]).output, + ); + return trim(print(ansi, 12, 3)); + } + + it("preserves combining accent (e + U+0301)", async () => { + expect(await render("cafe\u0301")).toEqual(`\ +┌──────────┐ +│cafe\u0301 │ +└──────────┘`); + }); + + it("preserves kitty-graphics placeholder cluster (base + 2 combining marks)", async () => { + expect(await render("\u{10EEEE}\u0305\u030D")).toEqual(`\ +┌──────────┐ +│\u{10EEEE}\u0305\u030D │ +└──────────┘`); + }); + + it("preserves ZWJ (ZWJ is per-cell combining; following emoji start new cells)", async () => { + // The trailing half of each wide emoji is the blank column after it. + expect(await render("👨\u200D👩\u200D👧\u200D👦")).toEqual(`\ +┌──────────┐ +│👨\u200D 👩\u200D 👧\u200D 👦 │ +└──────────┘`); + }); + + it("truncates excess combining marks from the end (first 8 survive)", async () => { + let grave = "\u0300"; + expect(await render("a" + grave.repeat(9))).toEqual(`\ +┌──────────┐ +│a${grave.repeat(8)} │ +└──────────┘`); + }); + + it("reports COMBINING_MARKS_EXCEEDED once per frame on truncation", async () => { + let t = await createTerm({ width: 12, height: 3 }); + let frame = (content: string) => [ + open("root", { + layout: { width: grow(), height: grow(), direction: "ttb" }, + }), + text(content), + close(), + ]; + let types = (content: string) => + t.render(frame(content)).errors.map((e) => e.type); + let grave = "̀"; + + expect(types("a" + grave.repeat(9) + "b" + grave.repeat(9))).toEqual([ + "COMBINING_MARKS_EXCEEDED", + ]); + expect(types("a" + grave.repeat(8))).toEqual([]); + expect(types("c" + grave.repeat(10))).toEqual([ + "COMBINING_MARKS_EXCEEDED", + ]); + }); + + it("drops combining marks whose base cell is clipped", async () => { + let t = await createTerm({ width: 6, height: 1 }); + let ansi = decode( + t.render([ + open("root", { + layout: { width: grow(), height: grow(), direction: "ltr" }, + }), + open("clip", { + layout: { width: fixed(2), height: fixed(1) }, + clip: { horizontal: true, vertical: true }, + }), + text("abe\u0301"), + close(), + close(), + ]).output, + ); + expect(print(ansi, 6, 1)).toEqual("ab "); + }); + }); + describe("caret placement", () => { // These tests use `print()`, which marks the terminal's final // cursor position by appending U+0332 COMBINING LOW LINE to that