fix: preserve grapheme clusters (combining marks) in cell emission

.changeset/grumpy-grapheme-fix.md

@@ -0,0 +1,5 @@
+---
+"@bomb.sh/tty": patch
+---
+
+Fixes combining marks (accents, ZWJ, variation selectors, kitty-graphics placeholder diacritics) being silently dropped from rendered output instead of attaching to their base character's cell. Cells that overflow the 8-mark-per-cell limit now also surface a `COMBINING_MARKS_EXCEEDED` render error instead of truncating silently.

.github/workflows/verify.yaml

@@ -43,6 +43,9 @@ jobs:
       - name: build wasm
         run: make
 
+      - name: test
+        run: deno task test
+
       - name: upload wasm artifact
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:

specs/renderer-spec.md

@@ -603,6 +603,15 @@ NOT override the background already present in each glyph cell; element
 backgrounds established by `open({ bg })` remain in effect, and the terminal
 default remains in effect where no element background applies.
 
+**Grapheme cluster preservation.** When `content` contains grapheme clusters — a
+base codepoint followed by one or more combining marks (Unicode codepoints with
+`wcwidth` ≤ 0) — the renderer MUST preserve and emit the full cluster. Combining
+marks MUST NOT be silently dropped. Combining marks do not advance the cursor
+position; they are attached to the preceding base codepoint's cell and emitted
+immediately after the base codepoint's bytes in the output stream. A cell
+occupied by a grapheme cluster with combining marks MUST be treated as changed
+(and thus emitted) when any mark in the cluster changes between frames.
+
 The set of styling properties accepted by `props` is part of the current
 implementation surface and may be extended.
 
@@ -1062,8 +1071,8 @@ The `errors` field contains any errors reported by the Clay layout engine during
 the most recent `render()` call. Each error is a `ClayError` object with:
 
 - `type`: a string identifying the error category. The following types are
-  defined. Most mirror Clay's error taxonomy; `"CLIP_DEPTH_EXCEEDED"` is
-  Clayterm-specific.
+  defined. Most mirror Clay's error taxonomy; `"CLIP_DEPTH_EXCEEDED"` and
+  `"COMBINING_MARKS_EXCEEDED"` are Clayterm-specific.
   - `"TEXT_MEASUREMENT_FUNCTION_NOT_PROVIDED"`
   - `"ARENA_CAPACITY_EXCEEDED"`
   - `"ELEMENTS_CAPACITY_EXCEEDED"`
@@ -1076,6 +1085,10 @@ the most recent `render()` call. Each error is a `ClayError` object with:
   - `"CLIP_DEPTH_EXCEEDED"` — A frame nested clip regions more deeply than the
     renderer could track. See §7.5 for the guarantees that still hold in this
     case. The `message` SHOULD identify the renderer's tracking limit.
+  - `"COMBINING_MARKS_EXCEEDED"` — A frame attached more combining marks to a
+    single cell than the cell can store. The excess marks are truncated (see
+    §13, Cell representation). Reported at most once per frame, on the first
+    truncation. The `message` SHOULD identify the per-cell limit.
 - `message`: a human-readable string describing the error in detail.
 
 Errors are collected per-render; each call to `render()` returns only the errors
@@ -1138,11 +1151,25 @@ elements; clip regions; and scroll containers.
 
 **Text measurement.** Text width measurement uses `wcwidth`-based character
 width computation, supporting ASCII, CJK wide characters, and other Unicode
-codepoints.
-
-**Cell representation.** Each cell in the buffer stores a Unicode codepoint, a
-foreground color (packed ARGB with attribute flags in the high byte), and a
-background color.
+codepoints. Combining marks (codepoints with `wcwidth` ≤ 0) contribute zero to
+measured width; they attach to the preceding base codepoint's cell and are not
+counted as separate cells in layout. Measurement and rendering MUST agree: if
+the measurer ignores a combining mark for width purposes, the renderer MUST
+still attach and emit it.
+
+**Cell representation.** Each cell in the buffer stores a grapheme cluster — a
+base Unicode codepoint plus up to 8 combining-mark codepoints — together with a
+foreground color (packed ARGB with attribute flags in the high byte) and a
+background color. The combining-mark slots are zero-terminated; a cell with no
+combining marks stores zero in every slot. When a text string produces more than
+8 combining marks for a single base codepoint, the excess marks are truncated
+from the end (marks 1–8 are kept, marks 9+ are discarded), ensuring that the
+first and most semantically significant marks always survive. The first
+truncation in a frame is reported as a `"COMBINING_MARKS_EXCEEDED"` error
+(§12.3); later truncations in the same frame are not reported again. Cell
+comparison for diffing considers combining marks: a cell is considered changed
+when any combining mark differs from the front buffer, not only when the base
+codepoint or color attributes differ.
 
 **Border junction resolution.** When bordered elements share edges, the renderer
 accumulates per-cell direction bitmasks and resolves them to correct box-drawing

src/cell.c

@@ -4,13 +4,18 @@
 
 void cells_fill(Cell *buf, int w, int h, uint32_t ch, uint32_t fg,
                 uint32_t bg) {
-  for (int i = 0; i < w * h; i++) {
-    buf[i].ch = ch;
-    buf[i].fg = fg;
-    buf[i].bg = bg;
-  }
+  /* Designated init zeros unspecified fields (including combining[]). */
+  Cell tmpl = {.ch = ch, .fg = fg, .bg = bg};
+  for (int i = 0; i < w * h; i++)
+    buf[i] = tmpl;
 }
 
 int cell_cmp(Cell *a, Cell *b) {
-  return a->ch != b->ch || a->fg != b->fg || a->bg != b->bg;
+  if (a->ch != b->ch || a->fg != b->fg || a->bg != b->bg)
+    return 1;
+  for (int i = 0; i < CELL_MAX_COMBINING; i++) {
+    if (a->combining[i] != b->combining[i])
+      return 1;
+  }
+  return 0;
 }

src/cell.h

@@ -5,10 +5,16 @@
 
 #include <stdint.h>
 
+/* Maximum combining marks stored per cell. Marks beyond this limit are
+ * silently truncated from the end; the first CELL_MAX_COMBINING are kept. */
+#define CELL_MAX_COMBINING 8
+
 typedef struct {
   uint32_t ch;
   uint32_t fg; /* 0xAARRGGBB — upper byte: attribute flags */
   uint32_t bg; /* 0xAARRGGBB — upper byte: attribute flags */
+  uint32_t combining[CELL_MAX_COMBINING]; /* zero-terminated combining-mark
+                                             codepoints */
 } Cell;
 
 /* Attribute flags (packed into high byte of fg) */

src/clayterm.c

@@ -51,9 +51,10 @@ struct Clayterm *ct_active_context = NULL;
 /* clip stack depth: nesting beyond this clamps to the deepest rect */
 #define CLIP_STACK_MAX 16
 
-/* Clayterm-specific error code, numbered past Clay's error enum (0..8).
+/* Clayterm-specific error codes, numbered past Clay's error enum (0..8).
  * Mirrored by ERROR_TYPES in term.ts. */
 #define CLAYTERM_ERR_CLIP_DEPTH_EXCEEDED 9
+#define CLAYTERM_ERR_COMBINING_MARKS_EXCEEDED 10
 
 #define CLAYTERM_STR_(x) #x
 #define CLAYTERM_STR(x) CLAYTERM_STR_(x)
@@ -80,6 +81,8 @@ struct Clayterm {
   int clipoverflow;
   /* set once per frame when nesting first exceeds the tracked depth */
   int clip_depth_exceeded;
+  /* set once per frame when a cell first runs out of combining-mark slots */
+  int combining_exceeded;
   /* error collection */
   Clay_ErrorData errors[MAX_ERRORS];
   int error_count;
@@ -109,30 +112,71 @@ struct Clayterm {
  * Output buffer is sized at 64 bytes per cell — enough for worst-case
  * full-screen redraws with truecolor SGR sequences on every cell.
  */
-#define OUT_BYTES_PER_CELL 64
+/* 128 bytes per cell: ~84 bytes worst-case for CUP + SGR sequences, plus up
+ * to 4 (base SMP char) + 8×4 (combining marks) = 36 bytes of cluster text. */
+#define OUT_BYTES_PER_CELL 128
 
 /* ── Cell buffer ops ──────────────────────────────────────────────── */
 
 static Cell *cell_at(struct Clayterm *ct, Cell *buf, int x, int y) {
   return &buf[y * ct->w + x];
 }
 
-static void setcell(struct Clayterm *ct, int x, int y, uint32_t ch, uint32_t fg,
-                    uint32_t bg) {
+static int setcell(struct Clayterm *ct, int x, int y, uint32_t ch, uint32_t fg,
+                   uint32_t bg) {
   if (x < 0 || x >= ct->w || y < 0 || y >= ct->h)
-    return;
+    return 0;
   if (ct->clipping) {
     if (x < ct->clipx || x >= ct->clipx + ct->clipw)
-      return;
+      return 0;
     if (y < ct->clipy || y >= ct->clipy + ct->cliph)
-      return;
+      return 0;
   }
   Cell *c = cell_at(ct, ct->back, x, y);
   c->ch = ch;
   c->fg = fg;
   if (!(bg & ATTR_DEFAULT)) {
     c->bg = bg;
   }
+  for (int i = 0; i < CELL_MAX_COMBINING; i++)
+    c->combining[i] = 0;
+  return 1;
+}
+
+/* Surface a COMBINING_MARKS_EXCEEDED error once per frame. */
+static void report_combining_exceeded(struct Clayterm *ct) {
+  if (ct->combining_exceeded)
+    return;
+  ct->combining_exceeded = 1;
+  if (ct->error_count >= MAX_ERRORS)
+    return;
+  static const char msg[] =
+      "cell exceeds combining-mark limit of " CLAYTERM_STR(
+          CELL_MAX_COMBINING) "; excess marks truncated";
+  ct->errors[ct->error_count++] = (Clay_ErrorData){
+      .errorType = (Clay_ErrorType)CLAYTERM_ERR_COMBINING_MARKS_EXCEEDED,
+      .errorText = {.isStaticallyAllocated = true,
+                    .length = (int32_t)(sizeof(msg) - 1),
+                    .chars = msg},
+      .userData = ct,
+  };
+}
+
+/* Append a combining-mark codepoint to the cell at (x, y) in the back buffer.
+ * Marks beyond CELL_MAX_COMBINING are dropped (truncation from end) and
+ * reported once per frame.
+ */
+static void append_combining(struct Clayterm *ct, int x, int y, uint32_t cp) {
+  if (x < 0 || x >= ct->w || y < 0 || y >= ct->h)
+    return;
+  Cell *c = cell_at(ct, ct->back, x, y);
+  for (int i = 0; i < CELL_MAX_COMBINING; i++) {
+    if (c->combining[i] == 0) {
+      c->combining[i] = cp;
+      return;
+    }
+  }
+  report_combining_exceeded(ct);
 }
 
 /* ── Escape sequence generation ───────────────────────────────────── */
@@ -238,6 +282,8 @@ static void present_cups(struct Clayterm *ct, int row) {
             emit_ch(ct, i, y, row, ' ');
         } else {
           emit_ch(ct, x, y, row, back->ch);
+          for (int ci = 0; ci < CELL_MAX_COMBINING && back->combining[ci]; ci++)
+            buf_char(&ct->out, back->combining[ci]);
           /* mark trailing cells of wide char as invalid in front
            * so they'll diff when overwritten by narrow chars */
           for (int i = 1; i < w; i++) {
@@ -298,6 +344,8 @@ static void present_lines(struct Clayterm *ct) {
         if (!iswprint(ch))
           ch = 0xfffd;
         buf_char(&ct->out, ch);
+        for (int ci = 0; ci < CELL_MAX_COMBINING && back->combining[ci]; ci++)
+          buf_char(&ct->out, back->combining[ci]);
         for (int i = 1; i < w; i++) {
           Cell *fw = cell_at(ct, ct->front, x + i, y);
           fw->ch = 0xffffffff;
@@ -391,6 +439,7 @@ static void render_text(struct Clayterm *ct, int x0, int y0,
   const char *p = slice;
   int rem = slice_len;
   int x = x0;
+  int last_x = -1; /* column of the most-recently written base cell */
 
   while (rem > 0) {
     /* Check at the top of each iteration: if the pointer we are about to
@@ -414,8 +463,11 @@ static void render_text(struct Clayterm *ct, int x0, int y0,
     if (cw < 0)
       cw = 1;
     if (cw > 0) {
-      setcell(ct, x, y0, cp, fg, bg);
+      last_x = setcell(ct, x, y0, cp, fg, bg) ? x : -1;
       x += cw;
+    } else if (last_x >= 0) {
+      /* combining mark: attach to the preceding base cell */
+      append_combining(ct, last_x, y0, cp);
     }
     p += n;
     rem -= n;
@@ -870,6 +922,7 @@ void reduce(struct Clayterm *ct, uint32_t *buf, int len, int mode, int row,
   ct->clipdepth = 0;
   ct->clipoverflow = 0;
   ct->clip_depth_exceeded = 0;
+  ct->combining_exceeded = 0;
   ct->clipping = 0;
 
   cells_fill(ct->back, ct->w, ct->h, ' ', ATTR_DEFAULT, ATTR_DEFAULT);

term.ts

@@ -69,6 +69,7 @@ const ERROR_TYPES = [
   "INTERNAL_ERROR",
   "UNBALANCED_OPEN_CLOSE",
   "CLIP_DEPTH_EXCEEDED",
+  "COMBINING_MARKS_EXCEEDED",
 ] as const;
 
 export interface ClayError {

test/print.ts

@@ -1,3 +1,5 @@
+const ZERO_WIDTH = /^[\p{Mn}\p{Me}\u200B-\u200F\uFE00-\uFE0F]$/u;
+
 /**
  * Interpret ANSI escape sequences into a plain text grid.
  * Handles CSI cursor positioning (row;colH), DECTCEM show/hide
@@ -7,6 +9,9 @@
  * cell is marked with U+0332 COMBINING LOW LINE appended to the base
  * character. The base char is preserved and the underline spans its
  * rendered width — including the full width of CJK/wide chars.
+ *
+ * Zero-width codepoints (combining marks, ZWJ, variation selectors)
+ * attach to the most recently written cell instead of advancing.
  */
 export function print(ansi: string, w: number, h: number): string {
   let grid: string[][] = [];
@@ -20,6 +25,7 @@ export function print(ansi: string, w: number, h: number): string {
   let x = 0;
   let y = 0;
   let i = 0;
+  let last: [number, number] | undefined;
   let cursorVisible = false;
 
   while (i < ansi.length) {
@@ -36,6 +42,7 @@ export function print(ansi: string, w: number, h: number): string {
       let cmd = ansi[i++];
 
       if (cmd === "H") {
+        last = undefined;
         // cursor position: row;col (1-indexed)
         let parts = params.split(";");
         y = (parseInt(parts[0]) || 1) - 1;
@@ -50,15 +57,23 @@ export function print(ansi: string, w: number, h: number): string {
       y++;
       x = 0;
       i++;
+      last = undefined;
     } else {
       // regular character — could be multi-byte UTF-8
       let cp = ansi.codePointAt(i)!;
       let ch = String.fromCodePoint(cp);
+      i += ch.length;
+      if (ZERO_WIDTH.test(ch)) {
+        if (last) grid[last[1]][last[0]] += ch;
+        continue;
+      }
       if (x >= 0 && x < w && y >= 0 && y < h) {
         grid[y][x] = ch;
+        last = [x, y];
+      } else {
+        last = undefined;
       }
       x++;
-      i += ch.length;
     }
   }