diff --git a/src/input/line-buffer.ts b/src/input/line-buffer.ts index 3edd987..9176a4c 100644 --- a/src/input/line-buffer.ts +++ b/src/input/line-buffer.ts @@ -1,20 +1,24 @@ +import {StringDecoder} from 'node:string_decoder'; + export class LineBuffer { + readonly #decoder = new StringDecoder('utf8'); #remainder = ''; push(chunk: string | Buffer): string[] { - const text = this.#remainder + chunk.toString('utf8'); + const decoded = typeof chunk === 'string' ? chunk : this.#decoder.write(chunk); + const text = this.#remainder + decoded; const parts = text.split(/\r?\n/); this.#remainder = parts.pop() ?? ''; return parts; } flush(): string[] { - if (this.#remainder.length === 0) { + const finalLine = this.#remainder + this.#decoder.end(); + this.#remainder = ''; + if (finalLine.length === 0) { return []; } - const finalLine = this.#remainder; - this.#remainder = ''; return [finalLine]; } } diff --git a/tests/input.test.ts b/tests/input.test.ts index 54019a7..b34f07f 100644 --- a/tests/input.test.ts +++ b/tests/input.test.ts @@ -9,6 +9,14 @@ describe('LineBuffer', () => { expect(buffer.flush()).toEqual(['three']); }); + it('keeps multibyte characters that are split across chunks', () => { + const buffer = new LineBuffer(); + const bytes = Buffer.from('café ✓\n', 'utf8'); + expect(buffer.push(bytes.subarray(0, 4))).toEqual([]); + expect(buffer.push(bytes.subarray(4, 7))).toEqual([]); + expect(buffer.push(bytes.subarray(7))).toEqual(['café ✓']); + }); + it('does not invent a line when the buffer is empty', () => { expect(new LineBuffer().flush()).toEqual([]); });