diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index b90e7f842..356a2fc26 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -7,15 +7,37 @@ on: branches: - main - release - paths-ignore: - - "**/*.md" - - "LICENSE" - - "benchmarks/**" + paths: + - "packages/**" + - "examples/**" + - "tests/**" + - "package.json" + - "package-lock.json" + - "tsconfig.base.json" + - "tsconfig.dev.json" + - "eslint.config.base.mjs" + - "docker-compose.yml" + - "vitest.node.config.ts" + - "vitest.node.setup.ts" + - "vitest.web.config.ts" + - "vitest.web.setup.ts" + - ".github/workflows/examples.yml" pull_request: - paths-ignore: - - "**/*.md" - - "LICENSE" - - "benchmarks/**" + paths: + - "packages/**" + - "examples/**" + - "tests/**" + - "package.json" + - "package-lock.json" + - "tsconfig.base.json" + - "tsconfig.dev.json" + - "eslint.config.base.mjs" + - "docker-compose.yml" + - "vitest.node.config.ts" + - "vitest.node.setup.ts" + - "vitest.web.config.ts" + - "vitest.web.setup.ts" + - ".github/workflows/examples.yml" concurrency: group: "${{ github.workflow }}-${{ github.ref }}" diff --git a/.github/workflows/publish-skill-rowbinary-parser.yml b/.github/workflows/publish-skill-rowbinary-parser.yml new file mode 100644 index 000000000..b5c2910ca --- /dev/null +++ b/.github/workflows/publish-skill-rowbinary-parser.yml @@ -0,0 +1,153 @@ +name: "publish: rowbinary parser" + +# Independent publish + release for the standalone @clickhouse/rowbinary +# package (the RowBinary parser skill). It is NOT part of the npm workspace +# lockstep release driven by publish.yml — it carries its own version in +# skills/clickhouse-js-node-rowbinary-parser/package.json and ships on its own +# cadence. Triggered manually, and — like publish.yml — must be dispatched from +# the `release` branch: the npm-publish environment is protected so only that +# branch may deploy (the repo's human-in-the-loop release gate). Dispatches from +# any other ref are skipped by the job-level `if` guard below. The `publish` job +# gates on typecheck/test/build, publishes the version currently in package.json +# with the "latest" tag using npm OIDC authentication and provenance, then pushes +# a matching git tag. The `e2e` job waits for the freshly published version to +# appear on the registry and verifies it installs and imports in a throwaway +# downstream project across the supported Node versions. + +permissions: + contents: read + id-token: write # Required for npm OIDC authentication and provenance + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +on: + workflow_dispatch: + +jobs: + publish: + # The npm-publish environment only permits the release branch to deploy; + # skip cleanly on any other ref instead of failing the protection check. + if: github.ref == 'refs/heads/release' + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: npm-publish + permissions: + contents: write # Required to push the release git tag + id-token: write # Required for npm OIDC authentication and provenance + defaults: + run: + working-directory: skills/clickhouse-js-node-rowbinary-parser + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Setup Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: 24 + registry-url: "https://registry.npmjs.org" + + - name: Install dependencies + run: npm ci + + - name: Typecheck + run: npm run typecheck + + - name: Test + run: npm test + + - name: Build + run: npm run build + + - name: Get the release version + id: version + run: | + VERSION=$(node -p "require('./package.json').version") + echo "Publishing @clickhouse/rowbinary@$VERSION" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + - name: Publish to npm + # prepack copies the repo-root LICENSE and rebuilds dist before packing. + run: npm publish --access public --provenance + + - name: Create and push release git tag + env: + RELEASE_TAG: rowbinary-v${{ steps.version.outputs.version }} + RELEASE_VERSION: ${{ steps.version.outputs.version }} + run: | + if git ls-remote --exit-code --tags origin "refs/tags/${RELEASE_TAG}" >/dev/null 2>&1; then + echo "Tag ${RELEASE_TAG} already exists on origin; skipping." + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git tag -a "${RELEASE_TAG}" -m "Release @clickhouse/rowbinary ${RELEASE_VERSION}" + git push origin "refs/tags/${RELEASE_TAG}" + + e2e: + name: e2e (node ${{ matrix.node }}) + needs: publish + if: needs.publish.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: true + matrix: + node: [20, 22, 24] + env: + PUBLISHED_VERSION: ${{ needs.publish.outputs.version }} + steps: + - name: Setup NodeJS ${{ matrix.node }} + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: ${{ matrix.node }} + registry-url: "https://registry.npmjs.org" + + - name: Wait for @clickhouse/rowbinary@${{ needs.publish.outputs.version }} on npm + run: | + set -euo pipefail + if [ -z "${PUBLISHED_VERSION}" ]; then + echo "PUBLISHED_VERSION is empty; cannot wait for npm publication." >&2 + exit 1 + fi + pkg="@clickhouse/rowbinary" + # Poll the registry for up to ~5 minutes. New versions usually surface + # in seconds, but the registry CDN can lag. + max_attempts=60 + sleep_seconds=5 + attempt=1 + echo "Waiting for ${pkg}@${PUBLISHED_VERSION} to be available on npm..." + while true; do + if npm view "${pkg}@${PUBLISHED_VERSION}" version >/dev/null 2>&1; then + echo " ${pkg}@${PUBLISHED_VERSION} is available." + break + fi + if [ "$attempt" -ge "$max_attempts" ]; then + echo "Timed out waiting for ${pkg}@${PUBLISHED_VERSION} on npm" >&2 + exit 1 + fi + echo " attempt ${attempt}/${max_attempts}: not available yet, sleeping ${sleep_seconds}s..." + attempt=$((attempt + 1)) + sleep "$sleep_seconds" + done + + - name: Install and import the published package + run: | + set -euo pipefail + work="$(mktemp -d)" + cd "$work" + npm init -y >/dev/null 2>&1 + npm install "@clickhouse/rowbinary@${PUBLISHED_VERSION}" + # Verify both the main barrel entry and a subpath export resolve and + # expose their parsers to a downstream consumer. + node --input-type=module -e " + import * as rb from '@clickhouse/rowbinary'; + import * as ints from '@clickhouse/rowbinary/integers'; + if (typeof rb.readRows !== 'function') throw new Error('readRows missing from main export'); + if (typeof ints.readUInt8 !== 'function') throw new Error('readUInt8 missing from subpath export'); + console.log('OK: @clickhouse/rowbinary@${PUBLISHED_VERSION} imports cleanly'); + " diff --git a/.github/workflows/tests-bun.yml b/.github/workflows/tests-bun.yml index b464da771..897bdffbf 100644 --- a/.github/workflows/tests-bun.yml +++ b/.github/workflows/tests-bun.yml @@ -6,15 +6,36 @@ on: push: branches: - main - paths-ignore: - - "**/*.md" - - "LICENSE" - - "benchmarks/**" + - release + paths: + - "packages/**" + - "tests/**" + - "package.json" + - "package-lock.json" + - "tsconfig.base.json" + - "tsconfig.dev.json" + - "eslint.config.base.mjs" + - "docker-compose.yml" + - "vitest.node.config.ts" + - "vitest.node.setup.ts" + - "vitest.web.config.ts" + - "vitest.web.setup.ts" + - ".github/workflows/tests-bun.yml" pull_request: - paths-ignore: - - "**/*.md" - - "LICENSE" - - "benchmarks/**" + paths: + - "packages/**" + - "tests/**" + - "package.json" + - "package-lock.json" + - "tsconfig.base.json" + - "tsconfig.dev.json" + - "eslint.config.base.mjs" + - "docker-compose.yml" + - "vitest.node.config.ts" + - "vitest.node.setup.ts" + - "vitest.web.config.ts" + - "vitest.web.setup.ts" + - ".github/workflows/tests-bun.yml" concurrency: group: "${{ github.workflow }}-${{ github.ref }}" diff --git a/.github/workflows/tests-node.yml b/.github/workflows/tests-node.yml index 260300722..cfb1d2911 100644 --- a/.github/workflows/tests-node.yml +++ b/.github/workflows/tests-node.yml @@ -6,15 +6,36 @@ on: push: branches: - main - paths-ignore: - - "**/*.md" - - "LICENSE" - - "benchmarks/**" + - release + paths: + - "packages/**" + - "tests/**" + - "package.json" + - "package-lock.json" + - "tsconfig.base.json" + - "tsconfig.dev.json" + - "eslint.config.base.mjs" + - "docker-compose.yml" + - "vitest.node.config.ts" + - "vitest.node.setup.ts" + - "vitest.web.config.ts" + - "vitest.web.setup.ts" + - ".github/workflows/tests-node.yml" pull_request: - paths-ignore: - - "**/*.md" - - "LICENSE" - - "benchmarks/**" + paths: + - "packages/**" + - "tests/**" + - "package.json" + - "package-lock.json" + - "tsconfig.base.json" + - "tsconfig.dev.json" + - "eslint.config.base.mjs" + - "docker-compose.yml" + - "vitest.node.config.ts" + - "vitest.node.setup.ts" + - "vitest.web.config.ts" + - "vitest.web.setup.ts" + - ".github/workflows/tests-node.yml" schedule: - cron: "0 9 * * *" diff --git a/.github/workflows/tests-oss-dependents.yml b/.github/workflows/tests-oss-dependents.yml index 38dc93ca7..14368c0c0 100644 --- a/.github/workflows/tests-oss-dependents.yml +++ b/.github/workflows/tests-oss-dependents.yml @@ -6,15 +6,36 @@ on: push: branches: - main - paths-ignore: - - "**/*.md" - - "LICENSE" - - "benchmarks/**" + - release + paths: + - "packages/**" + - "tests/**" + - "package.json" + - "package-lock.json" + - "tsconfig.base.json" + - "tsconfig.dev.json" + - "eslint.config.base.mjs" + - "docker-compose.yml" + - "vitest.node.config.ts" + - "vitest.node.setup.ts" + - "vitest.web.config.ts" + - "vitest.web.setup.ts" + - ".github/workflows/tests-oss-dependents.yml" pull_request: - paths-ignore: - - "**/*.md" - - "LICENSE" - - "benchmarks/**" + paths: + - "packages/**" + - "tests/**" + - "package.json" + - "package-lock.json" + - "tsconfig.base.json" + - "tsconfig.dev.json" + - "eslint.config.base.mjs" + - "docker-compose.yml" + - "vitest.node.config.ts" + - "vitest.node.setup.ts" + - "vitest.web.config.ts" + - "vitest.web.setup.ts" + - ".github/workflows/tests-oss-dependents.yml" schedule: - cron: "0 9 * * *" diff --git a/.github/workflows/tests-skill-rowbinary-parser.yml b/.github/workflows/tests-skill-rowbinary-parser.yml new file mode 100644 index 000000000..16aaccd26 --- /dev/null +++ b/.github/workflows/tests-skill-rowbinary-parser.yml @@ -0,0 +1,75 @@ +name: "skill: rowbinary parser" + +permissions: {} +on: + workflow_dispatch: + push: + branches: + - main + paths: + - .github/workflows/tests-skill-rowbinary-parser.yml + - skills/clickhouse-js-node-rowbinary-parser/** + pull_request: + paths: + - .github/workflows/tests-skill-rowbinary-parser.yml + - skills/clickhouse-js-node-rowbinary-parser/** + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true + +jobs: + typecheck: + timeout-minutes: 5 + runs-on: ubuntu-latest + defaults: + run: + working-directory: skills/clickhouse-js-node-rowbinary-parser + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Setup NodeJS + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: 24 + + - name: Install dependencies + run: npm ci + + - name: Typecheck + run: npm run typecheck + + unit-tests: + timeout-minutes: 5 + runs-on: ubuntu-latest + defaults: + run: + working-directory: skills/clickhouse-js-node-rowbinary-parser + strategy: + fail-fast: false + matrix: + node: [20, 22, 24] + clickhouse: [head, latest] + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Start ClickHouse (version - ${{ matrix.clickhouse }}) in Docker + uses: hoverkraft-tech/compose-action@11beaa1c2dae4e8ed7b1665aa074723b6cecb0e4 # v3.0.0 + env: + CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} + with: + # The skill suite only needs the single-node HTTP server on :8123. + compose-file: "docker-compose.yml" + services: "clickhouse" + down-flags: "--volumes" + + - name: Setup NodeJS ${{ matrix.node }} + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: ${{ matrix.node }} + + - name: Install dependencies + run: npm ci + + - name: Run unit tests + run: npm test diff --git a/.github/workflows/tests-web.yml b/.github/workflows/tests-web.yml index fd7d53dc2..129d93de9 100644 --- a/.github/workflows/tests-web.yml +++ b/.github/workflows/tests-web.yml @@ -6,15 +6,36 @@ on: push: branches: - main - paths-ignore: - - "**/*.md" - - "LICENSE" - - "benchmarks/**" + - release + paths: + - "packages/**" + - "tests/**" + - "package.json" + - "package-lock.json" + - "tsconfig.base.json" + - "tsconfig.dev.json" + - "eslint.config.base.mjs" + - "docker-compose.yml" + - "vitest.node.config.ts" + - "vitest.node.setup.ts" + - "vitest.web.config.ts" + - "vitest.web.setup.ts" + - ".github/workflows/tests-web.yml" pull_request: - paths-ignore: - - "**/*.md" - - "LICENSE" - - "benchmarks/**" + paths: + - "packages/**" + - "tests/**" + - "package.json" + - "package-lock.json" + - "tsconfig.base.json" + - "tsconfig.dev.json" + - "eslint.config.base.mjs" + - "docker-compose.yml" + - "vitest.node.config.ts" + - "vitest.node.setup.ts" + - "vitest.web.config.ts" + - "vitest.web.setup.ts" + - ".github/workflows/tests-web.yml" schedule: - cron: "0 9 * * *" diff --git a/CHANGELOG.md b/CHANGELOG.md index 3804223a5..a30fc3a0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# 1.23.0 + +## New features + +- (Node.js) Added a RowBinary reader library and agent skill under [`skills/clickhouse-js-node-rowbinary-parser`](./skills/clickhouse-js-node-rowbinary-parser). It ships type-specific, monomorphizable building blocks for decoding `RowBinary` / `RowBinaryWithNames` / `RowBinaryWithNamesAndTypes` streams (full-buffer and chunked), plus a skill that guides an agent to generate bespoke high-performance parsers from a query's column types. The skill is bundled into `@clickhouse/client` (registered in `agents.skills`) and is also published independently as the [`@clickhouse/rowbinary`](https://www.npmjs.com/package/@clickhouse/rowbinary) package. A matching RowBinary writer is planned. ([#864]) + # 1.22.0 ## New features @@ -64,6 +70,7 @@ await client.query({ [#825]: https://github.com/ClickHouse/clickhouse-js/pull/825 [#827]: https://github.com/ClickHouse/clickhouse-js/pull/827 [#828]: https://github.com/ClickHouse/clickhouse-js/pull/828 +[#864]: https://github.com/ClickHouse/clickhouse-js/pull/864 ## Bug Fixes diff --git a/README.md b/README.md index b49f1e2e0..2506505f7 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Official JS client for [ClickHouse](https://clickhouse.com/), written purely in The client has zero external dependencies and is optimized for maximum performance. -The repository consists of three packages: +The repository consists of four packages: - `@clickhouse/client` - a version of the client designed for Node.js platform only. It is built on top of [HTTP](https://nodejs.org/api/http.html) and [Stream](https://nodejs.org/api/stream.html) APIs; supports streaming for both selects and inserts. @@ -41,6 +41,7 @@ The repository consists of three packages: and [Web Streams](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API) APIs; supports streaming for selects. Compatible with Chrome/Firefox browsers and Cloudflare workers. - `@clickhouse/client-common` - shared common types and the base framework for building a custom client implementation. +- `@clickhouse/rowbinary` - a library for reading (and soon writing) ClickHouse RowBinary format. ## Installation diff --git a/demo/logs/.gitignore b/demo/logs/.gitignore new file mode 100644 index 000000000..9b7655a7b --- /dev/null +++ b/demo/logs/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +.next/ +next-env.d.ts +*.tsbuildinfo diff --git a/demo/logs/README.md b/demo/logs/README.md new file mode 100644 index 000000000..f2cbf983b --- /dev/null +++ b/demo/logs/README.md @@ -0,0 +1,98 @@ +# RowBinary Logs Demo + +Screenshot of the demo app + +A tiny, **server-only** Next.js app that pages through a ClickHouse logs table, +decoding each page from `RowBinary` with +[`@clickhouse/rowbinary`](../../). The browser receives only HTML — there are no +client components and no client-side data fetching. All decoding happens on the +server in [`lib/logs.ts`](lib/logs.ts). + +It's wired up so you can poke at the decoder yourself: change the schema, the +reader, the page size, or swap the API-combinator reader for a monomorphized one. + +## What it shows + +The `demo_logs` table is intentionally a mix of types that exercise the +decoder's interesting paths: + +| Column | ClickHouse type | Decoded as | +| ------------- | ------------------------- | ------------------------------- | +| `timestamp` | `DateTime64(3)` | JS `Date` (ms-lossless) | +| `level` | `Enum8('debug'..'error')` | `number` → name at the edge | +| `service` | `LowCardinality(String)` | plain `String` (transparent) | +| `host` | `IPv4` | dotted-quad `string` | +| `trace_id` | `UUID` | canonical `8-4-4-4-12` `string` | +| `status` | `UInt16` | `number` | +| `duration_ms` | `Float64` | `number` | +| `message` | `String` | `string` | + +The per-column reads live in `readLogRow` in [`lib/logs.ts`](lib/logs.ts) — one +leaf reader per column, in wire order. That's the clear, "correct by default" +combinator form; the comments point at where you'd monomorphize it if this were a +hot path. + +## Prerequisites + +- Node 18+ (built/tested on Node 24) +- A running ClickHouse. This demo ships a self-contained one — from **this + directory** (`demo/logs`): + + ```bash + docker compose up -d + ``` + + That exposes HTTP on `localhost:8123` with the default user and no password. + +## Run it + +From this directory (`demo/logs`): + +```bash +npm install # installs Next + the local @clickhouse/rowbinary tarball +npm run seed # create demo_logs and insert 1000 rows (pass a number to change: npm run seed -- 50000) +npm run dev # http://localhost:3000 +``` + +Then open and use **Newer / Older** to page through the +logs (25 per page). Pagination is plain `LIMIT`/`OFFSET` driven by the `?page=` +query param. + +## Configuration + +All connection settings come from the environment (defaults in parentheses): + +| Variable | Default | +| --------------------- | ----------------------- | +| `CLICKHOUSE_URL` | `http://localhost:8123` | +| `CLICKHOUSE_USER` | `default` | +| `CLICKHOUSE_PASSWORD` | _(empty)_ | +| `CLICKHOUSE_DATABASE` | `default` | + +## Layout + +``` +app/ + layout.tsx root layout + global styles + page.tsx the server component: reads ?page, renders the table + pager + globals.css styling +lib/ + clickhouse.ts minimal fetch-based ClickHouse HTTP access (server-only) + logs.ts the RowBinary reader + fetchLogsPage() ← the interesting bit +scripts/ + seed.mjs standalone seeder (INSERT ... SELECT FROM numbers(N)) +vendor/ + clickhouse-rowbinary-0.1.0.tgz the packed library this demo installs +``` + +## Updating the library + +This app installs `@clickhouse/rowbinary` from the packed tarball in `vendor/`. +To pick up changes you make in the parent package, repack and reinstall — from +the package root (`../../`): + +```bash +npm run build && npm pack +cp clickhouse-rowbinary-*.tgz demo/logs/vendor/clickhouse-rowbinary-0.1.0.tgz +cd demo/logs && npm install +``` diff --git a/demo/logs/app/globals.css b/demo/logs/app/globals.css new file mode 100644 index 000000000..58b5ce773 --- /dev/null +++ b/demo/logs/app/globals.css @@ -0,0 +1,156 @@ +:root { + color-scheme: dark; + --bg: #0b0e14; + --panel: #11151f; + --border: #222937; + --text: #d7dce5; + --muted: #8a93a6; + --debug: #6b7280; + --info: #3b82f6; + --warn: #f59e0b; + --error: #ef4444; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + background: var(--bg); + color: var(--text); + font: + 14px/1.5 ui-sans-serif, + system-ui, + -apple-system, + sans-serif; +} + +main { + max-width: 1100px; + margin: 0 auto; + padding: 32px 20px 64px; +} + +header h1 { + font-size: 20px; + margin: 0 0 4px; +} + +header p { + margin: 0 0 24px; + color: var(--muted); +} + +header code { + color: var(--text); + background: var(--panel); + border: 1px solid var(--border); + border-radius: 4px; + padding: 1px 5px; +} + +table { + width: 100%; + border-collapse: collapse; + background: var(--panel); + border: 1px solid var(--border); + border-radius: 8px; + overflow: hidden; + font-variant-numeric: tabular-nums; +} + +th, +td { + text-align: left; + padding: 8px 12px; + border-bottom: 1px solid var(--border); + white-space: nowrap; +} + +th { + font-size: 12px; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--muted); +} + +tr:last-child td { + border-bottom: none; +} + +td.msg { + white-space: normal; + color: var(--text); +} + +td.mono, +.mono { + font-family: ui-monospace, "SF Mono", Menlo, monospace; + font-size: 12.5px; + color: var(--muted); +} + +.level { + font-weight: 600; + text-transform: uppercase; + font-size: 11px; + letter-spacing: 0.03em; +} +.level.debug { + color: var(--debug); +} +.level.info { + color: var(--info); +} +.level.warn { + color: var(--warn); +} +.level.error { + color: var(--error); +} + +.pager { + display: flex; + align-items: center; + gap: 16px; + margin-top: 20px; +} + +.pager a, +.pager span.disabled { + padding: 6px 14px; + border: 1px solid var(--border); + border-radius: 6px; + background: var(--panel); + text-decoration: none; + color: var(--text); +} + +.pager span.disabled { + color: var(--muted); + opacity: 0.5; +} + +.pager .status { + color: var(--muted); + border: none; + background: none; + padding: 0; +} + +.empty { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 8px; + padding: 24px; +} + +.empty pre { + background: var(--bg); + border: 1px solid var(--border); + border-radius: 6px; + padding: 12px; + overflow-x: auto; + color: var(--text); +} diff --git a/demo/logs/app/layout.tsx b/demo/logs/app/layout.tsx new file mode 100644 index 000000000..3c2153271 --- /dev/null +++ b/demo/logs/app/layout.tsx @@ -0,0 +1,16 @@ +import "./globals.css"; +import type { ReactNode } from "react"; + +export const metadata = { + title: "RowBinary Logs Demo", + description: + "Server-rendered ClickHouse log viewer decoded with @clickhouse/rowbinary.", +}; + +export default function RootLayout({ children }: { children: ReactNode }) { + return ( + + {children} + + ); +} diff --git a/demo/logs/app/page.tsx b/demo/logs/app/page.tsx new file mode 100644 index 000000000..a8b52c81d --- /dev/null +++ b/demo/logs/app/page.tsx @@ -0,0 +1,126 @@ +import { fetchLogsPage, type LogsPage } from "@/lib/logs"; + +// This page hits ClickHouse on every request and must never be statically +// prerendered or cached — it is a live, server-rendered view. +export const dynamic = "force-dynamic"; + +const PAGE_SIZE = 25; + +function fmtTime(d: Date): string { + // YYYY-MM-DD HH:MM:SS.mmm + return d.toISOString().replace("T", " ").replace("Z", ""); +} + +export default async function LogsPageView({ + searchParams, +}: { + searchParams: Promise<{ page?: string }>; +}) { + const { page: pageParam } = await searchParams; + const page = Number(pageParam ?? "1"); + + let data: LogsPage | null = null; + let failed = false; + try { + data = await fetchLogsPage(page, PAGE_SIZE); + } catch (e) { + // The underlying error can carry ClickHouse SQL/server details, so log it + // server-side and show the user a generic message instead. + console.error("fetchLogsPage failed:", e); + failed = true; + } + + return ( +
+
+

RowBinary Logs

+

+ Server-rendered from ClickHouse, decoded with{" "} + @clickhouse/rowbinary. The browser only receives HTML — + all decoding happens in lib/logs.ts on the server. +

+
+ + {failed || !data ? ( + + ) : data.total === 0 ? ( + + ) : ( + + )} +
+ ); +} + +function LogsTable({ data }: { data: LogsPage }) { + return ( + <> + + + + + + + + + + + + + + + {data.rows.map((row) => ( + // Real logs can share a trace across rows; combine with the + // timestamp to keep React keys unique. + + + + + + + + + + + ))} + +
TimestampLevelServiceStatusDurationHostTrace IDMessage
{fmtTime(row.timestamp)} + {row.level} + {row.service}{row.status}{row.durationMs.toFixed(1)} ms{row.host}{row.traceId.slice(0, 8)}…{row.message}
+ + + ); +} + +function Pager({ data }: { data: LogsPage }) { + const { page, totalPages, total } = data; + const hasPrev = page > 1; + const hasNext = page < totalPages; + return ( + + ); +} + +function EmptyState({ message }: { message?: string | null }) { + return ( +
+

No logs to show{message ? `: ${message}` : "."}

+

Make sure ClickHouse is running and the table is seeded:

+
{`# from demo/logs\ndocker compose up -d\nnpm run seed`}
+
+ ); +} diff --git a/demo/logs/docker-compose.yml b/demo/logs/docker-compose.yml new file mode 100644 index 000000000..f1e27d70d --- /dev/null +++ b/demo/logs/docker-compose.yml @@ -0,0 +1,34 @@ +# Self-contained ClickHouse for the RowBinary logs demo. +# +# Brings up a single ClickHouse node exposing HTTP on localhost:8123 with the +# default user and no password — exactly what the app and seeder default to +# (see lib/clickhouse.ts / scripts/seed.mjs). No external config files needed: +# CLICKHOUSE_SKIP_USER_SETUP lets the default user connect without a password. +# +# docker compose up -d # start +# npm run seed # create demo_logs and insert rows +# npm run dev # http://localhost:3000 +# docker compose down # stop (add -v to also wipe the data volume) +services: + clickhouse: + image: "clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-latest}" + container_name: "rowbinary-logs-demo-clickhouse" + environment: + CLICKHOUSE_SKIP_USER_SETUP: 1 + ports: + - "8123:8123" # HTTP interface (used by the demo) + - "9000:9000" # native protocol + ulimits: + nofile: + soft: 262144 + hard: 262144 + volumes: + - "clickhouse-data:/var/lib/clickhouse" + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"] + interval: 5s + timeout: 3s + retries: 12 + +volumes: + clickhouse-data: diff --git a/demo/logs/lib/clickhouse.ts b/demo/logs/lib/clickhouse.ts new file mode 100644 index 000000000..953d2c497 --- /dev/null +++ b/demo/logs/lib/clickhouse.ts @@ -0,0 +1,53 @@ +import "server-only"; + +/** + * Tiny ClickHouse HTTP access layer for the demo — no client dependency, just + * `fetch` against the HTTP interface. We deliberately keep this minimal so the + * interesting part stays the RowBinary decode in `lib/logs.ts`. + * + * Connection comes from the environment, defaulting to the single-node + * `docker-compose` ClickHouse this repo ships (HTTP on 8123, default user, no + * password). Override with CLICKHOUSE_URL / CLICKHOUSE_USER / CLICKHOUSE_PASSWORD + * / CLICKHOUSE_DATABASE. + */ +const CONFIG = { + url: process.env.CLICKHOUSE_URL ?? "http://localhost:8123", + user: process.env.CLICKHOUSE_USER ?? "default", + password: process.env.CLICKHOUSE_PASSWORD ?? "", + database: process.env.CLICKHOUSE_DATABASE ?? "default", +}; + +function endpoint(): string { + const u = new URL(CONFIG.url); + u.searchParams.set("database", CONFIG.database); + return u.toString(); +} + +function authHeaders(): Record { + return { + "X-ClickHouse-User": CONFIG.user, + "X-ClickHouse-Key": CONFIG.password, + }; +} + +/** + * Run a query whose result should be decoded as raw `RowBinary` and return the + * complete response as a single `Buffer`. We page with `LIMIT`/`OFFSET`, so each + * response is small — buffering the whole thing and decoding it in one shot is + * both simpler and faster than streaming here (no `advance()` bounds check ever + * needs to fire). `FORMAT RowBinary` is appended by the caller's SQL. + */ +export async function queryRowBinary(sql: string): Promise { + const res = await fetch(endpoint(), { + method: "POST", + headers: authHeaders(), + body: sql, + }); + if (!res.ok) { + throw new Error( + `ClickHouse query failed (${res.status}): ${await res.text()}`, + ); + } + const arrayBuffer = await res.arrayBuffer(); + return Buffer.from(arrayBuffer); +} diff --git a/demo/logs/lib/logs.ts b/demo/logs/lib/logs.ts new file mode 100644 index 000000000..acd531455 --- /dev/null +++ b/demo/logs/lib/logs.ts @@ -0,0 +1,154 @@ +import "server-only"; + +import { + Cursor, + readRows, + readDateTime64P3, + readEnum8, + readString, + readIPv4, + formatIPv4, + readUUID, + formatUUID, + readUInt16, + readFloat64, + type Reader, +} from "@clickhouse/rowbinary"; + +import { queryRowBinary } from "./clickhouse"; + +export const LOGS_TABLE = "demo_logs"; + +/** + * One decoded log row, in the column order the table stores them — which is also + * the order the `SELECT` below lists them, which is the order they arrive on the + * RowBinary wire. Keep these three in lock-step. + */ +export interface LogRow { + timestamp: Date; + level: LogLevel; + service: string; + host: string; + traceId: string; + status: number; + durationMs: number; + message: string; +} + +export type LogLevel = "debug" | "info" | "warn" | "error"; + +/** + * `Enum8('debug'=1,'info'=2,'warn'=3,'error'=4)`. The wire carries only the + * underlying Int8; the name map lives in the column type, so we map it here. (The + * skill's preferred shape: keep the number on the hot path, resolve the name at + * the edge — which is exactly what this reader is.) + */ +const LEVEL_BY_ID: Record = { + 1: "debug", + 2: "info", + 3: "warn", + 4: "error", +}; + +/** + * Read exactly one log row from the cursor. + * + * This is the clear, API-combinator form the `@clickhouse/rowbinary` README calls + * "correct, clear, and a fine default" — one leaf read per column, in wire order. + * The row mixes fixed-width columns (DateTime64, Enum8, IPv4, UUID, UInt16, + * Float64) with variable-width ones (the two Strings), so it is a natural fit for + * the combinator style. If this became a hot path you'd ask the skill to + * monomorphize it: inline each leaf body and coalesce the bounds checks across + * the fixed-width run. For a paged UI it is nowhere near hot — clarity wins. + * + * Wire order matches the SELECT in `fetchLogsPage`: + * timestamp DateTime64(3) | level Enum8 | service LowCardinality(String) + * host IPv4 | trace_id UUID | status UInt16 | duration_ms Float64 | message String + */ +const readLogRow: Reader = (s) => { + // DateTime64(3) — P=3 is a JS Date's own millisecond resolution, lossless. + const timestamp = readDateTime64P3(s); + // Enum8 — underlying Int8, mapped to its name. + const level = LEVEL_BY_ID[readEnum8(s)] ?? "info"; + // LowCardinality(String) — transparent in RowBinary, decode as plain String. + const service = readString(s); + // IPv4 — 4 LE bytes as a UInt32; format to dotted-quad. + const host = formatIPv4(readIPv4(s)); + // UUID — two byte-reversed LE UInt64 halves; format to canonical 8-4-4-4-12. + const traceId = formatUUID(readUUID(s)); + // UInt16 + const status = readUInt16(s); + // Float64 + const durationMs = readFloat64(s); + // String (LEB128 length prefix + UTF-8 bytes) + const message = readString(s); + + return { + timestamp, + level, + service, + host, + traceId, + status, + durationMs, + message, + }; +}; + +/** Drive `readLogRow` over a full RowBinary buffer to an array of rows. */ +const readLogRows = readRows(readLogRow); + +export interface LogsPage { + rows: LogRow[]; + page: number; + pageSize: number; + total: number; + totalPages: number; +} + +/** + * Fetch one page of logs, newest first, decoded from RowBinary. + * + * Two queries: one for the page of rows (`FORMAT RowBinary`, decoded here), one + * for the total count (so the UI can render page N of M). The count is cheap on + * MergeTree and keeps the demo's pager honest. + */ +export async function fetchLogsPage( + page: number, + pageSize: number, +): Promise { + const safePage = Number.isFinite(page) && page > 0 ? Math.floor(page) : 1; + const safeSize = + Number.isFinite(pageSize) && pageSize > 0 ? Math.floor(pageSize) : 25; + const offset = (safePage - 1) * safeSize; + + const [buffer, total] = await Promise.all([ + queryRowBinary( + `SELECT timestamp, level, service, host, trace_id, status, duration_ms, message + FROM ${LOGS_TABLE} + ORDER BY timestamp DESC + LIMIT ${safeSize} OFFSET ${offset} + FORMAT RowBinary`, + ), + fetchTotal(), + ]); + + const rows = readLogRows(new Cursor(buffer)); + return { + rows, + page: safePage, + pageSize: safeSize, + total, + totalPages: Math.max(1, Math.ceil(total / safeSize)), + }; +} + +/** Total row count, decoded from a one-cell `UInt64` RowBinary response. */ +async function fetchTotal(): Promise { + const buffer = await queryRowBinary( + `SELECT count() FROM ${LOGS_TABLE} FORMAT RowBinary`, + ); + const s = new Cursor(buffer); + // count() is UInt64 → bigint on the wire; the table is demo-sized, so it fits a Number. + return Number(s.view.getBigUint64(0, true)); +} diff --git a/demo/logs/next.config.mjs b/demo/logs/next.config.mjs new file mode 100644 index 000000000..507113cdd --- /dev/null +++ b/demo/logs/next.config.mjs @@ -0,0 +1,15 @@ +import { fileURLToPath } from "node:url"; +import { dirname } from "node:path"; + +/** @type {import('next').NextConfig} */ +const nextConfig = { + // The decoder is a regular Node dependency we want to require at runtime on the + // server, not bundle/trace into the server build. Keep it external. + serverExternalPackages: ["@clickhouse/rowbinary"], + // This demo lives inside the clickhouse-js monorepo, which has its own + // lockfiles higher up. Pin the tracing root to this app so Next doesn't infer + // the wrong workspace root. + outputFileTracingRoot: dirname(fileURLToPath(import.meta.url)), +}; + +export default nextConfig; diff --git a/demo/logs/package-lock.json b/demo/logs/package-lock.json new file mode 100644 index 000000000..1f8045262 --- /dev/null +++ b/demo/logs/package-lock.json @@ -0,0 +1,1026 @@ +{ + "name": "rowbinary-logs-demo", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "rowbinary-logs-demo", + "version": "0.1.0", + "dependencies": { + "@clickhouse/rowbinary": "file:./vendor/clickhouse-rowbinary-0.1.0.tgz", + "next": "^15.1.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "server-only": "^0.0.1" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "@types/react": "^19.0.0", + "typescript": "^5.7.0" + } + }, + "node_modules/@clickhouse/rowbinary": { + "version": "0.1.0", + "resolved": "file:vendor/clickhouse-rowbinary-0.1.0.tgz", + "integrity": "sha512-DotolNbYJGi5T0WavNu8/+pME4KiKH/aLvPE4K9rndCKE23+AuxTpPRHiZJfHX73frkICtubSyRK56fll0iI7g==", + "license": "Apache-2.0" + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@next/env": { + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/env/-/env-15.5.19.tgz", + "integrity": "sha512-sWWluFvcv5v3Fxznmf2ZfjyoVQt/64oCnYqS90inQWGzMPK1VjvekPiz3OPHKmFT30EnHrjlbyaHLt3M0vWabw==", + "license": "MIT" + }, + "node_modules/@next/swc-darwin-arm64": { + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.5.19.tgz", + "integrity": "sha512-jx9wWlTKueHKPvVOndyr7WuaevWCkuYqsQ8gC0TMPKAVWG3MhcdMrjfo9tvIZNXd0QOUYXXvAcZ325y8Uq7uzg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-darwin-x64": { + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-15.5.19.tgz", + "integrity": "sha512-291KFcsIQ3OenRdiUDFOR6W3wezzH4auENXm1gbm1Bjd4ANMMRgxPrWTUztQN43BnVoVuMnHCrLeECIMwgFKbA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-gnu": { + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.5.19.tgz", + "integrity": "sha512-WeH+nelQyyMeE2f8FxBRZNrGipya5zHZV2vjzfCOAYyiI6am+NbnWAAldOBFQBB2w0DjJcsvrKqoFT2b7+5YoA==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-musl": { + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.5.19.tgz", + "integrity": "sha512-5xTOE0lDlDCSSfp+BAif7j17VRRCjWp//ZPZy6NI0QpdrhxtQnsZguSx0xAAZ0c9XZLrLLwCe/XVe5YPrRilKw==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-gnu": { + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.5.19.tgz", + "integrity": "sha512-LTxRmMgqqMv05Had879W00Fm53quiJd3Zuz8h1JSNJ3nGSlbZ/7Tjs1tKyScgN3Au3t3MyPsjPlq60fMmSHLsg==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-musl": { + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.5.19.tgz", + "integrity": "sha512-eoNQSpA5PQfB9wBO4RA47MTDXWz1fizy9Y3Z6e4DetYIF3dvjuu8sj7aIGn/bFCU6lnFzTK34NtCaffP4NsQ7Q==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-arm64-msvc": { + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.5.19.tgz", + "integrity": "sha512-6UNt2dFuCHOe446sm/Kp69nUe8/wIhnh9bm6Xcqw4qEWCOppLMOvhTBVgvM7invVUNr4SPpP6NOQsACtn2IN9Q==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-x64-msvc": { + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.5.19.tgz", + "integrity": "sha512-PhmojAHyqMne56HBLGu9dhDnHPuFmEjrXSQMM/nW0J6j849lk3ESrVtqNJcCk8CKOV7brpTTbaYAjwKPzKM69w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@swc/helpers": { + "version": "0.5.15", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.15.tgz", + "integrity": "sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.8.0" + } + }, + "node_modules/@types/node": { + "version": "22.20.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.0.tgz", + "integrity": "sha512-QWlFW2wf3nTjC13/DqRnBpR4ZO36VJH/JVBkA/vcnmbTBNQIlnObqyqZE1tUR7+Ni23Lda8R1BxMfbXRpCUx5g==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/react": { + "version": "19.2.17", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.17.tgz", + "integrity": "sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw==", + "dev": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001799", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001799.tgz", + "integrity": "sha512-hG1bReV+OUU+MOqK4t/ZWI0tZOyz3rqS9XuhOUz1cIcbwBKjOyJEJuw9ER5JuNyqxNk8u/JUVbGibBOL1yrjFw==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/client-only": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", + "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==", + "license": "MIT" + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/nanoid": { + "version": "3.3.15", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", + "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/next": { + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/next/-/next-15.5.19.tgz", + "integrity": "sha512-xNOW6tYshGX1/Oi3F8uuk4gpDeWsSUE/1Z0G5uUMekIxaQ0xc03UXd9II0VQHYMWviMeA0OHpJFAKsHf8bTYVg==", + "license": "MIT", + "dependencies": { + "@next/env": "15.5.19", + "@swc/helpers": "0.5.15", + "caniuse-lite": "^1.0.30001579", + "postcss": "8.4.31", + "styled-jsx": "5.1.6" + }, + "bin": { + "next": "dist/bin/next" + }, + "engines": { + "node": "^18.18.0 || ^19.8.0 || >= 20.0.0" + }, + "optionalDependencies": { + "@next/swc-darwin-arm64": "15.5.19", + "@next/swc-darwin-x64": "15.5.19", + "@next/swc-linux-arm64-gnu": "15.5.19", + "@next/swc-linux-arm64-musl": "15.5.19", + "@next/swc-linux-x64-gnu": "15.5.19", + "@next/swc-linux-x64-musl": "15.5.19", + "@next/swc-win32-arm64-msvc": "15.5.19", + "@next/swc-win32-x64-msvc": "15.5.19", + "sharp": "^0.34.3" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.1.0", + "@playwright/test": "^1.51.1", + "babel-plugin-react-compiler": "*", + "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", + "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", + "sass": "^1.3.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@playwright/test": { + "optional": true + }, + "babel-plugin-react-compiler": { + "optional": true + }, + "sass": { + "optional": true + } + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/postcss": { + "version": "8.4.31", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", + "integrity": "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.6", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/react": { + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.7.tgz", + "integrity": "sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.7.tgz", + "integrity": "sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ==", + "license": "MIT", + "dependencies": { + "scheduler": "^0.27.0" + }, + "peerDependencies": { + "react": "^19.2.7" + } + }, + "node_modules/scheduler": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", + "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "optional": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/server-only": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/server-only/-/server-only-0.0.1.tgz", + "integrity": "sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA==", + "license": "MIT" + }, + "node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/styled-jsx": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.6.tgz", + "integrity": "sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA==", + "license": "MIT", + "dependencies": { + "client-only": "0.0.1" + }, + "engines": { + "node": ">= 12.0.0" + }, + "peerDependencies": { + "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0 || ^19.0.0-0" + }, + "peerDependenciesMeta": { + "@babel/core": { + "optional": true + }, + "babel-plugin-macros": { + "optional": true + } + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/demo/logs/package.json b/demo/logs/package.json new file mode 100644 index 000000000..f01b188ba --- /dev/null +++ b/demo/logs/package.json @@ -0,0 +1,25 @@ +{ + "name": "rowbinary-logs-demo", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Server-only Next.js demo that pages through a ClickHouse logs table, decoded with @clickhouse/rowbinary.", + "scripts": { + "seed": "node scripts/seed.mjs", + "dev": "next dev -p 3000", + "build": "next build", + "start": "next start -p 3000" + }, + "dependencies": { + "@clickhouse/rowbinary": "file:./vendor/clickhouse-rowbinary-0.1.0.tgz", + "next": "^15.1.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "server-only": "^0.0.1" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "@types/react": "^19.0.0", + "typescript": "^5.7.0" + } +} diff --git a/demo/logs/screenshot.png b/demo/logs/screenshot.png new file mode 100644 index 000000000..33927d512 Binary files /dev/null and b/demo/logs/screenshot.png differ diff --git a/demo/logs/scripts/seed.mjs b/demo/logs/scripts/seed.mjs new file mode 100644 index 000000000..6139bf258 --- /dev/null +++ b/demo/logs/scripts/seed.mjs @@ -0,0 +1,96 @@ +#!/usr/bin/env node +/** + * Seed the demo logs table. + * + * Standalone Node script (no build step, no client dependency): talks to the + * ClickHouse HTTP interface with `fetch`, creates `demo_logs`, and fabricates a + * batch of realistic-looking rows with `INSERT ... SELECT FROM numbers(N)` so all + * the generation happens server-side. + * + * node scripts/seed.mjs # 1000 rows + * node scripts/seed.mjs 50000 # custom row count + * + * Via the package script, forward the count after `--`: + * + * npm run seed -- 50000 + * + * Connection comes from the same env vars the app uses (CLICKHOUSE_URL etc.), + * defaulting to this repo's docker-compose ClickHouse on localhost:8123. + */ + +const CONFIG = { + url: process.env.CLICKHOUSE_URL ?? "http://localhost:8123", + user: process.env.CLICKHOUSE_USER ?? "default", + password: process.env.CLICKHOUSE_PASSWORD ?? "", + database: process.env.CLICKHOUSE_DATABASE ?? "default", +}; + +const TABLE = "demo_logs"; +const ROWS = Number(process.argv[2] ?? 1000); + +function endpoint() { + const u = new URL(CONFIG.url); + u.searchParams.set("database", CONFIG.database); + return u.toString(); +} + +async function exec(sql) { + const res = await fetch(endpoint(), { + method: "POST", + headers: { + "X-ClickHouse-User": CONFIG.user, + "X-ClickHouse-Key": CONFIG.password, + }, + body: sql, + }); + if (!res.ok) { + throw new Error(`ClickHouse failed (${res.status}): ${await res.text()}`); + } + return res.text(); +} + +const CREATE = ` +CREATE TABLE IF NOT EXISTS ${TABLE} ( + timestamp DateTime64(3), + level Enum8('debug' = 1, 'info' = 2, 'warn' = 3, 'error' = 4), + service LowCardinality(String), + host IPv4, + trace_id UUID, + status UInt16, + duration_ms Float64, + message String +) +ENGINE = MergeTree +ORDER BY timestamp`; + +// Everything is generated server-side from `number`. `now64(3) - number seconds` +// spreads rows back in time so newest-first paging has something to walk. +const INSERT = ` +INSERT INTO ${TABLE} +SELECT + now64(3) - toIntervalSecond(number) AS timestamp, + ['info','info','info','debug','debug','warn','warn','error','info','debug'][(number % 10) + 1] AS level, + ['api-gateway','auth','payments','catalog','search','notifications'][(number % 6) + 1] AS service, + toIPv4(concat('10.0.', toString(number % 256), '.', toString((number * 37) % 256))) AS host, + generateUUIDv4() AS trace_id, + [200,200,200,201,204,301,400,404,500,503][(number % 10) + 1] AS status, + round(rand(number) / 4294967.295, 3) AS duration_ms, + ['request completed','request failed upstream','cache miss','cache hit','user login','user logout','slow db query','rate limit exceeded','payment authorized','token refreshed'][(number % 10) + 1] AS message +FROM numbers(${ROWS})`; + +async function main() { + if (!Number.isFinite(ROWS) || ROWS <= 0) { + throw new Error(`Invalid row count: ${process.argv[2]}`); + } + console.log(`Seeding ${ROWS} rows into ${CONFIG.database}.${TABLE} …`); + await exec(CREATE); + await exec(`TRUNCATE TABLE ${TABLE}`); + await exec(INSERT); + const count = (await exec(`SELECT count() FROM ${TABLE}`)).trim(); + console.log(`Done. ${TABLE} now has ${count} rows.`); +} + +main().catch((err) => { + console.error(err.message ?? err); + process.exit(1); +}); diff --git a/demo/logs/tsconfig.json b/demo/logs/tsconfig.json new file mode 100644 index 000000000..87b93b494 --- /dev/null +++ b/demo/logs/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2023"], + "module": "ESNext", + "moduleResolution": "Bundler", + "jsx": "preserve", + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "skipLibCheck": true, + "resolveJsonModule": true, + "isolatedModules": true, + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "paths": { + "@/*": ["./*"] + }, + "allowJs": true + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/demo/logs/vendor/clickhouse-rowbinary-0.1.0.tgz b/demo/logs/vendor/clickhouse-rowbinary-0.1.0.tgz new file mode 100644 index 000000000..5dbc9a328 Binary files /dev/null and b/demo/logs/vendor/clickhouse-rowbinary-0.1.0.tgz differ diff --git a/package-lock.json b/package-lock.json index b072addb7..84678bc89 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7191,16 +7191,16 @@ }, "packages/client-common": { "name": "@clickhouse/client-common", - "version": "1.22.0", + "version": "1.23.0", "license": "Apache-2.0", "devDependencies": {} }, "packages/client-node": { "name": "@clickhouse/client", - "version": "1.22.0", + "version": "1.23.0", "license": "Apache-2.0", "dependencies": { - "@clickhouse/client-common": "1.22.0" + "@clickhouse/client-common": "1.23.0" }, "devDependencies": { "simdjson": "^0.9.2" @@ -7211,15 +7211,15 @@ }, "packages/client-web": { "name": "@clickhouse/client-web", - "version": "1.22.0", + "version": "1.23.0", "license": "Apache-2.0", "dependencies": { - "@clickhouse/client-common": "1.22.0" + "@clickhouse/client-common": "1.23.0" } }, "tests/clickhouse-test-runner": { "name": "@clickhouse/clickhouse-test-runner", - "version": "1.22.0", + "version": "1.23.0", "dependencies": { "@clickhouse/client": "*" }, diff --git a/packages/client-common/package.json b/packages/client-common/package.json index 99aaecd3d..10c76ed79 100644 --- a/packages/client-common/package.json +++ b/packages/client-common/package.json @@ -2,7 +2,7 @@ "name": "@clickhouse/client-common", "description": "Official JS client for ClickHouse DB - common types", "homepage": "https://clickhouse.com", - "version": "1.22.0", + "version": "1.23.0", "license": "Apache-2.0", "keywords": [ "clickhouse", diff --git a/packages/client-common/src/version.ts b/packages/client-common/src/version.ts index cbfb1f9da..dff654268 100644 --- a/packages/client-common/src/version.ts +++ b/packages/client-common/src/version.ts @@ -1 +1 @@ -export default "1.22.0"; +export default "1.23.0"; diff --git a/packages/client-node/package.json b/packages/client-node/package.json index 6fc0b578e..bd592db8b 100644 --- a/packages/client-node/package.json +++ b/packages/client-node/package.json @@ -2,7 +2,7 @@ "name": "@clickhouse/client", "description": "Official JS client for ClickHouse DB - Node.js implementation", "homepage": "https://clickhouse.com", - "version": "1.22.0", + "version": "1.23.0", "license": "Apache-2.0", "keywords": [ "clickhouse", @@ -32,19 +32,23 @@ { "name": "clickhouse-js-node-troubleshooting", "path": "./skills/clickhouse-js-node-troubleshooting" + }, + { + "name": "clickhouse-js-node-rowbinary-parser", + "path": "./skills/clickhouse-js-node-rowbinary-parser" } ] }, "scripts": { "pack": "npm pack", - "prepack": "rm -rf skills && cp ../../README.md ../../LICENSE . && cp -r ../../skills .", + "prepack": "rm -rf skills && cp ../../README.md ../../LICENSE . && cp -r ../../skills . && RBP=skills/clickhouse-js-node-rowbinary-parser && rm -rf $RBP/tests $RBP/node_modules $RBP/dist $RBP/package.json $RBP/package-lock.json $RBP/tsconfig.json $RBP/tsconfig.build.json $RBP/vitest.config.ts $RBP/.gitignore $RBP/LICENSE $RBP/eval_result*.md", "typecheck": "tsc --noEmit", "lint": "eslint --max-warnings=0 .", "lint:fix": "eslint . --fix", "build": "rm -rf dist; tsc" }, "dependencies": { - "@clickhouse/client-common": "1.22.0" + "@clickhouse/client-common": "1.23.0" }, "devDependencies": { "simdjson": "^0.9.2" diff --git a/packages/client-node/src/version.ts b/packages/client-node/src/version.ts index cbfb1f9da..dff654268 100644 --- a/packages/client-node/src/version.ts +++ b/packages/client-node/src/version.ts @@ -1 +1 @@ -export default "1.22.0"; +export default "1.23.0"; diff --git a/packages/client-web/package.json b/packages/client-web/package.json index 15cc51367..a25d0a77e 100644 --- a/packages/client-web/package.json +++ b/packages/client-web/package.json @@ -2,7 +2,7 @@ "name": "@clickhouse/client-web", "description": "Official JS client for ClickHouse DB - Web API implementation", "homepage": "https://clickhouse.com", - "version": "1.22.0", + "version": "1.23.0", "license": "Apache-2.0", "keywords": [ "clickhouse", @@ -31,6 +31,6 @@ "build": "rm -rf dist; tsc" }, "dependencies": { - "@clickhouse/client-common": "1.22.0" + "@clickhouse/client-common": "1.23.0" } } diff --git a/packages/client-web/src/version.ts b/packages/client-web/src/version.ts index cbfb1f9da..dff654268 100644 --- a/packages/client-web/src/version.ts +++ b/packages/client-web/src/version.ts @@ -1 +1 @@ -export default "1.22.0"; +export default "1.23.0"; diff --git a/skills/clickhouse-js-node-rowbinary-parser/.gitignore b/skills/clickhouse-js-node-rowbinary-parser/.gitignore new file mode 100644 index 000000000..a2bb8401f --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +dist/ +# Copied from the repo root by `prepack` so the published tarball carries it. +LICENSE diff --git a/skills/clickhouse-js-node-rowbinary-parser/EXAMPLES.md b/skills/clickhouse-js-node-rowbinary-parser/EXAMPLES.md new file mode 100644 index 000000000..9a18b686b --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/EXAMPLES.md @@ -0,0 +1,48 @@ +# RowBinary reader examples + +Six end-to-end examples. Each `src/examples/*.ts` exports TWO readers for the +same row — `readXRow` (built from the generic combinator API; easiest to read) +and `readXRowFast` (the optimized, monomorphized form: leaf reads inlined, +combinators flattened to straight-line loops, `advance()` coalesced over +fixed-width runs — still streaming-safe). The matching `tests/X.example.test.ts` +runs the full create → populate → read-back round trip against a live ClickHouse +server (verified, not illustrative), and `tests/X.bench.ts` decodes a large +`numbers()`-generated buffer with both readers (equivalence-checked before +timing) to measure the speedup. + +To use one: find the example whose column types match your result, open its +reader, and adapt it. `readRows(readXRow)` drives a row reader over a whole +result; `streamRowBatches(chunks, readXRow)` drives it over a chunked HTTP stream. + +| Example | SQL schema (the trigger) | Speedup | Reader · Test | +| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| **orders** | `id UInt8, uid UUID, price Decimal64(2), status Enum8(...)` | **~3.4x** | [`src/examples/orders.ts`](src/examples/orders.ts) · [`tests/orders.example.test.ts`](tests/orders.example.test.ts) | +| **carts** | `cart_id UInt32, items Array(Tuple(sku String, qty UInt16)), discounts Array(Nullable(Int32))` | **~2.0x** | [`src/examples/carts.ts`](src/examples/carts.ts) · [`tests/carts.example.test.ts`](tests/carts.example.test.ts) | +| **telemetry** | `host String, tags Map(String,String), cpu Array(Float64), region Nullable(String), window Tuple(start UInt32, count UInt16)` | **~1.4x** | [`src/examples/telemetry.ts`](src/examples/telemetry.ts) · [`tests/telemetry.example.test.ts`](tests/telemetry.example.test.ts) | +| **observability** | `id UInt64, ts DateTime64(3), level Enum8, trace_id UUID, payload Variant(String,Int64,Float64), tags Map(LowCardinality(String),String), metrics Array(Tuple(LowCardinality(String),Float64)), attrs Array(Nullable(Int64))` | **~1.4x** | [`src/examples/observability.ts`](src/examples/observability.ts) · [`tests/observability.example.test.ts`](tests/observability.example.test.ts) | +| **profiles** | `id UInt32, tags Array(String), score Nullable(Int32)` | **~1.3x** | [`src/examples/profiles.ts`](src/examples/profiles.ts) · [`tests/profiles.example.test.ts`](tests/profiles.example.test.ts) | +| **events** | `id UInt64, name String, ts DateTime('UTC')` | **~1.05x — on par** | [`src/examples/events.ts`](src/examples/events.ts) · [`tests/events.example.test.ts`](tests/events.example.test.ts) | + +Speedups: Node 24 / V8, decoding a 20k-row buffer — read the ratio, not the +absolute hz, and run `npm run bench` for your own numbers. Two independent levers +drive them: **composite monomorphization** (removes per-row combinator closures — +`carts` / `telemetry` / `observability`) and **per-row formatting** (`orders` is +all-scalar yet the biggest win, almost entirely from the `formatUUIDTable` swap). +A flat scalar row with no hot formatter (`events`) is within noise, so prefer the +clearer API reader there. When in doubt, benchmark — the `*.bench.ts` files are +the template. + +The readers live under `src/examples/` and are excluded from the published build +(`tsconfig.build.json`): reference material and test fixtures, type-checked by the +base `tsconfig.json` and run by the suite, not part of the package's public API. + +## Columnar decode (struct-of-arrays) — the ~4x numeric path + +The examples above produce one object per row (array-of-structs). For a +**numeric, fixed-width result the consumer reads column-wise** (aggregate / scan +/ filter / plot, or hand off to a Worker / WASM kernel), decode the same +row-major bytes directly into **one typed array per column** in the same single +pass — no per-row object, no `Date`, no number boxing. That removes the +allocation that dominates a numeric decode for a **measured ~4.2x**. + +See example: [`decodeIotColumnar`](src/examples/iot.ts). diff --git a/skills/clickhouse-js-node-rowbinary-parser/README.md b/skills/clickhouse-js-node-rowbinary-parser/README.md new file mode 100644 index 000000000..441c5de10 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/README.md @@ -0,0 +1,248 @@ +# ClickHouse Node.js RowBinary Parser Generator + +**If JS had a -O3 compiler flag, this skill would be it.** (for RowBinary parsing) + +A skill and a library that lets a coding agent generate bespoke RowBinary parsers on the first pass from the column type definitions of a ClickHouse response. The [spirit](#the-spirit) behind the approach. + +**Reader only** for now. Today this covers reading (decoding) RowBinary streams. A matching RowBinary writer (encoding) is planned. + +## Status + +- ✅ Sonnet 4.6: 60% -> 94.0% pass rate +- ✅ Opus 4.8: 71% -> 94.7% pass rate +- ✅ Haiku 4.5: 52% -> 86.0% pass rate +- ✅ Composer 2.5 Fast: 3x parser performance +- ✅ 469/469 tests +- ✅ type-checked +- ✅ benchmarked + +## Example + +Take a small orders result: + +```sql +SELECT id, uid, price, status FROM orders +-- id UInt8 +-- uid UUID +-- price Decimal64(2) +-- status Enum8('new' = 1, 'shipped' = 2, 'done' = 3) +``` + +**The API-only reader** — what you write by composing the library's combinators. Correct, clear, and a fine default: + +```ts +const readOrderRow: Reader = (s) => ({ + id: readUInt8(s), + uid: formatUUID(readUUID(s)), + price: readDecimal64(2)(s), + status: readEnum8(s), +}); +``` + +**The optimized reader the skill generates** — same row, monomorphized to +straight-line code. The whole row is fixed-width (1 + 16 + 8 + 1 = 26 bytes), so +the four separate bounds checks coalesce into one `advance(s, 26)` and every leaf +read happens at a constant offset; the per-field combinators are gone: + +```ts +const readOrderRowFast: Reader = (s) => { + const { buf, view } = s; + const o = advance(s, 26); // one bounds check for the whole 26-byte row + const id = buf[o]!; + const uid = formatUUIDTable(buf.subarray(o + 1, o + 17)); + const price: DecimalValue = [view.getBigInt64(o + 17, true), 2]; + const status = view.getInt8(o + 25); + return { id, uid, price, status }; +}; +``` + +Same values, same streaming-safety — **~3.4x** faster. + +## How to use + +As a library (comes with the skill): + +```bash +npm install @clickhouse/rowbinary +npx skills-npm setup +``` + +As a skill only: + +```bash +npx skills add ClickHouse/clickhouse-js/skills/clickhouse-js-node-rowbinary-parser +``` + +```console +> Hey, Claude, tell me what the rowbinary parser skill can do for me. +> A lot! It generates custom, high-performance RowBinary parsers… +> Super, generate a parser for the queries in app/src/model.ts. +< Reading skill clickhouse-js-node-rowbinary-parser… +``` + +## Why it's worth it + +Four pillars — speed, correctness, judgment, and lifting smaller models: + +- **~2–3x faster code than the straightforward decoder.** The skill emits + monomorphized, flattened, straight-line code — inlined reads, bounds checks + coalesced across adjacent fixed-width columns, the right array layout — measured + at ~1.3–3.4x over the _same logic written with the plain combinator API_ + (`npm run bench`). This is why + - inlined JIT friendly code + - benchmarked hot paths + - minimal allocations + - v8 and Node.js specific optimizations +- **Correct on the gotchas that otherwise quietly break.** UUID byte + order, `Variant`'s sort-by-type-name discriminant, `DateTime64` sub-second + precision, signed-high-word wide integers, faithful decimals, `Dynamic`/`JSON` + self-description, transparent wrappers, opaque `AggregateFunction` — each + encoded with a live, server-verified test ([details below](#correctness-on-the-gotcha-heavy-types)). +- **Judgment, not just code.** The skill carries the working knowledge to make + the right call _before_ writing a line, so the agent neither over- nor + under-engineers: + - **Is RowBinary even right?** For string-heavy results read as text, a `JSON*` + format + V8's native `JSON.parse` (plus `gzip`/`zstd`) can beat a JS RowBinary + decoder — reach for RowBinary when the data is numeric / wide-integer / + binary-blob heavy. + - **Whole buffer or stream?** Drop the `advance()` bounds checks for a complete + in-memory buffer (faster); keep them for a chunked HTTP response that must + survive rows straddling chunk boundaries. + - **Drop the portability scaffolding.** RowBinary is little-endian and the + target is x86/ARM, so the skill steers away from big-endian / byte-swap + "portability" code a cautious one-shot pass tends to add. +- **Improves smaller models' performance.** Because the skill hands over the + hard-won answers up front, it lifts a weaker model the most. In a 24-eval + with-skill vs no-skill benchmark, the skill [raised](eval_result_sonnet.md) **Sonnet 4.6** from 60.4% to + **94.0%** (+34pp) — bringing it level with skill-equipped **Opus 4.8** (94.7%), + which itself [gained](eval_result.md) +23pp (71.5% → 94.7%). Composer 2.5 Fast + [got](eval_result_composer.md) a 3x parser performance boost, Haiku 4.5 + [raised](eval_result_haiku.md) from 52% to 86% — the skill closes + most of the model-capability gap on this task. + +## What it does + +Given the columns of a query result — their names and ClickHouse type +definitions (as returned by `RowBinaryWithNamesAndTypes`, or supplied by the +user) — the skill generates parser code tailored to exactly those types. Rather +than shipping a generic, runtime-driven decoder, it emits straight-line code +that reads each column in order, so the parser only contains the logic the +specific result shape needs. + +## Correctness on the gotcha-heavy types + +For a plain `UInt64, String, DateTime` result a strong model already writes fast, +correct code on its own. The skill earns its keep on the **long tail of RowBinary +traps** — the encodings where a from-scratch decoder is quietly wrong — each one +captured here with a live, server-verified test: + +- **UUID** — two little-endian `UInt64` halves, each byte-reversed vs. the text + form (not 16 bytes in order). +- **`Variant(...)`** — the 1-byte discriminant indexes the alternatives sorted by + **type name** (ClickHouse globally sorts them), NOT declaration order; `0xFF` + is NULL. +- **`DateTime64(P)`** — returned as `[Date, nanoseconds]` so the sub-second part + isn't lost to a `Date`'s millisecond resolution; `Time`/`Time64` are durations, + not instants. +- **Wide integers** — `Int128`/`Int256` compose from 64-bit words with the **high + word read signed**; 64-bit values stay `bigint`, never a lossy `number`. +- **Decimals** — kept as the exact `[unscaled, scale]` pair, not a lossy float. +- **`Dynamic` / `JSON`** — self-describing: a per-value binary type encoding, then + the value; declared typed `JSON` paths are written without a tag (need the + schema). Wrappers are erased (`Nullable`/`Variant` → concrete type). +- **Transparent wrappers** — `LowCardinality(T)` / `SimpleAggregateFunction(f, T)` + decode as the inner `T` (no dictionary layer in RowBinary); `Nested(...)` is + `Array(Tuple(...))` with no wire of its own. +- **`AggregateFunction(...)`** — opaque, unframed state: not decodable or even + skippable; finalize server-side instead. +- **`FixedString`** preserves trailing NUL padding; **`Enum`** decodes to the + underlying int (the name map is metadata); **`BFloat16`** is the top 16 bits of + a `Float32`. + +This is also where a raw model is most likely to go wrong. In a clean-room test +on a `Variant` / `UUID` / `DateTime64` / `LowCardinality` schema, a no-skill +Sonnet produced a **silently wrong UUID** (treated the bytes as plain, missing +the two-reversed-halves layout), and a no-skill Opus got it right only after +**three web searches**. The skill hands over these answers up front — correct by +construction, no lookups. See `baseline/README.md` for the full control. + +And the failure isn't a one-off — it's a coin-flip. Running the same no-skill +Sonnet on the `orders` schema (`UInt8, UUID, Decimal64(2), Enum8`) **5 times in +isolation**, only **3 of 5** runs decoded correctly; both failures were the same +UUID byte-order scramble. Even the passing runs varied ~1.9x in generated-code +throughput. With the skill, every run is correct. So a single A/B undersells the +gap: from scratch the model is right roughly 60% of the time and silently wrong +the rest, while the skill makes correctness deterministic. + +## Examples + +Six end-to-end examples live in [EXAMPLES.md](EXAMPLES.md). Each ships both an API-combinator +reader and an optimized, monomorphized one, with a runnable round-trip test and +a benchmark — so the speedups below are measured, not claimed (Node 24 / V8; +`npm run bench` for your own numbers): + +| Example | Columns | Optimized speedup | +| ----------------- | ---------------------------------------------------- | ------------------- | +| **orders** | `UUID`, `Decimal64`, `Enum8` | **~3.4x** | +| **carts** | nested `Array(Tuple(...))`, `Array(Nullable(...))` | **~2.0x** | +| **telemetry** | `Map`, `Array(Float64)`, `Nullable`, named `Tuple` | **~1.4x** | +| **observability** | `Variant`, `DateTime64(3)`, `LowCardinality`, nested | **~1.4x** | +| **profiles** | `Array(String)`, `Nullable(Int32)` | **~1.3x** | +| **events** | `UInt64`, `String`, `DateTime` scalars | **~1.05x — on par** | + +Two axes drive the win. **Composite structure** is one: monomorphization pays in +proportion to how many per-row combinator closures it removes (`carts` / +`telemetry` / `observability`). **Per-row formatting** is the other, independent +of composites: `orders` is all-scalar yet the biggest win (~3.4x), almost +entirely from swapping the BigInt UUID formatter for the lookup-table +`formatUUIDTable`. The genuinely flat case — a scalar row with no hot formatter +(`events`) — is on par, so the simpler API reader is the right call there. +Measure, don't assume. + +## Scope + +- **In scope:** `RowBinary`, `RowBinaryWithNames`, and + `RowBinaryWithNamesAndTypes` decoding for Node.js — full-buffer and streaming + (chunked) via `advance()`/`NeedMoreData`, `readRows()`, and the async + `streamRowBatches()` (with a built-in small-chunk warning and the optional + `coalesceChunks()` debounce filter). +- **Planned:** RowBinary **writing / encoding** (the inverse of everything above) +- **Out of scope (for now):** browsers and Edge runtimes, non-RowBinary formats + (JSON / CSV / TSV / Parquet), and big-endian hosts. + +## The spirit + +A RowBinary parser generator is a narrow thing. But it's built as an instance of +a broader bet about what libraries become once a capable LLM is part of the +toolchain. Three shifts, each already visible in this repo: + +- **Self-modifiable software.** The library deliberately ships _several_ + equivalent decoders for the same type — `readUUID` / `readUUIDBigInt` / + `readUUIDHiLo`, `formatUUID` / `formatUUIDTable`, `new Array(n)` vs `[]`+push, + streaming vs whole-buffer — because the fastest one depends on the workload, + not the type. Today the agent picks at generation time from measured + benchmarks. The next step is to pair the skill with a tracing layer that runs + variant A against variant B _on the live workload_ and keeps whichever wins for + this data shape and access pattern — a parser that re-tunes itself as the + traffic drifts, instead of freezing one author's guess into a release. + +- **Custom software.** The value here isn't a fixed high-level API; it's the + benchmarked building blocks plus the judgment to combine them. So the end user + doesn't bend their code to the authors' generic surface — they have the agent + assemble the high-level API _they_ actually want, shaped to their queries, row + shapes, and latency/memory budget. Two teams with different workloads grow two + different libraries from the same primitives, and neither inherits a design + decision that was only ever right for the original authors' use case. + +- **Read-write libraries.** For either of the above to be safe, the source has to + be legible to an LLM, not merely runnable. So this repo is written _read-write_: + every tradeoff is commented where it's made — the per-column ClickHouse type + annotations, the `SAFE TO TOGGLE` markers on the fast variants, each reader's + doc comment carrying its exact monomorphized form. An LLM can + read _why_ a decision was made and change it in depth with confidence — not + just call the public functions, but safely rework the internals. + +The through-line: the last mile is glue the LLM writes over stable, benchmarked +blocks, so the authors' job shrinks to exporting good primitives and documenting +their tradeoffs honestly — rather than trying to bake the right performance +constants for every possible workload into the library ahead of time. diff --git a/skills/clickhouse-js-node-rowbinary-parser/SKILL.md b/skills/clickhouse-js-node-rowbinary-parser/SKILL.md new file mode 100644 index 000000000..f424cd5e4 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/SKILL.md @@ -0,0 +1,190 @@ +--- +name: clickhouse-js-node-rowbinary-parser +description: > + Generate TypeScript/JavaScript code that reads and decodes ClickHouse + RowBinary streams from the ClickHouse HTTP server. + Use this skill whenever a user wants to parse `RowBinary`, + `RowBinaryWithNames`, or `RowBinaryWithNamesAndTypes`. + Node.js only, doesn't cover browsers. +--- + +# ClickHouse JS RowBinary Parser Generator for Node.js + +## First: is RowBinary even the right format? + +RowBinary exists for throughput, but it is **not automatically the fastest +path** — match the format to the shape of the data before committing to a +bespoke parser. + +**Prefer a `JSON*` format (e.g. `JSONEachRow`) when** the result is mostly +strings / JSON-like values that you consume wholesale — randomly accessing +essentially every field, running string/regexp methods on them, treating values +as text. V8's native `JSON.parse` is heavily optimized C++ and builds JS strings +and objects faster than a JS-level RowBinary decoder can; pair it with HTTP +response compression (`gzip` / `zstd`, which crushes JSON's repetitive keys) and +the wire cost shrinks too. + +**RowBinary clearly wins when** the result is dominated by: + +- **Wide numerics** — `Int128`/`Int256`/`UInt128`/`UInt256`, + `Decimal128`/`Decimal256`. +- **Binary / fixed-width blobs** — `IPv4`, `IPv6`, `UUID`, `FixedString`. +- **High-volume fixed-width numeric columns** generally, where each value is a + single `DataView` read. + +**Prefer the `Native` format when** columnar load and client-side analytics are +the main goal (fold/scan/filter columns, feed typed arrays to a Worker or WASM). +`Native` is column-major, so it loads straight into one typed array per column +with no transpose. + +For help choosing and consuming a `JSON*` format (or CSV / TSV) instead, use the +**`clickhouse-js-node-coding`** skill. + +## Second: complete buffer, or incremental stream? + +Decide this before writing the reader — it changes the shape of the code and is +a real performance fork. + +- **Incremental / streaming (the default here).** You consume the HTTP response + chunk by chunk as it arrives — low latency to the first row, bounded memory. + It is generally the best choice for large results, but slower per-row. + +- **Whole buffer in memory (faster, when it fits).** If you already hold the + entire response as one `Buffer`, the bounds check never fires — so you can drop + `advance()` entirely and read at a running offset in one monolithic loop. + This is 2-3x faster but introduces latency and unbounded memory use. + +The exposed API is streaming by default and requires an optimisation pass. + +## Third: row objects, or columnar (typed arrays)? + +The default output is one object per row (array-of-structs). For a **numeric, +fixed-width result that the consumer reads column-wise**, decode instead into one +typed array per column (struct-of-arrays) — it is **~4x faster and several times +smaller** because it removes the per-row object, `Date`, and number-boxing +allocations that dominate a numeric decode (the byte reads are already at memory +bandwidth). Measured in `tests/iot.columnar.bench.ts`; rationale in +`case-studies/wasm-vs-js.md`. + +- **Use columnar when** columns are numeric/fixed-width and the consumer + aggregates / filters / scans / plots them, or hands the buffers to a Worker or + WASM kernel (typed-array `ArrayBuffer`s are transferable — zero-copy). +- **The preallocation trick:** if EVERY column is fixed-width the row stride is + known, so the exact count is `buf.length / stride` — allocate each column once, + write at `[i]`, no growth, no per-row bounds check. +- **Streaming columnar is just that arithmetic per chunk.** Fixed width means + honoring a partial buffer needs no `advance()`/`NeedMoreData`/restart: the + complete-row count is `(chunk.length / stride) | 0`, and the leftover bytes + carry to the next chunk. Yield one typed-array batch per chunk, each owning a + fresh transferable `ArrayBuffer` (see `streamSensorColumns` in + `src/columnar.ts`). +- **Stay row-oriented when** downstream code is row-shaped, the row is + string-dominated (columnar's win is numeric — a JS string allocates either + way), or the schema is nested/heterogeneous (`Array`/`Map`/`Tuple`). +- **Hybrid:** store columnar, expose a lazy `rowAt(i)` accessor that builds an + object only for rows actually touched (see `iotRowAt` in `src/examples/iot.ts`). + +## Core guidance + +When generating a parser, follow these: + +- **Little-endian only.** RowBinary is little-endian; target x86/ARM. Read every + multi-byte number with `DataView` accessors passing a **literal** `true` for + the `littleEndian` flag. + +- **Correct first, then optimize.** First emit a correct reader built from the + plain per-type API. Only after it's correct (and tested) specialize it. Don't + bake performance assumptions in before correctness. + +- **Monomorphize generic/composite types.** Emit specialized, inlined code per + type combination instead of passing functions as arguments where the type + is known ahead of time. + +- **Streaming: throw + restart, not generators.** To signal "need more bytes", + a synchronous reader that throws a sentinel (`NeedMoreData`) and restarts the + row beats generators for realistic chunk sizes; + +- **Keep an eye on chunk sizes.** Partial trailing rows, small chunks are a silent + throughput killer: `streamRowBatches` warns once when + rows-per-chunk falls too low, and `coalesceChunks(source, { minSize, timeoutMs })` + merges small chunks in front of it when the source size isn't yours to raise. + +- **Shared scratch is not reentrant.** Some hot methods reuse a module-level + scratch buffer as a write-then-read pair — correct only because reads are fully + synchronous. An `async`/`yield` boundary between populating and reading it + corrupts the value. + +- **Hoist the cursor into locals.** Prefer the working buffer and view declared + once at the top of the generated reader, and keep the read offset in a **local variable**, + operating on it directly instead of re-reading from an object. + +- **Coalesce `advance()` across adjacent fixed-width columns.** A run of + neighbouring fixed-width columns has a known combined size, so bounds-check it + ONCE. + +- **Inline the leaf reads.** The per-type `readX` functions are the correct, + composable reference; the generated parser should INLINE their bodies, not call + them, so the row reader is straight-line with no per-field indirection (and so + the two points above can fold the offset arithmetic together). + +- **Annotate the decoded type per column.** Inlining erases the type structure, + so put a short comment above each column's decode block naming the ClickHouse + type it reads. + +- **Pre-allocate small result arrays.** RowBinary gives every array/map its + element count up front (the LEB128 prefix), so DEFAULT is to `new Array(n)`. + NOTE: for **large** arrays the application will iterate or compute over repeatedly, + prefer `[]` + `push` (faster to traverse in V8) — or a typed array (`Float64Array`…) + for numeric elements. + +- **TypeScript by default.** Generate TypeScript parsers and helpers unless the + user explicitly asks for plain JavaScript. + +## Type family references + +The readers live as real code under `src/`, split by type family. + +| Result contains (trigger) | Open | +| ---------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Always** — cursor state, `advance()`, `NeedMoreData`, `Reader` | `src/core.ts` | +| LEB128 length/count prefixes for `String`/`Array`/`Map` (`readUVarint`) | `src/varint.ts` | +| `Int8`–`Int256`, `UInt8`–`UInt256` | `src/integers.ts` | +| `Bool` | `src/bool.ts` | +| `Enum8`, `Enum16` | `src/enums.ts` | +| `Float32`, `Float64`, `BFloat16` | `src/floats.ts` | +| `Decimal32/64/128/256`, `Decimal(P, S)` | `src/decimals.ts` | +| `String`, `FixedString(N)` | `src/strings.ts` | +| `UUID` | `src/uuid.ts` | +| `IPv4`, `IPv6` | `src/ip.ts` | +| `Date`, `Date32`, `DateTime`, `DateTime(tz)`, `DateTime64(P[, tz])` | `src/datetime.ts` | +| `Time`, `Time64(P)` | `src/time.ts` | +| `IntervalNanosecond` … `IntervalYear` | `src/interval.ts` | +| `Array(T)`, `Map(K, V)`, `Tuple(...)`, `Nullable(T)`, `Variant(...)`, `QBit(...)` | `src/composite.ts` | +| `Point`, `Ring`, `LineString`, `MultiLineString`, `Polygon`, `MultiPolygon`, `Geometry` | `src/geo.ts` | +| `Dynamic` (and `Variant`/`Interval`/`Nested`/`Dynamic` nested inside it) | `src/dynamic.ts` | +| `JSON` | `src/json.ts` | +| The whole result — loop rows to EOF (`readRows`) | `src/rows.ts` | +| A chunked HTTP response — `streamRowBatches`, `coalesceChunks` | `src/stream.ts` | +| **Numeric/fixed-width result read column-wise** (aggregate/scan/plot, hand to a Worker/WASM) → decode into typed arrays, not row objects (~4x) | `src/columnar.ts` (`streamSensorColumns` — streaming, yields transferable typed-array batches); `decodeIotColumnar` in `src/examples/iot.ts` is the whole-buffer form | +| `LowCardinality(T)` — transparent, decode as `T` | `src/lowCardinality.ts` | +| `SimpleAggregateFunction(f, T)` — transparent, decode as `T` | `src/simpleAggregateFunction.ts` | +| `Nested(...)` — no wire of its own; `Array(Tuple(...))` | `src/nested.ts` | +| `Nothing` — zero-width, never decoded (only wrapped) | `src/nothing.ts` | +| `AggregateFunction(...)` — opaque state; finalize server-side | `src/aggregateFunction.ts` | + +## Worked examples + +Six end-to-end examples with real speedup are catalogued in [EXAMPLES.md](EXAMPLES.md). + +## Out of scope + +- **JSON / CSV / TSV / Parquet parsing** → use `clickhouse-js-node-coding`. +- **Connection errors, hangs, type mismatches** → use + `clickhouse-js-node-troubleshooting`. +- **Browser / Web Worker / Edge** → `@clickhouse/client-web`. + +## Still Stuck? + +- [ClickHouse RowBinary format](https://clickhouse.com/docs/interfaces/formats#rowbinary) +- [ClickHouse data types](https://clickhouse.com/docs/sql-reference/data-types) +- [ClickHouse JS client docs](https://clickhouse.com/docs/integrations/javascript) diff --git a/skills/clickhouse-js-node-rowbinary-parser/case-studies/iot-rowbinary-vs-json.md b/skills/clickhouse-js-node-rowbinary-parser/case-studies/iot-rowbinary-vs-json.md new file mode 100644 index 000000000..f3118ecaf --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/case-studies/iot-rowbinary-vs-json.md @@ -0,0 +1,83 @@ +# Case study: RowBinary vs JSON on a table of IoT readings + +**TL;DR** — On a dense fixed-width numeric row, the skill's optimized RowBinary +reader decodes **3.5x faster than the best JSON format** (`JSONCompactEachRow`) +and **5.4x faster than `JSONEachRow`**, over a wire that is **1.6–3.3x smaller**. +This is the workload shape the [SKILL's format-choice +guidance](../SKILL.md#first-is-rowbinary-even-the-right-format) points at +RowBinary for — and the numbers below are _measured_, not assumed. + +Reproduce: `npx vitest bench --run tests/iot.bench.ts` (against a live +ClickHouse server). Source: [`tests/iot.bench.ts`](../tests/iot.bench.ts), +reader: [`src/examples/iot.ts`](../src/examples/iot.ts). + +## The data + +A table of IoT sensor readings — every column fixed-width, not a string in the +row, so the whole record is a flat 41-byte run: + +```sql +sensor_id UInt32 -- 4 bytes +ts DateTime64(3) -- 8 bytes +temperature Float64 -- 8 bytes +humidity Float64 -- 8 bytes +pressure Float64 -- 8 bytes +battery Float32 -- 4 bytes +status UInt8 -- 1 byte +``` + +50,000 rows, fetched from a live server in three formats and decoded into +equivalent JS objects. A cross-format check asserts the RowBinary (binary +float) and JSON (decimal-text → float) decodes agree on every numeric column +before any timing is taken — so this measures the same work three ways, not +three different results. + +## What was compared + +- **RowBinary — optimized.** The skill's monomorphized reader: the seven column + bounds checks coalesce into one `advance(s, 41)`, every field read at a + constant offset off that base. +- **RowBinary — API combinators.** The same logic written with the plain + per-type readers (`readUInt32`, `readFloat64`, …) — the clear default. +- **JSONCompactEachRow — `JSON.parse`.** Newline-delimited _arrays_ (no repeated + keys). The strongest JSON contender a knowledgeable user would pick. +- **JSONEachRow — `JSON.parse`.** Newline-delimited _objects_ (keys repeated + every row) — the naive idiomatic choice. + +Both JSON paths use the fastest idiomatic decode: splice the rows into one +`[...]` document and hand it to V8's native `JSON.parse` in a single call. + +## Wire size (HTTP response bytes) + +| Format | Size | B/row | vs RowBinary | +| ------------------ | ------- | ----- | ------------ | +| RowBinary | 2.05 MB | 41.0 | 1.0x | +| JSONCompactEachRow | 3.38 MB | 67.6 | 1.6x | +| JSONEachRow | 6.68 MB | 133.6 | 3.3x | + +## Decode throughput (full 50k-row decode; higher = faster) + +| Decoder | ops/s | ms/decode | ≈ rows/s | speedup | +| --------------------------------- | ----- | --------- | -------- | -------- | +| **RowBinary — optimized** | 399 | 2.50 | ~20.0 M | **1.0x** | +| RowBinary — API combinators | 159 | 6.31 | ~7.9 M | 0.40x | +| JSONCompactEachRow — `JSON.parse` | 114 | 8.76 | ~5.7 M | 0.29x | +| JSONEachRow — `JSON.parse` | 74 | 13.47 | ~3.7 M | 0.19x | + +_Node 24 / V8. Your numbers will vary; run `npm run bench` on your own hardware._ + +## Takeaways + +- **This is the textbook RowBinary win.** High-volume fixed-width numerics where + each field is one `DataView` read and there is no text to tokenize or numbers + to parse from decimal strings. The monomorphization win (2.5x over the + combinator API) is unusually large here because the whole row coalesces into a + _single_ bounds check with constant-offset reads. +- **Format choice matters more than the optimization.** Even the plain + combinator-API RowBinary reader (~7.9 M rows/s) beats the best JSON option — + before any monomorphization. +- **The flip side still holds.** Had this been a string-heavy result (logs, JSON + blobs, text consumed wholesale), `JSON.parse`'s optimized C++ would likely + _win_, and the skill would steer you to `JSONEachRow` + compression instead. + For IoT telemetry, RowBinary is clearly right — match the format to the shape + of the data. diff --git a/skills/clickhouse-js-node-rowbinary-parser/case-studies/ledger-rowbinary-vs-json.md b/skills/clickhouse-js-node-rowbinary-parser/case-studies/ledger-rowbinary-vs-json.md new file mode 100644 index 000000000..c9c69ac40 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/case-studies/ledger-rowbinary-vs-json.md @@ -0,0 +1,103 @@ +# Case study: RowBinary vs JSON on a financial ledger (wide ints & decimals) + +**TL;DR** — When every column is wider than a JS `number` can hold (`UInt128`, +`Int64`, `Decimal128(18)`, `UInt256`), RowBinary wins _twice over_. Stock +`JSON.parse` is not merely slow here — it is **silently wrong**, rounding every +value to a float64. The only correct JSON path quotes the values server-side and +re-parses each string into a `bigint`/decimal pair by hand, which is **~5x +slower** than the optimized RowBinary reader over a **2.1–2.6x larger** wire. +RowBinary reads each value exactly, straight off the wire. + +This is the workload the [SKILL's format-choice +guidance](../SKILL.md#first-is-rowbinary-even-the-right-format) calls out +explicitly: "RowBinary clearly wins when the result is dominated by **wide +numerics** — `Int128`/`Int256`/`UInt128`/`UInt256`, `Decimal128`/`Decimal256`." + +Reproduce: `npx vitest bench --run tests/ledger.bench.ts` (against a live +ClickHouse server). Source: [`tests/ledger.bench.ts`](../tests/ledger.bench.ts), +reader: [`src/examples/ledger.ts`](../src/examples/ledger.ts). + +## The data + +A financial ledger — every column exceeds IEEE-754 double's 53-bit exact range: + +```sql +txn_id UInt128 -- 16 bytes +account Int64 -- 8 bytes (values past 2^53) +amount Decimal128(18) -- 16 bytes (~32 significant digits) +balance Decimal128(18) -- 16 bytes +fee Decimal64(4) -- 8 bytes +volume UInt256 -- 32 bytes +``` + +50,000 rows, fixed-width (96 bytes/row), fetched from a live server. + +## The correctness trap + +ClickHouse emits these types as **bare, unquoted JSON numbers**. So stock +`JSON.parse` parses them as float64 and silently corrupts every one — measured +on row 0 of the live result: + +| Column | Exact value (RowBinary) | `JSON.parse` of bare JSON | | +| --------- | ----------------------------------------- | ----------------------------------------- | ---------------- | +| `txn_id` | `340282366920938463463374607431768200000` | `340282366920938463463374607431768211456` | ✗ off by 11 456 | +| `account` | `9007199254740993` | `9007199254740992` | ✗ off by 1 | +| `amount` | `98765432109876.123456789012345678` | `98765432109876.12` | ✗ lost 16 digits | + +No exception, no warning — just wrong numbers. For money and IDs, that is a +correctness bug, not a performance footnote. + +### Making JSON correct costs extra work + +The only way to get exact values through JSON is to **quote them server-side** so +they arrive as strings, then re-parse each one: + +```sql +... SETTINGS output_format_json_quote_64bit_integers = 1, + output_format_json_quote_decimals = 1 +``` + +```ts +txn_id: BigInt(r.txn_id), // string -> bigint +amount: parseDecimal(r.amount, 18), // string -> [unscaled, scale] +``` + +That per-field `BigInt(...)` / decimal parse is work RowBinary doesn't do — it +reads the exact `bigint` directly with two `DataView` reads — and it lands on +top of a larger wire (strings are longer than the binary words). + +## Wire size (correct paths quote wide values as strings) + +| Format | Size | vs RowBinary | +| --------------------------- | -------- | ------------ | +| RowBinary | 4.80 MB | 1.0x | +| JSONCompactEachRow (quoted) | 9.88 MB | 2.1x | +| JSONEachRow (quoted) | 12.28 MB | 2.6x | + +## Decode throughput (full 50k-row decode; higher = faster) + +| Decoder | ops/s | ms/decode | ≈ rows/s | speedup | correct? | +| -------------------------------------------------- | ----- | --------- | -------- | -------- | -------------- | +| **RowBinary — optimized** | 130 | 7.71 | ~6.5 M | **1.0x** | ✅ | +| RowBinary — API combinators | 80 | 12.50 | ~4.0 M | 0.62x | ✅ | +| JSONEachRow bare — `JSON.parse` only | 44 | 22.74 | ~2.2 M | 0.34x | ❌ **corrupt** | +| JSONCompactEachRow quoted — parse + BigInt/decimal | 26 | 37.78 | ~1.3 M | 0.20x | ✅ | +| JSONEachRow quoted — parse + BigInt/decimal | 25 | 40.70 | ~1.2 M | 0.19x | ✅ | + +_Node 24 / V8. Your numbers will vary; run `npm run bench` on your own hardware._ + +## Takeaways + +- **The fast JSON path is the wrong one.** Bare `JSON.parse` is JSON's quickest + option and it is still 2.95x slower than RowBinary — _and_ it silently + corrupts every wide value. There is no "fast and correct" JSON here. +- **The correct JSON path is ~5x slower.** Quote + per-field `BigInt`/decimal + parsing is the price of correctness, on top of a 2.1–2.6x larger wire. +- **RowBinary is correct by construction.** Each value is composed from 64-bit + words read at constant offsets (high word signed for the signed types), + yielding an exact `bigint` or `[unscaled, scale]` pair — no rounding, no + string re-parsing. +- **Contrast with the [IoT case study](iot-rowbinary-vs-json.md):** there the + numbers fit a float64 and the win was purely throughput (3.5x). Here the values + don't fit, so the win is _correctness first_, throughput second. Match the + format to the shape of the data. diff --git a/skills/clickhouse-js-node-rowbinary-parser/case-studies/logs-json-wins.md b/skills/clickhouse-js-node-rowbinary-parser/case-studies/logs-json-wins.md new file mode 100644 index 000000000..951d84085 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/case-studies/logs-json-wins.md @@ -0,0 +1,86 @@ +# Case study: JSON beats RowBinary on a string-heavy log table + +**TL;DR** — This is the honest counter-case. When the result is mostly **text +consumed wholesale** (an application log table), `JSONCompactEachRow` + +`JSON.parse` decodes **1.4x faster** than the optimized RowBinary reader — and +once you turn on HTTP compression, RowBinary's raw-wire size advantage +**disappears**: gzip ties the two, and with **zstd the JSON response is actually +slightly smaller**. For this shape the skill steers you _away_ from RowBinary — +and proving that is what makes its "use RowBinary here" advice (see the +[IoT](iot-rowbinary-vs-json.md) and [ledger](ledger-rowbinary-vs-json.md) +studies) trustworthy. + +This is exactly what the [SKILL's format-choice +guidance](../SKILL.md#first-is-rowbinary-even-the-right-format) says: prefer a +`JSON*` format when the result is "mostly strings / JSON-like values that you +consume wholesale," because V8's native `JSON.parse` is heavily optimized C++ +and "pair it with HTTP response compression (`gzip` / `zstd`, which crushes +JSON's repetitive keys)." + +Reproduce: `npx vitest bench --run tests/logs.bench.ts` (against a live +ClickHouse server). Source: [`tests/logs.bench.ts`](../tests/logs.bench.ts), +reader: [`src/examples/logs.ts`](../src/examples/logs.ts). + +## The data + +An application log table — four of five columns are text consumed as text: + +```sql +ts DateTime +level LowCardinality(String) -- transparent in RowBinary -> plain String +service LowCardinality(String) +message String -- templated log line, varying values +trace_id String -- high-cardinality 32-char hex +``` + +50,000 rows, fetched from a live server. The two `LowCardinality` columns carry +no dictionary on the RowBinary wire — they decode as plain `String`. + +## Decode throughput (full 50k-row decode; higher = faster) + +| Decoder | ops/s | ms/decode | ≈ rows/s | speedup | +| ------------------------------------- | ----- | --------- | -------- | -------- | +| **JSONCompactEachRow — `JSON.parse`** | 93 | 10.73 | ~4.7 M | **1.0x** | +| JSONEachRow — `JSON.parse` | 72 | 13.89 | ~3.6 M | 0.77x | +| RowBinary — optimized (monomorphized) | 66 | 15.07 | ~3.3 M | 0.71x | +| RowBinary — API combinators | 54 | 18.68 | ~2.7 M | 0.57x | + +`JSONCompactEachRow` (arrays, no repeated keys) is the fastest JSON option and +beats even the optimized RowBinary reader by ~1.4x. A RowBinary string is a +varint length + `buf.toString("utf8", …)` decoded one field at a time in JS; +`JSON.parse` builds the same JS strings in one optimized C++ pass. + +## Wire size — raw, and compressed (gzip / zstd) + +| Format | raw | gzip | zstd | +| ------------------ | ------- | ------- | ------- | +| RowBinary | 5.04 MB | 1.46 MB | 1.35 MB | +| JSONCompactEachRow | 6.84 MB | 1.51 MB | 1.32 MB | +| JSONEachRow | 8.84 MB | 1.52 MB | 1.33 MB | + +RowBinary is 1.4–1.8x smaller **raw**, which is the usual argument for it. But +that edge is mostly JSON's repeated structure (keys, punctuation) — exactly what +a compressor removes. With `gzip` the three are within ~4% of each other, and +with `zstd` the JSON responses are _slightly smaller_ than RowBinary. Any +production HTTP path should have compression on, so the wire-size case for +RowBinary on this data effectively vanishes. + +_Node 24 / V8. Your numbers will vary; run `npm run bench` on your own hardware._ + +## Takeaways + +- **JSON wins both axes here.** Faster to decode (~1.4x) _and_, once compressed, + no larger on the wire. There is no reason to hand-write a RowBinary parser for + this shape. +- **`JSONCompactEachRow` is the one to reach for** — it drops the per-row + repeated keys, so it parses faster than `JSONEachRow` and compresses about the + same. +- **Compression erases RowBinary's raw-size advantage on text.** RowBinary's + smaller raw wire comes largely from not repeating keys; a compressor already + does that for JSON. Always compare _compressed_ sizes when the data is + string-heavy. +- **This is the boundary of the skill.** RowBinary earns its keep on + numeric/wide/binary data ([IoT](iot-rowbinary-vs-json.md), + [ledger](ledger-rowbinary-vs-json.md)); on string-heavy results read as text, + the right answer is `JSONCompactEachRow` + compression. Match the format to the + shape of the data — and measure. diff --git a/skills/clickhouse-js-node-rowbinary-parser/case-studies/wasm-vs-js.md b/skills/clickhouse-js-node-rowbinary-parser/case-studies/wasm-vs-js.md new file mode 100644 index 000000000..8083c03d4 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/case-studies/wasm-vs-js.md @@ -0,0 +1,172 @@ +# Case study: why JS, not WASM, for RowBinary parsing (and the one place WASM wins) + +**TL;DR** — A JIT-compiled JS RowBinary reader already streams bytes at **memory +bandwidth** (~16 GB/s), and the dominant cost of decoding is allocating the JS +values themselves (objects, `Date`, strings, `BigInt`) — which **WASM cannot do +and therefore cannot remove**. So for the skill's actual job, turning a RowBinary +response into usable JS data, WASM buys ~nothing (a wash, or a loss after the +copy-in tax). WASM wins decisively in exactly **one** different problem: +**in-place aggregation of wide integers / decimals** (and hash group-by), where +JS is forced onto heap `BigInt`/`Map`. There we measured a hand-written WASM +kernel at **27–38x** over JS. But that is _compute_, not parsing — and it is +usually pushable to ClickHouse anyway. And if you genuinely need heavier +**client-side analytics**, the lever isn't WASM-over-RowBinary at all — it's a +**columnar wire format (`Native`)**, coming soon to the JS client out of the +Python-client collaboration; RowBinary is row-major and fights every analytical +pass. + +Reproduce: + +- `npx vitest bench --run tests/iot.wasm-headroom.bench.ts` (the parsing headroom) +- `node tests/wasm-int128.experiment.mjs` (the hand-emitted WASM kernel) + +All numbers Node 24 / V8; yours will vary. + +## The idea under test + +A tempting architecture: a _dynamic WASM JIT inside the JS runtime_. A type +builder (`t.Int32()`, `t.Map(t.FixedString, t.Int32())`) plus a query DSL +(`q.sum(q.column(1))`) compile **on the fly** to a WASM module that parses the +raw network chunk sitting at address 0 in linear memory, computes the answer, +writes it to a result region, and returns the offset where the incomplete +trailing row begins (streaming resume). Elegant. The question is _what it would +win_ — and the honest answer needs three measurements. + +## Proof 1 — JIT-compiled JS reads at memory speed + +V8 compiles `DataView` accessors to native loads. Folding a 32 MB column of +native-width values (`Float64`) in a plain JS loop: + +| Read | ms / 32 MB | throughput | +| ---------------------- | ---------- | ------------- | +| JS `DataView` f64 fold | 1.94 ms | **16.5 GB/s** | + +That is essentially RAM bandwidth. **There is no headroom for a "faster +language" to read these bytes** — JS is already at the metal. A WASM parser +reading the same bytes lands in the same place (see Proof 3, where the WASM +kernel reads at 28 GB/s doing _integer_ loads — same order, also bandwidth-bound, +not 10x). + +## Proof 2 — the parsing bottleneck is allocation, which WASM can't touch + +On the best case for RowBinary (IoT, every column fixed-width numeric), three +decoders over the same buffer (`tests/iot.wasm-headroom.bench.ts`): + +| Decode | ms | vs current | what it isolates | +| ---------------------------------------------------- | ---- | ---------- | -------------------- | +| **rows** — current fast reader (objects + `Date`) | 3.48 | 1.0x | full materialization | +| **columnar** — into typed arrays, no per-row objects | 0.86 | 4.0x | drop the objects | +| **parseOnly** — reads only, zero allocation | 0.61 | 5.8x | the pure-read floor | + +**~83% of decode time is JS-side object/`Date` allocation**, not byte reading. +A WASM parser still has to produce those JS values across the boundary, so it +_cannot_ remove that 83%. Even if WASM made the parse slice instantaneous and the +copy-in free, the row-object decode would drop only `3.48 → 2.88 ms` — a **max +~1.2x**, and realistically a wash once you add the copy into linear memory. + +The 4.0x that _is_ on the table comes from the **output contract** (columnar +typed arrays), and it's available in **plain JS** — no WASM. (That columnar path +is worth shipping; it's the real win this whole investigation surfaced.) + +## Proof 3 — the one place WASM wins: wide-int / decimal aggregation + +Summing an `Int128` column forces JS onto heap `BigInt` (one allocation per +row). A hand-emitted WASM kernel (94 bytes; native `i64` add-with-carry) does it +in registers. Same 32 MB buffer, result verified equal to the BigInt sum +(`tests/wasm-int128.experiment.mjs`): + +| Sum of an `Int128` column | ms / 32 MB | throughput | | +| --------------------------------------- | ---------- | ---------- | -------------- | +| **JS BigInt-128 sum** (what JS must do) | 42.93 ms | 0.7 GB/s | correct | +| WASM `i64` add-carry — kernel only | 1.14 ms | 28.2 GB/s | correct | +| WASM + copy-in boundary tax | 1.62 ms | 19.7 GB/s | (copy 0.49 ms) | + +**WASM is 37.8x faster than JS (26.5x including the copy into linear memory).** +Note _why_: the win is escaping `BigInt`, not reading bytes faster — the WASM +kernel (28 GB/s) is the same order as the JS f64 floor (16.5 GB/s). JS pays a +**22x `BigInt` tax** purely to add 128-bit integers; WASM's native `i64` reclaims +it. The same logic applies to `Decimal128/256` accumulation and to hash group-by +(WASM open-addressing table in linear memory vs JS `Map` + GC). + +## Verdict on the dynamic-WASM-JIT + +The architecture is **sound for the aggregation regime and only that regime**. +It targets the one quadrant where WASM beats well-written JS: _parse and compute +in place, return a small result, never cross the boundary per value._ The design +answers its own open questions well: + +- **Where does the answer go?** Scalars return directly (`i128` via multi-value + or two `i64`s); group-by results go to a reserved linear-memory region that JS + reads as a typed-array view — only the small final result crosses. +- **Streaming.** Returning the resume offset (vs throwing across the FFI) is + clean, and accumulator state lives in linear memory across chunks — the module + _is_ the streaming aggregation state. + +But three caveats bound where it's worth building: + +1. **For parsing → JS values, use generated JS, not WASM.** Proofs 1–2: JS is + already at memory speed and the cost is materialization WASM can't remove. A + `DSL → new Function(generatedJS)` backend captures the parse + native-numeric + aggregation case with **zero toolchain**, debuggable. This is the skill's + existing monomorphization thesis. +2. **Reserve a WASM backend for the wide-int/decimal + group-by kernels only** — + gate it on the presence of `Int128/256`, `Decimal128/256`, or a `GROUP BY`, + where Proof 3's 27–38x is real. For `Float64` sums it would tie JS. +3. **SIMD won't help much** — RowBinary is row-major (AoS); strided columns + defeat Wasm SIMD (no gather) without a transpose pass. The WASM win here is + native `i64` + no GC, not vectorization. +4. **The elephant: push it down.** `q.sum(col)` is `SELECT sum(col)` — ClickHouse + will beat any client. Client-side aggregation only justifies itself when you + _can't_ push down: folding a stream you already receive for another reason, + combining across queries/sources, or compute SQL can't express. + +## If you need more client-side analytical strength: reach for Native columnar + +Step back from WASM and look at _why_ the wins above are so narrow. RowBinary is +**row-major (AoS)**: every row interleaves all columns, so any analytical pass — +fold a column, vectorize, build a column-at-a-time accumulator — has to stride +over the bytes it doesn't want and re-materialize a value at a time. That is the +same row-major tax that defeats SIMD (caveat 3) and that makes the free **4x in +Proof 2 cost a transpose** today (you decode rows, _then_ pack into typed +arrays). + +So the honest answer to _"I need real client-side analytical strength"_ is **not +a smarter parser over RowBinary, and not WASM** — it is a **columnar wire +format**. ClickHouse's **`Native`** format is **column-major (SoA)**: each block +arrives as contiguous per-column runs. That flips every constraint in this study: + +- The Proof-2 columnar typed-array path stops needing a transpose — the wire + _is_ already `Float64Array`-shaped, so you `subarray`/`set` a column in one + move instead of decoding rows first. +- Vectorization becomes real: a contiguous column is exactly what `v128.load` / + SIMD (and even auto-vectorized JS) want — the gather problem disappears. +- The wide-int/decimal aggregation win (Proof 3) keeps applying, now over + contiguous input, which is the friendliest possible layout for it. + +A columnar reader is **coming to the JS client soon**, out of the **collaboration +with the Python client** (which already ships a mature `Native`/columnar path — +the format and lessons port directly). When it lands, the order of preference for +client-side analytics becomes: **push down to ClickHouse → if you can't, decode +`Native` columnar → reserve WASM for the wide-int/decimal/group-by kernel on top +of those columns.** RowBinary stays the right tool for what this skill targets — +turning a result into JS _rows/values_ — not for analytics over them. + +## Takeaways + +- **Generated JS is the right engine for the parser.** It reads at memory + bandwidth; the remaining cost is JS-value materialization that no language + swap removes. WASM for parsing is a wash-to-loss. +- **The free 4x is a columnar (typed-array) output contract — in pure JS.** Worth + capturing as a first-class option for numeric results. +- **WASM earns its complexity in one place: in-place wide-int/decimal/group-by + aggregation** (27–38x measured), where JS is trapped in `BigInt`/`Map`. And + even then, prefer pushing the aggregation to ClickHouse unless you genuinely + can't. +- **For real client-side analytical strength, the answer is columnar, not WASM.** + RowBinary is row-major and taxes every analytical pass; a `Native` (SoA) + columnar reader — coming to the JS client soon via the Python-client + collaboration — removes the transpose, unlocks SIMD, and is the natural + substrate for the aggregation kernels above. +- Matches the rest of the studies' through-line: pick the tool for the shape of + the work, and **measure** — the 94-byte WASM kernel exists precisely so this + claim isn't hand-waved. diff --git a/skills/clickhouse-js-node-rowbinary-parser/eval_result.md b/skills/clickhouse-js-node-rowbinary-parser/eval_result.md new file mode 100644 index 000000000..76a32c1e2 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/eval_result.md @@ -0,0 +1,105 @@ +# Eval result — RowBinary skill (24 evals, with-skill vs no-skill) + +**Date:** 2026-06-20 +**Model:** Claude Opus 4.8 (1M context) — `claude-opus-4-8[1m]` (executors and grader) +**Harness:** Claude Code 2.1.183 +**Method:** skill-creator eval loop. For each of the 24 evals in +[`evals/evals.json`](evals/evals.json), an isolated subagent generated a parser +**with the skill** (told to read `SKILL.md` + the routed `src/` reader) and a +**no-skill control** (own knowledge, no access to the skill). Each output was +graded against that eval's assertions. 1 run per cell; graded by an LLM grader +against the assertions (no live-ClickHouse ground truth — that is what +[`skill-bench`](.claude/skills/skill-bench/SKILL.md) adds). + +## Headline + +| Metric | With skill | Without skill | Delta | +| ------------- | ------------- | ------------- | -------------- | +| Pass rate | **94.7%** | 71.5% | **+23pp** | +| Wall time | 90.4s ± 36.7s | 58.0s ± 21.4s | +32.4s (~1.6×) | +| Output tokens | 32066 ± 7096 | 17483 ± 2028 | +14583 (~1.8×) | + +The skill raised correctness on every knowledge-heavy type and never _lowered_ +it on the hard ones. The cost is reading `SKILL.md` + the routed reader and +emitting extra variants/self-tests. + +## Per-eval pass rate + +| Eval | With | Without | Δ | +| ----------------------------------------- | -------- | -------- | --------- | +| 0 fixed-width numerics (Buffer) | 1.00 | 0.40 | +0.60 | +| 1 DateTime64(3)/Float32 endianness | 1.00 | 0.50 | +0.50 | +| 2 varint length reader | 1.00 | 0.40 | +0.60 | +| 3 Int64/Int128 precision | 1.00 | 0.60 | +0.40 | +| 4 Buffer slice / DataView windowing | 1.00 | 0.80 | +0.20 | +| 5 Decimal64/IPv4 format separation | 1.00 | 0.67 | +0.33 | +| 6 UUID byte-order | 1.00 | 1.00 | 0 | +| 7 FixedString / binary String | 1.00 | 1.00 | 0 | +| 8 BFloat16 array | 1.00 | 1.00 | 0 | +| 9 Enum8 underlying int | 1.00 | 0.83 | +0.17 | +| 10 Date/DateTime tz metadata | 0.80 | 0.80 | 0 | +| 11 DateTime64(9) nanoseconds | 0.60 | 0.40 | +0.20 | +| 12 Time/Time64 durations | 1.00 | 0.60 | +0.40 | +| 13 LowCardinality/SimpleAggregateFunction | 1.00 | 1.00 | 0 | +| 14 Variant discriminant name-sort | 1.00 | 1.00 | 0 | +| 15 Nested = Array(Tuple) | 1.00 | 1.00 | 0 | +| 16 AggregateFunction opaque state | **1.00** | **0.00** | **+1.00** | +| 17 Dynamic runtime dispatch | 1.00 | 1.00 | 0 | +| 18 Dynamic nested type-headers | 1.00 | 0.67 | +0.33 | +| 19 JSON = paths + Dynamic | 1.00 | 0.33 | +0.67 | +| 20 hot UUID/IPv6/Array zero-copy | 0.83 | 0.83 | 0 | +| 21 Float32 array benchmark | **0.67** | **0.83** | **−0.17** | +| 22 documented String/Int64 toggles | 1.00 | 0.83 | +0.17 | +| 23 Array(Tuple) monomorphized | 0.83 | 0.67 | +0.17 | + +## Where the skill clearly earns its keep + +Correctness gaps the model gets wrong unaided: + +- **eval-16 AggregateFunction opaque state — 100% vs 0%.** Without the skill the + agent invents a byte-level decoder and claims the state is splittable / + round-trippable. The skill correctly refuses and points to server-side + finalization. +- **eval-19 JSON — 100% vs 33%.** Without the skill the agent falls back to + `CAST(col AS String)` + `JSON.parse`; only the skill decodes the + varuint-path-count + (String path, Dynamic value) wire and handles the + JSON-in-Dynamic `0x30` header and typed-path bail-out. +- **eval-18 Dynamic nested type-headers — 100% vs 67%.** The control invents + wrong type-encoding tag bytes and never consumes `max_dynamic_types`. +- **evals 0/2 (DataView windowing, varint unrolling) — 100% vs 40%**, plus + endianness scaffolding (1), Time64 ScaledTicks (12), signed Int128 high word + (3), Decimal scale preservation (5). + +## Genuine gaps the eval surfaced (candidate skill fixes) + +1. **Skill regressed on eval-21 (float32 benchmark): 67% vs 83%.** Both configs + emitted `.mjs` not TypeScript (TS-default assertion failed in _both_), and the + no-skill control added an independent source-byte oracle while the with-skill + run only cross-validated the two strategies against each other. The skill + teaches equivalence-before-timing but not an _independent_ oracle. +2. **TypeScript-default is unreliable on optimization/benchmark prompts** — + with-skill emitted plain `.mjs` on evals 21 and 23 despite the "TS by default" + assertion (eval-0 did produce `.ts`). +3. **Holey-array rule misfired on the one eval that targets it (eval-20, 5/6 + both):** both used `new Array(n)` + index for a "millions of rows" hot tag + array; the skill's small-vs-large `[]`+push heuristic didn't fire and the run + even justified `new Array(n)`. The guidance is ambiguous when an array is both + count-known _and_ large. +4. **Weakest with the skill: eval-10 (0.80) and eval-11 (0.60)** — the skill + version omitted the "one `Date` allocation per value, offer raw-count on a hot + path" note (10) and only partially delivered the `[Date, nanoseconds]` split + (11). + +## Caveats + +- Non-discriminating evals (6, 7, 8, 13, 14, 15, 17 all tie ≈100%) measure + baseline model competence, not skill lift. On 14/17 the no-skill run even got + the subtle name-sort / runtime-dispatch right; the assertions don't test the + concrete Dynamic tag bytes, which is where no-skill was actually shaky. +- 1 run per (eval, config); per-eval deltas are point estimates, not + variance-controlled. For server-truth correctness (decode vs what the live + ClickHouse server produced) use [`skill-bench`](.claude/skills/skill-bench/SKILL.md). + +_Raw per-assertion gradings, the benchmark JSON, and the interactive review +viewer live in the sibling `…-workspace/iteration-1/` directory +(`benchmark.json`, `benchmark.md`, `review.html`)._ diff --git a/skills/clickhouse-js-node-rowbinary-parser/eval_result_composer.md b/skills/clickhouse-js-node-rowbinary-parser/eval_result_composer.md new file mode 100644 index 000000000..e5a0d81fa --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/eval_result_composer.md @@ -0,0 +1,73 @@ +# Eval result (Composer) — RowBinary skill (skill-bench orders, with-skill vs no-skill) + +**Date:** 2026-06-21 +**Model:** Composer 2.5 Fast — `composer-2.5-fast` (executors). No separate grader — correctness scored against live ClickHouse server output (same fixture procedure as [`skill-bench`](.claude/skills/skill-bench/SKILL.md)). +**Harness:** Cursor Task subagents (isolated work dirs under `/tmp/rb-skill-bench/orders/`) +**Method:** The [`skill-bench` orders contract](.claude/skills/skill-bench/SKILL.md) — one fixed schema (`UInt8`, `UUID`, `Decimal64(2)`, `Enum8`) — run as a 2×1 matrix (Composer with-skill vs no-skill). Fixture bytes and expected fields built from a live ClickHouse server (64 rows); throughput measured on a 64×400 = 25,600-row concatenated buffer (equivalence-checked, best of 5 rounds). 1 run per cell. This is **server-truth** scoring, not the 24 LLM-graded evals in [`evals/evals.json`](evals/evals.json) (see [Sonnet](eval_result_sonnet.md) / [Opus](eval_result.md) for those). + +## Headline + +| Metric | With skill | Without skill | Delta | +| ------------------------- | ---------------- | ------------- | --------- | +| Correctness (64 rows) | **100%** | **100%** | 0 | +| Generated-code throughput | **14.0M rows/s** | 5.8M rows/s | **2.40×** | +| Parser size | 99 lines | 49 lines | +50 lines | +| Agent wall time | N/A | N/A | — | +| Output tokens | N/A | N/A | — | + +**Both cells decoded every field correctly** against what ClickHouse itself produced — including the three classic traps this schema is chosen for (UUID byte order, Decimal64 fixed scale, Enum8 underlying int). The skill's lift on Composer is **throughput and optimization patterns**, not correctness on this run: the no-skill parser got UUID/decimal/enum right without peeking at the skill. + +## With-skill vs without-skill by trap (orders schema) + +The orders skill-bench task exercises the same gotchas several evals target individually: + +| Trap | Eval analogue | With skill | Without skill | Δ | +| ----------------------------------------------------------- | ----------------- | ---------------- | ------------- | -------------------- | +| UUID two-LE-halves byte order | eval-6 | ✓ 64/64 | ✓ 64/64 | tie | +| Decimal64(2) → exactly 2 fractional digits | eval-5 | ✓ 64/64 | ✓ 64/64 | tie | +| Enum8 → underlying int (1/2/3), not name | eval-9 | ✓ 64/64 | ✓ 64/64 | tie | +| Hot-path codegen (lookup table, bigint decimal, row stride) | eval-20 / eval-23 | **14.0M rows/s** | 5.8M rows/s | **+140% throughput** | + +## Composer vs Sonnet (skill-bench orders, same fixture shape) + +Cross-run comparison uses the same orders contract and scoring rules; Sonnet numbers are from the prior claude-CLI skill-bench cell in [`.claude/skills/skill-bench/results/orders/`](.claude/skills/skill-bench/results/orders/). + +| | With skill | Without skill | Skill throughput lift | +| --------------------------------------- | -------------- | ------------- | --------------------- | +| **Composer 2.5 Fast** | ✓ 14.0M rows/s | ✓ 5.8M rows/s | **2.40×** | +| **Sonnet 4.6** (no-skill only recorded) | — | ✓ 8.0M rows/s | — | + +Composer-no-skill is **slower** than Sonnet-no-skill on this one run (5.8M vs 8.0M rows/s) despite equal correctness — the Sonnet baseline used a tighter hand-rolled loop, while Composer-no-skill used `readUInt32LE`/`readInt32LE` for decimal and string-built UUID hex. Composer-with-skill more than closes that gap and beats Sonnet-no-skill by **1.76×** on generated-code speed. + +## Where the skill clearly earned its keep (Composer) + +Correctness gaps the skill closes on weaker models (see [Sonnet eval-6 at 0.20](eval_result_sonnet.md)) did **not** appear here — Composer-unaided passed server truth. What the skill _did_ deliver: + +- **`formatUUIDTable` lookup-table path** — adapted from the orders example (`src/examples/orders.ts` / skill UUID guidance) instead of per-byte string concatenation in the no-skill cell. +- **Bigint + `DataView.getBigInt64` decimal path** — faithful signed Int64 unscaled units with scale-2 padding; no-skill used JS number arithmetic on 32-bit limbs. +- **Flattened 26-byte fixed row** — single stride (`1 + 16 + 8 + 1`), pre-sized `new Array(rowCount)`, column comments — the "flatten the assembled row reader" tier from `SKILL.md`. + +**Isolation audit (no-skill):** clean — hand-rolled `formatUUID` with per-half byte reversal, no `formatUUIDTable` / `UUID_HEX16` / skill module names; verified the agent did not read paths under `skills/clickhouse-js-node-rowbinary-parser-generator`. + +## Findings specific to this Composer run + +1. **Composer-unaided correctness on orders is strong.** One run, but both UUID and decimal formatting matched ClickHouse `toString()` output — unlike Sonnet-no-skill on eval-6 (0.20 pass rate across the 24-eval suite). Skill-bench still recommends multiple no-skill trials before claiming stability; this run is a point estimate only. +2. **Skill value here is performance, not rescue.** The 2.40× throughput gap is the headline; the skill cell is also ~2× the line count because it inlines the optimized UUID table and bigint formatters. +3. **Agent cost not measured.** Cursor Task subagents do not emit the `stream-json` transcript the claude-CLI skill-bench procedure uses for turns/tokens/USD — only generated-code metrics are reported. + +## Findings that align with the Opus / Sonnet eval runs + +These skill-bench observations are consistent with themes from the 24-eval A/B runs, even though Composer did not execute that suite: + +- **Gotcha types are where no-skill breaks on weaker models** — Composer passed this small schema; Sonnet's 24-eval no-skill pass rate was 60.4% with UUID/JSON/AggregateFunction as the big holes. +- **Optimization tier is not automatic without the skill** — no-skill Composer reached for readable `Buffer.read*LE` helpers; with-skill reached for the benchmarked hot path. Same pattern as eval-23 (monomorphized / inlined) and eval-20 (zero-copy / packed arrays). +- **The orders schema is a weak correctness discriminator for strong models** — both Composer cells correct; the schema discriminates **code quality and speed** instead (as [`skill-bench` expects](.claude/skills/skill-bench/SKILL.md) for Sonnet-no-skill UUID flakiness, not for every model). + +## Caveats + +- **Server-truth scoring on one schema**, not the 24 LLM-graded evals — for assertion-level coverage across all type families, see [eval_result_sonnet.md](eval_result_sonnet.md) and [eval_result.md](eval_result.md). +- **1 run per (cell)** — per-cell results are point estimates; Sonnet-no-skill UUID failure is non-deterministic across runs. +- **Agent cost metrics unavailable** in the Cursor Task harness (wall time, turns, tokens, USD all N/A). +- **Cross-model throughput comparison is provisional** — Sonnet-no-skill is from a different fixture build (fresh INSERT each run); row bytes differ, but schema and scoring rules match. + +_Raw parsers, fixture, and machine-readable scores: [`.claude/skills/skill-bench/results/orders/`](.claude/skills/skill-bench/results/orders/) (`composer-{noskill,skill}.parser.mjs`, `fixture.json`, `results.json`, `report.md`). Work dirs: `/tmp/rb-skill-bench/orders/composer-{noskill,skill}/`._ diff --git a/skills/clickhouse-js-node-rowbinary-parser/eval_result_haiku.md b/skills/clickhouse-js-node-rowbinary-parser/eval_result_haiku.md new file mode 100644 index 000000000..02fcc806f --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/eval_result_haiku.md @@ -0,0 +1,80 @@ +# Eval result (Haiku) — RowBinary skill (26 evals, with-skill vs no-skill) + +**Date:** 2026-06-22 +**Model:** Claude Haiku 4.5 — `claude-haiku-4-5` (executors). Grader: Claude Opus 4.8 — `claude-opus-4-8[1m]` (held constant as the measurement instrument, same as the [Opus](eval_result.md) and [Sonnet](eval_result_sonnet.md) runs). +**Harness:** Claude Code Workflow (background `pipeline`) — a 52-cell generate→grade pipeline (26 evals × {with-skill, no-skill}); 104 subagents total, ~1.94M subagent tokens, ~296s wall. +**Method:** same A/B as the Opus/Sonnet runs — for each eval in [`evals/evals.json`](evals/evals.json) an isolated **Haiku** subagent generated a parser **with the skill** and a **no-skill control** (instructed not to read repo files), scored by an **Opus** grader against the eval's assertions (1 = met, 0.5 = partial, 0 = miss/violation; overall = mean). 1 run per cell; no live-ClickHouse ground truth (use [`skill-bench`](.claude/skills/skill-bench/SKILL.md) for server truth). This run covers **26** evals — the original 24 plus the two columnar streaming cases (24, 25) added after the Opus/Sonnet runs; a 0–23 subset is reported below for apples-to-apples cross-model comparison. + +## Headline + +| Metric | With skill | Without skill | Delta | +| ----------------------------- | ---------- | ------------- | --------- | +| Pass rate (26 evals) | **86.2%** | 53.2% | **+33pp** | +| Pass rate (0–23 subset) | **86.0%** | 52.2% | **+34pp** | +| Wall time / tokens (per cell) | N/A | N/A | — | + +**On Haiku the skill is worth +33pp** — it lifts a weak 53% unaided baseline to 86%. The delta matches Sonnet's (+34pp) and is larger than Opus's (+23pp), for the same reason the Sonnet run gave: the skill mostly closes the _baseline_ gap. But unlike Sonnet, **Haiku-with-skill does not reach the Opus/Sonnet with-skill ceiling** (86% vs ~94%) — Haiku sometimes can't faithfully _apply_ the skill on the hardest cases (it emits prose instead of code on eval-18, and silently drops the skill's signature no-bigint trick on eval-25). Per-cell wall/token are N/A: the Workflow harness reports run-level totals only, not the per-cell `stream-json` the Sonnet run used. + +## With-skill vs without-skill by eval (Haiku) + +| Eval | With | Without | Δ | +| ------------------------------------------------- | -------- | -------- | --------- | +| 0 fixed-width numerics (Buffer) | 0.80 | 0.60 | +0.20 | +| 1 DateTime64(3)/Float32 endianness | 1.00 | 1.00 | 0 | +| 2 varint length reader | **1.00** | **0.00** | **+1.00** | +| 3 Int64/Int128 precision | 1.00 | 1.00 | 0 | +| 4 Buffer slice / DataView windowing | 1.00 | 0.40 | +0.60 | +| 5 Decimal64/IPv4 format separation | 0.92 | 0.25 | +0.67 | +| 6 UUID byte-order | 1.00 | 0.90 | +0.10 | +| 7 FixedString / binary String | 0.83 | 0.00 | +0.83 | +| 8 BFloat16 array | 1.00 | 0.90 | +0.10 | +| 9 Enum8 underlying int | 0.92 | 0.75 | +0.17 | +| 10 Date/DateTime tz metadata | 0.80 | 0.50 | +0.30 | +| 11 DateTime64(9) nanoseconds | 0.80 | 0.20 | +0.60 | +| 12 Time/Time64 durations | 0.90 | 0.00 | +0.90 | +| 13 LowCardinality/SimpleAggregateFunction | 1.00 | 0.58 | +0.42 | +| 14 Variant discriminant name-sort | 1.00 | 0.70 | +0.30 | +| 15 Nested = Array(Tuple) | 0.83 | 0.50 | +0.33 | +| 16 AggregateFunction opaque state | **1.00** | **0.00** | **+1.00** | +| 17 Dynamic runtime dispatch | 1.00 | 1.00 | 0 | +| 18 Dynamic nested type-headers | **0.25** | **0.42** | **−0.17** | +| 19 JSON = paths + Dynamic | 0.67 | 0.08 | +0.58 | +| 20 hot UUID/IPv6/Array zero-copy | 0.67 | 0.67 | 0 | +| 21 Float32 array benchmark | **0.33** | **0.83** | **−0.50** | +| 22 documented String/Int64 toggles | 0.92 | 0.58 | +0.33 | +| 23 Array(Tuple) monomorphized | 1.00 | 0.67 | +0.33 | +| 24 streaming columnar (sensor, no per-row bigint) | 0.93 | 0.71 | +0.21 | +| 25 streaming columnar (trades, 2×64-bit) | 0.86 | 0.58 | +0.27 | + +## Haiku vs Opus vs Sonnet (grader = Opus 4.8, 0–23 subset) + +| Executors | With skill | Without skill | Delta | +| -------------- | ---------- | ------------- | ----- | +| **Opus 4.8** | 94.7% | 71.5% | +23pp | +| **Sonnet 4.6** | 94.0% | 60.4% | +34pp | +| **Haiku 4.5** | 86.0% | 52.2% | +34pp | + +Haiku's _unaided_ baseline is the weakest of the three (52.2%), and the skill rescues it by the same magnitude it rescues Sonnet — but the **with-skill ceiling is ~8pp below** Opus/Sonnet. Two things hold Haiku-with-skill back that don't hold the bigger models back: + +- **eval-18 Dynamic nested type-headers — 0.25, a regression below its own no-skill 0.42.** With the skill, Haiku produced _prose describing_ the header layout but almost no decoding code; the no-skill cell at least emitted (wrong) code that scored partial. Haiku couldn't operationalize the skill's most complex section. +- **eval-25 columnar trades — with-skill **violated** the headline no-per-row-bigint expectation.** Haiku read both 64-bit columns with `getBigInt64`/`getBigUint64` per row instead of the two-`getUint32`-words-into-a-`Uint32Array`-view trick — exactly the manual finding from the case-25 spot-check. It adapted the schema (stride 29, `BigUint64Array`, offsets) correctly but dropped the optimization the moment it couldn't copy it verbatim. + +## Findings that reproduce across ALL THREE models (highest-priority skill fixes) + +1. **eval-21 (float32 benchmark) regression — Haiku 0.33 vs 0.83 no-skill.** Identical shape to Sonnet (0.33 vs 0.83) and Opus (0.67 vs 0.83): the with-skill run times the two decoders without an equivalence check, no disqualification statement, no runnable test. **This defect now reproduces on every model tested** — the skill teaches equivalence-before-timing but doesn't make the agent actually wire up the guard. Clearest, most reproducible skill defect. +2. **eval-10 / eval-11 are among the weakest with the skill (0.80 / 0.80).** Same gaps as Opus/Sonnet: the per-`Date` allocation note is missing (10), and the `[Date, nanoseconds]` split / `Nanoseconds` alias / P3-vs-P9 note is only partial (11). + +## Findings specific to the Haiku run + +1. **No-skill _refusals_, not just wrong answers.** eval-7 (FixedString/binary String) and eval-12 (Time/Time64) scored **0.00 no-skill because Haiku refused / emitted no code**; eval-12 also asserted the types are unsigned. The skill turns both into 0.83 / 0.90. Weaker models don't just guess wrong unaided — they sometimes don't attempt the decoder at all. +2. **The skill's signature optimization doesn't fully transfer to Haiku.** On the two new columnar evals the no-bigint word-copy trick is **skill-exclusive** (no-skill Haiku used `readBigInt64LE` per row on both 24 and 25), but even _with_ the skill Haiku reproduced it only on the matching schema (eval-24, raw ticks kept) and dropped it on the novel two-64-bit schema (eval-25). This is the inverse of a strong model: Opus/Sonnet-with-skill carried the trick to the new schema; Haiku needs the example to match. +3. **Faithful where the skill is concrete and copyable.** eval-2 (varint), eval-16 (AggregateFunction = don't decode), eval-13 (transparent wrappers), eval-23 (monomorphized Array(Tuple)) all hit 1.00 with skill from 0.00–0.67 without — Haiku reliably reproduces well-scoped, single-pattern guidance. + +## Caveats + +- LLM-graded against assertions; no live-server ground truth (same as the Opus/Sonnet runs). +- 1 run per (eval, config) — per-eval deltas are point estimates; Haiku's no-skill refusals (7, 12) may not reproduce every run. +- Per-cell wall-time/token metrics unavailable in the Workflow harness (run-level only: 104 agents, ~1.94M subagent tokens, ~296s wall). +- No-skill isolation relied on a prompt instruction not to read repo files (not sandbox-enforced); a peeking agent could have seen `evals.json`. Same method caveat as the Sonnet run. +- Cross-model headline uses the **0–23 subset** so it matches the 24-eval Opus/Sonnet runs; the 26-eval figure (86.2% / 53.2%) includes the two columnar cases. +- Non-discriminating evals on Haiku (tie): 1, 3, 17 (tie at 100%), 20 (tie at 0.67) — and two with-skill **regressions** (18, 21), more than Sonnet had. diff --git a/skills/clickhouse-js-node-rowbinary-parser/eval_result_sonnet.md b/skills/clickhouse-js-node-rowbinary-parser/eval_result_sonnet.md new file mode 100644 index 000000000..6b1c7e39f --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/eval_result_sonnet.md @@ -0,0 +1,99 @@ +# Eval result (Sonnet) — RowBinary skill (24 evals, with-skill vs no-skill) + +**Date:** 2026-06-20 +**Model:** Claude Sonnet 4.6 — `claude-sonnet-4-6` (executors). Grader: Claude Opus 4.8 — `claude-opus-4-8[1m]` (held constant as the measurement instrument, same as the [Opus run](eval_result.md)) +**Harness:** Claude Code 2.1.183 +**Method:** identical to the [Opus run](eval_result.md) — for each of the 24 evals in +[`evals/evals.json`](evals/evals.json), an isolated Sonnet subagent generated a parser +**with the skill** and a **no-skill control**, scored by an Opus grader against the +eval's assertions. 1 run per cell; no live-ClickHouse ground truth (use +[`skill-bench`](.claude/skills/skill-bench/SKILL.md) for server truth). + +## Headline + +| Metric | With skill | Without skill | Delta | +| ------------- | ------------- | ------------- | --------------- | +| Pass rate | **94.0%** | 60.4% | **+34pp** | +| Wall time | 93.3s ± 26.9s | 73.4s ± 24.8s | +19.9s (~1.27×) | +| Output tokens | 30529 ± 5335 | 18416 ± 1987 | +12113 (~1.66×) | + +**The skill delta is larger on Sonnet (+34pp) than on Opus (+23pp)** — not because +with-skill is better (94.0% vs Opus's 94.7%, essentially tied), but because Sonnet's +_unaided_ baseline is weaker (60.4% vs Opus's 71.5%). In other words, the skill brings +Sonnet up to roughly the same place Opus-with-skill reaches, closing most of the +model-capability gap. + +## With-skill vs without-skill by eval (Sonnet) + +| Eval | With | Without | Δ | +| ----------------------------------------- | -------- | -------- | --------- | +| 0 fixed-width numerics (Buffer) | 1.00 | 0.80 | +0.20 | +| 1 DateTime64(3)/Float32 endianness | 1.00 | 0.25 | +0.75 | +| 2 varint length reader | 1.00 | 0.40 | +0.60 | +| 3 Int64/Int128 precision | 1.00 | 0.60 | +0.40 | +| 4 Buffer slice / DataView windowing | 1.00 | 0.80 | +0.20 | +| 5 Decimal64/IPv4 format separation | 1.00 | 0.50 | +0.50 | +| 6 UUID byte-order | 1.00 | 0.20 | +0.80 | +| 7 FixedString / binary String | 1.00 | 1.00 | 0 | +| 8 BFloat16 array | 1.00 | 1.00 | 0 | +| 9 Enum8 underlying int | 0.83 | 0.67 | +0.16 | +| 10 Date/DateTime tz metadata | 0.80 | 0.80 | 0 | +| 11 DateTime64(9) nanoseconds | 0.60 | 0.40 | +0.20 | +| 12 Time/Time64 durations | 1.00 | 0.40 | +0.60 | +| 13 LowCardinality/SimpleAggregateFunction | 1.00 | 1.00 | 0 | +| 14 Variant discriminant name-sort | 1.00 | 1.00 | 0 | +| 15 Nested = Array(Tuple) | 1.00 | 1.00 | 0 | +| 16 AggregateFunction opaque state | **1.00** | **0.00** | **+1.00** | +| 17 Dynamic runtime dispatch | 1.00 | 0.67 | +0.33 | +| 18 Dynamic nested type-headers | 1.00 | 0.50 | +0.50 | +| 19 JSON = paths + Dynamic | **1.00** | **0.00** | **+1.00** | +| 20 hot UUID/IPv6/Array zero-copy | 1.00 | 0.50 | +0.50 | +| 21 Float32 array benchmark | **0.33** | **0.83** | **−0.50** | +| 22 documented String/Int64 toggles | 1.00 | 0.67 | +0.33 | +| 23 Array(Tuple) monomorphized | 1.00 | 0.50 | +0.50 | + +## Sonnet vs Opus (both grader = Opus 4.8) + +| | With skill | Without skill | Delta | +| ------------------------ | ---------- | ------------- | ----- | +| **Opus 4.8** executors | 94.7% | 71.5% | +23pp | +| **Sonnet 4.6** executors | 94.0% | 60.4% | +34pp | + +Where Sonnet-unaided falls down harder than Opus-unaided (and the skill rescues it): + +- **eval-6 UUID — 0.20 vs Opus-noskill 1.00.** Sonnet misdiagnoses the layout as + big-endian and hexes bytes in wire order — exactly the scrambling the prompt describes. +- **eval-19 JSON — 0.00 vs Opus 0.33.** Sonnet insists the column is plain UTF-8 JSON text. +- **eval-16 AggregateFunction — 0.00.** Invents a LEB128 length prefix for the unframed state. +- **eval-1 endianness — 0.25**, **eval-12 Time — 0.40**, **eval-5 Decimal — 0.50**, + **eval-17/18 Dynamic — 0.67/0.50.** All lifted to 1.00 with the skill. + +## Findings that reproduce across BOTH models (highest-priority skill fixes) + +1. **eval-21 (float32 benchmark) regression — and worse on Sonnet: 0.33 vs 0.83 + (Opus: 0.67 vs 0.83).** Same root cause both times: the with-skill run omits the + equivalence guard, timing the two decoders without ever comparing their outputs. + The skill teaches equivalence-before-timing but not an _independent_ correctness + oracle. This is the clearest, most reproducible skill defect. +2. **eval-10 / eval-11 are the weakest with the skill on both models** (Date-allocation + note missing; `[Date, nanoseconds]` split only partial, no `Nanoseconds` alias / P3-vs-P9 note). + +## Findings that differ from the Opus run + +- **Holey-array rule (eval-20):** on Sonnet the with-skill run correctly used `[]`+push + (1.00) while no-skill used `new Array(n)` (0.50) — here the skill _helped_. On Opus both + used `new Array(n)` and tied at 0.83. The rule is followed inconsistently across + models; tightening the large-vs-count-known guidance would make it reliable. +- **TypeScript-default:** Sonnet-with-skill followed it better than Opus-with-skill + (emitted `.ts` on the optimization evals 21/23), so the "TS by default" gap is + more an Opus-with-skill issue. + +## Caveats + +- LLM-graded against assertions; no live-server ground truth. +- 1 run per (eval, config) — per-eval deltas are point estimates. +- Non-discriminating evals on Sonnet (tie at 100%): 7, 8, 13, 14, 15 — fewer than the + Opus run, i.e. the eval set discriminates skill value more sharply at Sonnet's level. + +_Raw gradings, `benchmark.json`, and the interactive `review.html` (with the Opus run as +the "previous" comparison) live in the sibling `…-workspace/iteration-2/` directory._ diff --git a/skills/clickhouse-js-node-rowbinary-parser/package-lock.json b/skills/clickhouse-js-node-rowbinary-parser/package-lock.json new file mode 100644 index 000000000..334a4d8d8 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/package-lock.json @@ -0,0 +1,1328 @@ +{ + "name": "@clickhouse/rowbinary", + "version": "0.1.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@clickhouse/rowbinary", + "version": "0.1.1", + "license": "Apache-2.0", + "devDependencies": { + "@types/node": "^26.0.0", + "typescript": "^6.0.3", + "vitest": "^4.1.9" + } + }, + "node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.5.tgz", + "integrity": "sha512-AWPoBRJ9tsnVhor4sjO7rkni+7p+2IAEFj6cx06UgP10jkQHqay/36uRV/bFkgrh18D9vb4cr8Q0Pthskgzy+Q==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.133.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.133.0.tgz", + "integrity": "sha512-KzkdCd6Uxqnf6l3HOw1xfatAlUURA0g14cvBYFyJ5SaNOQbOUvBr9PKArcPcrNIeRsBdgcUzOGrhKveVpvOIGA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.3.tgz", + "integrity": "sha512-454rs7jHngixp/NMxd5srYD57OnzSlZ/eFTETjORQHLwJG1lRtmNOJcBerZlfu4GjKqeq8aCCIQrMdHyhI51Hw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.3.tgz", + "integrity": "sha512-PcAhP+ynjURNyy8SKGl5DQP94aGuB/7JrXJb/t7P+hanXvQVMWzUvRRhBAcg/lNRadBhoUPqSoP4xw5tR/KBEA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.3.tgz", + "integrity": "sha512-9YpfeUvSE2RS7wysJ81uOZkXJz7f7Q55H2Gvp3VEw/EsahqDtrphrZ0EwDLK5vvKOzaCrBsjF8JmnMLcUt78Gg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.3.tgz", + "integrity": "sha512-yB1IlAsSNHncV6SCTL27/MVGR5htvQsoGxIv5KMGXALp+Ll1wYsn+x98M9MW7qa+NdSbvrrY7ANI4wLJ0n1e6g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.3.tgz", + "integrity": "sha512-Yi30IVAAfLUCy2MseFjbB1jAMDl1VMCAas5StnYp8da9+CKvMd2H2cbEjWcw5NPaPqzvYkVIaF1nNUG+b7u/sw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.3.tgz", + "integrity": "sha512-jsO7R8To+AdlYgUmN5sHSCZbfhtMBkO0WUx8iORQnPcMMdgr7qM2DQmMwgabs3GhNztdmoKkMKQFHD6DTMCIQw==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.3.tgz", + "integrity": "sha512-VWkUHwWriDciit80wleYwKILoR/KMvxh/IdwS/paX+ZgpuRpCrKLUdadJbc0NpBEiyhpYawsJ73j9aCvOH+f7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.3.tgz", + "integrity": "sha512-5f1laC0SlIR0yDbFCd8acUhvJIag6N3zC5P7oUPN6wX0aOma+uKJ0wBDH5aq7I1PVI2ttTlhJwzwRIBnLiSGEg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.3.tgz", + "integrity": "sha512-Iq4ko0r4XsgbrF/LunNgHtAGLRRVE2kXonAXQ/MV0mC6jQpMOhW1SvtZja2EhC/kd05++bP78dsqBeIQyYJ6Yg==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.3.tgz", + "integrity": "sha512-B8m6tD5+/N5FeNQFbKlLA/2yVq9ycQP1SeedyEYYKWBNR3ZQbkvIUcNnDNM03lO1l5F2roiiFJGgvoLLyZXtSg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.3.tgz", + "integrity": "sha512-pSdpdUJHkuCxun9LE7jvgUB9qsRgaiyNNCX7m/AvHTcq67AiT/Yhoxvw5zPfhrM8k/BfP8ce/hMOpthKDpEUow==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.3.tgz", + "integrity": "sha512-OXXS3RKJgX2uLwM+gYyuH5omcH8fL1LJs96pZGgtetVCahON57+d4SJHzTgZiOjxgGkSnpXpOsWuPDGAKAigEg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.3.tgz", + "integrity": "sha512-JTtb8BWFynicNSoPrehsCzBtOKjZ6jhMiPFEmOiuXg1Fl8dn2KHQob+GuPSGR0dryQa1PQJbzjF3dqO/whhjLg==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.10.0", + "@emnapi/runtime": "1.10.0", + "@napi-rs/wasm-runtime": "^1.1.4" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.3.tgz", + "integrity": "sha512-gEdFFEN70A/jxb2svrWsN3aDL7OUtmvlOy+6fa2jxG8K0wQ1ZbdeLGnidov6Yu5/733dI5ySfzFlQ/cb0bSz1g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.3.tgz", + "integrity": "sha512-eXB7CHuaQdqmJcc3koCNtNPmT/bj2gc999kUFgBxG8Ac0NdgXc4rkCHhqrgrhN3zddvvvrgzj1e90SuSfmyIXA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", + "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "26.0.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.0.0.tgz", + "integrity": "sha512-vf2YFi1iY9lHGwNJMs01biZFbKJkrZR1T6/MlzjhJLPdntOHLhTrDSnSVcdtvjihi4VQNlrFRIxLsDBlQpAipA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/@vitest/expect": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.9.tgz", + "integrity": "sha512-vl/rYsUKcBr3SnQn166+XR5ZQcgMx3DQhFWdfli/cWpLnLUmbxZvyrJZotLFUryib+LtArYMSTJ5RbQ57ZqrlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.9", + "@vitest/utils": "4.1.9", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.9.tgz", + "integrity": "sha512-EVkXzBjrPGM+cK8/ANWgBrkUCfJfb38/EfTSO8h7pWvKkyPkpWxvR7BkD2MyItMF62C97zAEoqdpUixwR/e+Rw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.9", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.9.tgz", + "integrity": "sha512-s0iufns3iIFitdgm+YR7g1whCAaGtXz459VS9/PqyKDEEFgYIhsHOQmXgIgDuYCt7DeQmiZT0Qe2OA2p4ZPu5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.9.tgz", + "integrity": "sha512-KXLMDtc7oe70+3mJfGrPUWPesswH+3sTxAMAMl8DG7I8IUQT4XW718dY5ID3vPUcmlu27CcKfY4P3h3I29SLJg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.1.9", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.9.tgz", + "integrity": "sha512-Jc7RKGNBo8Z28WYIm0Niej4xdSPByRf6mU58VpHQkd6Zh05rlnA+twjbK5HyeIGHxrzsc3mJgS43uM0CZKzaIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.9", + "@vitest/utils": "4.1.9", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.9.tgz", + "integrity": "sha512-fHpsS6mIi+PiEW+vcRVOMkX1oSaPKne3VOclSFICPcGOmfKgXPU5iAah+wcNcj2xPrCCmfq99IDGf+EojhhvhA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.9.tgz", + "integrity": "sha512-A51o8ymO5PpqlWNnBP9ZHPXDIpuMtTLlGSjN7la4US+LJzoUMyhwjA5QXlm39JexgwHKW4Xjs8Z2d3dLCXOeuA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.9", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/es-module-lexer": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.1.0.tgz", + "integrity": "sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", + "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/nanoid": { + "version": "3.3.13", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.13.tgz", + "integrity": "sha512-sPdqC6ByMVVGvF1ynvvMo0/o+oD1VX7DaHhijt1bFgjvBkHBib4t49GoNDhf2NDta4oeUNlaGbSt5K7qjZ955Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/obug": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.3.tgz", + "integrity": "sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.15", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", + "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.12", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/rolldown": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.3.tgz", + "integrity": "sha512-i00lAJ2ks1BYr7rjNjKC7BcqAS7nVfiT3QX1SI5aY+AFHblCmaUf9OE9dbdzDvW6dJxbi2ZCZiy9v3CcwOiX3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.133.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.0.3", + "@rolldown/binding-darwin-arm64": "1.0.3", + "@rolldown/binding-darwin-x64": "1.0.3", + "@rolldown/binding-freebsd-x64": "1.0.3", + "@rolldown/binding-linux-arm-gnueabihf": "1.0.3", + "@rolldown/binding-linux-arm64-gnu": "1.0.3", + "@rolldown/binding-linux-arm64-musl": "1.0.3", + "@rolldown/binding-linux-ppc64-gnu": "1.0.3", + "@rolldown/binding-linux-s390x-gnu": "1.0.3", + "@rolldown/binding-linux-x64-gnu": "1.0.3", + "@rolldown/binding-linux-x64-musl": "1.0.3", + "@rolldown/binding-openharmony-arm64": "1.0.3", + "@rolldown/binding-wasm32-wasi": "1.0.3", + "@rolldown/binding-win32-arm64-msvc": "1.0.3", + "@rolldown/binding-win32-x64-msvc": "1.0.3" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.1.0.tgz", + "integrity": "sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", + "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyrainbow": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", + "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD", + "optional": true + }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/vite": { + "version": "8.0.16", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.16.tgz", + "integrity": "sha512-h9bXPmJichP5fLmVQo3PyaGSDE2n3aPuomeAlVRm0JLmt4rY6zmPKd59HYI4LNW8oTK7tlTsuC7l/m7awx9Jcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "lightningcss": "^1.32.0", + "picomatch": "^4.0.4", + "postcss": "^8.5.15", + "rolldown": "1.0.3", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.1.18", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.9.tgz", + "integrity": "sha512-nE3/LEyc0z87uHYLZebqCUOaJr2hdtuPp7BQ4BosVFnfltxgAvMG08NyrSGlPpOUWvR27c5flSmYFTNr78L9GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.9", + "@vitest/mocker": "4.1.9", + "@vitest/pretty-format": "4.1.9", + "@vitest/runner": "4.1.9", + "@vitest/snapshot": "4.1.9", + "@vitest/spy": "4.1.9", + "@vitest/utils": "4.1.9", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.9", + "@vitest/browser-preview": "4.1.9", + "@vitest/browser-webdriverio": "4.1.9", + "@vitest/coverage-istanbul": "4.1.9", + "@vitest/coverage-v8": "4.1.9", + "@vitest/ui": "4.1.9", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + } + } +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/package.json b/skills/clickhouse-js-node-rowbinary-parser/package.json new file mode 100644 index 000000000..8966ce9f6 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/package.json @@ -0,0 +1,69 @@ +{ + "name": "@clickhouse/rowbinary", + "version": "0.1.1", + "description": "RowBinary building blocks for Node.js — read/decode ClickHouse RowBinary / RowBinaryWithNames(AndTypes) streams (a matching writer is planned). Ships with the clickhouse-js-node-rowbinary-parser agent skill.", + "homepage": "https://github.com/ClickHouse/clickhouse-js/tree/main/skills/clickhouse-js-node-rowbinary-parser", + "license": "Apache-2.0", + "keywords": [ + "clickhouse", + "rowbinary", + "parser", + "decoder", + "streaming", + "skill" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/ClickHouse/clickhouse-js.git", + "directory": "skills/clickhouse-js-node-rowbinary-parser" + }, + "type": "module", + "sideEffects": false, + "engines": { + "node": ">=20" + }, + "publishConfig": { + "access": "public" + }, + "main": "./dist/reader.js", + "module": "./dist/reader.js", + "types": "./dist/reader.d.ts", + "exports": { + ".": { + "types": "./dist/reader.d.ts", + "import": "./dist/reader.js" + }, + "./*": { + "types": "./dist/*.d.ts", + "import": "./dist/*.js" + } + }, + "files": [ + "dist", + "src", + "SKILL.md", + "README.md", + "EXAMPLES.md" + ], + "agents": { + "skills": [ + { + "name": "clickhouse-js-node-rowbinary-parser", + "path": "." + } + ] + }, + "scripts": { + "build": "rm -rf dist && tsc -p tsconfig.build.json", + "test": "vitest run", + "test:watch": "vitest", + "bench": "vitest bench --run", + "typecheck": "tsc --noEmit", + "prepack": "cp ../../LICENSE . && npm run build" + }, + "devDependencies": { + "@types/node": "^26.0.0", + "typescript": "^6.0.3", + "vitest": "^4.1.9" + } +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/aggregateFunction.ts b/skills/clickhouse-js-node-rowbinary-parser/src/aggregateFunction.ts new file mode 100644 index 000000000..2778b25d4 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/aggregateFunction.ts @@ -0,0 +1,34 @@ +import { type Reader } from "./core.js"; + +/** + * `AggregateFunction(func, T…)` holds an OPAQUE serialized aggregation STATE + * (what `-State` combinators produce). In RowBinary this state is written RAW, + * with **NO length prefix** and a layout entirely specific to `func` (and to the + * ClickHouse version): `sumState(UInt64)` is 8 bytes, `uniqState(...)` is a + * variable-length hash-set blob, etc. + * + * So it cannot be decoded generically (there is no schema in the bytes) and + * cannot even be SKIPPED generically (there is no length to skip past) — without + * knowing `func`'s exact byte layout you cannot find where it ends, and every + * later column in the row misaligns. There is therefore NO generic reader. + * + * Fix it server-side (RECOMMENDED): finalize with the `-Merge` combinator or + * `finalizeAggregation()` in SQL so the column becomes a normal value + * (`sum` -> `UInt64`, `uniq` -> `UInt64`, `avg` -> `Float64`, …) and use the + * matching reader. Never ship raw `-State` columns to the client unless you + * intend to merge them later. + * + * ESCAPE HATCH: a few functions' state IS just a value of a known type (e.g. + * `sumState(UInt64)` is literally that `UInt64`), so you may decode it as that + * type — fragile and version-specific; only when you truly know the layout. See + * `tests/aggregateFunction.test.ts`. + * + * This reader throws to stop a generic parser from silently misaligning the row. + */ +export const readAggregateFunction: Reader = () => { + throw new Error( + "RowBinary: AggregateFunction is opaque, unframed aggregation state with no " + + "length prefix — not generically decodable or skippable. Finalize server-side " + + "(-Merge / finalizeAggregation()) and decode the concrete result type instead.", + ); +}; diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/bool.ts b/skills/clickhouse-js-node-rowbinary-parser/src/bool.ts new file mode 100644 index 000000000..52113cc9e --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/bool.ts @@ -0,0 +1,10 @@ +import { Cursor } from "./core.js"; +import { readUInt8 } from "./integers.js"; + +/** + * Read a `Bool`: 1 byte, stored as `UInt8` (`0` = false, `1` = true). Treats any + * non-zero byte as true. + */ +export function readBool(state: Cursor): boolean { + return readUInt8(state) !== 0; +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/columnar.ts b/skills/clickhouse-js-node-rowbinary-parser/src/columnar.ts new file mode 100644 index 000000000..4749d6b29 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/columnar.ts @@ -0,0 +1,125 @@ +/** + * Streaming COLUMNAR decode for an all-numeric, fixed-width RowBinary result — + * one concrete example reader that ties together the three wins this skill keeps + * pointing at. The schema is hard-coded on purpose: a real columnar reader is + * MONOMORPHIZED to its result, so the row loop is straight-line constant-offset + * reads with no per-field dispatch. Generate one shaped like this per schema. + * + * The example schema (`sensor_id UInt32, ts DateTime64(3), value Float64, + * quality Float32, status UInt8`) — every column fixed-width, stride 25 bytes: + * + * sensor_id UInt32 @ o+0 getUint32 -> Uint32Array + * ts DateTime64(3) @ o+4 2x getUint32 -> BigInt64Array (raw ms ticks) + * value Float64 @ o+12 getFloat64 -> Float64Array + * quality Float32 @ o+20 getFloat32 -> Float32Array + * status UInt8 @ o+24 buf[o+24] -> Uint8Array + * + * The three wins: + * + * 1. COLUMNAR (struct-of-arrays). One typed array per column, not one object + * per row — removes the per-row object / `Date` / number-boxing allocation + * that dominates a numeric decode (~4x in plain JS; see `src/examples/iot.ts` + * and `tests/iot.columnar.bench.ts`). Keep `ts` as raw `BigInt64Array` ticks + * and make a `Date` lazily, per displayed row — never allocate 50k `Date`s. + * The `Int64` column is itself filled WITHOUT allocating a bigint per row: + * copy the two little-endian 32-bit words straight into a `Uint32Array` view + * over its buffer (`getBigInt64` would box a bigint each row); the bigint is + * materialized lazily, only when the consumer reads `ts[i]`. + * + * 2. TRANSFERABLE. Each column is a fresh, exactly-sized typed array that OWNS + * its `ArrayBuffer` at offset 0, so a batch ships to a Worker / WASM kernel + * zero-copy: `postMessage(batch, columns.map(c => c.buffer))`. + * + * 3. RESPECTS INCOMPLETE BUFFERS (streaming). Because the stride is constant, + * honoring a partial trailing row is pure ARITHMETIC: the number of complete + * rows in the buffer is `(work.length / STRIDE) | 0`. No `advance()`, no + * `NeedMoreData`, no throw/restart — the leftover `work.length % STRIDE` bytes + * just carry to the next chunk. Strictly cheaper than the row-oriented + * `streamRowBatches`, which re-decodes the partial row on every boundary. + * + * SCOPE: fixed-width numeric columns only — the ClickHouse types with a 1:1 + * native TypedArray (`Int8/16/32/64`, `UInt8/16/32/64`, `Float32/64`). Anything + * whose value isn't one native-typed number has no constant stride to divide by + * (`String`/`Array`/`Map`/`Tuple`) or no 1:1 array (`Int128`+, `Decimal*`, + * `BFloat16`); decode those row-wise. `Bool`/`Enum`/`Date*`/`DateTime*` ride + * their underlying int here for the RAW value. + */ + +/** One decoded batch of the example schema: `rows` complete rows, one typed array per column. */ +export interface SensorColumnBatch { + /** Number of complete rows decoded in this batch. */ + rows: number; + columns: { + sensor_id: Uint32Array; + ts: BigInt64Array; // raw DateTime64(3) ms ticks + value: Float64Array; + quality: Float32Array; + status: Uint8Array; + }; +} + +/** Byte stride of one fixed-width row: 4 + 8 + 8 + 4 + 1. */ +const STRIDE = 25; + +const EMPTY_CHUNK = Buffer.alloc(0); + +/** + * Stream a chunked RowBinary response of the example schema into columnar + * batches: one `{ rows, columns }` per incoming chunk, holding exactly the rows + * that completed within it. + * + * BACKPRESSURE: a pull stream — the next chunk is requested only when the + * consumer asks for the next batch. SMALL CHUNKS: tiny chunks mean tiny batches + * (more allocations, worse Worker amortization); compose `coalesceChunks` (from + * `./stream.js`) in front to merge them up to a target size first. + */ +export async function* streamSensorColumns( + chunks: AsyncIterable, +): AsyncGenerator { + let carry: Buffer = EMPTY_CHUNK; + for await (const chunk of chunks) { + // Wrap as a Buffer VIEW over the chunk's bytes — no copy (a Buffer made from + // an ArrayBuffer slice shares it). We own the chunk for the life of this + // generator, so holding a view into it is safe. + const incoming = Buffer.from( + chunk.buffer, + chunk.byteOffset, + chunk.byteLength, + ); + const work = + carry.length === 0 ? incoming : Buffer.concat([carry, incoming]); + + // Complete rows available right now — pure arithmetic, since STRIDE is fixed. + const n = (work.length / STRIDE) | 0; + if (n > 0) { + const view = new DataView(work.buffer, work.byteOffset, work.byteLength); + const sensor_id = new Uint32Array(n); + const ts = new BigInt64Array(n); + // Uint32 view over ts's OWN bytes: 2 little-endian words per Int64, + // [lo, hi, lo, hi, ...]. Filling ts through this view copies the raw bytes + // and skips the per-row bigint allocation `getBigInt64` would force; the + // bigint is materialized lazily, only for rows the consumer indexes. + const tsWords = new Uint32Array(ts.buffer); + const value = new Float64Array(n); + const quality = new Float32Array(n); + const status = new Uint8Array(n); + for (let i = 0, o = 0; i < n; i++, o += STRIDE) { + sensor_id[i] = view.getUint32(o, true); // UInt32 @ o+0 + // DateTime64(3) Int64 @ o+4: two LE 32-bit words, no bigint allocated. + tsWords[i * 2] = view.getUint32(o + 4, true); // low word + tsWords[i * 2 + 1] = view.getUint32(o + 8, true); // high word + value[i] = view.getFloat64(o + 12, true); // Float64 @ o+12 + quality[i] = view.getFloat32(o + 20, true); // Float32 @ o+20 + status[i] = work[o + 24]!; // UInt8 @ o+24 + } + yield { rows: n, columns: { sensor_id, ts, value, quality, status } }; + } + // Carry the partial trailing row (if any) to the next chunk. + carry = work.subarray(n * STRIDE); + } + if (carry.length > 0) { + throw new Error( + `RowBinary stream ended mid-row: ${carry.length} trailing byte(s) left undecoded`, + ); + } +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/composite.ts b/skills/clickhouse-js-node-rowbinary-parser/src/composite.ts new file mode 100644 index 000000000..b91a25aff --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/composite.ts @@ -0,0 +1,181 @@ +import { type Reader } from "./core.js"; +import { readUInt8 } from "./integers.js"; +import { readUVarint } from "./varint.js"; + +/** + * Read a `Nullable(T)`: a 1-byte null flag (0 = present, non-zero = NULL). + * Curried: pass the inner reader, get a `Reader`. + * + * GOTCHA: the inner value bytes follow ONLY when the flag is 0. A NULL is the + * single `0x01` flag byte with nothing after it — so do NOT read the inner value + * when the flag is set, or the cursor desyncs. + * + * `readValue` decodes the inner `T`. This generic combinator is the reference + * shape; when generating code, MONOMORPHIZE — emit a dedicated `readNullableX` + * that inlines the inner read: + * + * const readNullableUInt32 = (s) => readUInt8(s) !== 0 ? null : readUInt32(s); + */ +export function readNullable(readValue: Reader): Reader { + return (state) => (readUInt8(state) !== 0 ? null : readValue(state)); +} + +/** + * Read an `Array(T)`: a LEB128 element count, then that many `T` values + * back-to-back. An empty array is just the count byte `0x00`. Curried: pass the + * element reader, get a `Reader`. + * + * ARRAY LAYOUT: the count is known up front (the LEB128 prefix), so a generated + * reader can pre-size. Pick by how the result is used: + * - small / consumed-as-is (the common case) → DEFAULT to `new Array(n)` + + * index assignment; it skips `push`'s repeated capacity growth. A clean-room + * benchmark found this edged out `push` on the small composite arrays here + * (`baseline/README.md`). + * - large + iterated/computed-over downstream → `[]` + `push` keeps it a PACKED + * elements kind (faster to traverse; a pre-sized array is HOLEY), or use a + * typed array (`Float64Array`…) for numeric elements. + * This generic combinator uses `push` for simplicity; the monomorphized + * `readArrayX` below should choose per the rule above. + * + * `readElement` decodes one element. This generic combinator is the reference + * shape; when generating code, MONOMORPHIZE — emit a dedicated `readArrayX` that + * inlines the element read in the loop (and pre-sizes for the common small case): + * + * function readArrayUInt32(s) { + * const n = readUVarint(s); + * const out = new Array(n); + * for (let i = 0; i < n; i++) out[i] = readUInt32(s); + * return out; + * } + */ +export function readArray(readElement: Reader): Reader { + return (state) => { + const n = readUVarint(state); + const out: T[] = []; + for (let i = 0; i < n; i++) out.push(readElement(state)); + return out; + }; +} + +/** + * Read a `QBit(element_type, dimension)` vector. `QBit` is a vector-search type + * whose ON-DISK layout is quantized and bit-transposed — but that is a STORAGE / + * Native-format concern. In RowBinary a `QBit` is fully TRANSPARENT: it is the + * plain vector, encoded byte-for-byte like `Array(element_type)` (a LEB128 + * length, then `dimension` element values). So this is just {@link readArray}. + * + * `element_type` is one of `BFloat16` / `Float32` / `Float64`, so `readElement` + * is the matching float reader. When generating code, MONOMORPHIZE — inline the + * element read in the loop. + */ +export function readQBit(readElement: Reader): Reader { + return readArray(readElement); +} + +/** + * Read a `Tuple(...)` into a positional array: each element's value back-to-back, + * with NO count and NO delimiter. Curried: pass one reader per element (in + * order), get a `Reader` of the tuple. For a named tuple as an object, use + * {@link readTupleNamed} (identical wire). + * + * Reference shape; when generating code, MONOMORPHIZE — emit the inline sequence + * with no array-of-readers and no loop: + * + * [readUInt32(s), readString(s)] + */ +export function readTuple(readers: { + [K in keyof T]: Reader; +}): Reader { + return (state) => { + const out: unknown[] = []; + for (const read of readers as ReadonlyArray>) { + out.push(read(state)); + } + return out as unknown as T; + }; +} + +/** + * Read a named `Tuple(name1 T1, ...)` into an object. The wire is identical to + * an unnamed tuple — values back-to-back, no count or delimiter — so the + * `readers` object's keys MUST be listed in the tuple's declared field order + * (JS iterates string keys in insertion order), and each reader runs in that + * order. Curried: pass the readers object, get a `Reader` of the result object. + * + * Reference shape; when generating code, MONOMORPHIZE — emit the inline object + * literal instead of looping over entries: + * + * { id: readUInt32(s), name: readString(s) } + */ +export function readTupleNamed>(readers: { + [K in keyof T]: Reader; +}): Reader { + const fns = readers as Record>; + const keys = Object.keys(fns); + return (state) => { + const out: Record = {}; + for (const key of keys) out[key] = fns[key]!(state); + return out as T; + }; +} + +/** + * Read a `Map(K, V)`: a LEB128 pair count, then that many key/value pairs with + * key and value interleaved (k, v, k, v, ...) — a flattened `Array(Tuple(K, V))`. + * An empty map is just the count byte `0x00`. Curried: pass the key and value + * readers, get a `Reader>`. + * + * The key is read BEFORE the value in each pair. Returns a JS `Map`, which keeps + * insertion order and accepts any key type. + * + * Reference shape; when generating code, MONOMORPHIZE — inline both reads in the + * loop. + */ +export function readMap( + readKey: Reader, + readValue: Reader, +): Reader> { + return (state) => { + const n = readUVarint(state); + const out = new Map(); + for (let i = 0; i < n; i++) { + const key = readKey(state); + out.set(key, readValue(state)); + } + return out; + }; +} + +/** + * Read a `Variant(T1, ..., Tn)`: a 1-byte discriminant selecting the active + * alternative, then that alternative's value. Discriminant `0xFF` means NULL. + * Curried: pass the alternative readers (in sorted-type-name order), get a + * `Reader`. + * + * GOTCHA: the discriminant indexes the alternatives sorted by type NAME + * (ClickHouse globally sorts them), NOT their declaration order. So `readers` + * MUST be ordered by sorted type name. E.g. `Variant(UInt8, String)` sorts to + * ["String", "UInt8"], so discriminant 0 = String and 1 = UInt8. + * + * Reference shape; when generating code, MONOMORPHIZE — emit a `switch` over the + * discriminant with each branch inlined, alternatives in sorted order, `0xFF` + * -> null. + */ +export function readVariant(readers: { + [K in keyof T]: Reader; +}): Reader { + const fns = readers as ReadonlyArray>; + return (state) => { + const discriminant = readUInt8(state); + if (discriminant === 0xff) return null; + const fn = fns[discriminant]; + if (fn === undefined) { + // Out-of-range discriminant (corrupted/truncated input): fail loudly + // instead of throwing a cryptic "fns[discriminant] is not a function". + throw new RangeError( + `RowBinary Variant: discriminant ${discriminant} out of range (${fns.length} alternatives)`, + ); + } + return fn(state); + }; +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/core.ts b/skills/clickhouse-js-node-rowbinary-parser/src/core.ts new file mode 100644 index 000000000..415da3d30 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/core.ts @@ -0,0 +1,77 @@ +/** + * Thrown by {@link advance} when the buffer lacks the bytes a read needs — the + * "need more bytes" signal for incremental decoding over a still-filling buffer. + * A driver catches it (`err === NeedMoreData`), waits for more input, and retries + * the row from its last committed position. + * + * A bare sentinel, NOT an `Error` subclass, on purpose: constructing an Error + * captures a stack trace — the expensive part of throwing — and on a path that + * starves once per chunk that cost is pure waste. Throwing a constant skips it, + * which is why throw + restart beats a generator's yield for realistic chunks + * (see `streamingRow.bench.ts`). + */ +export const NeedMoreData = Symbol("RowBinary.NeedMoreData"); + +/** + * The cursor state every reader threads through: the input `Buffer`, the current + * position, and a `DataView` over the same bytes. + * + * Deliberately STATE only — no read methods. Decoding lives in the free + * `readX(state, ...)` functions in the sibling modules, so a generated parser + * pulls in only the per-type readers a result needs. `view`/`buf` are public so + * those free functions can reach them. + */ +export class Cursor { + pos = 0; + + /** + * Node-only skill, so the input is a `Buffer`: number reads go through + * {@link Cursor.view} (DataView), while `String`/`FixedString` use the + * fast `buf.toString("utf8", ...)`. + */ + readonly buf: Buffer; + + /** + * `DataView` over the same bytes, for fixed-width integer/float reads. Built + * with the buffer's own `byteOffset`/`byteLength`: a `Buffer` is often a window + * into a larger pooled `ArrayBuffer`, so `new DataView(buf.buffer)` alone would + * point at the wrong bytes. + */ + readonly view: DataView; + + constructor(buf: Buffer) { + this.buf = buf; + this.view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength); + } +} + +/** + * A `Reader` decodes one value of type `T` from the cursor, advancing it. Leaf + * readers (e.g. `readUInt32`) are `Reader`s directly; combinators (e.g. + * `readArray`) take sub-`Reader`s and return a `Reader`, so types compose with no + * per-element closures. + */ +export type Reader = (state: Cursor) => T; + +/** + * Reserve `n` bytes for the next read: bounds-check them, advance the cursor past + * them, and return the offset the read starts at (the value BEFORE advancing). + * Every fixed-width read goes through this, so the length check and cursor + * bookkeeping live in one place: + * + * function readInt32(s) { return s.view.getInt32(advance(s, 4), true); } + * + * Throws {@link NeedMoreData} when fewer than `n` bytes remain, WITHOUT moving the + * cursor, so a driver can rewind to its last committed row and retry. + * + * SAFE TO TOGGLE: for a complete in-memory buffer the check never fires — a parser + * for that case can drop `advance` and read against `state.pos` directly, trading + * streaming tolerance for one fewer compare per read. + */ +export function advance(state: Cursor, n: number): number { + const start = state.pos; + const next = start + n; + if (next > state.buf.length) throw NeedMoreData; + state.pos = next; + return start; +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/datetime.ts b/skills/clickhouse-js-node-rowbinary-parser/src/datetime.ts new file mode 100644 index 000000000..658a4313a --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/datetime.ts @@ -0,0 +1,113 @@ +import { type Reader, Cursor } from "./core.js"; +import { readInt32, readInt64, readUInt16, readUInt32 } from "./integers.js"; + +/** + * Semantic aliases for `number` that mark the unit of a temporal value in a + * return type. They are plain `number`s (no runtime brand) — purely to make + * `[Date, Nanoseconds]` etc. self-documenting at the call site. + */ +export type Milliseconds = number; +export type Microseconds = number; +export type Nanoseconds = number; + +/** + * Read a `Date`: 2-byte `UInt16` count of days since 1970-01-01 (UTC), returned + * as a JS `Date` at UTC midnight. A ClickHouse `Date` has no time or timezone; + * `.toISOString().slice(0, 10)` gives "YYYY-MM-DD". + * + * SAFE TO TOGGLE: a `Date` is an object allocation per value. On a hot path that + * only needs the calendar number, read the raw `UInt16` (days) instead. + */ +export function readDate(state: Cursor): Date { + return new Date(readUInt16(state) * 86_400_000); +} + +/** + * Read a `Date32`: 4-byte signed `Int32` count of days since 1970-01-01 (UTC), + * returned as a JS `Date` at UTC midnight (pre-1970 dates are negative day + * counts, which `Date` handles). + */ +export function readDate32(state: Cursor): Date { + return new Date(readInt32(state) * 86_400_000); +} + +/** + * Read a `DateTime` (and `DateTime(tz)`): 4-byte `UInt32` Unix seconds, returned + * as a JS `Date` (exact at second resolution). The instant is UTC-based; a + * column's timezone is display metadata, not in the bytes. + */ +export function readDateTime(state: Cursor): Date { + return new Date(readUInt32(state) * 1000); +} + +/** + * Read a `DateTime64(P)` (and `DateTime64(P, tz)`): 8-byte signed `Int64` count + * of `10^-P`-second ticks since the epoch. Curried: `readDateTime64(P)` returns + * the reader. + * + * Returns a pair `[date, nanoseconds]`: `date` is a JS `Date` truncated to whole + * seconds, and `nanoseconds` is the sub-second remainder in nanoseconds + * (0..999_999_999). The split keeps full precision a `Date` alone (millisecond + * resolution) can't hold. `nanoseconds` is always in ns regardless of P. Timezone + * is metadata. + * + * For the typical precisions, prefer the specialized variants + * {@link readDateTime64P3} (ms — returns a plain `Date`), + * {@link readDateTime64P6} (µs), and {@link readDateTime64P9} (ns). + */ +export function readDateTime64(precision: number): Reader<[Date, Nanoseconds]> { + return (state) => { + const ticks = readInt64(state); + const scale = 10n ** BigInt(precision); + let sec = ticks / scale; + let frac = ticks % scale; + if (frac < 0n) { + // Floor toward -inf so the fractional remainder stays in [0, scale). + frac += scale; + sec -= 1n; + } + return [new Date(Number(sec) * 1000), Number(frac) * 10 ** (9 - precision)]; + }; +} + +/** + * Read a `DateTime64(3)` ({@link Milliseconds}) — the most common precision — as + * a plain JS `Date`. P=3 is exactly a `Date`'s own millisecond resolution, so the + * instant is represented losslessly with no separate fraction. Specialized + * variant of {@link readDateTime64} with the scale baked in. + */ +export function readDateTime64P3(state: Cursor): Date { + return new Date(Number(readInt64(state))); +} + +/** + * Read a `DateTime64(6)` (microseconds) as `[date, microseconds]`: a JS `Date` + * truncated to whole seconds plus the sub-second remainder in microseconds. + * Specialized variant of {@link readDateTime64}. + */ +export function readDateTime64P6(state: Cursor): [Date, Microseconds] { + const ticks = readInt64(state); + let sec = ticks / 1_000_000n; + let frac = ticks % 1_000_000n; // microseconds within the second + if (frac < 0n) { + frac += 1_000_000n; + sec -= 1n; + } + return [new Date(Number(sec) * 1000), Number(frac)]; +} + +/** + * Read a `DateTime64(9)` (nanoseconds) as `[date, nanoseconds]`: a JS `Date` + * truncated to whole seconds plus the sub-second remainder in nanoseconds. + * Specialized variant of {@link readDateTime64} with the scale baked in. + */ +export function readDateTime64P9(state: Cursor): [Date, Nanoseconds] { + const ticks = readInt64(state); + let sec = ticks / 1_000_000_000n; + let frac = ticks % 1_000_000_000n; // nanoseconds within the second + if (frac < 0n) { + frac += 1_000_000_000n; + sec -= 1n; + } + return [new Date(Number(sec) * 1000), Number(frac)]; +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/decimals.ts b/skills/clickhouse-js-node-rowbinary-parser/src/decimals.ts new file mode 100644 index 000000000..5e0e499a1 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/decimals.ts @@ -0,0 +1,57 @@ +import { type Reader } from "./core.js"; +import { readInt32, readInt64, readInt128, readInt256 } from "./integers.js"; + +/** + * A decimal kept lossless as its raw parts: `value = unscaled / 10 ** scale`. + * The `readDecimal*` readers return this so no precision or scale information is + * thrown away at decode time. + */ +export type DecimalValue = readonly [unscaled: bigint, scale: number]; + +/** + * Format a {@link DecimalValue} as a fixed-point decimal string with `scale` + * fractional digits (e.g. `[15000n, 4]` -> `"1.5000"`). Plug in only when you + * need a string. + * + * Trailing zeros are preserved to reflect the declared scale, deliberately unlike + * ClickHouse's text output, which trims them (`"1.5"`) and drops the point for + * integers (`"10"`). + */ +export function formatDecimal([unscaled, scale]: DecimalValue): string { + if (scale === 0) return unscaled.toString(); + if (unscaled < 0n) { + const digits = (-unscaled).toString().padStart(scale + 1, "0"); + const point = digits.length - scale; + return `-${digits.slice(0, point)}.${digits.slice(point)}`; + } + const digits = unscaled.toString().padStart(scale + 1, "0"); + const point = digits.length - scale; + return `${digits.slice(0, point)}.${digits.slice(point)}`; +} + +/** + * Read a `Decimal32(P, S)`: a 4-byte little-endian signed integer (same wire + * shape as `Int32`) scaled by 10^S. Pass the column's scale `S`; returns a + * `Reader` of the raw `[unscaled, scale]` pair (see {@link formatDecimal}). + * + * `Decimal(P, S)` is an alias: pick the width reader by precision P — P<=9 -> + * Decimal32, <=18 -> Decimal64, <=38 -> Decimal128, <=76 -> Decimal256. + */ +export function readDecimal32(scale: number): Reader { + return (state) => [BigInt(readInt32(state)), scale]; +} + +/** Read a `Decimal64(P, S)`: 8-byte LE signed integer scaled by 10^S. Returns `[unscaled, scale]`; see {@link formatDecimal}. */ +export function readDecimal64(scale: number): Reader { + return (state) => [readInt64(state), scale]; +} + +/** Read a `Decimal128(P, S)`: 16-byte LE signed integer scaled by 10^S. Returns `[unscaled, scale]`; see {@link formatDecimal}. */ +export function readDecimal128(scale: number): Reader { + return (state) => [readInt128(state), scale]; +} + +/** Read a `Decimal256(P, S)`: 32-byte LE signed integer scaled by 10^S. Returns `[unscaled, scale]`; see {@link formatDecimal}. */ +export function readDecimal256(scale: number): Reader { + return (state) => [readInt256(state), scale]; +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/dynamic.ts b/skills/clickhouse-js-node-rowbinary-parser/src/dynamic.ts new file mode 100644 index 000000000..336ea8e7e --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/dynamic.ts @@ -0,0 +1,328 @@ +import { type Reader, Cursor } from "./core.js"; +import { readUVarint } from "./varint.js"; +import { + readInt8, + readInt16, + readInt32, + readInt64, + readInt128, + readInt256, + readUInt8, + readUInt16, + readUInt32, + readUInt64, + readUInt128, + readUInt256, +} from "./integers.js"; +import { readBool } from "./bool.js"; +import { readFloat32, readFloat64 } from "./floats.js"; +import { readString, readFixedString } from "./strings.js"; +import { readUUID } from "./uuid.js"; +import { readIPv4, readIPv6 } from "./ip.js"; +import { + readDate, + readDate32, + readDateTime, + readDateTime64, +} from "./datetime.js"; +import { + readDecimal32, + readDecimal64, + readDecimal128, + readDecimal256, +} from "./decimals.js"; +import { + INTERVAL_UNITS, + type IntervalValue, + readInterval, +} from "./interval.js"; +import { + readArray, + readMap, + readNullable, + readTuple, + readTupleNamed, + readVariant, +} from "./composite.js"; +import { readJSON } from "./json.js"; + +/** + * Read one `Dynamic` value. A `Dynamic` is SELF-DESCRIBING: every value is a + * binary TYPE ENCODING followed by the value's RowBinary bytes. So unlike every + * other reader, the type is not known until runtime — this is the one place a + * generic runtime dispatch is correct and unavoidable. {@link readDynamicType} + * parses the type header into a value `Reader`; here we just invoke it. + * + * GOTCHA — wrappers are erased. `Dynamic` stores the CONCRETE type of the stored + * value, never a wrapper: a non-null `Nullable(UInt8)` is stored as plain + * `UInt8`, an active `Variant(...)` as just its current alternative's type. A + * NULL (from any source) is stored as `Nothing` (tag 0x00) and decodes to + * `null`. So you never see Nullable/Variant tags here. + * + * For a Dynamic-heavy hot path where the same few types recur, parse the type + * once and reuse the returned reader across rows instead of re-parsing. + */ +export function readDynamic(state: Cursor): unknown { + return readDynamicType(state)(state); +} + +/** + * Parse a binary TYPE ENCODING (the header ClickHouse writes before each + * `Dynamic` value) and return a `Reader` that reads ONE value of that type. The + * type bytes are consumed now; the returned reader consumes only value bytes + * when called. Composites recurse: element/key/field types are parsed eagerly + * into inner readers, then composed with the existing combinators + * ({@link readArray}/{@link readTuple}/{@link readMap}/...). + * + * The leading byte is a 1-byte tag; parameterized types (FixedString, Enum, + * Decimal, DateTime64, timezone'd DateTime) carry extra LEB128/varint and string + * fields in the header, which we consume to reach the value reader. + * + * Only the tags ClickHouse actually emits for stored `Dynamic` values are + * handled (wrappers are erased — see {@link readDynamic}). Unknown tags throw + * with the tag value so you can extend this switch for the types your data + * actually contains. + */ +export function readDynamicType(state: Cursor): Reader { + const tag = readUInt8(state); + switch (tag) { + // Nothing — a stored NULL. Zero value bytes. + case 0x00: + return () => null; + // Unsigned integers. + case 0x01: + return readUInt8; + case 0x02: + return readUInt16; + case 0x03: + return readUInt32; + case 0x04: + return readUInt64; + case 0x05: + return readUInt128; + case 0x06: + return readUInt256; + // Signed integers. + case 0x07: + return readInt8; + case 0x08: + return readInt16; + case 0x09: + return readInt32; + case 0x0a: + return readInt64; + case 0x0b: + return readInt128; + case 0x0c: + return readInt256; + // Floats. + case 0x0d: + return readFloat32; + case 0x0e: + return readFloat64; + // Dates and times. The timezone'd variants carry a tz string in the header + // (metadata only — identical value wire to the untimezoned form). + case 0x0f: + return readDate; + case 0x10: + return readDate32; + case 0x11: + return readDateTime; + case 0x12: + readString(state); // timezone name (metadata) + return readDateTime; + case 0x13: + return readDateTime64(readUVarint(state)); + case 0x14: { + const precision = readUVarint(state); + readString(state); // timezone name (metadata) + return readDateTime64(precision); + } + // String / FixedString(N). + case 0x15: + return readString; + case 0x16: + return readFixedString(readUVarint(state)); + // Enum8 / Enum16: a count then (name String, value Int8/Int16) pairs. The + // name<->value map is metadata; the stored value is the underlying int. + case 0x17: { + const n = readUVarint(state); + for (let i = 0; i < n; i++) { + readString(state); + readInt8(state); + } + return readInt8; + } + case 0x18: { + const n = readUVarint(state); + for (let i = 0; i < n; i++) { + readString(state); + readInt16(state); + } + return readInt16; + } + // Decimals: header carries precision P then scale S (both varint). Only S + // matters for decoding; P is consumed and dropped. Returns [unscaled, S]. + case 0x19: { + readUVarint(state); + return readDecimal32(readUVarint(state)); + } + case 0x1a: { + readUVarint(state); + return readDecimal64(readUVarint(state)); + } + case 0x1b: { + readUVarint(state); + return readDecimal128(readUVarint(state)); + } + case 0x1c: { + readUVarint(state); + return readDecimal256(readUVarint(state)); + } + case 0x1d: + return readUUID; + // Array(T): parse the element type once, then read a length-prefixed run. + case 0x1e: + return readArray(readDynamicType(state)); + // Tuple(...): a field count, then that many element type encodings. + case 0x1f: { + const n = readUVarint(state); + const fields: Array> = []; + for (let i = 0; i < n; i++) fields.push(readDynamicType(state)); + return readTuple(fields); + } + // Named Tuple: a count, then (name String, type) pairs. Names shape the + // result object; the value wire is identical to an unnamed tuple. + case 0x20: { + const n = readUVarint(state); + const fields: Record> = {}; + for (let i = 0; i < n; i++) { + const name = readString(state); + fields[name] = readDynamicType(state); + } + return readTupleNamed(fields); + } + // Set (0x21): a type used inside IN-expressions, not a stored column value. + case 0x21: + throw new RangeError( + "RowBinary: Dynamic type 0x21 (Set) has no decodable value form", + ); + // Interval (0x22): the header carries a 1-byte unit kind (0x00 Nanosecond + // ... 0x0a Year), then the value is a signed Int64 count of that unit. Here + // — unlike a standalone Interval* column — the unit IS in the wire, so we + // pair it with the count as an IntervalValue rather than dropping it. + case 0x22: { + const kind = readUInt8(state); + const unit = INTERVAL_UNITS[kind]; + if (unit === undefined) { + throw new RangeError( + `RowBinary: unknown Interval kind ${kind} in Dynamic type encoding`, + ); + } + return (s): IntervalValue => [readInterval(s), unit]; + } + // Nullable(T): a NULL flag byte then (if not null) the inner value. At the + // TOP level Dynamic erases Nullable, but NESTED inside Array/Tuple/Map the + // element type really is Nullable(T) — e.g. Array(Nullable(UInt8)) — so the + // tag does appear here. + case 0x23: + return readNullable(readDynamicType(state)); + // Function (0x24): a higher-order function type (lambda), header-only with no + // stored value form. + case 0x24: + throw new RangeError( + "RowBinary: Dynamic type 0x24 (Function) has no decodable value form", + ); + // AggregateFunction (0x25): an opaque, UNFRAMED aggregation state with a + // function-specific layout and no length prefix, so it cannot be decoded OR + // skipped generically. Finalize server-side before putting it in a Dynamic. + case 0x25: + throw new RangeError( + "RowBinary: Dynamic type 0x25 (AggregateFunction) is an opaque unframed state — finalize it server-side", + ); + // LowCardinality(T): transparent — keep the inner type's reader as-is. + case 0x26: + return readDynamicType(state); + // Map(K, V): parse the key type then the value type. + case 0x27: { + const key = readDynamicType(state); + const value = readDynamicType(state); + return readMap(key, value); + } + case 0x28: + return readIPv4; + case 0x29: + return readIPv6; + // Variant (0x2a): the header is (count, then each alternative's type + // encoding). ClickHouse writes the alternatives ALREADY SORTED by type name, + // so the parsed readers line up with the discriminant directly. The value is + // a 1-byte discriminant (0xff = NULL) then the chosen value. NOTE: top-level + // Dynamic erases Variant, so this tag only appears NESTED. + case 0x2a: { + const n = readUVarint(state); + const alternatives: Array> = []; + for (let i = 0; i < n; i++) alternatives.push(readDynamicType(state)); + return readVariant(alternatives); + } + // Dynamic (0x2b): a Dynamic nested inside a Dynamic. The header is a single + // max_dynamic_types byte; the value is itself a type-encoding + value, so it + // is just a recursive readDynamic. We skip max_dynamic_types because it does + // NOT affect value decoding — it is a storage/Native-format overflow + // threshold; in RowBinary every value is normalized to a plain (tag, value). + case 0x2b: + readUInt8(state); // max_dynamic_types — storage threshold, not used to decode + return readDynamic; + // Custom type (0x2c): the type name is written as a String and must be + // re-parsed to learn the real type — we don't have a type-name parser. + case 0x2c: + throw new RangeError( + "RowBinary: Dynamic type 0x2c (custom type, name-encoded) is not supported — requires parsing the type name string", + ); + case 0x2d: + return readBool; + // SimpleAggregateFunction (0x2e): transparent — the value is just its + // underlying type T. The header is (function_name String, argument types); + // extend here by consuming those, then returning T's reader. + case 0x2e: + throw new RangeError( + "RowBinary: Dynamic type 0x2e (SimpleAggregateFunction) is not supported yet — consume the header, then read the inner T", + ); + // Nested(...) (0x2f): on the wire it IS Array(Tuple(...)). The header is + // identical to a named Tuple's (count, then (name String, type) pairs), and + // the value is an Array of those tuples, so compose readArray + readTupleNamed. + case 0x2f: { + const n = readUVarint(state); + const fields: Record> = {}; + for (let i = 0; i < n; i++) { + const name = readString(state); + fields[name] = readDynamicType(state); + } + return readArray(readTupleNamed(fields)); + } + // JSON (0x30): the type-encoding header is a version byte, max_dynamic_paths + // (varuint), max_dynamic_types (uint8), then the typed-path / skip-path / + // skip-regexp lists. We consume it to reach the value body. Typed paths are + // serialized WITHOUT a Dynamic tag, so a schema-less reader can't decode them + // — bail if any are declared. + case 0x30: { + readUInt8(state); // serialization version (observed 0x00) + readUVarint(state); // max_dynamic_paths + readUInt8(state); // max_dynamic_types + const typedPaths = readUVarint(state); + if (typedPaths !== 0) { + throw new RangeError( + "RowBinary: JSON with declared typed paths is not supported — read each typed path with its known type", + ); + } + const skipPaths = readUVarint(state); + for (let i = 0; i < skipPaths; i++) readString(state); + const skipRegexps = readUVarint(state); + for (let i = 0; i < skipRegexps; i++) readString(state); + return readJSON; + } + default: + throw new RangeError( + `RowBinary: unknown Dynamic type tag 0x${tag.toString(16)} (not in the binary type encoding table)`, + ); + } +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/enums.ts b/skills/clickhouse-js-node-rowbinary-parser/src/enums.ts new file mode 100644 index 000000000..cc2513520 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/enums.ts @@ -0,0 +1,28 @@ +import { Cursor, advance } from "./core.js"; + +/** + * Read an `Enum8`: the value's underlying signed `Int8`. The name<->value map + * lives in the column's type, not the bytes. Two strategies, both better than one + * shared name-resolving reader: + * + * - Keep the number: carry the raw Int8 and map to a name only where needed — + * most hot loops never need it. + * - Or generate a per-enum reader with a baked-in constant map, so the JIT can + * monomorphize each enum's decode: + * + * const STATUS = { 1: "active", 2: "closed" } as const; + * const readStatusEnum = (s) => STATUS[readInt8(s) as keyof typeof STATUS]; + */ +export function readEnum8(state: Cursor): number { + return state.view.getInt8(advance(state, 1)); +} + +/** + * Read an `Enum16`: the value's underlying signed `Int16` (2 bytes). The + * name<->value map lives in the column's type definition, not the bytes. Prefer + * keeping the number, or a generated per-enum reader with a baked-in constant + * map so the JIT can optimize each enum's decode independently. + */ +export function readEnum16(state: Cursor): number { + return state.view.getInt16(advance(state, 2), true); +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/examples/carts.ts b/skills/clickhouse-js-node-rowbinary-parser/src/examples/carts.ts new file mode 100644 index 000000000..902412be5 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/examples/carts.ts @@ -0,0 +1,71 @@ +import { readArray, readNullable, readTupleNamed } from "../composite.js"; +import { type Reader, advance } from "../core.js"; +import { readInt32, readUInt16, readUInt32 } from "../integers.js"; +import { readString } from "../strings.js"; +import { readUVarint } from "../varint.js"; + +/** + * Example: a carts table — nested generics. + * + * Columns (the trigger): + * cart_id UInt32 + * items Array(Tuple(sku String, qty UInt16)) + * discounts Array(Nullable(Int32)) + * + * The element reader of an `Array` can itself be a combinator: `items` is an + * `Array` whose element is a named `Tuple`, `discounts` an `Array` whose element + * is a `Nullable`. Combinators nest to any depth, matching the column type. + */ +export type CartRow = { + cartId: number; + items: { sku: string; qty: number }[]; + discounts: (number | null)[]; +}; + +export const readCartRow: Reader = (s) => ({ + cartId: readUInt32(s), + items: readArray(readTupleNamed({ sku: readString, qty: readUInt16 }))(s), + discounts: readArray(readNullable(readInt32))(s), +}); + +/** + * Optimized {@link readCartRow}, monomorphized through the nesting: the API + * version rebuilds the outer `readArray`, the inner `readTupleNamed`, and the + * `readNullable` closures on every row (and the tuple reader iterates a keys + * array per element). Here both arrays are inlined loops, the tuple element is a + * flat object literal, and the nullable is an inline branch — no closures, no key + * iteration, at either nesting level. + * + * MEASURED (Node 24 / V8, `carts.bench.ts`): ~2x faster — nested combinators + * (outer `readArray`, inner `readTupleNamed` / `readNullable`) rebuilt per row in + * the API version, all flattened here. + */ +export const readCartRowFast: Reader = (s) => { + const { buf, view } = s; + + // cart_id UInt32. + const cartId = view.getUint32(advance(s, 4), true); + + // items Array(Tuple(sku String, qty UInt16)): count, then per element a + // length-prefixed string and a 2-byte int. + const itemsN = readUVarint(s); + const items = new Array<{ sku: string; qty: number }>(itemsN); + for (let i = 0; i < itemsN; i++) { + const len = readUVarint(s); + const start = advance(s, len); + const sku = buf.toString("utf8", start, start + len); + const qty = view.getUint16(advance(s, 2), true); + items[i] = { sku, qty }; + } + + // discounts Array(Nullable(Int32)): count, then per element a null-flag byte + // and, if non-null, a 4-byte int. + const discN = readUVarint(s); + const discounts = new Array(discN); + for (let i = 0; i < discN; i++) { + discounts[i] = + buf[advance(s, 1)]! !== 0 ? null : view.getInt32(advance(s, 4), true); + } + + return { cartId, items, discounts }; +}; diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/examples/events.ts b/skills/clickhouse-js-node-rowbinary-parser/src/examples/events.ts new file mode 100644 index 000000000..b07614523 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/examples/events.ts @@ -0,0 +1,51 @@ +import { type Reader, advance } from "../core.js"; +import { readDateTime } from "../datetime.js"; +import { readUInt64 } from "../integers.js"; +import { readString } from "../strings.js"; +import { readUVarint } from "../varint.js"; + +/** + * Example: a plain events table — the scalar baseline. + * + * Columns (the trigger — generate this reader when a result has these types): + * id UInt64 + * name String + * ts DateTime('UTC') + * + * A wide integer (returned as `bigint`, never a lossy `number`), an arbitrary + * `String`, and a `DateTime` rendered to an ISO-8601 string here for a stable, + * timezone-independent result (the raw reader returns a JS `Date`). Drive it over + * a whole result with `readRows(readEventRow)`. + */ +export type EventRow = { id: bigint; name: string; ts: string }; + +export const readEventRow: Reader = (s) => ({ + id: readUInt64(s), + name: readString(s), + ts: readDateTime(s).toISOString(), +}); + +/** + * Optimized {@link readEventRow}: the same three reads inlined into one function + * body — no per-field reader calls, the `String` length + slice and the + * `DateTime` math written out in place. All scalars, so there is little for the + * monomorphization to remove (the JIT already inlines the leaf readers); see + * `events.bench.ts`. Still goes through `advance()`, so it stays streaming-safe. + * + * MEASURED (Node 24 / V8, `events.bench.ts`): ~1.05x — essentially ON PAR, within + * run-to-run noise. A purely scalar row has no per-row closures to remove and V8 + * already inlines the leaf readers, so there is no real win here: prefer the + * clearer API `readEventRow` unless your own profiling says otherwise. (Contrast + * the composite examples, where monomorphization removes per-row closures and + * wins 1.3x–2.7x.) + */ +export const readEventRowFast: Reader = (s) => { + const id = s.view.getBigUint64(advance(s, 8), true); + const len = readUVarint(s); + const start = advance(s, len); + const name = s.buf.toString("utf8", start, start + len); + const ts = new Date( + s.view.getUint32(advance(s, 4), true) * 1000, + ).toISOString(); + return { id, name, ts }; +}; diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/examples/iot.ts b/skills/clickhouse-js-node-rowbinary-parser/src/examples/iot.ts new file mode 100644 index 000000000..24da52536 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/examples/iot.ts @@ -0,0 +1,158 @@ +import { type Reader, advance } from "../core.js"; +import { readDateTime64P3 } from "../datetime.js"; +import { readFloat32, readFloat64 } from "../floats.js"; +import { readUInt8, readUInt32 } from "../integers.js"; + +/** + * Example: a table of IoT sensor readings — the dense, fixed-width NUMERIC case + * that RowBinary is built for, and the headline of the RowBinary-vs-JSON + * comparison in `iot.bench.ts`. + * + * Columns (the trigger — generate this reader when a result has these types): + * sensor_id UInt32 + * ts DateTime64(3) + * temperature Float64 + * humidity Float64 + * pressure Float64 + * battery Float32 + * status UInt8 + * + * Every column is fixed-width and there is not a single string or composite in + * the row, so the whole record is a flat 4 + 8 + 8 + 8 + 8 + 4 + 1 = 41-byte + * run. This is the shape where a JS RowBinary decoder beats `JSON.parse`: the + * wire is ~1/3 the size and each field is one `DataView` read, versus JSON's + * tokenize-and-number-parse over a much larger, key-repeating text. + */ +export type IotRow = { + sensor_id: number; + ts: Date; + temperature: number; + humidity: number; + pressure: number; + battery: number; + status: number; +}; + +/** + * API-combinator reader: correct and clear, one leaf reader per column. A fine + * default; `readIotRowFast` is the monomorphized form `iot.bench.ts` measures. + */ +export const readIotRow: Reader = (s) => ({ + sensor_id: readUInt32(s), + ts: readDateTime64P3(s), + temperature: readFloat64(s), + humidity: readFloat64(s), + pressure: readFloat64(s), + battery: readFloat32(s), + status: readUInt8(s), +}); + +/** + * Optimized {@link readIotRow}: every column is fixed-width, so the seven + * separate bounds checks coalesce into one `advance(s, 41)` and each field is + * read at a constant offset off that base — no per-field reader calls, no cursor + * write-back between fields. Stays streaming-safe (one `advance`), so a row that + * straddles a chunk boundary still rewinds and retries cleanly. + * + * sensor_id UInt32 @ o+0 getUint32 + * ts DateTime64(3) @ o+4 getBigInt64 (ms ticks -> Date) + * temperature Float64 @ o+12 getFloat64 + * humidity Float64 @ o+20 getFloat64 + * pressure Float64 @ o+28 getFloat64 + * battery Float32 @ o+36 getFloat32 + * status UInt8 @ o+40 buf[o+40] + */ +export const readIotRowFast: Reader = (s) => { + const { buf, view } = s; + const o = advance(s, 41); // one bounds check for the whole 41-byte row + const sensor_id = view.getUint32(o, true); + // DateTime64(3): Int64 millisecond ticks; ms fits a JS number, so Number() is exact here. + const ts = new Date(Number(view.getBigInt64(o + 4, true))); + const temperature = view.getFloat64(o + 12, true); + const humidity = view.getFloat64(o + 20, true); + const pressure = view.getFloat64(o + 28, true); + const battery = view.getFloat32(o + 36, true); + const status = buf[o + 40]!; + return { sensor_id, ts, temperature, humidity, pressure, battery, status }; +}; + +/** Byte width of one fixed-width IoT row: 4 + 8 + 8 + 8 + 8 + 4 + 1. */ +export const IOT_ROW_BYTES = 41; + +/** + * Columnar (struct-of-arrays) form of the IoT result: one typed array per + * column instead of one object per row. `ts` is kept as epoch milliseconds in a + * `Float64Array` (format the few you display; don't allocate 50k `Date`s). + */ +export type IotColumns = { + sensor_id: Uint32Array; + ts: Float64Array; // epoch ms + temperature: Float64Array; + humidity: Float64Array; + pressure: Float64Array; + battery: Float32Array; + status: Uint8Array; +}; + +/** + * Decode the whole IoT result into columns (SoA) rather than row objects (AoS). + * + * MEASURED (`iot.columnar.bench.ts`): ~4x faster than `readIotRowFast` over the + * same buffer, and several times smaller in memory. The win is entirely from + * what it does NOT do — no per-row object, no `Date`, no number boxing — so the + * cost drops to one unboxed store per field. It is a NUMERIC win; it would not + * help a string column (a JS string must be allocated either way). + * + * WHOLE-BUFFER ONLY: this needs the complete response in one `Buffer`. Because + * every IoT column is fixed-width the row stride is known, so the exact row + * count is `buf.length / IOT_ROW_BYTES` — one exact allocation per column, no + * growth, no bounds check in the loop. + * + * Reach for this when the consumer is column-oriented (aggregate / filter / + * scan / plot / feed a Worker or WASM kernel via the transferable + * `ArrayBuffer`s). Prefer the row reader when downstream code is row-shaped. + */ +export function decodeIotColumnar(buf: Buffer): IotColumns { + const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength); + const n = (buf.length / IOT_ROW_BYTES) | 0; + // Hold each column in a LOCAL so the fill loop writes through a register-held + // typed-array reference, not a property load off a result object every + // iteration; assemble the object once, on return. + const sensor_id = new Uint32Array(n); + const ts = new Float64Array(n); + const temperature = new Float64Array(n); + const humidity = new Float64Array(n); + const pressure = new Float64Array(n); + const battery = new Float32Array(n); + const status = new Uint8Array(n); + let o = 0; + for (let i = 0; i < n; i++) { + sensor_id[i] = view.getUint32(o, true); // UInt32 + ts[i] = Number(view.getBigInt64(o + 4, true)); // DateTime64(3) ms ticks + temperature[i] = view.getFloat64(o + 12, true); // Float64 + humidity[i] = view.getFloat64(o + 20, true); // Float64 + pressure[i] = view.getFloat64(o + 28, true); // Float64 + battery[i] = view.getFloat32(o + 36, true); // Float32 + status[i] = buf[o + 40]!; // UInt8 + o += IOT_ROW_BYTES; + } + return { sensor_id, ts, temperature, humidity, pressure, battery, status }; +} + +/** + * Hybrid accessor: reconstruct a single {@link IotRow} object from columns on + * demand (here is where `ts` becomes a `Date`). Store columnar, and pay the + * object/`Date` cost only for the rows a caller actually touches — best when + * row access is sparse. + */ +export function iotRowAt(c: IotColumns, i: number): IotRow { + return { + sensor_id: c.sensor_id[i]!, + ts: new Date(c.ts[i]!), + temperature: c.temperature[i]!, + humidity: c.humidity[i]!, + pressure: c.pressure[i]!, + battery: c.battery[i]!, + status: c.status[i]!, + }; +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/examples/ledger.ts b/skills/clickhouse-js-node-rowbinary-parser/src/examples/ledger.ts new file mode 100644 index 000000000..3e0dbc63c --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/examples/ledger.ts @@ -0,0 +1,98 @@ +import { type Reader, advance } from "../core.js"; +import { + type DecimalValue, + readDecimal64, + readDecimal128, +} from "../decimals.js"; +import { readInt64, readUInt128, readUInt256 } from "../integers.js"; + +/** + * Example: a financial ledger — the WIDE-NUMERIC case where RowBinary wins on + * *correctness*, not merely speed, and the headline of the wide-int/decimal + * comparison in `ledger.bench.ts`. + * + * Columns (the trigger — generate this reader when a result has these types): + * txn_id UInt128 + * account Int64 + * amount Decimal128(18) + * balance Decimal128(18) + * fee Decimal64(4) + * volume UInt256 + * + * Every value here exceeds what a JS `number` (IEEE-754 double, 53-bit mantissa) + * can hold exactly, and ClickHouse emits them as **bare JSON numbers**. So + * `JSON.parse` silently rounds every field — the only correct JSON path is to + * quote the values server-side and re-parse each string into a `bigint` / + * decimal pair by hand. RowBinary reads each as an exact `bigint` straight off + * the wire. The whole row is fixed-width (16+8+16+16+8+32 = 96 bytes). + */ +export type LedgerRow = { + txn_id: bigint; + account: bigint; + amount: DecimalValue; + balance: DecimalValue; + fee: DecimalValue; + volume: bigint; +}; + +/** + * API-combinator reader: correct and clear, one leaf reader per column. A fine + * default; `readLedgerRowFast` is the monomorphized form `ledger.bench.ts` + * measures. Both return identical values. + */ +export const readLedgerRow: Reader = (s) => ({ + txn_id: readUInt128(s), + account: readInt64(s), + amount: readDecimal128(18)(s), + balance: readDecimal128(18)(s), + fee: readDecimal64(4)(s), + volume: readUInt256(s), +}); + +/** + * Optimized, monomorphized reader: the six column bounds checks coalesce into + * one `advance(s, 96)`; each wide value is composed from 64-bit words read at + * constant offsets off that base, with the high word read **signed** for the + * signed types (`Int64`, the `Decimal128` unscaled value, no high word needed + * for the unsigned `UInt128`/`UInt256`). Stays streaming-safe (one `advance`). + * + * txn_id UInt128 @ o+0 lo + (hi<<64) unsigned + * account Int64 @ o+16 getBigInt64 + * amount Decimal128(18) @ o+24 lo + (hiSigned<<64) -> [v, 18] + * balance Decimal128(18) @ o+40 lo + (hiSigned<<64) -> [v, 18] + * fee Decimal64(4) @ o+56 getBigInt64 -> [v, 4] + * volume UInt256 @ o+64 w0 + w1<<64 + w2<<128 + w3<<192 unsigned + */ +export const readLedgerRowFast: Reader = (s) => { + const { view } = s; + const o = advance(s, 96); // one bounds check for the whole 96-byte row + + // UInt128 — unsigned, both words unsigned. + const txn_id = + view.getBigUint64(o, true) + (view.getBigUint64(o + 8, true) << 64n); + + // Int64 — signed. + const account = view.getBigInt64(o + 16, true); + + // Decimal128(18) — Int128 unscaled (low word unsigned, high word signed), scale 18. + const amount: DecimalValue = [ + view.getBigUint64(o + 24, true) + (view.getBigInt64(o + 32, true) << 64n), + 18, + ]; + const balance: DecimalValue = [ + view.getBigUint64(o + 40, true) + (view.getBigInt64(o + 48, true) << 64n), + 18, + ]; + + // Decimal64(4) — Int64 unscaled, scale 4. + const fee: DecimalValue = [view.getBigInt64(o + 56, true), 4]; + + // UInt256 — unsigned, four unsigned words. + const volume = + view.getBigUint64(o + 64, true) + + (view.getBigUint64(o + 72, true) << 64n) + + (view.getBigUint64(o + 80, true) << 128n) + + (view.getBigUint64(o + 88, true) << 192n); + + return { txn_id, account, amount, balance, fee, volume }; +}; diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/examples/logs.ts b/skills/clickhouse-js-node-rowbinary-parser/src/examples/logs.ts new file mode 100644 index 000000000..4fd3b233f --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/examples/logs.ts @@ -0,0 +1,73 @@ +import { type Reader, advance } from "../core.js"; +import { readDateTime } from "../datetime.js"; +import { readString } from "../strings.js"; +import { readUVarint } from "../varint.js"; + +/** + * Example: an application log table — the STRING-HEAVY case where the skill + * steers you AWAY from RowBinary and toward a `JSON*` format. The honest + * counter-case to the IoT and ledger studies; see `logs.bench.ts`. + * + * Columns (the trigger — generate this reader when a result has these types): + * ts DateTime + * level LowCardinality(String) + * service LowCardinality(String) + * message String + * trace_id String + * + * Four of the five columns are text consumed wholesale, and `LowCardinality(T)` + * is transparent in RowBinary (decodes as the inner `String`, no dictionary on + * the wire). A RowBinary string read is a varint length + `buf.toString("utf8", + * …)` per field in JS; V8's native `JSON.parse` builds the same JS strings in + * optimized C++ and tends to WIN here. This reader exists so the comparison is + * apples-to-apples — not because RowBinary is the right call for this shape. + */ +export type LogRow = { + ts: Date; + level: string; + service: string; + message: string; + trace_id: string; +}; + +/** + * API-combinator reader: one leaf reader per column, the clear default. There is + * little to monomorphize on a mostly-string row — the only fixed-width field is + * `ts` — so `readLogRowFast` below is barely different and barely faster; the + * real lesson (see `logs.bench.ts`) is that JSON beats both. + */ +export const readLogRow: Reader = (s) => ({ + ts: readDateTime(s), + level: readString(s), // LowCardinality(String) — transparent, decode as String + service: readString(s), // LowCardinality(String) — transparent, decode as String + message: readString(s), + trace_id: readString(s), +}); + +/** + * Optimized {@link readLogRow}: the four string reads inlined (varint length + + * `buf.toString` in place) and the `DateTime` read written out. Note how little + * monomorphization can do when the row is dominated by variable-length strings — + * there are no adjacent fixed-width columns to coalesce, so this stays close to + * the API version. Included to make `logs.bench.ts` a fair fight; the takeaway + * is to pick `JSONEachRow` for this shape, not to tune this reader. + */ +export const readLogRowFast: Reader = (s) => { + const { buf } = s; + // DateTime: 4-byte LE Unix seconds. + const ts = new Date(s.view.getUint32(advance(s, 4), true) * 1000); + // Four UTF-8 Strings (the two LowCardinality columns are plain String on the wire). + let len = readUVarint(s); + let o = advance(s, len); + const level = buf.toString("utf8", o, o + len); + len = readUVarint(s); + o = advance(s, len); + const service = buf.toString("utf8", o, o + len); + len = readUVarint(s); + o = advance(s, len); + const message = buf.toString("utf8", o, o + len); + len = readUVarint(s); + o = advance(s, len); + const trace_id = buf.toString("utf8", o, o + len); + return { ts, level, service, message, trace_id }; +}; diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/examples/observability.ts b/skills/clickhouse-js-node-rowbinary-parser/src/examples/observability.ts new file mode 100644 index 000000000..7897af584 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/examples/observability.ts @@ -0,0 +1,142 @@ +import { + readArray, + readMap, + readNullable, + readTupleNamed, + readVariant, +} from "../composite.js"; +import { type Reader, advance } from "../core.js"; +import { readDateTime64P3 } from "../datetime.js"; +import { readEnum8 } from "../enums.js"; +import { readFloat64 } from "../floats.js"; +import { readInt64, readUInt64 } from "../integers.js"; +import { readString } from "../strings.js"; +import { formatUUID, formatUUIDTable, readUUID } from "../uuid.js"; +import { readUVarint } from "../varint.js"; + +/** + * Example: an observability/events table — the gotcha-heavy one. It packs the + * traps that trip a from-scratch decoder; the skill's job is getting them right. + * + * Columns (the trigger): + * id UInt64 + * ts DateTime64(3, 'UTC') + * level Enum8('debug'=1, 'info'=2, 'warn'=3, 'error'=4) + * trace_id UUID + * payload Variant(String, Int64, Float64) + * tags Map(LowCardinality(String), String) + * metrics Array(Tuple(name LowCardinality(String), value Float64)) + * attrs Array(Nullable(Int64)) + * + * Gotchas exercised, all in one row: + * - `Variant(String, Int64, Float64)`: the discriminant indexes the alternatives + * SORTED BY TYPE NAME — ["Float64", "Int64", "String"] → 0=Float64, 1=Int64, + * 2=String (NOT declaration order); `0xFF` = NULL. `readVariant` takes the + * readers in that sorted order. + * - `DateTime64(3)`: 8-byte Int64 of millisecond ticks; P=3 is exactly a `Date`'s + * resolution, so `readDateTime64P3` returns a plain `Date` (here ISO-stringed). + * - `LowCardinality(String)` (in the Map key and the Tuple field) is TRANSPARENT + * in RowBinary — decode as plain `String`, no dictionary layer. + * - `UUID` is two little-endian `UInt64` halves, byte-reversed vs the text form. + * - `Array(Nullable(Int64))`: per element a null flag then (if present) an Int64, + * kept as `bigint`. + */ +export type ObsRow = { + id: bigint; + ts: string; + level: number; + traceId: string; + payload: number | bigint | string | null; + tags: Map; + metrics: { name: string; value: number }[]; + attrs: (bigint | null)[]; +}; + +/** + * API-combinator reader. Note the `Variant` readers are in sorted-type-name + * order (Float64, Int64, String), and `LowCardinality` columns just use the + * inner `String` reader. + */ +export const readObsRow: Reader = (s) => ({ + id: readUInt64(s), + ts: readDateTime64P3(s).toISOString(), + level: readEnum8(s), + traceId: formatUUID(readUUID(s)), + payload: readVariant([readFloat64, readInt64, readString])(s), + tags: readMap(readString, readString)(s), + metrics: readArray(readTupleNamed({ name: readString, value: readFloat64 }))( + s, + ), + attrs: readArray(readNullable(readInt64))(s), +}); + +/** + * Optimized {@link readObsRow}, flattened per the SKILL.md guidance: + * - `buf`/`view` hoisted to locals. + * - the leading run of FIXED-WIDTH columns — `id` UInt64 (8) + `ts` DateTime64 (8) + * + `level` Enum8 (1) + `trace_id` UUID (16) = 33 bytes — is bounds-checked ONCE + * (`advance(s, 33)`) and read at constant offsets, instead of four `advance`s. + * - the `Variant` is an inlined `switch` over the discriminant (sorted order). + * - leaf reads inlined, `formatUUIDTable` for the UUID, pre-sized arrays. + * The variable-width columns (`payload`/`tags`/`metrics`/`attrs`) each start a new + * `advance` run because their size isn't known until decoded. + */ +export const readObsRowFast: Reader = (s) => { + const { buf, view } = s; + + // One bounds check for the 33-byte fixed-width head. + const o = advance(s, 33); + const id = view.getBigUint64(o, true); + const ts = new Date(Number(view.getBigInt64(o + 8, true))).toISOString(); + const level = view.getInt8(o + 16); + const traceId = formatUUIDTable(buf.subarray(o + 17, o + 33)); + + // payload Variant(String, Int64, Float64): 1-byte discriminant (sorted names: + // 0=Float64, 1=Int64, 2=String), 0xFF = NULL. + let payload: number | bigint | string | null; + const disc = buf[advance(s, 1)]!; + if (disc === 0xff) { + payload = null; + } else if (disc === 0) { + payload = view.getFloat64(advance(s, 8), true); + } else if (disc === 1) { + payload = view.getBigInt64(advance(s, 8), true); + } else { + const len = readUVarint(s); + const st = advance(s, len); + payload = buf.toString("utf8", st, st + len); + } + + // tags Map(LowCardinality(String) -> String): count, then key/value strings. + const tagN = readUVarint(s); + const tags = new Map(); + for (let i = 0; i < tagN; i++) { + let len = readUVarint(s); + let st = advance(s, len); + const k = buf.toString("utf8", st, st + len); + len = readUVarint(s); + st = advance(s, len); + tags.set(k, buf.toString("utf8", st, st + len)); + } + + // metrics Array(Tuple(name LowCardinality(String), value Float64)). + const mN = readUVarint(s); + const metrics = new Array<{ name: string; value: number }>(mN); + for (let i = 0; i < mN; i++) { + const len = readUVarint(s); + const st = advance(s, len); + const name = buf.toString("utf8", st, st + len); + const value = view.getFloat64(advance(s, 8), true); + metrics[i] = { name, value }; + } + + // attrs Array(Nullable(Int64)). + const aN = readUVarint(s); + const attrs = new Array(aN); + for (let i = 0; i < aN; i++) { + attrs[i] = + buf[advance(s, 1)]! !== 0 ? null : view.getBigInt64(advance(s, 8), true); + } + + return { id, ts, level, traceId, payload, tags, metrics, attrs }; +}; diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/examples/orders.ts b/skills/clickhouse-js-node-rowbinary-parser/src/examples/orders.ts new file mode 100644 index 000000000..ddf862457 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/examples/orders.ts @@ -0,0 +1,65 @@ +import { type Reader, advance } from "../core.js"; +import { type DecimalValue, readDecimal64 } from "../decimals.js"; +import { readEnum8 } from "../enums.js"; +import { readUInt8 } from "../integers.js"; +import { formatUUID, formatUUIDTable, readUUID } from "../uuid.js"; + +/** + * Example: an orders table — UUID, Decimal, and Enum (awkward-as-JSON types). + * + * Columns (the trigger): + * id UInt8 + * uid UUID + * price Decimal64(2) + * status Enum8('new' = 1, 'shipped' = 2, 'done' = 3) + * + * Shows the parse/format split and faithful values: `uid` is read as raw bytes + * then formatted with `formatUUID`; `price` stays the exact `[unscaled, scale]` + * pair (`[1234n, 2]` == 12.34), not a lossy float; `status` decodes to the + * underlying `Int8` value (1/2/3), the name<->value map being type metadata, not + * on the wire. The declared scale `2` is baked into `readDecimal64(2)`. + */ +export type OrderRow = { + id: number; + uid: string; + price: DecimalValue; + status: number; +}; + +export const readOrderRow: Reader = (s) => ({ + id: readUInt8(s), + uid: formatUUID(readUUID(s)), + price: readDecimal64(2)(s), + status: readEnum8(s), +}); + +/** + * Optimized {@link readOrderRow}, flattened per the SKILL.md guidance: + * - `buf`/`view` hoisted to locals (one property load, not one per field). + * - every column here is FIXED-WIDTH, so the whole row — `id` UInt8 (1) + `uid` + * UUID (16) + `price` Decimal64 (8) + `status` Enum8 (1) = 26 bytes — is + * bounds-checked ONCE (`advance(s, 26)`) and read at constant offsets, instead + * of four separate `advance`s. (This is the exact worked example in SKILL.md.) + * - the four leaf reads are inlined, and the BigInt `formatUUID` is swapped for + * the lookup-table `formatUUIDTable` (~1.6x on its own; see `readUUID.bench.ts`). + * Since this example formats every UUID to a string, that swap is the dominant + * win — the `readDecimal64(2)` closure (rebuilt per row above) is inlined too. + * + * MEASURED (Node 24 / V8, `orders.bench.ts`): ~2.6x faster — the largest win of + * the examples, dominated by the `formatUUIDTable` swap (every row stringifies a + * UUID; the table formatter is ~1.7x on its own and the row is otherwise cheap). + * + * `formatUUIDTable` uses a shared scratch buffer, so it is non-reentrant — fine + * here because the bytes are copied into the returned string synchronously before + * the next call. + */ +export const readOrderRowFast: Reader = (s) => { + const { buf, view } = s; + // One bounds check for the whole 26-byte fixed-width row. + const o = advance(s, 26); + const id = buf[o]!; + const uid = formatUUIDTable(buf.subarray(o + 1, o + 17)); + const price: DecimalValue = [view.getBigInt64(o + 17, true), 2]; + const status = view.getInt8(o + 25); + return { id, uid, price, status }; +}; diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/examples/profiles.ts b/skills/clickhouse-js-node-rowbinary-parser/src/examples/profiles.ts new file mode 100644 index 000000000..7a09c3148 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/examples/profiles.ts @@ -0,0 +1,60 @@ +import { readArray, readNullable } from "../composite.js"; +import { type Reader, advance } from "../core.js"; +import { readInt32, readUInt32 } from "../integers.js"; +import { readString } from "../strings.js"; +import { readUVarint } from "../varint.js"; + +/** + * Example: a profiles table — Array and Nullable wrappers. + * + * Columns (the trigger): + * id UInt32 + * tags Array(String) + * score Nullable(Int32) + * + * `readArray(elem)` reads a LEB128 length then that many elements; `readNullable` + * reads a 1-byte present/NULL flag then the value. Both are combinators: pass the + * inner reader and they return a `Reader`. Empty array and NULL are the sharp + * cases (a single byte each). + */ +export type ProfileRow = { id: number; tags: string[]; score: number | null }; + +export const readProfileRow: Reader = (s) => ({ + id: readUInt32(s), + tags: readArray(readString)(s), + score: readNullable(readInt32)(s), +}); + +/** + * Optimized {@link readProfileRow}, monomorphized: `readArray(readString)` and + * `readNullable(readInt32)` each allocate a fresh combinator closure on EVERY + * row in the version above; here the array loop and the null-flag branch are + * inlined, so no per-row closures are created and the element/inner reads are + * straight-line. This is the kind of win the SKILL's "monomorphize" step targets; + * see `profiles.bench.ts`. + * + * MEASURED (Node 24 / V8, `profiles.bench.ts`): ~1.3x faster — removing the two + * per-row combinator closures (`readArray(readString)`, `readNullable(readInt32)`) + * is the win. + */ +export const readProfileRowFast: Reader = (s) => { + const { buf, view } = s; + + // id UInt32. + const id = view.getUint32(advance(s, 4), true); + + // tags Array(String): count, then each a length-prefixed UTF-8 string. + const n = readUVarint(s); + const tags = new Array(n); + for (let i = 0; i < n; i++) { + const len = readUVarint(s); + const start = advance(s, len); + tags[i] = buf.toString("utf8", start, start + len); + } + + // score Nullable(Int32): null-flag byte, then if non-null a 4-byte int. + const score = + buf[advance(s, 1)]! !== 0 ? null : view.getInt32(advance(s, 4), true); + + return { id, tags, score }; +}; diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/examples/telemetry.ts b/skills/clickhouse-js-node-rowbinary-parser/src/examples/telemetry.ts new file mode 100644 index 000000000..3cc38a5af --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/examples/telemetry.ts @@ -0,0 +1,102 @@ +import { + readArray, + readMap, + readNullable, + readTupleNamed, +} from "../composite.js"; +import { type Reader, advance } from "../core.js"; +import { readFloat64 } from "../floats.js"; +import { readUInt16, readUInt32 } from "../integers.js"; +import { readString } from "../strings.js"; +import { readUVarint } from "../varint.js"; + +/** + * Example: a telemetry table — composite readers that nest. + * + * Columns (the trigger): + * host String + * tags Map(String, String) + * cpu Array(Float64) + * region Nullable(String) + * window Tuple(start UInt32, count UInt16) + * + * The combinators compose exactly the way the column type nests: + * `readMap(k, v)`, `readArray(elem)`, `readNullable(inner)`, and + * `readTupleNamed({...})` each take sub-readers and return a `Reader`. This is the + * generic (closure-per-element) API; a generated parser would monomorphize these + * into inlined per-type loops, but the result shape is exactly this. + */ +export type TelemetryRow = { + host: string; + tags: Map; + cpu: number[]; + region: string | null; + window: { start: number; count: number }; +}; + +export const readTelemetryRow: Reader = (s) => ({ + host: readString(s), + tags: readMap(readString, readString)(s), + cpu: readArray(readFloat64)(s), + region: readNullable(readString)(s), + window: readTupleNamed({ start: readUInt32, count: readUInt16 })(s), +}); + +/** + * Optimized {@link readTelemetryRow}, fully monomorphized: the API version above + * builds FOUR combinator closures per row (`readMap(...)`, `readArray(...)`, + * `readNullable(...)`, `readTupleNamed(...)`) and, for the named tuple, iterates + * a keys array building an object field by field. Here every loop and branch is + * inlined and the `window` object is a flat literal — no per-row closures, no + * key iteration. The most composite-heavy example. + * + * MEASURED (Node 24 / V8, `telemetry.bench.ts`): ~1.4x faster — four per-row + * combinator closures and the named-tuple key iteration removed. + */ +export const readTelemetryRowFast: Reader = (s) => { + const { buf, view } = s; + + // host String: length prefix, then the bytes. + let len = readUVarint(s); + let start = advance(s, len); + const host = buf.toString("utf8", start, start + len); + + // tags Map(String, String): count, then key/value strings. + const mapN = readUVarint(s); + const tags = new Map(); + for (let i = 0; i < mapN; i++) { + len = readUVarint(s); + start = advance(s, len); + const k = buf.toString("utf8", start, start + len); + len = readUVarint(s); + start = advance(s, len); + tags.set(k, buf.toString("utf8", start, start + len)); + } + + // cpu Array(Float64): count, then 8 bytes each. + const cpuN = readUVarint(s); + const cpu = new Array(cpuN); + for (let i = 0; i < cpuN; i++) { + cpu[i] = view.getFloat64(advance(s, 8), true); + } + + // region Nullable(String): null-flag byte, then if non-null a string. + let region: string | null; + if (buf[advance(s, 1)]! !== 0) { + region = null; + } else { + len = readUVarint(s); + start = advance(s, len); + region = buf.toString("utf8", start, start + len); + } + + // window Tuple(start UInt32, count UInt16): two adjacent fixed-width fields, + // bounds-checked once (6 bytes), then read at literal offsets. + const w = advance(s, 6); + const window = { + start: view.getUint32(w, true), + count: view.getUint16(w + 4, true), + }; + + return { host, tags, cpu, region, window }; +}; diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/floats.ts b/skills/clickhouse-js-node-rowbinary-parser/src/floats.ts new file mode 100644 index 000000000..05bc32b6f --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/floats.ts @@ -0,0 +1,32 @@ +import { Cursor, advance } from "./core.js"; + +/** + * Scratch view for widening a `BFloat16`: its 16 bits are the top half of an + * IEEE 754 float32, so we stage them into a 4-byte buffer and read a float32. + */ +const bf16Scratch = new DataView(new ArrayBuffer(4)); + +/** Read a `Float32`: 4 bytes, little-endian IEEE 754 single precision. */ +export function readFloat32(state: Cursor): number { + return state.view.getFloat32(advance(state, 4), true); +} + +/** Read a `Float64`: 8 bytes, little-endian IEEE 754 double precision. */ +export function readFloat64(state: Cursor): number { + return state.view.getFloat64(advance(state, 8), true); +} + +/** + * Read a `BFloat16`: 2 bytes, little-endian. BFloat16 is the high 16 bits of a + * float32 (same 8-bit exponent, 7-bit mantissa), so placing the bits in the top + * half of a 32-bit float and reading it back is exact. + * + * NOTE: `bf16Scratch` is module-level shared state written-then-read in this + * function. That is safe because the read is synchronous; do NOT introduce an + * `await`/`yield` between the `setUint32` and the `getFloat32`. + */ +export function readBFloat16(state: Cursor): number { + const bits = state.view.getUint16(advance(state, 2), true); + bf16Scratch.setUint32(0, bits * 0x10000, true); + return bf16Scratch.getFloat32(0, true); +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/geo.ts b/skills/clickhouse-js-node-rowbinary-parser/src/geo.ts new file mode 100644 index 000000000..619442ec3 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/geo.ts @@ -0,0 +1,109 @@ +import { Cursor } from "./core.js"; +import { readFloat64 } from "./floats.js"; +import { readUInt8 } from "./integers.js"; +import { readUVarint } from "./varint.js"; + +/** A geo `Point`: `[x, y]`, the base of every ClickHouse geo type. */ +export type Point = [x: number, y: number]; + +// Geo types are concrete compositions of Point = Tuple(Float64, Float64). They +// are monomorphic (no sub-readers) — the generator can emit them as-is. + +/** Read a `Point`: `Tuple(Float64, Float64)` -> `[x, y]`. */ +export function readPoint(state: Cursor): Point { + const x = readFloat64(state); + const y = readFloat64(state); + return [x, y]; +} + +/** + * Read a `Ring`: `Array(Point)` — a LEB128 point count, then that many points. + * `LineString` has the identical wire (see {@link readLineString}). `readPoint` + * is inlined here (two `readFloat64`s) to drop a call per point on this hot path. + */ +export function readRing(state: Cursor): Point[] { + const n = readUVarint(state); + const out: Point[] = []; + for (let i = 0; i < n; i++) { + const x = readFloat64(state); + const y = readFloat64(state); + out.push([x, y]); + } + return out; +} + +/** + * Read a `LineString`: `Array(Point)` (identical wire to a `Ring`). Points are + * inlined (two `readFloat64`s) to drop a call per point on this hot path. + */ +export function readLineString(state: Cursor): Point[] { + const n = readUVarint(state); + const out: Point[] = []; + for (let i = 0; i < n; i++) { + const x = readFloat64(state); + const y = readFloat64(state); + out.push([x, y]); + } + return out; +} + +/** Read a `Polygon`: `Array(Ring)` — the outer ring first, then any holes. */ +export function readPolygon(state: Cursor): Point[][] { + const n = readUVarint(state); + const out: Point[][] = []; + for (let i = 0; i < n; i++) out.push(readRing(state)); + return out; +} + +/** Read a `MultiLineString`: `Array(LineString)` (identical wire to a `Polygon`). */ +export function readMultiLineString(state: Cursor): Point[][] { + const n = readUVarint(state); + const out: Point[][] = []; + for (let i = 0; i < n; i++) out.push(readLineString(state)); + return out; +} + +/** Read a `MultiPolygon`: `Array(Polygon)`. */ +export function readMultiPolygon(state: Cursor): Point[][][] { + const n = readUVarint(state); + const out: Point[][][] = []; + for (let i = 0; i < n; i++) out.push(readPolygon(state)); + return out; +} + +/** + * Read a `Geometry`: a named `Variant` over the six geo types. This is the + * MONOMORPHIZED form of `readVariant` for a concrete variant — a switch over the + * discriminant with each branch inlined, no reader array. The alternatives, + * sorted by type name (so in discriminant order), are LineString(0), + * MultiLineString(1), MultiPolygon(2), Point(3), Polygon(4), Ring(5); 0xFF is NULL. + * + * NOTE: the value shapes overlap — LineString and Ring are both `Point[]`, + * MultiLineString and Polygon both `Point[][]` — so the value alone does not say + * which geo type it was. If you need the kind, branch on the discriminant. + */ +export function readGeometry( + state: Cursor, +): Point | Point[] | Point[][] | Point[][][] | null { + const discriminant = readUInt8(state); + switch (discriminant) { + case 0: + return readLineString(state); + case 1: + return readMultiLineString(state); + case 2: + return readMultiPolygon(state); + case 3: + return readPoint(state); + case 4: + return readPolygon(state); + case 5: + return readRing(state); + case 0xff: + return null; + default: + throw new RangeError( + `RowBinary: unknown Geometry discriminant ${discriminant}`, + ); + } +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/integers.ts b/skills/clickhouse-js-node-rowbinary-parser/src/integers.ts new file mode 100644 index 000000000..cf733811c --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/integers.ts @@ -0,0 +1,95 @@ +import { Cursor, advance } from "./core.js"; + +/** Read a single unsigned byte and advance. */ +export function readUInt8(state: Cursor): number { + return state.buf[advance(state, 1)]!; +} + +/** Read an `Int8`: 1 byte, two's-complement signed (-128 .. 127). */ +export function readInt8(state: Cursor): number { + return state.view.getInt8(advance(state, 1)); +} + +/** Read a `UInt16`: 2 bytes, little-endian (0 .. 65535). */ +export function readUInt16(state: Cursor): number { + return state.view.getUint16(advance(state, 2), true); +} + +/** + * Read an `Int16`: 2 bytes, little-endian, two's-complement signed (-32768 .. + * 32767). `DataView` reads from any offset and decodes explicitly little-endian, + * so the value never depends on host byte order. + */ +export function readInt16(state: Cursor): number { + return state.view.getInt16(advance(state, 2), true); +} + +/** Read a `UInt32`: 4 bytes, little-endian (0 .. 4294967295). */ +export function readUInt32(state: Cursor): number { + return state.view.getUint32(advance(state, 4), true); +} + +/** Read an `Int32`: 4 bytes, little-endian, two's-complement signed. */ +export function readInt32(state: Cursor): number { + return state.view.getInt32(advance(state, 4), true); +} + +/** + * Read a `UInt64`: 8 bytes, little-endian. Returns a `bigint`. + * SAFE TO TOGGLE: if the values fit in 53 bits, wrap in `Number(...)`. + */ +export function readUInt64(state: Cursor): bigint { + return state.view.getBigUint64(advance(state, 8), true); +} + +/** + * Read an `Int64`: 8 bytes, little-endian, two's-complement. Returns a `bigint` + * (range exceeds `Number.MAX_SAFE_INTEGER`). + * SAFE TO TOGGLE: if the values fit in 53 bits, wrap in `Number(...)`. + */ +export function readInt64(state: Cursor): bigint { + return state.view.getBigInt64(advance(state, 8), true); +} + +/** Read a `UInt128`: 16 bytes, little-endian. Always a `bigint`. */ +export function readUInt128(state: Cursor): bigint { + const start = advance(state, 16); + const lo = state.view.getBigUint64(start, true); + const hi = state.view.getBigUint64(start + 8, true); + return (hi << 64n) + lo; +} + +/** + * Read an `Int128`: 16 bytes, little-endian, two's-complement. Always a + * `bigint`, composed from the low (unsigned) and high (signed) 64-bit words — + * reading the high word signed extends the sign across all 128 bits. + */ +export function readInt128(state: Cursor): bigint { + const start = advance(state, 16); + const lo = state.view.getBigUint64(start, true); + const hi = state.view.getBigInt64(start + 8, true); + return (hi << 64n) + lo; +} + +/** Read a `UInt256`: 32 bytes, little-endian. Always a `bigint`. */ +export function readUInt256(state: Cursor): bigint { + const start = advance(state, 32); + const w0 = state.view.getBigUint64(start, true); + const w1 = state.view.getBigUint64(start + 8, true); + const w2 = state.view.getBigUint64(start + 16, true); + const w3 = state.view.getBigUint64(start + 24, true); + return w0 + (w1 << 64n) + (w2 << 128n) + (w3 << 192n); +} + +/** + * Read an `Int256`: 32 bytes, little-endian, two's-complement. Always a + * `bigint`. The most-significant 64-bit word is read signed. + */ +export function readInt256(state: Cursor): bigint { + const start = advance(state, 32); + const w0 = state.view.getBigUint64(start, true); + const w1 = state.view.getBigUint64(start + 8, true); + const w2 = state.view.getBigUint64(start + 16, true); + const w3 = state.view.getBigInt64(start + 24, true); + return (w3 << 192n) + (w2 << 128n) + (w1 << 64n) + w0; +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/interval.ts b/skills/clickhouse-js-node-rowbinary-parser/src/interval.ts new file mode 100644 index 000000000..fbcb7d744 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/interval.ts @@ -0,0 +1,54 @@ +import { Cursor } from "./core.js"; +import { readInt64 } from "./integers.js"; + +/** The 11 `Interval` units, in ClickHouse's ascending order. */ +export type IntervalUnit = + | "Nanosecond" + | "Microsecond" + | "Millisecond" + | "Second" + | "Minute" + | "Hour" + | "Day" + | "Week" + | "Month" + | "Quarter" + | "Year"; + +/** + * `Interval` units indexed by the kind byte the binary type encoding writes + * after the `0x22` tag (`0x00` = Nanosecond ... `0x0a` = Year). Exported because + * the `Dynamic` reader needs it to decode an `Interval` nested in a `Dynamic`. + */ +export const INTERVAL_UNITS: readonly IntervalUnit[] = [ + "Nanosecond", + "Microsecond", + "Millisecond", + "Second", + "Minute", + "Hour", + "Day", + "Week", + "Month", + "Quarter", + "Year", +]; + +/** + * An `Interval` decoded where the unit is carried IN the wire (inside a + * `Dynamic`): the signed `Int64` count plus its unit. A standalone `Interval*` + * column has no unit byte — there, use {@link readInterval} and take the unit + * from the column type instead. + */ +export type IntervalValue = readonly [count: bigint, unit: IntervalUnit]; + +/** + * Read an `Interval` — any of `IntervalNanosecond` ... `IntervalYear`: a signed + * `Int64` count of the unit. The unit is in the type name, not the bytes, and + * all 11 interval types share this exact wire, so this one reader covers them + * all; the caller knows the unit from the column type. Returns a `bigint`; wrap + * in `Number(...)` if the counts are known to fit in 53 bits. + */ +export function readInterval(state: Cursor): bigint { + return readInt64(state); +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/ip.ts b/skills/clickhouse-js-node-rowbinary-parser/src/ip.ts new file mode 100644 index 000000000..086102d76 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/ip.ts @@ -0,0 +1,93 @@ +import { Cursor, advance } from "./core.js"; + +/** + * Read an `IPv4`: stored as a 4-byte little-endian `UInt32`. Returns the raw + * 32-bit value (the little-endian load already orders the octets); pass it to + * {@link formatIPv4} for the dotted-quad string. + */ +export function readIPv4(state: Cursor): number { + return state.view.getUint32(advance(state, 4), true); +} + +/** + * Read an `IPv6`: 16 bytes in network (big-endian) order. Returns the raw bytes + * as a zero-copy view; pass them to {@link formatIPv6} for the canonical string. + * + * The view shares memory with the response buffer, so keeping it alive pins the + * whole response chunk in memory. If the value must outlive the row/response, + * copy it with `Buffer.from(...)`. + */ +export function readIPv6(state: Cursor): Buffer { + const start = advance(state, 16); + return state.buf.subarray(start, start + 16); +} + +/** + * Format an `IPv4` (the raw 32-bit value from {@link readIPv4}) as a dotted-quad + * string. Kept aside so the hot read path can skip building a string when the + * numeric value is all the caller needs. + */ +export function formatIPv4(value: number): string { + return `${(value >>> 24) & 0xff}.${(value >>> 16) & 0xff}.${(value >>> 8) & 0xff}.${value & 0xff}`; +} + +/** + * Join groups `[from, to)` as colon-separated lowercase hex, by concatenating + * into a string in a loop. Benchmarks faster than `slice().map().join(":")`, + * which allocates an intermediate array. Returns `""` for an empty range. + */ +function joinGroupsHex(g: number[], from: number, to: number): string { + if (from >= to) return ""; + let s = g[from]!.toString(16); + for (let i = from + 1; i < to; i++) s += ":" + g[i]!.toString(16); + return s; +} + +/** + * Format an `IPv6` (the raw 16 bytes from {@link readIPv6}) as the canonical + * RFC 5952 string: lowercase, no leading zeros, the longest run of zero groups + * (>= 2) collapsed to `::` (leftmost on a tie), and the `::ffff:a.b.c.d` form + * for IPv4-mapped addresses (matching ClickHouse). + * + * Kept aside from the read so the hot path only formats when a string is + * actually needed. + */ +export function formatIPv6(b: Buffer): string { + // IPv4-mapped (::ffff:a.b.c.d): first 10 bytes zero, then 0xffff. + let mapped = b[10] === 0xff && b[11] === 0xff; + for (let i = 0; mapped && i < 10; i++) { + if (b[i] !== 0) mapped = false; + } + if (mapped) { + return `::ffff:${b[12]}.${b[13]}.${b[14]}.${b[15]}`; + } + + // Eight 16-bit groups, big-endian. + const g: number[] = []; + for (let i = 0; i < 8; i++) { + g.push((b[2 * i]! << 8) | b[2 * i + 1]!); + } + + // Longest run of >= 2 zero groups becomes "::" (leftmost wins on a tie). + let bestStart = -1; + let bestLen = 0; + let curStart = -1; + let curLen = 0; + for (let i = 0; i < 8; i++) { + if (g[i] === 0) { + if (curStart < 0) curStart = i; + curLen++; + if (curLen > bestLen) { + bestLen = curLen; + bestStart = curStart; + } + } else { + curStart = -1; + curLen = 0; + } + } + if (bestLen < 2) { + return joinGroupsHex(g, 0, 8); + } + return `${joinGroupsHex(g, 0, bestStart)}::${joinGroupsHex(g, bestStart + bestLen, 8)}`; +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/json.ts b/skills/clickhouse-js-node-rowbinary-parser/src/json.ts new file mode 100644 index 000000000..c4bfa1c8a --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/json.ts @@ -0,0 +1,33 @@ +import { Cursor } from "./core.js"; +import { readUVarint } from "./varint.js"; +import { readString } from "./strings.js"; +import { readDynamic } from "./dynamic.js"; + +/** + * Read a `JSON` value. ClickHouse's `JSON` is NOT JSON text and NOT BSON — it is + * a list of (path, value) pairs built on the same machinery as `Dynamic`: + * + * then pathCount x ( ) + * + * Nested objects are FLATTENED to dotted paths (`{a:{b:2}}` -> path `"a.b"`), and + * each leaf value is a self-describing `Dynamic`, so this just loops + * {@link readString} + {@link readDynamic}. Returns a `Map` keyed by the flat + * dotted path. Path order on the wire is not significant. + * + * GOTCHA: a null-valued path is NOT stored at all — `{"a":null}` serializes as + * zero paths, identical to `{}`. JSON arrays come back as `Array(Nullable(T))`. + * + * LIMITATION — typed paths only. This reads a plain `JSON` column, where every + * path is dynamic (tagged). A `JSON(a T, ...)` with DECLARED typed paths + * serializes those paths' values WITHOUT a type tag, so they cannot be decoded + * without the schema; read each typed path with its known `T` reader instead. + */ +export function readJSON(state: Cursor): Map { + const n = readUVarint(state); + const out = new Map(); + for (let i = 0; i < n; i++) { + const path = readString(state); + out.set(path, readDynamic(state)); + } + return out; +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/lowCardinality.ts b/skills/clickhouse-js-node-rowbinary-parser/src/lowCardinality.ts new file mode 100644 index 000000000..c4c150818 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/lowCardinality.ts @@ -0,0 +1,18 @@ +import { type Reader } from "./core.js"; + +/** + * `LowCardinality(T)` is TRANSPARENT in RowBinary: it is encoded byte-for-byte + * the same as `T`, with NO dictionary/index layer. (The dictionary encoding + * exists only in the Native format — do not look for it here.) So there is + * nothing to decode at this level: use `T`'s own reader directly. + * + * This identity combinator exists only to document that, and to let a generated + * parser name the wrapper at the call site if it wants the type to read + * literally — it returns the inner reader unchanged: + * + * readLowCardinality(readString) === readString + * + * Prefer just calling the inner reader. + */ +export const readLowCardinality = (readValue: Reader): Reader => + readValue; diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/nested.ts b/skills/clickhouse-js-node-rowbinary-parser/src/nested.ts new file mode 100644 index 000000000..c4e1e69fa --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/nested.ts @@ -0,0 +1,23 @@ +import { readArray, readTupleNamed } from "./composite.js"; +import { type Reader } from "./core.js"; + +/** + * `Nested(a T1, b T2, …)` has NO wire format of its own: + * - `flatten_nested = 1` (the default): the column expands into separate + * columns `a Array(T1)`, `b Array(T2)`, … — decode each with `readArray`. + * - `flatten_nested = 0`: the column is `Array(Tuple(a T1, b T2, …))` — decode + * with `readArray` + `readTupleNamed`. + * + * Either way it reuses existing readers; there is no dedicated Nested wire. This + * thin alias just composes the two for the `flatten_nested = 0` shape, as + * documentation that "Nested === Array(Tuple(...))": + * + * readNested({ a: readUInt8, b: readString }) + * === readArray(readTupleNamed({ a: readUInt8, b: readString })) + * + * When generating code, prefer inlining (monomorphize the array + tuple) over + * this generic composition. + */ +export const readNested = >(fields: { + [K in keyof T]: Reader; +}): Reader => readArray(readTupleNamed(fields)); diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/nothing.ts b/skills/clickhouse-js-node-rowbinary-parser/src/nothing.ts new file mode 100644 index 000000000..75921fbc7 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/nothing.ts @@ -0,0 +1,29 @@ +import { type Reader } from "./core.js"; + +/** + * `Nothing` is the empty type: it has NO values and occupies ZERO bytes. It is + * never a column on its own (you cannot materialize a value of it) — it only + * appears wrapped, as the inferred element of an untyped literal: + * + * [] -> Array(Nothing) -> always the empty array (varint length 0x00) + * NULL -> Nullable(Nothing) -> always NULL (lone flag byte 0x01) + * + * So a `Nothing` value is NEVER read: `readArray`'s element reader and + * `readNullable`'s inner reader are not called in those cases (the array is + * empty / the value is NULL). There is nothing to decode. + * + * Wire this in as the inner reader to make that invariant loud: it throws if it + * is ever actually invoked, which would mean a `Nothing` reader was placed where + * a real element/inner type was expected. + * + * readArray(readNothing) // [] — readNothing never runs + * readNullable(readNothing) // null — readNothing never runs + */ +export const readNothing: Reader = () => { + throw new Error( + "RowBinary: Nothing is zero-width and is never decoded — it only appears as " + + "an empty Array(Nothing) or a NULL Nullable(Nothing), where the inner reader " + + "is not called. Reaching here means a Nothing reader was wired where a real " + + "element/inner type was expected.", + ); +}; diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/reader.ts b/skills/clickhouse-js-node-rowbinary-parser/src/reader.ts new file mode 100644 index 000000000..bc20844cb --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/reader.ts @@ -0,0 +1,51 @@ +/** + * Barrel re-export of the RowBinary reader, split by type family into the + * sibling modules. Import from here for everything in one place, or from a + * specific module (e.g. `./integers.js`, `./strings.js`) to pull in only the + * sub-parsers a given result actually needs — the latter is what a generated + * parser should do, copying just the modules its column types require. + * + * - core — Cursor, Reader, advance, NeedMoreData + * - varint — readUVarint + * - integers — readUInt8..readUInt256, readInt8..readInt256 + * - bool / enums / floats + * - decimals — DecimalValue, formatDecimal, readDecimal32..256 + * - strings — readString, readFixedString, readFixedStringBytes + * - uuid — readUUID(+BigInt/HiLo), formatUUID(+Table) + * - ip — readIPv4/6, formatIPv4/6 + * - datetime / time / interval + * - composite — readArray/Map/Tuple/TupleNamed/Nullable/Variant/QBit + * - rows — readRows + * - geo — Point, readPoint/Ring/LineString/Polygon/MultiLineString/MultiPolygon/Geometry + * - dynamic — readDynamic, readDynamicType + * - json — readJSON + * - stream — streamRowBatches, coalesceChunks + * - transparent / special wrappers (mostly documentation; see each file): + * lowCardinality (readLowCardinality), simpleAggregateFunction + * (readSimpleAggregateFunction), nested (readNested), nothing (readNothing), + * aggregateFunction (readAggregateFunction) + */ +export * from "./core.js"; +export * from "./varint.js"; +export * from "./integers.js"; +export * from "./bool.js"; +export * from "./enums.js"; +export * from "./floats.js"; +export * from "./decimals.js"; +export * from "./strings.js"; +export * from "./uuid.js"; +export * from "./ip.js"; +export * from "./datetime.js"; +export * from "./time.js"; +export * from "./interval.js"; +export * from "./composite.js"; +export * from "./rows.js"; +export * from "./geo.js"; +export * from "./dynamic.js"; +export * from "./json.js"; +export * from "./stream.js"; +export * from "./lowCardinality.js"; +export * from "./simpleAggregateFunction.js"; +export * from "./nested.js"; +export * from "./nothing.js"; +export * from "./aggregateFunction.js"; diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/rows.ts b/skills/clickhouse-js-node-rowbinary-parser/src/rows.ts new file mode 100644 index 000000000..f397c4cf3 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/rows.ts @@ -0,0 +1,58 @@ +import { NeedMoreData, type Reader } from "./core.js"; + +/** + * Drive `readRow` over every row of a plain `RowBinary` result into an array. + * Curried: `readRows(readRow)` returns a `Reader`. Rows are concatenated on + * the wire with no count, length prefix, or delimiter, so the result is exhausted + * only when the cursor reaches the buffer end. + * + * `readRow` must consume EXACTLY one row's bytes — a byte short or long compounds + * across rows and the cursor overshoots or never lands on `buf.length`. Returns + * `[]` for an empty buffer. When generating code, inline the per-column reads + * into the loop body: + * + * function readRowsUser(s) { + * const out = []; + * while (s.pos < s.buf.length) { + * out.push({ id: readUInt64(s), name: readString(s) }); + * } + * return out; + * } + * + * STREAMING (partial trailing row): a chunk of a still-arriving response may end + * mid-row. `pos` is committed only AFTER a row reads cleanly, so when a row + * starves and `readRow` throws {@link NeedMoreData}, this catches it, rewinds + * `pos` to the last complete row boundary, and returns the rows so far — never a + * half-built row. The cursor is left at the straddling row, a commit point the + * driver carries forward: + * + * const drive = readRows(readRow); + * let committed = 0; + * for (const chunk of chunks) { // chunk = growing prefix + * const s = new Cursor(chunk); + * s.pos = committed; + * emit(drive(s)); // complete rows in this chunk + * committed = s.pos; // start of the straddling row + * } + * + * On a complete buffer no read starves, so the catch never runs. Errors other + * than {@link NeedMoreData} are real decode faults and propagate. See also + * `streamRowBatches`, the async driver built on this. + */ +export function readRows(readRow: Reader): Reader { + return (state) => { + const out: T[] = []; + let committed = state.pos; + try { + while (state.pos < state.buf.length) { + const row = readRow(state); + committed = state.pos; // row read cleanly — advance the commit point + out.push(row); + } + } catch (e) { + if (e !== NeedMoreData) throw e; + state.pos = committed; // drop the partial trailing row; resume next chunk + } + return out; + }; +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/simpleAggregateFunction.ts b/skills/clickhouse-js-node-rowbinary-parser/src/simpleAggregateFunction.ts new file mode 100644 index 000000000..86d18b756 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/simpleAggregateFunction.ts @@ -0,0 +1,20 @@ +import { type Reader } from "./core.js"; + +/** + * `SimpleAggregateFunction(func, T)` is TRANSPARENT in RowBinary: the column + * already holds a finished value of the underlying type `T` (the partial + * aggregate of a "simple" function — sum / min / max / groupArrayArray / … — is + * just a value of `T`), so it is encoded byte-for-byte the same as `T`. Decode + * the inner `T` directly. + * + * Do NOT confuse it with `AggregateFunction(func, T)`, whose value is an opaque + * serialized aggregation STATE with a function-specific binary layout — see + * `./aggregateFunction.js`. + * + * Identity combinator, documentation only: + * + * readSimpleAggregateFunction(readUInt64) === readUInt64 + */ +export const readSimpleAggregateFunction = ( + readValue: Reader, +): Reader => readValue; diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/stream.ts b/skills/clickhouse-js-node-rowbinary-parser/src/stream.ts new file mode 100644 index 000000000..174c9393c --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/stream.ts @@ -0,0 +1,276 @@ +import { type Reader, Cursor } from "./core.js"; +import { readRows } from "./rows.js"; + +/** Empty buffer reused as the "no carry" sentinel between chunks. */ +const EMPTY_CHUNK = Buffer.alloc(0); + +/** Stats captured at the moment the small-chunk warning fires. */ +export interface SmallChunkStats { + /** Chunks consumed so far. */ + chunks: number; + /** Rows decoded so far. */ + rows: number; + /** `rows / chunks` — the ratio that tripped the threshold. */ + rowsPerChunk: number; +} + +/** + * Tuning for {@link streamRowBatches}'s small-chunk warning. Pass `false` to + * disable it, `true` / omit for the defaults, or an object to tune. + */ +export type WarnOnSmallChunks = + | boolean + | { + /** + * Warn when the running `rows / chunks` average drops below this. Default + * `2`: throw + restart re-decodes the partial trailing row on EVERY chunk, + * so once a chunk barely covers a row or two the re-scan dominates — the + * regime where `streamingRow.bench.ts` shows throw+restart losing to a lean + * generator. Keep it low so the warning only fires when chunks are + * genuinely too small, never on a healthy hundreds-of-rows-per-chunk stream. + */ + minRowsPerChunk?: number; + /** + * Don't evaluate until this many chunks have been seen. Default `16`: + * lets the average settle and suppresses the warning on small results, + * where the gotcha doesn't bite (it only matters at megabytes / millions + * of rows). A stream that ends before this never warns. + */ + warmupChunks?: number; + /** Where the warning goes. Default `console.warn`. */ + warn?: (message: string, stats: SmallChunkStats) => void; + }; + +/** Options for {@link streamRowBatches}. */ +export interface StreamRowBatchesOptions { + /** + * Diagnostic that catches a silent throughput killer: chunks so small that the + * throw+restart streaming strategy spends most of its time re-decoding the + * partial trailing row instead of making progress. Fires AT MOST ONCE per + * stream. On by default; see {@link WarnOnSmallChunks} to tune or disable. + * + * The fix it points at is usually upstream — raise the HTTP response's read + * size (Node sets the socket/stream `highWaterMark`; a fetch `Response.body` + * reader delivers larger chunks than a hand-rolled tiny read) into the + * tens–hundreds of KB range — or, when chunk size isn't yours to control, + * compose {@link coalesceChunks} in front to merge small chunks first. + */ + warnOnSmallChunks?: WarnOnSmallChunks; +} + +/** + * Stream a chunked `RowBinary` response into batches of decoded rows. This is + * the async front door built on {@link readRows}: feed it the byte chunks of an + * HTTP response (anything async-iterable — a Node `Readable`, `response.body`, + * etc.) and a per-row `Reader`, and `for await` the batches. + * + * One batch is yielded per incoming chunk — exactly the rows that completed + * within it — so batch size tracks chunk size, which the caller controls. A + * chunk that doesn't complete a new row yields nothing; its bytes are carried + * into the next chunk. Empty batches are never yielded. + * + * How it works (the carry-buffer driver): + * - Join the leftover `carry` from the previous chunk to the new chunk, build a + * state over the join, and run `readRows`. It decodes whole rows, stops cleanly + * on the partial trailing row (catching `NeedMoreData`), and leaves `pos` at + * that row's start. + * - The unread tail `pos..end` becomes the next `carry` as a `subarray` VIEW, + * NOT a copy. The joined buffer is owned entirely by this generator — it is + * never yielded to the caller — so there is no aliasing hazard in keeping a + * view into it, and we skip a per-chunk copy of the tail. The view is also + * short-lived: the next chunk's `Buffer.concat` copies these bytes into a + * fresh buffer, after which the old one is released. + * - When the stream ends, any non-empty carry means the response was truncated + * mid-row — a malformed stream — so it throws rather than silently dropping + * bytes. + * + * `readRow` is a `Reader` — write it as `(s) => ({ id: readUInt64(s), + * name: readString(s) })`. Build any configured/combinator readers ONCE (e.g. + * `const readRow = readTupleNamed({...})`) and reuse, rather than rebuilding them + * per chunk. + * + * ZERO-COPY NOTE: raw-bytes readers (`readUUID`/`readIPv6`/`readFixedStringBytes` + * and binary `String`) return views into the current chunk's joined buffer. Those + * stay valid as long as you hold the row objects, but are NOT views into one + * stable buffer across batches. If you retain them long-term, copy in `readRow`. + * + * BACKPRESSURE: this is a pull stream — the next chunk is only requested when the + * consumer asks for the next batch, so a slow consumer naturally throttles reading. + * + * The per-chunk bookkeeping for the small-chunk warning (two integer adds and a + * compare) runs once per CHUNK, not per row, so it is off every hot path; the + * default-on warning is documented in {@link StreamRowBatchesOptions}. + */ +export async function* streamRowBatches( + chunks: AsyncIterable, + readRow: Reader, + options?: StreamRowBatchesOptions, +): AsyncGenerator { + const drive = readRows(readRow); + let carry: Buffer = EMPTY_CHUNK; + + // Resolve the warning config once, outside the loop. + const warnCfg = options?.warnOnSmallChunks; + const warnEnabled = warnCfg !== false; + const warnObj = typeof warnCfg === "object" ? warnCfg : undefined; + const minRowsPerChunk = warnObj?.minRowsPerChunk ?? 2; + const warmupChunks = warnObj?.warmupChunks ?? 16; + const warn = warnObj?.warn ?? ((message: string) => console.warn(message)); + let chunkCount = 0; + let rowCount = 0; + let warned = false; + + for await (const chunk of chunks) { + // Normalize to a Buffer without copying (a Uint8Array shares its ArrayBuffer). + const incoming = Buffer.isBuffer(chunk) + ? chunk + : Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength); + const work = + carry.length === 0 ? incoming : Buffer.concat([carry, incoming]); + + const state = new Cursor(work); + const rows = drive(state); + if (rows.length > 0) yield rows; + + // Carry the unread tail (the partial trailing row, if any) to the next + // chunk. A view, not a copy: we own `work` and never expose it, so keeping a + // subarray into it is safe; the next concat copies these bytes out. + carry = state.pos < work.length ? work.subarray(state.pos) : EMPTY_CHUNK; + + if (warnEnabled && !warned) { + chunkCount++; + rowCount += rows.length; + const rowsPerChunk = rowCount / chunkCount; + if (chunkCount >= warmupChunks && rowsPerChunk < minRowsPerChunk) { + warned = true; + warn( + `RowBinary stream: chunks look too small — ${rowsPerChunk.toFixed(2)} rows/chunk over ${chunkCount} chunks. ` + + `Streaming throws + restarts the partial trailing row on every chunk, so tiny chunks spend most of their ` + + `time re-decoding instead of advancing. Increase the upstream read/highWaterMark to tens–hundreds of KB, ` + + `or compose coalesceChunks() in front of this stream to merge small chunks first.`, + { chunks: chunkCount, rows: rowCount, rowsPerChunk }, + ); + } + } + } + if (carry.length > 0) { + throw new Error( + `RowBinary stream ended mid-row: ${carry.length} trailing byte(s) left undecoded`, + ); + } +} + +/** A timeout result distinct from any `IteratorResult`. */ +const TIMED_OUT = Symbol("coalesceChunks.timeout"); + +/** + * Coalesce (debounce) a chunk stream so each emitted chunk is at least `minSize` + * bytes — a filter you compose IN FRONT of {@link streamRowBatches} when the + * source delivers chunks too small to stream efficiently and you can't enlarge + * them upstream: + * + * streamRowBatches(coalesceChunks(httpChunks, { minSize: 64 * 1024, timeoutMs: 50 }), readRow) + * + * WHY: the throw+restart streaming strategy re-decodes the partial trailing row + * on every chunk boundary, so the smaller the chunks the more time is wasted + * re-scanning (see `streamingRow.bench.ts`). Merging small chunks up front cuts + * the number of boundaries — and the backtracking with it. + * + * THE TRADE-OFF (latency vs. reallocation vs. backtracking): merging holds bytes + * back until enough accumulate, so it ADDS up to `timeoutMs` of latency to data + * that arrives in a trickle, and it COPIES via `Buffer.concat` to join the parts + * (one extra allocation per emitted chunk). In return the downstream parser + * backtracks far less. Tune `minSize` to the downstream sweet spot (tens–hundreds + * of KB) and `timeoutMs` to the latency you can spare. + * + * SEMANTICS: + * - Accumulates incoming chunks until their total reaches `minSize`, then emits + * the join immediately. + * - A batch below `minSize` is flushed early when `timeoutMs` elapses from the + * moment its FIRST byte arrived (the deadline is anchored, not reset per + * chunk — a steady trickle of tiny chunks can't defer the flush forever). + * - While nothing is buffered it blocks indefinitely for the next chunk: an idle + * or finished stream is never charged the timeout. + * - End of stream flushes whatever remains (possibly below `minSize`); a single + * already-large-enough chunk passes straight through with no copy. + * + * It keeps exactly ONE outstanding pull on the source at a time (never calls + * `next()` while a prior result is still in flight), reads one chunk ahead so it + * can race arrival against the timer, and releases the source via `return()` if + * the consumer abandons it early. + */ +export async function* coalesceChunks( + source: AsyncIterable, + { minSize, timeoutMs }: { minSize: number; timeoutMs: number }, +): AsyncGenerator { + const it = source[Symbol.asyncIterator](); + // The single in-flight pull. Read one ahead so we always have a promise to + // race the timer against; never start a second next() before this resolves. + let pull = it.next(); + let parts: Buffer[] = []; + let buffered = 0; + let deadline = 0; // ms timestamp; armed when the first byte enters an empty batch + + const asBuffer = (u8: Uint8Array): Buffer => + Buffer.isBuffer(u8) + ? u8 + : Buffer.from(u8.buffer, u8.byteOffset, u8.byteLength); + + const flush = (): Buffer => { + // One part: hand it back as-is (no concat, no copy). Many: join them. + const out = parts.length === 1 ? parts[0]! : Buffer.concat(parts, buffered); + parts = []; + buffered = 0; + return out; + }; + + const take = (u8: Uint8Array): void => { + const b = asBuffer(u8); + parts.push(b); + buffered += b.length; + }; + + try { + while (true) { + if (buffered === 0) { + // Nothing buffered: block for the next chunk with no timeout. + const r = await pull; + if (r.done) return; + take(r.value); + deadline = Date.now() + timeoutMs; + pull = it.next(); + if (buffered >= minSize) yield flush(); + continue; + } + + // Below minSize with bytes in hand: race the next chunk against the time + // left on this batch's anchored deadline. + const remaining = deadline - Date.now(); + if (remaining <= 0) { + yield flush(); + continue; + } + let timer: ReturnType | undefined; + const timeout = new Promise((resolve) => { + timer = setTimeout(() => resolve(TIMED_OUT), remaining); + }); + const r = await Promise.race([pull, timeout]); + clearTimeout(timer); // no-op if it already fired; frees the loop otherwise + if (r === TIMED_OUT) { + // pull is STILL outstanding — keep it; just flush what we have so far. + yield flush(); + continue; + } + if (r.done) { + yield flush(); // emit the tail; stream is over + return; + } + take(r.value); + pull = it.next(); + if (buffered >= minSize) yield flush(); + } + } finally { + // Consumer broke out early (break/throw): let the source clean up. + if (typeof it.return === "function") await it.return(); + } +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/strings.ts b/skills/clickhouse-js-node-rowbinary-parser/src/strings.ts new file mode 100644 index 000000000..681686173 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/strings.ts @@ -0,0 +1,55 @@ +import { type Reader, Cursor, advance } from "./core.js"; +import { readUVarint } from "./varint.js"; + +/** + * Read a `String`: a varint byte-length prefix followed by that many bytes, + * decoded as UTF-8. + * + * NOTE: ClickHouse `String` is arbitrary bytes, not guaranteed UTF-8. For binary + * columns, read `state.buf.subarray(start, start + len)` and skip the decode to + * keep the raw bytes. + */ +export function readString(state: Cursor): string { + const len = readUVarint(state); + const start = advance(state, len); + return state.buf.toString("utf8", start, start + len); +} + +/** + * Read a `FixedString(N)`: exactly `size` raw bytes, decoded as UTF-8. Curried: + * `readFixedString(N)` returns the reader. + * + * The value is right-padded with NUL bytes to `size`; those trailing `\x00` are + * part of the stored value and are preserved here. Trim them + * (`.replace(/\x00+$/, "")`) only if your column holds NUL-terminated text. + * + * ClickHouse server returns `FixedString`s in JSON with the trailing NULs, + * therefore this reader preserves them as well. + */ +export function readFixedString(size: number): Reader { + return (state) => { + const start = advance(state, size); + return state.buf.toString("utf8", start, start + size); + }; +} + +/** + * Read a `FixedString(N)` as raw bytes (no UTF-8 decode) — for binary columns. + * Curried: `readFixedStringBytes(N)` returns the reader. Returns a zero-copy + * view: no allocation, but the slice shares memory with the response, so + * retaining any one slice pins the entire chunk buffer in memory. + * + * SAFE TO TOGGLE — if the bytes outlive the row/response, return an independent + * copy instead so the chunk can be freed: + * + * // return Buffer.from(state.buf.subarray(start, start + size)); + * + * Make an educated tradeoff: view (default) when consumed immediately, a copy + * when retained. + */ +export function readFixedStringBytes(size: number): Reader { + return (state) => { + const start = advance(state, size); + return state.buf.subarray(start, start + size); + }; +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/time.ts b/skills/clickhouse-js-node-rowbinary-parser/src/time.ts new file mode 100644 index 000000000..87ea47a26 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/time.ts @@ -0,0 +1,61 @@ +import { type Reader, Cursor } from "./core.js"; +import { readInt32, readInt64 } from "./integers.js"; + +/** Semantic alias for `number` marking a seconds value (see {@link readTime}). */ +export type Seconds = number; + +/** + * A signed sub-second duration kept lossless as its raw parts: the value is + * `ticks / 10 ** precision` seconds. Used by `Time64` (a time-of-day duration, + * which has no natural JS type), carrying the precision so nothing is lost. + */ +export type ScaledTicks = readonly [ticks: bigint, precision: number]; + +/** + * Read a `Time`: 4-byte signed `Int32` seconds-of-day (range ±999:59:59). + * Returns the raw seconds; pass it to {@link formatTime}. + */ +export function readTime(state: Cursor): Seconds { + return readInt32(state); +} + +/** + * Read a `Time64(P)`: 8-byte signed `Int64` count of `10^-P`-second ticks. + * Curried: `readTime64(P)` returns the reader. Returns `[ticks, precision]` (a + * {@link ScaledTicks}); pass it to {@link formatTime64}. + */ +export function readTime64(precision: number): Reader { + return (state) => [readInt64(state), precision]; +} + +/** + * Format a `Time` value (signed seconds-of-day) as "[-]HH:MM:SS". The hour + * field can exceed two digits (the range is ±999:59:59). + */ +export function formatTime(seconds: Seconds): string { + const sign = seconds < 0 ? "-" : ""; + const s = Math.abs(seconds); + const hh = Math.floor(s / 3600); + const mm = Math.floor((s % 3600) / 60); + const ss = s % 60; + return `${sign}${String(hh).padStart(2, "0")}:${String(mm).padStart(2, "0")}:${String(ss).padStart(2, "0")}`; +} + +/** + * Format a `Time64` [ticks, precision] (signed sub-second time-of-day) as + * "[-]HH:MM:SS[.fff]". + */ +export function formatTime64([ticks, precision]: ScaledTicks): string { + const sign = ticks < 0n ? "-" : ""; + const t = ticks < 0n ? -ticks : ticks; + const scale = 10n ** BigInt(precision); + const totalSec = Number(t / scale); + const frac = t % scale; + const hh = Math.floor(totalSec / 3600); + const mm = Math.floor((totalSec % 3600) / 60); + const ss = totalSec % 60; + const base = `${sign}${String(hh).padStart(2, "0")}:${String(mm).padStart(2, "0")}:${String(ss).padStart(2, "0")}`; + return precision > 0 + ? `${base}.${frac.toString().padStart(precision, "0")}` + : base; +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/uuid.ts b/skills/clickhouse-js-node-rowbinary-parser/src/uuid.ts new file mode 100644 index 000000000..75f7c2360 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/uuid.ts @@ -0,0 +1,153 @@ +import { Cursor, advance } from "./core.js"; + +/** + * `UUID_HEX16[b]` packs the two lowercase ASCII hex chars of byte `b`, low char + * in the low byte. Drives the lookup-table UUID formatter {@link formatUUIDTable}. + */ +const UUID_HEX16 = new Uint16Array(256); +for (let b = 0; b < 256; b++) { + const hex = b.toString(16).padStart(2, "0"); + UUID_HEX16[b] = hex.charCodeAt(0) | (hex.charCodeAt(1) << 8); +} + +/** + * Reusable 36-byte scratch for {@link formatUUIDTable}. The four `-` separators + * are written once and never touched again; each call overwrites only the 32 + * hex slots, then copies the bytes out as a string. + */ +const UUID_OUT = Buffer.alloc(36); +UUID_OUT[8] = UUID_OUT[13] = UUID_OUT[18] = UUID_OUT[23] = 0x2d; // '-' + +/** + * Read a `UUID`: 16 raw bytes (two little-endian `UInt64` halves on the wire). + * Returns a zero-copy view; pass it to {@link formatUUID} for the canonical + * `xxxxxxxx-...` string. + * + * The view shares memory with the response buffer, so keeping it alive pins the + * whole chunk; copy with `Buffer.from(...)` if it must outlive the row. + * + * FAST ALTERNATIVE: if you stringify every UUID, use {@link formatUUIDTable} + * (lookup table, no BigInt, ~1.6x faster). + */ +export function readUUID(state: Cursor): Buffer { + const start = advance(state, 16); + return state.buf.subarray(start, start + 16); +} + +/** + * Read a `UUID` as a single 128-bit `bigint` (`hi << 64 | lo`) — useful for + * numeric storage, comparison, or de-duplication without a string. + * + * Reads the halves with `DataView.getBigUint64` rather than + * `Buffer.readBigUInt64LE`: V8 inlines the DataView accessors, measurably faster + * for 8-byte reads. For the canonical string, use {@link readUUID} + {@link formatUUID}. + */ +export function readUUIDBigInt(state: Cursor): bigint { + const start = advance(state, 16); + const hi = state.view.getBigUint64(start, true); + const lo = state.view.getBigUint64(start + 8, true); + return (hi << 64n) | lo; +} + +/** + * Read a `UUID` as its two raw little-endian `UInt64` halves, `[hi, lo]` — the + * faithful wire split with no combining work. Cheaper than {@link readUUIDBigInt} + * (skips `hi << 64 | lo`) and a compact two-value key for comparison/dedup. For + * the canonical string, use {@link readUUID} + {@link formatUUID}. + */ +export function readUUIDHiLo(state: Cursor): [hi: bigint, lo: bigint] { + const start = advance(state, 16); + const hi = state.view.getBigUint64(start, true); + const lo = state.view.getBigUint64(start + 8, true); + return [hi, lo]; +} + +/** + * Format a `UUID` (raw 16 bytes from {@link readUUID}) as the canonical + * `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` string. + * + * THE TRAP: ClickHouse stores a UUID as two little-endian `UInt64` halves (high + * then low), so each half is byte-reversed vs the text form. Reading each half + * with `readBigUInt64LE` undoes that; concatenating high then low gives the 32 + * canonical hex digits. (Hexing the 16 bytes in wire order scrambles the value.) + * Kept aside from the read so the hot path can skip stringifying when raw bytes + * suffice. + * + * FAST ALTERNATIVE: to format every value, {@link formatUUIDTable} does the same + * via a byte->hex lookup table with no BigInt (~1.6x faster). + */ +export function formatUUID(b: Buffer): string { + const hex = ((b.readBigUInt64LE(0) << 64n) | b.readBigUInt64LE(8)) + .toString(16) + .padStart(32, "0"); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +} + +/** + * Fast {@link formatUUID}: same canonical string via a byte -> two-hex-char + * lookup table (`UUID_HEX16`) written into a reused 36-byte buffer (`UUID_OUT`, + * dashes preset), no BigInt, no slicing. ~1.6x faster (see `readUUID.bench.ts`). + * Takes the raw 16 bytes from {@link readUUID}. + * + * Same byte-reversal as formatUUID: emit the high half in reverse (`b[7]..b[0]`) + * then the low half (`b[15]..b[8]`). + * + * SAFE TO TOGGLE — opt-in fast formatter, not the default. `UUID_OUT` is shared + * scratch, so NOT reentrant; safe for synchronous formatting because the bytes + * are copied into the returned string before the next call (don't alias + * `UUID_OUT`). Worth it only when you stringify every UUID. + */ +export function formatUUIDTable(b: Buffer): string { + let p: number; + // High half: bytes b[7]..b[0] -> hex positions 0..7 (chars 0..15). + p = UUID_HEX16[b[7]!]!; + UUID_OUT[0] = p & 0xff; + UUID_OUT[1] = p >>> 8; + p = UUID_HEX16[b[6]!]!; + UUID_OUT[2] = p & 0xff; + UUID_OUT[3] = p >>> 8; + p = UUID_HEX16[b[5]!]!; + UUID_OUT[4] = p & 0xff; + UUID_OUT[5] = p >>> 8; + p = UUID_HEX16[b[4]!]!; + UUID_OUT[6] = p & 0xff; + UUID_OUT[7] = p >>> 8; + p = UUID_HEX16[b[3]!]!; + UUID_OUT[9] = p & 0xff; + UUID_OUT[10] = p >>> 8; + p = UUID_HEX16[b[2]!]!; + UUID_OUT[11] = p & 0xff; + UUID_OUT[12] = p >>> 8; + p = UUID_HEX16[b[1]!]!; + UUID_OUT[14] = p & 0xff; + UUID_OUT[15] = p >>> 8; + p = UUID_HEX16[b[0]!]!; + UUID_OUT[16] = p & 0xff; + UUID_OUT[17] = p >>> 8; + // Low half: bytes b[15]..b[8] -> hex positions 8..15 (chars 19..35). + p = UUID_HEX16[b[15]!]!; + UUID_OUT[19] = p & 0xff; + UUID_OUT[20] = p >>> 8; + p = UUID_HEX16[b[14]!]!; + UUID_OUT[21] = p & 0xff; + UUID_OUT[22] = p >>> 8; + p = UUID_HEX16[b[13]!]!; + UUID_OUT[24] = p & 0xff; + UUID_OUT[25] = p >>> 8; + p = UUID_HEX16[b[12]!]!; + UUID_OUT[26] = p & 0xff; + UUID_OUT[27] = p >>> 8; + p = UUID_HEX16[b[11]!]!; + UUID_OUT[28] = p & 0xff; + UUID_OUT[29] = p >>> 8; + p = UUID_HEX16[b[10]!]!; + UUID_OUT[30] = p & 0xff; + UUID_OUT[31] = p >>> 8; + p = UUID_HEX16[b[9]!]!; + UUID_OUT[32] = p & 0xff; + UUID_OUT[33] = p >>> 8; + p = UUID_HEX16[b[8]!]!; + UUID_OUT[34] = p & 0xff; + UUID_OUT[35] = p >>> 8; + return UUID_OUT.toString("latin1"); +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/src/varint.ts b/skills/clickhouse-js-node-rowbinary-parser/src/varint.ts new file mode 100644 index 000000000..323b1f2d9 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/src/varint.ts @@ -0,0 +1,70 @@ +import { Cursor, advance } from "./core.js"; + +/** + * Read a LEB128 unsigned varint (used for string/array lengths). + * + * Returns a JS `number`, so it is NOT bigint-friendly: only values up to + * `Number.MAX_SAFE_INTEGER` (2^53 - 1) are representable exactly. A varint + * larger than that throws rather than silently losing precision. RowBinary + * lengths never approach this in practice. + * + * The loop is unrolled: each byte carries 7 bits, so its place value is the + * constant 2^(7*k). The overwhelmingly common 1–2 byte case costs one or two + * reads and a compare. + * + * Multipliers must stay as `*` (not `<<`): JS bitwise shift is 32-bit and would wrap past bit 31. + * + * SAFE TO TOGGLE — how many bytes to handle: + * - If you know the maximum blob/array size, keep only the steps you need and + * delete the rest along with the overflow guard. E.g. lengths < 2^28 fit in + * 4 bytes, so everything below the `* 268435456` step can go. + * - Keep all eight steps (the default) when lengths are untrusted. + * If you genuinely need lengths beyond 2^53, create a bigint version of this + * function with a bigint accumulator instead of removing the guard. + * + * OPTIMIZATION HINT — for a known invariant, emit a dedicated named variant + * rather than toggling here. E.g. a `readUVarint32` for lengths guaranteed to be + * 32-bit would unroll only the first five bytes and throw past 2^32 - 1. + */ +export function readUVarint(state: Cursor): number { + // Each byte reserves its space through `advance(1)` (the bounds check), but + // the read itself stays inlined as `state.buf[...]` rather than calling + // readUInt8 — this is the hottest loop in the reader. + let byte = state.buf[advance(state, 1)]!; + if (byte < 0x80) return byte; // 1 byte -> 2^0 + let result = byte & 0x7f; + + byte = state.buf[advance(state, 1)]!; + if (byte < 0x80) return result + byte * 128; // 2^7 + result += (byte & 0x7f) * 128; + + byte = state.buf[advance(state, 1)]!; + if (byte < 0x80) return result + byte * 16384; // 2^14 + result += (byte & 0x7f) * 16384; + + byte = state.buf[advance(state, 1)]!; + if (byte < 0x80) return result + byte * 2097152; // 2^21 + result += (byte & 0x7f) * 2097152; + + byte = state.buf[advance(state, 1)]!; + if (byte < 0x80) return result + byte * 268435456; // 2^28 + result += (byte & 0x7f) * 268435456; + + byte = state.buf[advance(state, 1)]!; + if (byte < 0x80) return result + byte * 34359738368; // 2^35 + result += (byte & 0x7f) * 34359738368; + + byte = state.buf[advance(state, 1)]!; + if (byte < 0x80) return result + byte * 4398046511104; // 2^42 + result += (byte & 0x7f) * 4398046511104; + + // 8th byte: only its low 4 payload bits (bits 49..52) fit under 2^53. A larger + // payload, or a continuation bit signalling a 9th byte, overflows MAX_SAFE_INTEGER. + byte = state.buf[advance(state, 1)]!; + if (byte > 0x0f) { + throw new RangeError( + "RowBinary: varint exceeds Number.MAX_SAFE_INTEGER (2^53 - 1)", + ); + } + return result + byte * 562949953421312; // 2^49 +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Array.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Array.test.ts new file mode 100644 index 000000000..e76647f78 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Array.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { readArray, readNullable } from "../src/composite.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUInt32, readUInt8 } from "../src/integers.js"; +import { readString } from "../src/strings.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readArray", () => { + it("decodes a fixed-width element array", async () => { + const r = await reader("CAST([1, 2, 3] AS Array(UInt32))"); + expect(readArray(readUInt32)(r)).toEqual([1, 2, 3]); + expect(r.pos).toBe(13); // 1 count + 3 * 4 + }); + + it("decodes the empty array (just the count byte)", async () => { + const r = await reader("CAST([] AS Array(UInt32))"); + expect(readArray(readUInt32)(r)).toEqual([]); + expect(r.pos).toBe(1); + }); + + it("decodes a variable-length element array", async () => { + const r = await reader("CAST(['a', 'bb'] AS Array(String))"); + expect(readArray(readString)(r)).toEqual(["a", "bb"]); + }); + + // Nesting composes by nesting the element reader. + it("decodes Array(Array(UInt8))", async () => { + const r = await reader("CAST([[1], [2, 3]] AS Array(Array(UInt8)))"); + expect(readArray(readArray(readUInt8))(r)).toEqual([[1], [2, 3]]); + }); + + // Composes with Nullable: the NULL element is just its flag byte. + it("decodes Array(Nullable(UInt8)) with a NULL element", async () => { + const r = await reader("CAST([1, NULL, 3] AS Array(Nullable(UInt8)))"); + expect(readArray(readNullable(readUInt8))(r)).toEqual([1, null, 3]); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST([1, 2, 3] AS Array(UInt32)) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readArray(readUInt32)(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/BFloat16.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/BFloat16.test.ts new file mode 100644 index 000000000..b375dae05 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/BFloat16.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readBFloat16 } from "../src/floats.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readBFloat16", () => { + it("decodes 0", async () => { + const r = await reader("toBFloat16(0)"); + expect(readBFloat16(r)).toBe(0); + expect(r.pos).toBe(2); + }); + + // Values whose float32 mantissa fits in BFloat16's 7 bits, so they survive + // the round-trip exactly. + it("decodes 1.5", async () => { + expect(readBFloat16(await reader("toBFloat16(1.5)"))).toBe(1.5); + }); + + it("decodes -2.5", async () => { + expect(readBFloat16(await reader("toBFloat16(-2.5)"))).toBe(-2.5); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toBFloat16(1.5) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readBFloat16(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Bool.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Bool.test.ts new file mode 100644 index 000000000..afedd3eed --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Bool.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { readBool } from "../src/bool.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readBool", () => { + it("decodes true", async () => { + const r = await reader("true"); + expect(readBool(r)).toBe(true); + expect(r.pos).toBe(1); + }); + + it("decodes false", async () => { + expect(readBool(await reader("false"))).toBe(false); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT true FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readBool(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Date.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Date.test.ts new file mode 100644 index 000000000..7128474ea --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Date.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readDate } from "../src/datetime.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readDate", () => { + it("decodes to a JS Date at UTC midnight", async () => { + const r = await reader("toDate('2021-03-15')"); + const d = readDate(r); + expect(d.toISOString()).toBe("2021-03-15T00:00:00.000Z"); + expect(r.pos).toBe(2); + }); + + it("decodes the epoch", async () => { + const d = readDate(await reader("toDate('1970-01-01')")); + expect(d.toISOString()).toBe("1970-01-01T00:00:00.000Z"); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toDate('2021-03-15') FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readDate(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Date32.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Date32.test.ts new file mode 100644 index 000000000..05b0ec414 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Date32.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readDate32 } from "../src/datetime.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readDate32", () => { + it("decodes a pre-1970 date (negative days) to a JS Date", async () => { + const r = await reader("toDate32('1950-01-01')"); + const d = readDate32(r); + expect(d.toISOString()).toBe("1950-01-01T00:00:00.000Z"); + expect(r.pos).toBe(4); + }); + + it("decodes a post-1970 date", async () => { + const d = readDate32(await reader("toDate32('2021-03-15')")); + expect(d.toISOString()).toBe("2021-03-15T00:00:00.000Z"); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT toDate32('1950-01-01') FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readDate32(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/DateTime.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/DateTime.test.ts new file mode 100644 index 000000000..c65ca1c98 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/DateTime.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readDateTime } from "../src/datetime.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readDateTime", () => { + it("decodes Unix seconds to a JS Date", async () => { + const r = await reader("toDateTime('2021-01-01 00:00:00', 'UTC')"); + const d = readDateTime(r); + expect(d.toISOString()).toBe("2021-01-01T00:00:00.000Z"); + expect(d.getTime()).toBe(1609459200000); + expect(r.pos).toBe(4); + }); + + it("decodes the epoch", async () => { + const d = readDateTime( + await reader("toDateTime('1970-01-01 00:00:00', 'UTC')"), + ); + expect(d.getTime()).toBe(0); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT toDateTime('2021-01-01 00:00:00', 'UTC') FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readDateTime(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/DateTime64.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/DateTime64.test.ts new file mode 100644 index 000000000..0857d474f --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/DateTime64.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readDateTime64 } from "../src/datetime.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readDateTime64", () => { + it("decodes P=3 to [Date (whole seconds), nanoseconds]", async () => { + const r = await reader("toDateTime64('2021-01-01 00:00:00.123', 3, 'UTC')"); + const [date, nanos] = readDateTime64(3)(r); + expect(date.toISOString()).toBe("2021-01-01T00:00:00.000Z"); + expect(nanos).toBe(123_000_000); + expect(r.pos).toBe(8); + }); + + it("keeps nanosecond precision (P=9) that a Date alone can't hold", async () => { + const [date, nanos] = readDateTime64(9)( + await reader("toDateTime64('2021-01-01 00:00:00.123456789', 9, 'UTC')"), + ); + expect(date.toISOString()).toBe("2021-01-01T00:00:00.000Z"); + expect(nanos).toBe(123456789); + }); + + it("decodes P=0 with a zero fraction", async () => { + const [date, nanos] = readDateTime64(0)( + await reader("toDateTime64('2021-01-01 00:00:00', 0, 'UTC')"), + ); + expect(date.toISOString()).toBe("2021-01-01T00:00:00.000Z"); + expect(nanos).toBe(0); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT toDateTime64('2021-01-01 00:00:00.123', 3, 'UTC') FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readDateTime64(3)(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/DateTime64P3.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/DateTime64P3.test.ts new file mode 100644 index 000000000..1c9fa5745 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/DateTime64P3.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readDateTime64P3 } from "../src/datetime.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readDateTime64P3", () => { + // P=3 is exactly Date's resolution: the millisecond instant is lossless in + // a single Date, no separate fraction. + it("decodes milliseconds straight into a JS Date", async () => { + const r = await reader("toDateTime64('2021-01-01 00:00:00.123', 3, 'UTC')"); + const d = readDateTime64P3(r); + expect(d.toISOString()).toBe("2021-01-01T00:00:00.123Z"); + expect(r.pos).toBe(8); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT toDateTime64('2021-01-01 00:00:00.123', 3, 'UTC') FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readDateTime64P3(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/DateTime64P6.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/DateTime64P6.test.ts new file mode 100644 index 000000000..7c92da007 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/DateTime64P6.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readDateTime64P6 } from "../src/datetime.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readDateTime64P6", () => { + it("decodes microseconds to [Date (whole seconds), microseconds]", async () => { + const r = await reader( + "toDateTime64('2021-01-01 00:00:00.123456', 6, 'UTC')", + ); + const [date, micros] = readDateTime64P6(r); + expect(date.toISOString()).toBe("2021-01-01T00:00:00.000Z"); + expect(micros).toBe(123456); // native microseconds + expect(r.pos).toBe(8); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT toDateTime64('2021-01-01 00:00:00.123456', 6, 'UTC') FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readDateTime64P6(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/DateTime64P9.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/DateTime64P9.test.ts new file mode 100644 index 000000000..59e646b0a --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/DateTime64P9.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readDateTime64P9 } from "../src/datetime.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readDateTime64P9", () => { + it("decodes nanoseconds to [Date (whole seconds), nanoseconds]", async () => { + const r = await reader( + "toDateTime64('2021-01-01 00:00:00.123456789', 9, 'UTC')", + ); + const [date, nanos] = readDateTime64P9(r); + expect(date.toISOString()).toBe("2021-01-01T00:00:00.000Z"); + expect(nanos).toBe(123456789); + expect(r.pos).toBe(8); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT toDateTime64('2021-01-01 00:00:00.123456789', 9, 'UTC') FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readDateTime64P9(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Decimal128.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Decimal128.test.ts new file mode 100644 index 000000000..465169f29 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Decimal128.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { formatDecimal, readDecimal128 } from "../src/decimals.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readDecimal128", () => { + it("decodes -123.456789 at scale 6", async () => { + const r = await reader("toDecimal128('-123.456789', 6)"); + const dec = readDecimal128(6)(r); + expect(dec).toEqual([-123456789n, 6]); + expect(r.pos).toBe(16); + expect(formatDecimal(dec)).toBe("-123.456789"); + }); + + // Unscaled = 2^63, beyond Int64 range — exercises the 128-bit composition. + it("decodes a value whose unscaled int exceeds 64 bits", async () => { + const r = await reader("toDecimal128('92233720368547758.08', 2)"); + const dec = readDecimal128(2)(r); + expect(dec).toEqual([9223372036854775808n, 2]); + expect(formatDecimal(dec)).toBe("92233720368547758.08"); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT toDecimal128('-123.456789', 6) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readDecimal128(6)(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Decimal256.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Decimal256.test.ts new file mode 100644 index 000000000..fca855973 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Decimal256.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { formatDecimal, readDecimal256 } from "../src/decimals.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readDecimal256", () => { + it("decodes -1 at scale 0", async () => { + const r = await reader("toDecimal256('-1', 0)"); + const dec = readDecimal256(0)(r); + expect(dec).toEqual([-1n, 0]); + expect(r.pos).toBe(32); + expect(formatDecimal(dec)).toBe("-1"); + }); + + // A large unscaled magnitude that only fits in 256 bits. + it("decodes a large value at scale 10", async () => { + const r = await reader( + "toDecimal256('123456789012345678901234567890.0123456789', 10)", + ); + const dec = readDecimal256(10)(r); + expect(dec).toEqual([1234567890123456789012345678900123456789n, 10]); + expect(formatDecimal(dec)).toBe( + "123456789012345678901234567890.0123456789", + ); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toDecimal256('-1', 0) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readDecimal256(0)(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Decimal32.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Decimal32.test.ts new file mode 100644 index 000000000..170954007 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Decimal32.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { formatDecimal, readDecimal32 } from "../src/decimals.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readDecimal32", () => { + it("decodes 1.5 at scale 4 as [unscaled, scale]", async () => { + const r = await reader("toDecimal32(1.5, 4)"); + const dec = readDecimal32(4)(r); + expect(dec).toEqual([15000n, 4]); + expect(r.pos).toBe(4); + // formatDecimal keeps the trailing zeros (CH text would show "1.5"). + expect(formatDecimal(dec)).toBe("1.5000"); + }); + + it("keeps a pure fraction lossless", async () => { + const dec = readDecimal32(3)(await reader("toDecimal32(0.005, 3)")); + expect(dec).toEqual([5n, 3]); + expect(formatDecimal(dec)).toBe("0.005"); + }); + + it("decodes a negative value", async () => { + const dec = readDecimal32(4)(await reader("toDecimal32(-1.5, 4)")); + expect(dec).toEqual([-15000n, 4]); + expect(formatDecimal(dec)).toBe("-1.5000"); + }); + + it("decodes scale 0 (formats with no decimal point)", async () => { + const dec = readDecimal32(0)(await reader("toDecimal32(42, 0)")); + expect(dec).toEqual([42n, 0]); + expect(formatDecimal(dec)).toBe("42"); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toDecimal32(1.5, 4) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readDecimal32(4)(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Decimal64.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Decimal64.test.ts new file mode 100644 index 000000000..b82275a83 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Decimal64.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { formatDecimal, readDecimal64 } from "../src/decimals.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readDecimal64", () => { + it("decodes -12.34 at scale 2", async () => { + const r = await reader("toDecimal64(-12.34, 2)"); + const dec = readDecimal64(2)(r); + expect(dec).toEqual([-1234n, 2]); + expect(r.pos).toBe(8); + expect(formatDecimal(dec)).toBe("-12.34"); + }); + + it("keeps the declared scale's trailing zero in the raw value", async () => { + const dec = readDecimal64(2)(await reader("toDecimal64(1.20, 2)")); + expect(dec).toEqual([120n, 2]); + // CH text would show "1.2"; formatDecimal keeps the scale: "1.20". + expect(formatDecimal(dec)).toBe("1.20"); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT toDecimal64(-12.34, 2) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readDecimal64(2)(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Dynamic.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Dynamic.test.ts new file mode 100644 index 000000000..4b5242688 Binary files /dev/null and b/skills/clickhouse-js-node-rowbinary-parser/tests/Dynamic.test.ts differ diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Enum16.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Enum16.test.ts new file mode 100644 index 000000000..932b3aada --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Enum16.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readEnum16 } from "../src/enums.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readEnum16", () => { + it("decodes a 16-bit underlying value", async () => { + const r = await reader("CAST('big' AS Enum16('small' = 1, 'big' = 300))"); + const value = readEnum16(r); + expect(value).toBe(300); + expect(r.pos).toBe(2); + }); + + it("decodes a negative enum value", async () => { + const value = readEnum16( + await reader("CAST('lo' AS Enum16('lo' = -1000, 'hi' = 1000))"), + ); + expect(value).toBe(-1000); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST('big' AS Enum16('small' = 1, 'big' = 300)) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readEnum16(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Enum8.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Enum8.test.ts new file mode 100644 index 000000000..84fe3e605 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Enum8.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readEnum8 } from "../src/enums.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readEnum8", () => { + it("decodes the underlying value and resolves the name via a lookup", async () => { + const r = await reader("CAST('b' AS Enum8('a' = 1, 'b' = 2))"); + const value = readEnum8(r); + expect(value).toBe(2); + expect(r.pos).toBe(1); + // The name map comes from the column's type definition, not the wire. + const NAMES: Record = { 1: "a", 2: "b" }; + expect(NAMES[value]).toBe("b"); + }); + + it("decodes a negative enum value", async () => { + const value = readEnum8( + await reader("CAST('x' AS Enum8('x' = -1, 'y' = 2))"), + ); + expect(value).toBe(-1); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST('b' AS Enum8('a' = 1, 'b' = 2)) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readEnum8(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/FixedString.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/FixedString.test.ts new file mode 100644 index 000000000..eab2e985f --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/FixedString.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readFixedString } from "../src/strings.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readFixedString", () => { + it("decodes a full-width value (no padding)", async () => { + const r = await reader("toFixedString('abcd', 4)"); + expect(readFixedString(4)(r)).toBe("abcd"); + expect(r.pos).toBe(4); + }); + + // Shorter content is right-padded with NUL bytes, which are preserved. + it("preserves trailing NUL padding", async () => { + expect(readFixedString(4)(await reader("toFixedString('ab', 4)"))).toBe( + "ab\x00\x00", + ); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT toFixedString('ab', 4) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readFixedString(4)(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/FixedStringBytes.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/FixedStringBytes.test.ts new file mode 100644 index 000000000..99a30fe91 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/FixedStringBytes.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readFixedStringBytes } from "../src/strings.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readFixedStringBytes", () => { + it("returns the raw bytes, padding included", async () => { + const r = await reader("toFixedString('ab', 4)"); + expect(readFixedStringBytes(4)(r)).toEqual(Buffer.from([0x61, 0x62, 0, 0])); + expect(r.pos).toBe(4); + }); + + // The default is a zero-copy view, so it shares memory with the source. + it("returns a zero-copy view sharing memory", async () => { + const r = await reader("toFixedString('ab', 4)"); + const bytes = readFixedStringBytes(4)(r); + r.buf[0] = 0xff; + expect(bytes[0]).toBe(0xff); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT toFixedString('ab', 4) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readFixedStringBytes(4)(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Float32.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Float32.test.ts new file mode 100644 index 000000000..6ce0dae1a --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Float32.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readFloat32 } from "../src/floats.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readFloat32", () => { + it("decodes 0", async () => { + const r = await reader("toFloat32(0)"); + expect(readFloat32(r)).toBe(0); + expect(r.pos).toBe(4); + }); + + // Values exactly representable in float32, so no rounding to account for. + it("decodes 1.5", async () => { + expect(readFloat32(await reader("toFloat32(1.5)"))).toBe(1.5); + }); + + it("decodes -2.5", async () => { + expect(readFloat32(await reader("toFloat32(-2.5)"))).toBe(-2.5); + }); + + it("decodes inf", async () => { + expect(readFloat32(await reader("toFloat32(inf)"))).toBe(Infinity); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toFloat32(1.5) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readFloat32(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Float64.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Float64.test.ts new file mode 100644 index 000000000..7de12973a --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Float64.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readFloat64 } from "../src/floats.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readFloat64", () => { + it("decodes 0", async () => { + const r = await reader("toFloat64(0)"); + expect(readFloat64(r)).toBe(0); + expect(r.pos).toBe(8); + }); + + it("decodes 1.5", async () => { + expect(readFloat64(await reader("toFloat64(1.5)"))).toBe(1.5); + }); + + // float64 represents 0.1 exactly as the same double JS uses. + it("decodes 0.1", async () => { + expect(readFloat64(await reader("toFloat64(0.1)"))).toBe(0.1); + }); + + it("decodes -inf", async () => { + expect(readFloat64(await reader("toFloat64(-inf)"))).toBe(-Infinity); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toFloat64(1.5) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readFloat64(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Geometry.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Geometry.test.ts new file mode 100644 index 000000000..28cd86d3b --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Geometry.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readGeometry } from "../src/geo.js"; + +// Geometry's variant has "similar" alternatives (LineString/Ring), so the type +// needs allow_suspicious_variant_types; the value still casts through a geo type. +async function reader(expr: string): Promise { + return new Cursor( + await query( + `SELECT ${expr} SETTINGS allow_suspicious_variant_types = 1 FORMAT RowBinary`, + ), + ); +} + +describe("readGeometry", () => { + it("decodes a Point (discriminant 3)", async () => { + const r = await reader("CAST(CAST((1.5, 2.5) AS Point) AS Geometry)"); + expect(readGeometry(r)).toEqual([1.5, 2.5]); + }); + + it("decodes a LineString (discriminant 0)", async () => { + const r = await reader( + "CAST(CAST([(0, 0), (1, 2)] AS LineString) AS Geometry)", + ); + expect(readGeometry(r)).toEqual([ + [0, 0], + [1, 2], + ]); + }); + + it("decodes a MultiPolygon (discriminant 2)", async () => { + const r = await reader( + "CAST(CAST([[[(0, 0), (1, 0), (1, 1)]]] AS MultiPolygon) AS Geometry)", + ); + expect(readGeometry(r)).toEqual([ + [ + [ + [0, 0], + [1, 0], + [1, 1], + ], + ], + ]); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST(CAST((1.5, 2.5) AS Point) AS Geometry) SETTINGS allow_suspicious_variant_types = 1 FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readGeometry(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/IPv4.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/IPv4.test.ts new file mode 100644 index 000000000..90db4a2e8 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/IPv4.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { formatIPv4, readIPv4 } from "../src/ip.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readIPv4", () => { + it("decodes the raw UInt32 and formats a dotted quad", async () => { + const r = await reader("toIPv4('1.2.3.4')"); + const value = readIPv4(r); + expect(value).toBe(0x01020304); + expect(r.pos).toBe(4); + expect(formatIPv4(value)).toBe("1.2.3.4"); + }); + + it("decodes 0.0.0.0", async () => { + const value = readIPv4(await reader("toIPv4('0.0.0.0')")); + expect(value).toBe(0); + expect(formatIPv4(value)).toBe("0.0.0.0"); + }); + + it("decodes 255.255.255.255", async () => { + const value = readIPv4(await reader("toIPv4('255.255.255.255')")); + expect(value).toBe(0xffffffff); + expect(formatIPv4(value)).toBe("255.255.255.255"); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toIPv4('1.2.3.4') FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readIPv4(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/IPv6.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/IPv6.test.ts new file mode 100644 index 000000000..672e23f9f --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/IPv6.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { formatIPv6, readIPv6 } from "../src/ip.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readIPv6", () => { + it("returns the raw 16 bytes and formats loopback ::1", async () => { + const r = await reader("toIPv6('::1')"); + const bytes = readIPv6(r); + expect(r.pos).toBe(16); + expect(bytes).toEqual( + Buffer.from([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]), + ); + expect(formatIPv6(bytes)).toBe("::1"); + }); + + it("formats the all-zero address ::", async () => { + expect(formatIPv6(readIPv6(await reader("toIPv6('::')")))).toBe("::"); + }); + + it("collapses the longest zero run", async () => { + expect(formatIPv6(readIPv6(await reader("toIPv6('2001:db8::1')")))).toBe( + "2001:db8::1", + ); + }); + + // Two zero runs: the longer one (positions 4-6) is collapsed, not the first. + it("collapses the longest run, not the leftmost", async () => { + expect( + formatIPv6(readIPv6(await reader("toIPv6('1:0:0:2:0:0:0:3')"))), + ).toBe("1:0:0:2::3"); + }); + + it("leaves a fully-populated address uncompressed", async () => { + expect( + formatIPv6(readIPv6(await reader("toIPv6('2001:db8:1:2:3:4:5:6')"))), + ).toBe("2001:db8:1:2:3:4:5:6"); + }); + + it("renders IPv4-mapped addresses as ::ffff:a.b.c.d", async () => { + expect(formatIPv6(readIPv6(await reader("toIPv6('::ffff:1.2.3.4')")))).toBe( + "::ffff:1.2.3.4", + ); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toIPv6('2001:db8::1') FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readIPv6(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Int128.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Int128.test.ts new file mode 100644 index 000000000..81d69d7af --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Int128.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readInt128 } from "../src/integers.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +const MAX = 170141183460469231731687303715884105727n; // 2^127 - 1 +const MIN = -170141183460469231731687303715884105728n; // -2^127 + +describe("readInt128", () => { + it("decodes 0n", async () => { + const r = await reader("toInt128(0)"); + expect(readInt128(r)).toBe(0n); + expect(r.pos).toBe(16); + }); + + it("decodes -1n (all 0xff, sign spans both words)", async () => { + expect(readInt128(await reader("toInt128('-1')"))).toBe(-1n); + }); + + it("decodes the max (2^127 - 1)", async () => { + expect(readInt128(await reader(`toInt128('${MAX}')`))).toBe(MAX); + }); + + it("decodes the min (-2^127)", async () => { + expect(readInt128(await reader(`toInt128('${MIN}')`))).toBe(MIN); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toInt128(-5) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readInt128(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Int16.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Int16.test.ts new file mode 100644 index 000000000..0d1641d3e --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Int16.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readInt16 } from "../src/integers.js"; + +/** + * Int16 is 2 bytes, little-endian, two's-complement. Each case selects the + * value with `FORMAT RowBinary` and decodes the bytes the server produces. + */ +async function int16Reader(expr: string): Promise { + return new Cursor(await query(`SELECT toInt16(${expr}) FORMAT RowBinary`)); +} + +describe("readInt16", () => { + it("decodes 0", async () => { + const r = await int16Reader("0"); + expect(readInt16(r)).toBe(0); + expect(r.pos).toBe(2); + }); + + it("decodes 1", async () => { + const r = await int16Reader("1"); + expect(readInt16(r)).toBe(1); + }); + + it("decodes -1 (0xffff)", async () => { + const r = await int16Reader("-1"); + expect(readInt16(r)).toBe(-1); + }); + + // Confirms little-endian byte order: 258 = 0x0102 -> bytes 02 01. + it("decodes 258 (little-endian byte order)", async () => { + const r = await int16Reader("258"); + expect(readInt16(r)).toBe(258); + }); + + it("decodes 32767 (max)", async () => { + const r = await int16Reader("32767"); + expect(readInt16(r)).toBe(32767); + }); + + it("decodes -32768 (min)", async () => { + const r = await int16Reader("-32768"); + expect(readInt16(r)).toBe(-32768); + }); + + // Guards the byteOffset handling: a Buffer that is a window into a larger + // ArrayBuffer (nonzero byteOffset) must still decode correctly. + it("decodes from a buffer window with a nonzero byteOffset", () => { + const ab = Uint8Array.from([0xaa, 0xbb, 0xcc, 0x02, 0x01]).buffer; // 258 at offset 3 + const sub = Buffer.from(ab, 3, 2); + expect(sub.byteOffset).toBe(3); + expect(readInt16(new Cursor(sub))).toBe(258); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toInt16(-12345) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readInt16(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Int256.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Int256.test.ts new file mode 100644 index 000000000..52af1bcb6 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Int256.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readInt256 } from "../src/integers.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +// 2^255 - 1 and -2^255 +const MAX = + 57896044618658097711785492504343953926634992332820282019728792003956564819967n; +const MIN = + -57896044618658097711785492504343953926634992332820282019728792003956564819968n; + +describe("readInt256", () => { + it("decodes 0n", async () => { + const r = await reader("toInt256(0)"); + expect(readInt256(r)).toBe(0n); + expect(r.pos).toBe(32); + }); + + it("decodes -1n (all 0xff, sign spans all four words)", async () => { + expect(readInt256(await reader("toInt256('-1')"))).toBe(-1n); + }); + + it("decodes the max (2^255 - 1)", async () => { + expect(readInt256(await reader(`toInt256('${MAX}')`))).toBe(MAX); + }); + + it("decodes the min (-2^255)", async () => { + expect(readInt256(await reader(`toInt256('${MIN}')`))).toBe(MIN); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toInt256(-5) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readInt256(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Int32.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Int32.test.ts new file mode 100644 index 000000000..8bacd22c7 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Int32.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readInt32 } from "../src/integers.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readInt32", () => { + it("decodes 0", async () => { + const r = await reader("toInt32(0)"); + expect(readInt32(r)).toBe(0); + expect(r.pos).toBe(4); + }); + + it("decodes -1", async () => { + expect(readInt32(await reader("toInt32(-1)"))).toBe(-1); + }); + + it("decodes 2147483647 (max)", async () => { + expect(readInt32(await reader("toInt32(2147483647)"))).toBe(2147483647); + }); + + it("decodes -2147483648 (min)", async () => { + expect(readInt32(await reader("toInt32(-2147483648)"))).toBe(-2147483648); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toInt32(-5) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readInt32(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Int64.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Int64.test.ts new file mode 100644 index 000000000..260a0553f --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Int64.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readInt64 } from "../src/integers.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readInt64", () => { + it("decodes 0n", async () => { + const r = await reader("toInt64(0)"); + expect(readInt64(r)).toBe(0n); + expect(r.pos).toBe(8); + }); + + it("decodes -1n", async () => { + expect(readInt64(await reader("toInt64(-1)"))).toBe(-1n); + }); + + it("decodes 9223372036854775807n (max)", async () => { + expect(readInt64(await reader("toInt64(9223372036854775807)"))).toBe( + 9223372036854775807n, + ); + }); + + it("decodes -9223372036854775808n (min)", async () => { + expect(readInt64(await reader("toInt64(-9223372036854775808)"))).toBe( + -9223372036854775808n, + ); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toInt64(-5) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readInt64(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Int8.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Int8.test.ts new file mode 100644 index 000000000..c521dcf5d --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Int8.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readInt8 } from "../src/integers.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readInt8", () => { + it("decodes 0", async () => { + const r = await reader("toInt8(0)"); + expect(readInt8(r)).toBe(0); + expect(r.pos).toBe(1); + }); + + it("decodes 127 (max)", async () => { + expect(readInt8(await reader("toInt8(127)"))).toBe(127); + }); + + it("decodes -1", async () => { + expect(readInt8(await reader("toInt8(-1)"))).toBe(-1); + }); + + it("decodes -128 (min)", async () => { + expect(readInt8(await reader("toInt8(-128)"))).toBe(-128); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toInt8(-1) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readInt8(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Interval.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Interval.test.ts new file mode 100644 index 000000000..d60d4700b --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Interval.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readInterval } from "../src/interval.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +// All 11 Interval* types share one Int64 wire; the unit lives in the type name. +describe("readInterval", () => { + it("decodes a positive count (IntervalSecond)", async () => { + const r = await reader("toIntervalSecond(5)"); + expect(readInterval(r)).toBe(5n); + expect(r.pos).toBe(8); + }); + + it("decodes a negative count (IntervalDay)", async () => { + expect(readInterval(await reader("toIntervalDay(-3)"))).toBe(-3n); + }); + + it("decodes a large count (IntervalNanosecond)", async () => { + expect(readInterval(await reader("toIntervalNanosecond(1000000000)"))).toBe( + 1000000000n, + ); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toIntervalSecond(5) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readInterval(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/JSON.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/JSON.test.ts new file mode 100644 index 000000000..05505feea --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/JSON.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readDynamic } from "../src/dynamic.js"; +import { readJSON } from "../src/json.js"; + +const J = "SETTINGS allow_experimental_json_type = 1, enable_json_type = 1"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} ${J} FORMAT RowBinary`)); +} + +describe("readJSON", () => { + it("reads (path, Dynamic value) pairs into a Map", async () => { + const r = await reader(`'{"a":1}'::JSON`); + expect(readJSON(r)).toEqual(new Map([["a", 1n]])); // a -> Int64 1 + }); + + it("reads multiple paths, each with its own inline type", async () => { + const r = await reader(`'{"a":1,"b":"hi"}'::JSON`); + expect(readJSON(r)).toEqual( + new Map([ + ["b", "hi"], // String + ["a", 1n], // Int64 + ]), + ); + }); + + it("FLATTENS nested objects to dotted paths", async () => { + const r = await reader(`'{"a":{"b":2}}'::JSON`); + expect(readJSON(r)).toEqual(new Map([["a.b", 2n]])); + }); + + it("an empty object is zero paths", async () => { + const r = await reader(`'{}'::JSON`); + expect(readJSON(r)).toEqual(new Map()); + }); + + it("a null-valued path is NOT stored — same as an empty object", async () => { + const r = await reader(`'{"a":null}'::JSON`); + expect(readJSON(r)).toEqual(new Map()); // {"a":null} serializes as 0 paths + }); + + it("a JSON array path decodes as Array(Nullable(T)) via Dynamic", async () => { + const r = await reader(`'{"a":[1,2]}'::JSON`); + expect(readJSON(r)).toEqual(new Map([["a", [1n, 2n]]])); + }); + + it("decodes mixed scalar types (Float64, Bool)", async () => { + const r = await reader(`'{"x":1.5,"y":true}'::JSON`); + expect(readJSON(r)).toEqual( + new Map([ + ["y", true], + ["x", 1.5], + ]), + ); + }); + + // JSON nested inside a Dynamic: the 0x30 tag's type-encoding header precedes + // the body, which readDynamicType consumes before delegating to readJSON. + describe("inside a Dynamic (tag 0x30, with the type-encoding header)", () => { + async function dyn(expr: string): Promise { + return new Cursor( + await query( + `SELECT CAST(${expr} AS Dynamic) ${J}, allow_experimental_dynamic_type = 1 FORMAT RowBinary`, + ), + ); + } + + it("decodes a JSON value, skipping the parameter header", async () => { + const r = await dyn(`'{"a":1}'::JSON`); + expect(readDynamic(r)).toEqual(new Map([["a", 1n]])); + }); + + it("recurses through Array(JSON)", async () => { + const r = await dyn(`['{"a":1}'::JSON, '{"b":2}'::JSON]`); + expect(readDynamic(r)).toEqual([ + new Map([["a", 1n]]), + new Map([["b", 2n]]), + ]); + }); + + it("throws on declared typed paths (need the schema to read them)", async () => { + const r = await dyn(`'{"a":1}'::JSON(b UInt32)`); + expect(() => readDynamic(r)).toThrow(/typed paths/); + }); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query(`SELECT '{"a":1}'::JSON ${J} FORMAT RowBinary`); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readJSON(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/LineString.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/LineString.test.ts new file mode 100644 index 000000000..d646d2a58 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/LineString.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readLineString } from "../src/geo.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readLineString", () => { + it("decodes a LineString as an array of points", async () => { + const r = await reader("CAST([(3, 4), (5, 6)] AS LineString)"); + expect(readLineString(r)).toEqual([ + [3, 4], + [5, 6], + ]); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST([(3, 4), (5, 6)] AS LineString) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readLineString(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Map.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Map.test.ts new file mode 100644 index 000000000..1f6783a23 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Map.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { readMap, readNullable } from "../src/composite.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUInt32, readUInt8 } from "../src/integers.js"; +import { readString } from "../src/strings.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readMap", () => { + it("decodes key/value pairs into a JS Map", async () => { + const r = await reader("CAST(map('a', 1, 'b', 2) AS Map(String, UInt32))"); + const m = readMap(readString, readUInt32)(r); + expect(m).toEqual( + new Map([ + ["a", 1], + ["b", 2], + ]), + ); + expect(r.pos).toBe(13); // 1 count + 2 * (2-byte key + 4-byte value) + }); + + it("decodes the empty map (just the count byte)", async () => { + const r = await reader("CAST(map() AS Map(String, UInt32))"); + expect(readMap(readString, readUInt32)(r)).toEqual(new Map()); + expect(r.pos).toBe(1); + }); + + // Composes: a Nullable value (NULL is just its flag byte). + it("decodes Map(UInt8, Nullable(String)) with a NULL value", async () => { + const r = await reader( + "CAST(map(1, 'x', 2, NULL) AS Map(UInt8, Nullable(String)))", + ); + const m = readMap(readUInt8, readNullable(readString))(r); + expect(m).toEqual( + new Map([ + [1, "x"], + [2, null], + ]), + ); + expect(r.pos).toBe(7); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST(map('a', 1, 'b', 2) AS Map(String, UInt8)) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readMap(readString, readUInt8)(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/MultiLineString.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/MultiLineString.test.ts new file mode 100644 index 000000000..cae87a61d --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/MultiLineString.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readMultiLineString } from "../src/geo.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readMultiLineString", () => { + it("decodes a MultiLineString as an array of line strings", async () => { + const r = await reader("CAST([[(0, 0), (1, 1)]] AS MultiLineString)"); + expect(readMultiLineString(r)).toEqual([ + [ + [0, 0], + [1, 1], + ], + ]); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST([[(0, 0), (1, 1)]] AS MultiLineString) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readMultiLineString(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/MultiPolygon.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/MultiPolygon.test.ts new file mode 100644 index 000000000..0ae3fe4a2 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/MultiPolygon.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readMultiPolygon } from "../src/geo.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readMultiPolygon", () => { + it("decodes a MultiPolygon as an array of polygons", async () => { + const r = await reader( + "CAST([[[(0, 0), (1, 0), (1, 1)]]] AS MultiPolygon)", + ); + expect(readMultiPolygon(r)).toEqual([ + [ + [ + [0, 0], + [1, 0], + [1, 1], + ], + ], + ]); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST([[[(0, 0), (1, 0), (1, 1)]]] AS MultiPolygon) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readMultiPolygon(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Nullable.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Nullable.test.ts new file mode 100644 index 000000000..ab631439a --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Nullable.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { readNullable } from "../src/composite.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUInt32, readUInt8 } from "../src/integers.js"; +import { readString } from "../src/strings.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readNullable", () => { + it("decodes a present value (flag 0 + value)", async () => { + const r = await reader("CAST(42 AS Nullable(UInt32))"); + expect(readNullable(readUInt32)(r)).toBe(42); + expect(r.pos).toBe(5); // 1 flag + 4 value + }); + + it("decodes NULL as the lone flag byte (no value follows)", async () => { + const r = await reader("CAST(NULL AS Nullable(UInt32))"); + expect(readNullable(readUInt32)(r)).toBeNull(); + expect(r.pos).toBe(1); // just the flag — the inner reader must not run + }); + + // Framing across rows: NULL then 1 -> bytes 01 00 01. + it("keeps the cursor aligned across NULL and non-NULL rows", async () => { + const r = await reader( + "CAST(number = 0 ? NULL : number AS Nullable(UInt8)) FROM numbers(2)", + ); + expect(readNullable(readUInt8)(r)).toBeNull(); + expect(r.pos).toBe(1); + expect(readNullable(readUInt8)(r)).toBe(1); + expect(r.pos).toBe(3); + }); + + // A variable-length inner type: the inner reader must not run on NULL. + it("works with a variable-length inner type (String)", async () => { + const r = await reader("CAST('hi' AS Nullable(String))"); + expect(readNullable(readString)(r)).toBe("hi"); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST(42 AS Nullable(UInt32)) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readNullable(readUInt32)(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Point.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Point.test.ts new file mode 100644 index 000000000..bb47c19f4 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Point.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readPoint } from "../src/geo.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readPoint", () => { + it("decodes a Point as [x, y]", async () => { + const r = await reader("CAST((1.5, 2.5) AS Point)"); + expect(readPoint(r)).toEqual([1.5, 2.5]); + expect(r.pos).toBe(16); // two Float64 + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST((1.5, 2.5) AS Point) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readPoint(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Polygon.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Polygon.test.ts new file mode 100644 index 000000000..31fd6a10b --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Polygon.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readPolygon } from "../src/geo.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readPolygon", () => { + it("decodes a Polygon as an array of rings", async () => { + const r = await reader("CAST([[(0, 0), (1, 0), (1, 1)]] AS Polygon)"); + expect(readPolygon(r)).toEqual([ + [ + [0, 0], + [1, 0], + [1, 1], + ], + ]); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST([[(0, 0), (1, 0), (1, 1)]] AS Polygon) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readPolygon(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Ring.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Ring.test.ts new file mode 100644 index 000000000..1c9bfcf12 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Ring.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readRing } from "../src/geo.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readRing", () => { + it("decodes a Ring as an array of points", async () => { + const r = await reader("CAST([(0, 0), (1, 2)] AS Ring)"); + expect(readRing(r)).toEqual([ + [0, 0], + [1, 2], + ]); + expect(r.pos).toBe(33); // 1 count + 2 * 16 + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST([(0, 0), (1, 2)] AS Ring) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readRing(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/String.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/String.test.ts new file mode 100644 index 000000000..335a247f1 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/String.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readString } from "../src/strings.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readString", () => { + it("decodes a short ASCII string", async () => { + const r = await reader("'hello'"); + expect(readString(r)).toBe("hello"); + // 1-byte varint length (5) + 5 payload bytes. + expect(r.pos).toBe(6); + }); + + it("decodes the empty string", async () => { + const r = await reader("''"); + expect(readString(r)).toBe(""); + expect(r.pos).toBe(1); // just the 0x00 length byte + }); + + it("decodes multi-byte UTF-8", async () => { + expect(readString(await reader("'héllo · 日本'"))).toBe("héllo · 日本"); + }); + + // A string longer than 127 bytes uses a 2-byte varint length prefix. + it("decodes a string with a multi-byte length prefix", async () => { + const r = await reader("repeat('x', 300)"); + expect(readString(r)).toBe("x".repeat(300)); + expect(r.pos).toBe(302); // 2-byte length + 300 payload + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT 'hello' FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readString(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Time.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Time.test.ts new file mode 100644 index 000000000..95d67712c --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Time.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { formatTime, readTime } from "../src/time.js"; + +// Time / Time64 need enable_time_time64_type; pass it inline via SETTINGS. +async function reader(expr: string): Promise { + return new Cursor( + await query( + `SELECT ${expr} SETTINGS enable_time_time64_type = 1 FORMAT RowBinary`, + ), + ); +} + +describe("readTime", () => { + it("decodes seconds-of-day", async () => { + const r = await reader("CAST('12:34:56' AS Time)"); + const secs = readTime(r); + expect(secs).toBe(45296); + expect(r.pos).toBe(4); + expect(formatTime(secs)).toBe("12:34:56"); + }); + + it("decodes a negative time", async () => { + const secs = readTime(await reader("CAST('-01:00:00' AS Time)")); + expect(secs).toBe(-3600); + expect(formatTime(secs)).toBe("-01:00:00"); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST('12:34:56' AS Time) SETTINGS enable_time_time64_type = 1 FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readTime(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Time64.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Time64.test.ts new file mode 100644 index 000000000..c7dccf71c --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Time64.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { formatTime64, readTime64 } from "../src/time.js"; + +async function reader(expr: string): Promise { + return new Cursor( + await query( + `SELECT ${expr} SETTINGS enable_time_time64_type = 1 FORMAT RowBinary`, + ), + ); +} + +describe("readTime64", () => { + it("decodes millisecond ticks (P=3)", async () => { + const r = await reader("toTime64('12:34:56.123', 3)"); + const t = readTime64(3)(r); + expect(t).toEqual([45296123n, 3]); + expect(r.pos).toBe(8); + expect(formatTime64(t)).toBe("12:34:56.123"); + }); + + it("decodes a negative time with fractional seconds", async () => { + const t = readTime64(3)(await reader("toTime64('-01:00:00.500', 3)")); + expect(t).toEqual([-3600500n, 3]); + expect(formatTime64(t)).toBe("-01:00:00.500"); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT toTime64('12:34:56.123', 3) SETTINGS enable_time_time64_type = 1 FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readTime64(3)(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Tuple.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Tuple.test.ts new file mode 100644 index 000000000..5b37552b1 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Tuple.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { readNullable, readTuple } from "../src/composite.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUInt32, readUInt8 } from "../src/integers.js"; +import { readString } from "../src/strings.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readTuple", () => { + it("decodes heterogeneous elements back-to-back (no count)", async () => { + const r = await reader("CAST((42, 'hi') AS Tuple(UInt32, String))"); + const t = readTuple([readUInt32, readString])(r); + expect(t).toEqual([42, "hi"]); + expect(r.pos).toBe(7); // 4 (UInt32) + 1 (len) + 2 ("hi") + }); + + // Composes with Nullable: (1, NULL) -> bytes 01 01. + it("composes with Nullable elements", async () => { + const r = await reader("CAST((1, NULL) AS Tuple(UInt8, Nullable(UInt8)))"); + const t = readTuple([readUInt8, readNullable(readUInt8)])(r); + expect(t).toEqual([1, null]); + expect(r.pos).toBe(2); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST((1, 'x') AS Tuple(UInt8, String)) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readTuple([readUInt8, readString])(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/TupleNamed.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/TupleNamed.test.ts new file mode 100644 index 000000000..c31d55f64 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/TupleNamed.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { readNullable, readTupleNamed } from "../src/composite.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUInt32, readUInt8 } from "../src/integers.js"; +import { readString } from "../src/strings.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readTupleNamed", () => { + it("decodes a named tuple into an object (same wire as unnamed)", async () => { + const r = await reader("CAST((42, 'hi') AS Tuple(a UInt32, b String))"); + const obj = readTupleNamed({ + a: readUInt32, + b: readString, + })(r); + expect(obj).toEqual({ a: 42, b: "hi" }); + expect(r.pos).toBe(7); // 4 (UInt32) + 1 (len) + 2 ("hi") + }); + + // Keys are read in listed order; the result object carries the names. + it("composes with Nullable and preserves field names", async () => { + const r = await reader( + "CAST((1, NULL) AS Tuple(id UInt8, parent Nullable(UInt8)))", + ); + const obj = readTupleNamed({ + id: readUInt8, + parent: readNullable(readUInt8), + })(r); + expect(obj).toEqual({ id: 1, parent: null }); + expect(r.pos).toBe(2); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST((1, 'x') AS Tuple(a UInt8, b String)) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readTupleNamed({ a: readUInt8, b: readString })(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/UInt128.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/UInt128.test.ts new file mode 100644 index 000000000..b90fc3b71 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/UInt128.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUInt128 } from "../src/integers.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +const MAX = 340282366920938463463374607431768211455n; // 2^128 - 1 + +describe("readUInt128", () => { + it("decodes 0n", async () => { + const r = await reader("toUInt128(0)"); + expect(readUInt128(r)).toBe(0n); + expect(r.pos).toBe(16); + }); + + // Value in the high word confirms the low/high composition. + it("decodes 2^64 (only the high word set)", async () => { + expect(readUInt128(await reader("toUInt128('18446744073709551616')"))).toBe( + 18446744073709551616n, + ); + }); + + it("decodes the max (2^128 - 1)", async () => { + expect(readUInt128(await reader(`toUInt128('${MAX}')`))).toBe(MAX); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toUInt128(5) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readUInt128(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/UInt16.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/UInt16.test.ts new file mode 100644 index 000000000..0d1b0ab3a --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/UInt16.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUInt16 } from "../src/integers.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readUInt16", () => { + it("decodes 0", async () => { + const r = await reader("toUInt16(0)"); + expect(readUInt16(r)).toBe(0); + expect(r.pos).toBe(2); + }); + + // Confirms little-endian byte order: 258 = 0x0102 -> bytes 02 01. + it("decodes 258 (little-endian byte order)", async () => { + expect(readUInt16(await reader("toUInt16(258)"))).toBe(258); + }); + + it("decodes 65535 (max)", async () => { + expect(readUInt16(await reader("toUInt16(65535)"))).toBe(65535); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toUInt16(258) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readUInt16(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/UInt256.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/UInt256.test.ts new file mode 100644 index 000000000..83478d7fd --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/UInt256.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUInt256 } from "../src/integers.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +const MAX = + 115792089237316195423570985008687907853269984665640564039457584007913129639935n; // 2^256 - 1 + +describe("readUInt256", () => { + it("decodes 0n", async () => { + const r = await reader("toUInt256(0)"); + expect(readUInt256(r)).toBe(0n); + expect(r.pos).toBe(32); + }); + + // Set the third word (bits 128..191) to confirm word ordering. + it("decodes 2^128 (only the third word set)", async () => { + expect( + readUInt256( + await reader("toUInt256('340282366920938463463374607431768211456')"), + ), + ).toBe(340282366920938463463374607431768211456n); + }); + + it("decodes the max (2^256 - 1)", async () => { + expect(readUInt256(await reader(`toUInt256('${MAX}')`))).toBe(MAX); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toUInt256(5) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readUInt256(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/UInt32.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/UInt32.test.ts new file mode 100644 index 000000000..a0fd2025e --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/UInt32.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUInt32 } from "../src/integers.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readUInt32", () => { + it("decodes 0", async () => { + const r = await reader("toUInt32(0)"); + expect(readUInt32(r)).toBe(0); + expect(r.pos).toBe(4); + }); + + it("decodes 4294967295 (max)", async () => { + expect(readUInt32(await reader("toUInt32(4294967295)"))).toBe(4294967295); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toUInt32(258) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readUInt32(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/UInt64.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/UInt64.test.ts new file mode 100644 index 000000000..d2809929a --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/UInt64.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUInt64 } from "../src/integers.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readUInt64", () => { + it("decodes 0n", async () => { + const r = await reader("toUInt64(0)"); + expect(readUInt64(r)).toBe(0n); + expect(r.pos).toBe(8); + }); + + it("decodes 1n", async () => { + expect(readUInt64(await reader("toUInt64(1)"))).toBe(1n); + }); + + it("decodes 18446744073709551615n (max)", async () => { + expect(readUInt64(await reader("toUInt64(18446744073709551615)"))).toBe( + 18446744073709551615n, + ); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toUInt64(5) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readUInt64(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/UInt8.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/UInt8.test.ts new file mode 100644 index 000000000..0921192e6 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/UInt8.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUInt8 } from "../src/integers.js"; + +describe("readUInt8", () => { + it("reads sequential unsigned bytes", () => { + const r = new Cursor(Buffer.from([1, 2, 255])); + expect(readUInt8(r)).toBe(1); + expect(readUInt8(r)).toBe(2); + expect(readUInt8(r)).toBe(255); + expect(r.pos).toBe(3); + }); + + it("decodes a UInt8 straight from ClickHouse", async () => { + const bytes = await query("SELECT toUInt8(255) FORMAT RowBinary"); + const r = new Cursor(bytes); + expect(readUInt8(r)).toBe(255); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query("SELECT toUInt8(255) FORMAT RowBinary"); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readUInt8(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/UUID.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/UUID.test.ts new file mode 100644 index 000000000..a28da0705 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/UUID.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { formatUUID, formatUUIDTable, readUUID } from "../src/uuid.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readUUID", () => { + it("returns the raw 16 bytes and formats them (per-half byte order)", async () => { + const r = await reader("toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')"); + const bytes = readUUID(r); + expect(r.pos).toBe(16); + // Wire layout: each LE UInt64 half is byte-reversed vs the text form. + expect(bytes).toEqual( + Buffer.from([ + 0xe7, 0x11, 0xb3, 0x5c, 0x04, 0xc4, 0xf0, 0x61, 0xa0, 0xdb, 0xd3, 0x6a, + 0x00, 0xa6, 0x7b, 0x90, + ]), + ); + expect(formatUUID(bytes)).toBe("61f0c404-5cb3-11e7-907b-a6006ad3dba0"); + }); + + it("formats the nil UUID", async () => { + expect( + formatUUID( + readUUID( + await reader("toUUID('00000000-0000-0000-0000-000000000000')"), + ), + ), + ).toBe("00000000-0000-0000-0000-000000000000"); + }); + + // Smallest non-zero value: exercises zero-padding of leading hex digits. + it("zero-pads leading digits", async () => { + expect( + formatUUID( + readUUID( + await reader("toUUID('00000000-0000-0000-0000-000000000001')"), + ), + ), + ).toBe("00000000-0000-0000-0000-000000000001"); + }); + + it("formats the all-ones UUID", async () => { + expect( + formatUUID( + readUUID( + await reader("toUUID('ffffffff-ffff-ffff-ffff-ffffffffffff')"), + ), + ), + ).toBe("ffffffff-ffff-ffff-ffff-ffffffffffff"); + }); + + // The fast formatter: formatUUIDTable must match formatUUID exactly on the + // same raw bytes (lookup table instead of BigInt). + describe("formatUUIDTable (fast lookup-table formatter)", () => { + it("matches formatUUID for a typical value", async () => { + const b = readUUID( + await reader("toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')"), + ); + expect(formatUUIDTable(b)).toBe("61f0c404-5cb3-11e7-907b-a6006ad3dba0"); + expect(formatUUIDTable(b)).toBe(formatUUID(b)); + }); + + it("zero-pads leading digits and formats the nil UUID", async () => { + expect( + formatUUIDTable( + readUUID( + await reader("toUUID('00000000-0000-0000-0000-000000000001')"), + ), + ), + ).toBe("00000000-0000-0000-0000-000000000001"); + expect( + formatUUIDTable( + readUUID( + await reader("toUUID('00000000-0000-0000-0000-000000000000')"), + ), + ), + ).toBe("00000000-0000-0000-0000-000000000000"); + }); + + it("reuses the shared output buffer across calls without corruption", async () => { + const a = formatUUIDTable( + readUUID( + await reader("toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')"), + ), + ); + const b = formatUUIDTable( + readUUID( + await reader("toUUID('ffffffff-ffff-ffff-ffff-ffffffffffff')"), + ), + ); + expect(a).toBe("61f0c404-5cb3-11e7-907b-a6006ad3dba0"); // earlier result is a copied string, not clobbered + expect(b).toBe("ffffffff-ffff-ffff-ffff-ffffffffffff"); + }); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0') FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readUUID(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/UUIDBigInt.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/UUIDBigInt.test.ts new file mode 100644 index 000000000..3902b6adb --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/UUIDBigInt.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUUIDBigInt } from "../src/uuid.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readUUIDBigInt", () => { + it("decodes a UUID as its 128-bit value", async () => { + const r = await reader("toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')"); + expect(readUUIDBigInt(r)).toBe(0x61f0c4045cb311e7907ba6006ad3dba0n); + expect(r.pos).toBe(16); + }); + + it("decodes the nil UUID as 0n", async () => { + expect( + readUUIDBigInt( + await reader("toUUID('00000000-0000-0000-0000-000000000000')"), + ), + ).toBe(0n); + }); + + it("decodes ...0001 as 1n", async () => { + expect( + readUUIDBigInt( + await reader("toUUID('00000000-0000-0000-0000-000000000001')"), + ), + ).toBe(1n); + }); + + it("decodes the all-ones UUID as 2^128 - 1", async () => { + expect( + readUUIDBigInt( + await reader("toUUID('ffffffff-ffff-ffff-ffff-ffffffffffff')"), + ), + ).toBe((1n << 128n) - 1n); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0') FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readUUIDBigInt(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/UUIDHiLo.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/UUIDHiLo.test.ts new file mode 100644 index 000000000..f01ff23f2 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/UUIDHiLo.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { + formatUUID, + readUUID, + readUUIDBigInt, + readUUIDHiLo, +} from "../src/uuid.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +describe("readUUIDHiLo", () => { + it("returns the two little-endian UInt64 halves [hi, lo]", async () => { + const r = await reader("toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')"); + expect(readUUIDHiLo(r)).toEqual([0x61f0c4045cb311e7n, 0x907ba6006ad3dba0n]); + expect(r.pos).toBe(16); + }); + + it("composes back to the same value as readUUIDBigInt / formatUUID", async () => { + const expr = "toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')"; + const [hi, lo] = readUUIDHiLo(await reader(expr)); + // hi << 64 | lo is exactly what readUUIDBigInt returns. + expect((hi << 64n) | lo).toBe(readUUIDBigInt(await reader(expr))); + // The 32 hex digits are hi then lo, zero-padded — matching formatUUID. + const hex = + hi.toString(16).padStart(16, "0") + lo.toString(16).padStart(16, "0"); + expect(hex).toBe( + formatUUID(readUUID(await reader(expr))).replaceAll("-", ""), + ); + }); + + it("decodes the nil UUID as [0, 0]", async () => { + const r = await reader("toUUID('00000000-0000-0000-0000-000000000000')"); + expect(readUUIDHiLo(r)).toEqual([0n, 0n]); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0') FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readUUIDHiLo(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/UVarint.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/UVarint.test.ts new file mode 100644 index 000000000..8229c530c --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/UVarint.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUVarint } from "../src/varint.js"; + +/** + * RowBinary prefixes every String with its length as a LEB128 unsigned varint. + * So a real, server-encoded varint of value N is just the leading bytes of + * `SELECT repeat('a', N) FORMAT RowBinary`, followed by N payload bytes. + * + * Edge cases worth covering are the byte-width boundaries of LEB128: + * 0 — empty, single 0x00 byte + * 127 / 128 — 1-byte max -> first 2-byte value + * 16383/16384 — 2-byte max -> first 3-byte value + */ +async function repeatReader(n: number): Promise { + return new Cursor(await query(`SELECT repeat('a', ${n}) FORMAT RowBinary`)); +} + +describe("readUVarint", () => { + it("decodes 0 (single 0x00 byte)", async () => { + const r = await repeatReader(0); + expect(readUVarint(r)).toBe(0); + expect(r.pos).toBe(r.buf.length - 0); + }); + + it("decodes 1", async () => { + const r = await repeatReader(1); + expect(readUVarint(r)).toBe(1); + expect(r.pos).toBe(r.buf.length - 1); + }); + + it("decodes 127 (1-byte max)", async () => { + const r = await repeatReader(127); + expect(readUVarint(r)).toBe(127); + expect(r.pos).toBe(r.buf.length - 127); + }); + + it("decodes 128 (first 2-byte value)", async () => { + const r = await repeatReader(128); + expect(readUVarint(r)).toBe(128); + expect(r.pos).toBe(r.buf.length - 128); + }); + + it("decodes 16383 (2-byte max)", async () => { + const r = await repeatReader(16383); + expect(readUVarint(r)).toBe(16383); + expect(r.pos).toBe(r.buf.length - 16383); + }); + + it("decodes 16384 (first 3-byte value)", async () => { + const r = await repeatReader(16384); + expect(readUVarint(r)).toBe(16384); + expect(r.pos).toBe(r.buf.length - 16384); + }); + + // The upper bound of what readUVarint can represent exactly: values past + // Number.MAX_SAFE_INTEGER (2^53 - 1) would lose precision (it returns a JS + // number, not a bigint). Such a length is far larger than any string we + // could SELECT, so the bytes are constructed directly rather than fetched. + it("decodes Number.MAX_SAFE_INTEGER (2^53 - 1)", () => { + // 53 bits set: seven full 7-bit groups (0xff) plus a 4-bit top group (0x0f). + const r = new Cursor( + Buffer.from([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f]), + ); + expect(readUVarint(r)).toBe(Number.MAX_SAFE_INTEGER); + expect(r.pos).toBe(8); + }); + + // One past the safe ceiling: 2^53 (MAX_SAFE_INTEGER + 1). The top group's + // bit 4 (0x10) sets bit 53; everything below is zero. Must throw rather than + // return an imprecise number. + it("throws when a varint exceeds Number.MAX_SAFE_INTEGER", () => { + const r = new Cursor( + Buffer.from([0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x10]), + ); + expect(() => readUVarint(r)).toThrow(RangeError); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix of a multi-byte varint", () => { + // 16384 -> 3-byte LEB128 [0x80, 0x80, 0x01]; each shorter prefix ends on a + // byte with the continuation bit set and no following byte, so it must starve. + const full = Buffer.from([0x80, 0x80, 0x01]); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readUVarint(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len}`).toBe(NeedMoreData); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/Variant.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/Variant.test.ts new file mode 100644 index 000000000..98e86a90f --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/Variant.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { readVariant } from "../src/composite.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readFloat64 } from "../src/floats.js"; +import { readUInt64, readUInt8 } from "../src/integers.js"; +import { readString } from "../src/strings.js"; + +async function reader(expr: string): Promise { + return new Cursor( + await query( + `SELECT ${expr} SETTINGS allow_experimental_variant_type = 1 FORMAT RowBinary`, + ), + ); +} + +describe("readVariant", () => { + // Variant(UInt8, String) types sort to ["String", "UInt8"], so the readers + // are listed in that sorted order: String first (discriminant 0), UInt8 (1). + it("picks the alternative by sorted-order discriminant (UInt8 = 1)", async () => { + const r = await reader("CAST(42 AS Variant(UInt8, String))"); + expect(readVariant([readString, readUInt8])(r)).toBe(42); + }); + + it("picks the String alternative (discriminant 0)", async () => { + const r = await reader("CAST('hi' AS Variant(UInt8, String))"); + expect(readVariant([readString, readUInt8])(r)).toBe("hi"); + }); + + it("decodes NULL (discriminant 0xFF)", async () => { + const r = await reader("CAST(NULL AS Variant(UInt8, String))"); + expect(readVariant([readString, readUInt8])(r)).toBeNull(); + expect(r.pos).toBe(1); + }); + + // Three alternatives sort to [Float64, String, UInt64] -> discriminants 0,1,2. + it("handles three sorted alternatives", async () => { + const t = "Variant(Float64, String, UInt64)"; + + const a = await reader(`CAST(toUInt64(9) AS ${t})`); + expect(readVariant([readFloat64, readString, readUInt64])(a)).toBe(9n); // discriminant 2 + + const b = await reader(`CAST(toFloat64(1.5) AS ${t})`); + expect(readVariant([readFloat64, readString, readUInt64])(b)).toBe(1.5); // discriminant 0 + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST(42 AS Variant(UInt8, String)) SETTINGS allow_experimental_variant_type = 1 FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readVariant([readString, readUInt8])(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/advance.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/advance.test.ts new file mode 100644 index 000000000..ae39a2e4a --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/advance.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUInt64 } from "../src/integers.js"; +import { readString } from "../src/strings.js"; + +/** + * `advance` / `NeedMoreData` tests: the per-read "need more bytes" throw that is + * the foundation for streaming. A read that would cross the end of the buffer + * throws `NeedMoreData` instead of reading garbage, WITHOUT moving the cursor — + * so a driver can rewind to its last committed row and retry once more bytes + * arrive. (This is just the per-read throw; chunk reassembly and commit tracking + * are the driver's job, modelled here in the tests but not yet in the reader.) + * + * The data is real ClickHouse output; the truncation is simulated by viewing a + * prefix of the response buffer (`buf.subarray(0, avail)`), which is exactly + * what the reader treats as "all the bytes there are so far". + */ +describe("advance() and NeedMoreData", () => { + it("throws NeedMoreData when a fixed-width read crosses the end, leaving pos put", async () => { + const full = await query("SELECT toUInt64(1) FORMAT RowBinary"); // 8 bytes + const r = new Cursor(full.subarray(0, 5)); // one byte short of nothing + let thrown: unknown; + try { + readUInt64(r); + } catch (e) { + thrown = e; + } + expect(thrown).toBe(NeedMoreData); + expect(r.pos).toBe(0); // cursor not advanced on a starved read + }); + + it("throws NeedMoreData when a String body is truncated past its length prefix", async () => { + // "hello" -> 1 varint length byte (0x05) + 5 bytes. Reveal length + 2 body + // bytes: the varint read succeeds, the body read starves. + const full = await query("SELECT 'hello' FORMAT RowBinary"); // 6 bytes + const r = new Cursor(full.subarray(0, 3)); + let thrown: unknown; + try { + readString(r); + } catch (e) { + thrown = e; + } + expect(thrown).toBe(NeedMoreData); + }); + + it("a throw+restart driver reassembles every row from a chunked stream", async () => { + // (UInt64, String) rows of varying width, so chunk boundaries land mid-field + // and mid-row, exercising the throw on both the number and the string read. + const full = await query( + "SELECT number AS id, repeat('ab', number) AS s FROM numbers(20) FORMAT RowBinary", + ); + + const expected = Array.from({ length: 20 }, (_, i) => ({ + id: BigInt(i), + s: "ab".repeat(i), + })); + + // Drive the reader the way a streaming consumer would: reveal `chunk` more + // bytes whenever a read starves, and restart the row from the last commit. + for (const chunk of [1, 3, 7, 64, 4096]) { + const rows: Array<{ id: bigint; s: string }> = []; + let committed = 0; + let avail = 0; + while (committed < full.length) { + avail = Math.min(full.length, avail + chunk); + const r = new Cursor(full.subarray(0, avail)); + r.pos = committed; + try { + while (r.pos < r.buf.length) { + const id = readUInt64(r); + const s = readString(r); + rows.push({ id, s }); // only reached once BOTH reads succeed + committed = r.pos; // commit the row boundary + } + } catch (e) { + if (e !== NeedMoreData) throw e; + // starved: loop, reveal more, retry from `committed` (no double-push, + // because the row is pushed only after a clean id+s read). + } + } + expect(rows, `chunk size ${chunk}`).toEqual(expected); + } + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/aggregateFunction.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/aggregateFunction.test.ts new file mode 100644 index 000000000..b1d22654b --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/aggregateFunction.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { readAggregateFunction } from "../src/aggregateFunction.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUInt64, readUInt8 } from "../src/integers.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +/** + * `AggregateFunction(func, T...)` holds an OPAQUE serialized aggregation STATE + * (what `-State` combinators produce). In RowBinary this state is written RAW, + * with **NO length prefix** and a layout that is entirely specific to `func` + * (and to the ClickHouse version). For example `sumState(UInt64)` is 8 bytes + * (the running sum), while `uniqState(...)` is a variable-length hash-set blob. + * + * Consequences for a generic parser: + * - It cannot be decoded generically — there is no schema in the bytes. + * - It cannot even be SKIPPED generically — there is no length to skip past; + * you must know `func`'s exact byte layout to find where it ends, otherwise + * every column after it in the row is misaligned. + * + * So there is NO generic reader. Two real options: + * + * 1. RECOMMENDED — finalize server-side, decode the concrete result type. + * Apply the `-Merge` combinator or `finalizeAggregation()` in SQL so the + * column becomes a normal value (`sum` -> `UInt64`, `uniq` -> `UInt64`, + * `avg` -> `Float64`, ...) and use the matching reader. Never ship raw + * `-State` columns to the client unless you intend to merge them later. + * + * 2. ESCAPE HATCH — known fixed layout only. A few functions' state IS just a + * value of a known type (e.g. `sumState(UInt64)` is literally that UInt64), + * so you may decode it as that type. This is fragile and version-specific; + * only do it when you truly know the internal layout. See below. + */ +describe("AggregateFunction (opaque state — finalize server-side)", () => { + it("RECOMMENDED: finalize with -Merge / finalizeAggregation, then decode normally", async () => { + // uniqMerge collapses the uniq state to a concrete UInt64 cardinality. + const merged = await reader( + "uniqMerge(s) FROM (SELECT uniqState(number) AS s FROM numbers(5))", + ); + expect(readUInt64(merged)).toBe(5n); + + // finalizeAggregation does the same inline without a GROUP BY context. + const finalized = await reader( + "finalizeAggregation(sumState(toUInt64(42)))", + ); + expect(readUInt64(finalized)).toBe(42n); // type is now plain UInt64 + }); + + it("ESCAPE HATCH: sumState(UInt64) state happens to be the raw UInt64 (fragile, layout-specific)", async () => { + // sumState's state is just the running sum, with NO length prefix: the next + // column begins immediately after the 8 sum bytes. We decode it as UInt64 + // only because we know this exact layout — do not generalize this. + const r = await reader("sumState(toUInt64(42)) AS a, toUInt8(255) AS b"); + expect(readUInt64(r)).toBe(42n); // the state, read as its known UInt64 shape + expect(r.pos).toBe(8); // no framing — column b starts right here + expect(readUInt8(r)).toBe(255); // proves the 8-byte state was exact + }); + + it("readAggregateFunction is a guard: it always throws (never decode opaque state)", () => { + const r = new Cursor(Buffer.alloc(0)); + expect(() => readAggregateFunction(r)).toThrow(/opaque/i); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + // sumState(UInt64) is a raw 8-byte sum with no length prefix. + const full = await query( + "SELECT sumState(toUInt64(42)) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readUInt64(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/carts.bench.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/carts.bench.ts new file mode 100644 index 000000000..d93884897 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/carts.bench.ts @@ -0,0 +1,47 @@ +import { bench, describe } from "vitest"; +import { query } from "./clickhouse.js"; +import { type Reader, Cursor } from "../src/core.js"; +import { + type CartRow, + readCartRow, + readCartRowFast, +} from "../src/examples/carts.js"; + +/** + * Benchmark: API-combinator `readCartRow` vs monomorphized `readCartRowFast` + * over the same large buffer. Nested generics — `Array(Tuple(...))` and + * `Array(Nullable(...))` — so the API version rebuilds nested closures per row; + * the inlined version flattens both levels. + */ +const N = 20_000; +const BUF = await query( + `SELECT toUInt32(number) AS cart_id, ` + + `arrayMap(x -> CAST(tuple(concat('s', toString(x)), toUInt16(x)) AS Tuple(sku String, qty UInt16)), range(number % 3)) AS items, ` + + `arrayMap(x -> CAST(if(x % 2 = 0, toInt32(x), NULL) AS Nullable(Int32)), range(number % 4)) AS discounts ` + + `FROM numbers(${N}) FORMAT RowBinary`, +); + +function decodeAll(read: Reader): CartRow[] { + const s = new Cursor(BUF); + const out: CartRow[] = []; + while (s.pos < s.buf.length) out.push(read(s)); + return out; +} + +const norm = (rows: CartRow[]): string => JSON.stringify(rows); +{ + const a = decodeAll(readCartRow); + const b = decodeAll(readCartRowFast); + if (a.length !== N) + throw new Error(`carts: decoded ${a.length} rows, expected ${N}`); + if (norm(a) !== norm(b)) throw new Error("carts: API vs fast mismatch"); +} + +describe("example carts: API vs optimized", () => { + bench("API (combinators)", () => { + decodeAll(readCartRow); + }); + bench("optimized (monomorphized)", () => { + decodeAll(readCartRowFast); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/carts.example.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/carts.example.test.ts new file mode 100644 index 000000000..32229373c --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/carts.example.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { Cursor } from "../src/core.js"; +import { type CartRow, readCartRow } from "../src/examples/carts.js"; +import { readRows } from "../src/rows.js"; + +/** + * Runs the `carts` example end to end (nested generics): an Array of named + * Tuples and an Array of Nullables. Populated via `JSONEachRow`; the second row + * is fully empty so both arrays read as a single count byte. + */ +describe("example: carts (nested generics via JSONEachRow)", () => { + it("creates, populates, and reads back through readCartRow", async () => { + const t = "rb_example_carts"; + await query(`DROP TABLE IF EXISTS ${t}`); + await query( + `CREATE TABLE ${t} (` + + `cart_id UInt32, ` + + `items Array(Tuple(sku String, qty UInt16)), ` + + `discounts Array(Nullable(Int32))` + + `) ENGINE = Memory`, + ); + try { + const rows = [ + { + cart_id: 1, + items: [ + { sku: "A", qty: 2 }, + { sku: "B", qty: 1 }, + ], + discounts: [10, null, 5], + }, + { cart_id: 2, items: [], discounts: [] }, + ]; + await query( + `INSERT INTO ${t} FORMAT JSONEachRow\n` + + rows.map((r) => JSON.stringify(r)).join("\n"), + ); + + const r = new Cursor( + await query( + `SELECT cart_id, items, discounts FROM ${t} ORDER BY cart_id FORMAT RowBinary`, + ), + ); + const out: CartRow[] = readRows(readCartRow)(r); + expect(out).toEqual([ + { + cartId: 1, + items: [ + { sku: "A", qty: 2 }, + { sku: "B", qty: 1 }, + ], + discounts: [10, null, 5], + }, + { cartId: 2, items: [], discounts: [] }, + ]); + expect(r.pos).toBe(r.buf.length); + } finally { + await query(`DROP TABLE IF EXISTS ${t}`); + } + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/clickhouse.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/clickhouse.ts new file mode 100644 index 000000000..16577b0ff --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/clickhouse.ts @@ -0,0 +1,39 @@ +/** + * The one shared test helper: run SQL against a live ClickHouse server over + * HTTP and return the raw response bytes. + * + * Unit tests query ClickHouse directly (no static fixtures) so the bytes under + * test are always exactly what the server produces. Pass a complete statement + * including the format, e.g. `SELECT toUInt8(255) FORMAT RowBinary`. + * + * Assumes a ClickHouse server is already running. Override the connection with + * env vars: + * CLICKHOUSE_URL (default http://localhost:8123) + * CLICKHOUSE_USER (default default) + * CLICKHOUSE_PASSWORD (default empty) + * + * Note: the suite hard-depends on a reachable server — with none running, every + * test errors rather than skips. Intentional for now; if a friendlier + * "no server -> skip with a clear message" behavior is wanted later, add it here. + */ +const URL_BASE = process.env.CLICKHOUSE_URL ?? "http://localhost:8123"; +const USER = process.env.CLICKHOUSE_USER ?? "default"; +const PASSWORD = process.env.CLICKHOUSE_PASSWORD ?? ""; + +export async function query(sql: string): Promise { + const res = await fetch(URL_BASE, { + method: "POST", + headers: { + "X-ClickHouse-User": USER, + "X-ClickHouse-Key": PASSWORD, + }, + body: sql, + }); + if (!res.ok) { + throw new Error( + `ClickHouse ${res.status} for query: ${sql}\n${await res.text()}`, + ); + } + // Buffer.from(ArrayBuffer) is a no-copy view over the response bytes. + return Buffer.from(await res.arrayBuffer()); +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/coalesceChunks.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/coalesceChunks.test.ts new file mode 100644 index 000000000..2f1b946d7 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/coalesceChunks.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { Cursor } from "../src/core.js"; +import { readUInt64 } from "../src/integers.js"; +import { coalesceChunks, streamRowBatches } from "../src/stream.js"; +import { readString } from "../src/strings.js"; + +/** + * `coalesceChunks` merges a too-small chunk stream into chunks of at least + * `minSize` bytes (flushing early on `timeoutMs` or end-of-stream). It is a pure + * byte-level filter — no ClickHouse needed for most of it — composed in front of + * `streamRowBatches`. These tests pin both the size-based and timeout-based + * flush paths and prove the bytes survive the round-trip unchanged. + */ +const delay = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)); + +/** A buffer holding the sequential bytes `[from, to)` (mod 256). */ +const seq = (from: number, to: number): Buffer => + Buffer.from(Array.from({ length: to - from }, (_, i) => (from + i) & 0xff)); + +async function collect(src: AsyncIterable): Promise { + const out: Buffer[] = []; + for await (const c of src) out.push(c); + return out; +} + +describe("coalesceChunks (debounce small chunks before streaming)", () => { + it("accumulates tiny chunks up to minSize, preserving byte order", async () => { + async function* tiny(): AsyncGenerator { + for (let i = 0; i < 20; i++) yield seq(i, i + 1); // 20 × 1 byte + } + // Long timeout: chunks arrive back-to-back, so only the size rule fires. + const out = await collect( + coalesceChunks(tiny(), { minSize: 5, timeoutMs: 10_000 }), + ); + expect(out.map((c) => c.length)).toEqual([5, 5, 5, 5]); + expect(Buffer.concat(out)).toEqual(seq(0, 20)); + }); + + it("passes a single already-large chunk straight through without copying", async () => { + const big = seq(0, 100); + async function* one(): AsyncGenerator { + yield big; + } + const out = await collect( + coalesceChunks(one(), { minSize: 16, timeoutMs: 10_000 }), + ); + expect(out).toHaveLength(1); + expect(out[0]).toBe(big); // same reference: no concat when one part suffices + }); + + it("flushes the remainder below minSize at end of stream", async () => { + async function* short(): AsyncGenerator { + yield seq(0, 3); + yield seq(3, 7); // 7 bytes total, never reaches minSize + } + const out = await collect( + coalesceChunks(short(), { minSize: 1000, timeoutMs: 10_000 }), + ); + expect(out).toHaveLength(1); + expect(out[0]).toEqual(seq(0, 7)); + }); + + it("flushes early on the timeout when data arrives in a trickle", async () => { + async function* trickle(): AsyncGenerator { + yield seq(0, 10); // below minSize + await delay(80); // > timeoutMs, so the buffered 10 bytes flush first + yield seq(10, 20); + } + const out = await collect( + coalesceChunks(trickle(), { minSize: 1000, timeoutMs: 20 }), + ); + // First 10 bytes flushed by the timer, last 10 by end-of-stream. + expect(out).toHaveLength(2); + expect(out[0]).toEqual(seq(0, 10)); + expect(out[1]).toEqual(seq(10, 20)); + }); + + it("anchors the deadline at first byte — a steady trickle can't defer forever", async () => { + async function* drip(): AsyncGenerator { + // Five 1-byte chunks, each ~15ms apart; deadline is 25ms from the FIRST. + for (let i = 0; i < 5; i++) { + yield seq(i, i + 1); + await delay(15); + } + } + const out = await collect( + coalesceChunks(drip(), { minSize: 1000, timeoutMs: 25 }), + ); + // The flush must land mid-trickle (not swallow all five into one), proving + // the deadline is anchored and not reset by each arriving chunk. + expect(out.length).toBeGreaterThan(1); + expect(Buffer.concat(out)).toEqual(seq(0, 5)); + }); + + it("releases the source when the consumer breaks out early", async () => { + let returned = false; + async function* infinite(): AsyncGenerator { + try { + for (let i = 0; ; i++) yield seq(i, i + 1); + } finally { + returned = true; // source cleanup ran + } + } + // minSize 1 → the first chunk flushes immediately; then we bail. + for await (const _ of coalesceChunks(infinite(), { + minSize: 1, + timeoutMs: 10_000, + })) { + break; + } + expect(returned).toBe(true); + }); + + it("composes in front of streamRowBatches: 1-byte chunks decode correctly", async () => { + type Row = { id: bigint; s: string }; + const readRow = (s: Cursor): Row => ({ + id: readUInt64(s), + s: readString(s), + }); + const full = await query( + "SELECT number AS id, repeat('q', number % 7) AS s FROM numbers(40) FORMAT RowBinary", + ); + const expected: Row[] = Array.from({ length: 40 }, (_, i) => ({ + id: BigInt(i), + s: "q".repeat(i % 7), + })); + + async function* oneByteAtATime(): AsyncGenerator { + for (let i = 0; i < full.length; i++) yield full.subarray(i, i + 1); + } + + const rows: Row[] = []; + for await (const batch of streamRowBatches( + coalesceChunks(oneByteAtATime(), { minSize: 32, timeoutMs: 10_000 }), + readRow, + )) { + rows.push(...batch); + } + expect(rows).toEqual(expected); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/columnar.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/columnar.test.ts new file mode 100644 index 000000000..d04d5a53b --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/columnar.test.ts @@ -0,0 +1,150 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { streamSensorColumns } from "../src/columnar.js"; + +/** + * Eval for the streaming columnar decoder (`streamSensorColumns`). The example + * schema — `sensor_id UInt32, ts DateTime64(3), value Float64, quality Float32, + * status UInt8` — is a 25-byte fixed-width row, generated here by the live + * ClickHouse server so we decode against its OWN RowBinary bytes. + * + * Three things under test: + * 1. correctness of every column, checked against a reference decode of the + * same buffer; + * 2. the streaming contract — chunk boundaries that fall mid-row must not + * corrupt or drop rows, and a truncated stream must throw; + * 3. THE COLUMNAR INVARIANT THAT MATTERS HERE: the Int64 (`ts`) column is filled + * by copying two 32-bit words, NOT via `getBigInt64`, so no bigint is + * allocated per row on the decode path. We prove it by spying on + * `DataView.prototype.getBigInt64` and asserting it is never called while + * decoding. + */ + +const STRIDE = 25; +const N = 1000; + +// Deterministic, exactly representable values per column. `ts` is a DateTime64(3), +// whose wire form is Int64 millisecond ticks — here (1700000000 + i) * 1000. +const SELECT = + `SELECT toUInt32(number) AS sensor_id, ` + + `toDateTime64(1700000000 + number, 3) AS ts, ` + + `toFloat64(number) / 2 AS value, ` + + `toFloat32(number) AS quality, ` + + `toUInt8(number % 256) AS status ` + + `FROM numbers(${N})`; + +const BUF = await query(`${SELECT} FORMAT RowBinary`); + +/** Reference decode of the whole buffer — the test is free to allocate bigints. */ +function reference(buf: Buffer) { + const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength); + const sensor_id: number[] = []; + const ts: bigint[] = []; + const value: number[] = []; + const quality: number[] = []; + const status: number[] = []; + for (let o = 0; o + STRIDE <= buf.length; o += STRIDE) { + sensor_id.push(view.getUint32(o, true)); + ts.push(view.getBigInt64(o + 4, true)); + value.push(view.getFloat64(o + 12, true)); + quality.push(view.getFloat32(o + 20, true)); + status.push(buf[o + 24]!); + } + return { sensor_id, ts, value, quality, status }; +} + +/** Yield `buf` in chunks of the given repeating sizes (deliberately mid-row). */ +async function* chunked( + buf: Buffer, + sizes: number[], +): AsyncGenerator { + let o = 0; + let k = 0; + while (o < buf.length) { + const len = sizes[k++ % sizes.length]!; + yield buf.subarray(o, Math.min(o + len, buf.length)); + o += len; + } +} + +/** Drain the columnar stream into flat per-column arrays. */ +async function collect(chunks: AsyncIterable) { + const sensor_id: number[] = []; + const ts: bigint[] = []; + const value: number[] = []; + const quality: number[] = []; + const status: number[] = []; + let batches = 0; + for await (const b of streamSensorColumns(chunks)) { + batches++; + for (let i = 0; i < b.rows; i++) { + sensor_id.push(b.columns.sensor_id[i]!); + ts.push(b.columns.ts[i]!); + value.push(b.columns.value[i]!); + quality.push(b.columns.quality[i]!); + status.push(b.columns.status[i]!); + } + } + return { sensor_id, ts, value, quality, status, batches }; +} + +describe("streamSensorColumns", () => { + it("matches a reference decode of the live RowBinary buffer", async () => { + const ref = reference(BUF); + // One chunk = whole buffer. + const got = await collect(chunked(BUF, [BUF.length])); + expect(got.sensor_id).toEqual(ref.sensor_id); + expect(got.ts).toEqual(ref.ts); + expect(got.value).toEqual(ref.value); + expect(got.quality).toEqual(ref.quality); + expect(got.status).toEqual(ref.status); + expect(got.sensor_id.length).toBe(N); + }); + + it("survives chunk boundaries that split rows mid-field", async () => { + const ref = reference(BUF); + // Sizes coprime-ish to STRIDE (25) so boundaries land inside every field. + const got = await collect(chunked(BUF, [1, 7, 13, 100, 3])); + expect(got.batches).toBeGreaterThan(1); + expect(got.sensor_id).toEqual(ref.sensor_id); + expect(got.ts).toEqual(ref.ts); + expect(got.value).toEqual(ref.value); + expect(got.quality).toEqual(ref.quality); + expect(got.status).toEqual(ref.status); + }); + + it("throws on a stream truncated mid-row", async () => { + const truncated = BUF.subarray(0, BUF.length - 3); + await expect(collect(chunked(truncated, [256]))).rejects.toThrow(/mid-row/); + }); + + describe("Int64 column is not transferred through a bigint allocation", () => { + const original = DataView.prototype.getBigInt64; + afterEach(() => { + DataView.prototype.getBigInt64 = original; + }); + + it("never calls getBigInt64 while decoding", async () => { + let calls = 0; + // Spy that ALSO returns a correct value, so if the decoder regressed to + // using it the column would still be right — the test would fail only on + // the call count, pinpointing the allocation, not on a value mismatch. + DataView.prototype.getBigInt64 = function ( + this: DataView, + byteOffset: number, + littleEndian?: boolean, + ): bigint { + calls++; + return original.call(this, byteOffset, littleEndian); + }; + + const got = await collect(chunked(BUF, [1, 7, 13, 100, 3])); + + expect(calls).toBe(0); + // sanity: ts still decoded correctly via the two-word copy + expect(got.ts.length).toBe(N); + expect(got.ts[0]).toBe(1700000000n * 1000n); + expect(got.ts[N - 1]).toBe(BigInt(1700000000 + N - 1) * 1000n); + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/combinations.generated.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/combinations.generated.test.ts new file mode 100644 index 000000000..84dd4421c --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/combinations.generated.test.ts @@ -0,0 +1,298 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { + readArray, + readMap, + readNullable, + readTuple, + readVariant, +} from "../src/composite.js"; +import { NeedMoreData, type Reader, Cursor } from "../src/core.js"; +import { readInt32, readUInt8 } from "../src/integers.js"; +import { readString } from "../src/strings.js"; + +/** + * GENERATED type-combination coverage — the systematic companion to the curated, + * hand-written cases in `framing-nested.test.ts`. + * + * The bug surface for composite readers is boundary DESYNC: an inner reader + * miscounts its bytes and silently shifts everything after it. The space of + * nestings (every combinator wrapping every inner, at every depth, with every + * edge payload) is exponential, so we don't take the Cartesian product. + * + * Instead, following the "one-leaf-at-a-time" strategy from + * type-predicate-generator issue #18: build small case generators that compose + * like the readers do, then vary ONE position at a time — each combinator is + * shown wrapping each *category* of inner (fixed-width / variable-length / + * nullable / NULL / self-describing / composite) plus the bug-prone edge + * payloads (empty string, zero, empty array, NULL-flag-only). That turns N^M into + * ~combinators x categories while still exercising each edge IN a nesting + * context. ClickHouse is the byte oracle — a case only declares its SQL + * expression and expected JS value; the server produces the bytes. + * + * Each case is checked two ways: + * 1. FRAMED `i32(LEAD), X, i32(TRAIL)` — reading TRAIL back is only possible if + * X consumed EXACTLY its bytes (same harness as framing-nested.test.ts). + * 2. TRUNCATION SWEEP — every incomplete prefix `0 .. full.length-1` must throw + * `NeedMoreData` (generalizes Array.test.ts's prefix sweep to the whole + * matrix, covering the streaming/`advance()` half of the bug surface). + */ + +const LEAD = 123456789; +const TRAIL = 987654321; + +const SETTINGS = [ + "enable_time_time64_type = 1", + "allow_experimental_variant_type = 1", + "allow_suspicious_variant_types = 1", + "allow_experimental_dynamic_type = 1", + "allow_experimental_json_type = 1", + "enable_json_type = 1", + "allow_experimental_qbit_type = 1", + "allow_suspicious_low_cardinality_types = 1", +].join(", "); + +/** + * One self-checking node of a generated case. `expr` is a fully type-annotated + * ClickHouse expression (every level casts, so server-side type inference is + * never ambiguous); `read` decodes it; `expected` is the decoded JS value. + * The flags gate which outer combinators may legally wrap this node. + */ +type Gen = { + type: string; + expr: string; + read: Reader; + expected: unknown; + label: string; + /** plain scalar — the only thing `Nullable(...)` is allowed to wrap */ + leaf: boolean; + /** already a `Nullable` — can't be re-wrapped by Nullable or put in a Variant */ + nullableRoot: boolean; + /** already a `Variant` — can't be an alternative of another Variant */ + variantRoot: boolean; +}; + +// ---- leaf generators: the white-box edge set, not arbitrary values ---------- + +const u8 = (n: number): Gen => ({ + type: "UInt8", + expr: `${n}::UInt8`, + read: readUInt8, + expected: n, + label: `u8=${n}`, + leaf: true, + nullableRoot: false, + variantRoot: false, +}); + +const str = (s: string): Gen => ({ + type: "String", + expr: `'${s}'::String`, + read: readString, + expected: s, + label: s === "" ? "str=''" : `str='${s}'`, + leaf: true, + nullableRoot: false, + variantRoot: false, +}); + +// ---- combinator generators: compose like the readers do --------------------- + +const nullablePresent = (inner: Gen): Gen => ({ + type: `Nullable(${inner.type})`, + expr: `${inner.expr}::Nullable(${inner.type})`, + read: readNullable(inner.read), + expected: inner.expected, + label: `Nullable(${inner.label})`, + leaf: false, + nullableRoot: true, + variantRoot: false, +}); + +const nullValue = (inner: Gen): Gen => ({ + type: `Nullable(${inner.type})`, + expr: `NULL::Nullable(${inner.type})`, + read: readNullable(inner.read), + expected: null, + label: `Nullable(${inner.label})=NULL`, + leaf: false, + nullableRoot: true, + variantRoot: false, +}); + +const array = (inner: Gen, n: number): Gen => { + const elems = Array.from({ length: n }, () => inner.expr); + return { + type: `Array(${inner.type})`, + expr: `[${elems.join(", ")}]::Array(${inner.type})`, + read: readArray(inner.read), + expected: Array.from({ length: n }, () => inner.expected), + label: `Array[len=${n}](${inner.label})`, + leaf: false, + nullableRoot: false, + variantRoot: false, + }; +}; + +const tuple2 = (a: Gen, b: Gen): Gen => ({ + type: `Tuple(${a.type}, ${b.type})`, + expr: `(${a.expr}, ${b.expr})::Tuple(${a.type}, ${b.type})`, + read: readTuple([a.read, b.read]), + expected: [a.expected, b.expected], + label: `Tuple(${a.label}, ${b.label})`, + leaf: false, + nullableRoot: false, + variantRoot: false, +}); + +const map = (key: Gen, value: Gen): Gen => ({ + type: `Map(${key.type}, ${value.type})`, + expr: `map(${key.expr}, ${value.expr})::Map(${key.type}, ${value.type})`, + read: readMap(key.read, value.read), + expected: new Map([[key.expected, value.expected]]), + label: `Map(${key.label} => ${value.label})`, + leaf: false, + nullableRoot: false, + variantRoot: false, +}); + +/** + * Hold `chosen` inside a `Variant(chosen, marker)`. ClickHouse sorts alternatives + * by type NAME and the discriminant indexes that sorted order, so the reader list + * must be sorted the same way (see `readVariant`'s gotcha). + */ +const variantHolding = (chosen: Gen, marker: Gen): Gen => { + const sorted = [chosen, marker].sort((x, y) => (x.type < y.type ? -1 : 1)); + const type = `Variant(${sorted.map((g) => g.type).join(", ")})`; + return { + type, + expr: `${chosen.expr}::${type}`, + read: readVariant(sorted.map((g) => g.read)), + expected: chosen.expected, + label: `Variant<${chosen.label}>`, + leaf: false, + nullableRoot: false, + variantRoot: true, + }; +}; + +// ---- the matrix: combinators (rows) x inner categories (columns) ------------ + +const INNERS: Gen[] = [ + u8(1), // fixed-width + u8(0), // edge: zero + str("hi"), // variable-length + str(""), // edge: empty string + nullablePresent(u8(1)), // nullable, present + nullValue(u8(1)), // edge: NULL flag only + variantHolding(str("hi"), u8(7)), // self-describing + array(u8(1), 2), // composite -> drives depth-2 nesting +]; + +// A distinct-typed Variant marker so the two alternatives never collide. +const markerFor = (inner: Gen): Gen => + inner.type === "String" ? u8(7) : str("zz"); + +const COMBINATORS: Array<{ + label: string; + accepts: (inner: Gen) => boolean; + build: (inner: Gen) => Gen; +}> = [ + { + label: "Array(len=0)", // edge: count byte only, inner never read + accepts: () => true, + build: (inner) => array(inner, 0), + }, + { + label: "Array(len=2)", + accepts: () => true, + build: (inner) => array(inner, 2), + }, + { + label: "Nullable", // only legal around a plain scalar + accepts: (inner) => inner.leaf, + build: (inner) => nullablePresent(inner), + }, + { + label: "Tuple2", // inner adjacent to a fixed sibling + accepts: () => true, + build: (inner) => tuple2(inner, u8(1)), + }, + { + label: "Map(String,_)", + accepts: () => true, + build: (inner) => map(str("k"), inner), + }, + { + label: "Variant", // can't wrap a Nullable or another Variant + accepts: (inner) => !inner.nullableRoot && !inner.variantRoot, + build: (inner) => variantHolding(inner, markerFor(inner)), + }, +]; + +const cases: Gen[] = []; +const skipped: string[] = []; +for (const comb of COMBINATORS) { + for (const inner of INNERS) { + if (comb.accepts(inner)) { + cases.push({ + ...comb.build(inner), + label: `${comb.label} / ${inner.label}`, + }); + } else { + skipped.push(`${comb.label} / ${inner.label}`); + } + } +} + +async function framedBytes(expr: string): Promise { + const sql = + `SELECT toInt32(${LEAD}) AS a, ${expr} AS x, toInt32(${TRAIL}) AS b` + + ` SETTINGS ${SETTINGS} FORMAT RowBinary`; + return query(sql); +} + +describe("type combinations (generated, one-leaf-at-a-time)", () => { + for (const c of cases) { + it(c.label, async () => { + const full = await framedBytes(c.expr); + + // 1. framed: decodes correctly AND consumes exactly its bytes. + const r = new Cursor(full); + expect(readInt32(r)).toBe(LEAD); + expect(c.read(r)).toEqual(c.expected); + expect(readInt32(r), `${c.label}: x over/under-read`).toBe(TRAIL); + + // 2. truncation sweep: every incomplete prefix must starve, never desync. + for (let len = 0; len < full.length; len++) { + const p = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readInt32(p); + c.read(p); + readInt32(p); + } catch (e) { + thrown = e; + } + expect(thrown, `${c.label}: prefix ${len}/${full.length}`).toBe( + NeedMoreData, + ); + } + }); + } + + // No silent caps: lock the skip list so an accidental new gap fails the test. + it("skips only the type-system-illegal combinations", () => { + expect(skipped.sort()).toEqual( + [ + "Nullable / Array[len=2](u8=1)", + "Nullable / Nullable(u8=1)", + "Nullable / Nullable(u8=1)=NULL", + "Nullable / Variant", + "Variant / Nullable(u8=1)", + "Variant / Nullable(u8=1)=NULL", + "Variant / Variant", + ].sort(), + ); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/events.bench.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/events.bench.ts new file mode 100644 index 000000000..a08a9b5fd --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/events.bench.ts @@ -0,0 +1,48 @@ +import { bench, describe } from "vitest"; +import { query } from "./clickhouse.js"; +import { type Reader, Cursor } from "../src/core.js"; +import { + type EventRow, + readEventRow, + readEventRowFast, +} from "../src/examples/events.js"; + +/** + * Benchmark: the API-combinator `readEventRow` vs the inlined `readEventRowFast`, + * both decoding the SAME large `FORMAT RowBinary` buffer (N rows from numbers()). + * Each `bench` op decodes the whole buffer, so the number is rows/iteration; the + * ratio between the two contenders is the takeaway. Equivalence is checked once + * before timing — a faster wrong answer is worthless. + */ +const N = 20_000; +const BUF = await query( + `SELECT toUInt64(number) AS id, concat('name', toString(number)) AS name, ` + + `toDateTime('2021-01-01 00:00:00', 'UTC') + number AS ts ` + + `FROM numbers(${N}) FORMAT RowBinary`, +); + +function decodeAll(read: Reader): EventRow[] { + const s = new Cursor(BUF); + const out: EventRow[] = []; + while (s.pos < s.buf.length) out.push(read(s)); + return out; +} + +const norm = (rows: EventRow[]): string => + JSON.stringify(rows, (_k, v) => (typeof v === "bigint" ? `${v}n` : v)); +{ + const a = decodeAll(readEventRow); + const b = decodeAll(readEventRowFast); + if (a.length !== N) + throw new Error(`events: decoded ${a.length} rows, expected ${N}`); + if (norm(a) !== norm(b)) throw new Error("events: API vs fast mismatch"); +} + +describe("example events: API vs optimized", () => { + bench("API (combinators)", () => { + decodeAll(readEventRow); + }); + bench("optimized (inlined)", () => { + decodeAll(readEventRowFast); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/events.example.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/events.example.test.ts new file mode 100644 index 000000000..65ec4b2da --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/events.example.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { Cursor } from "../src/core.js"; +import { type EventRow, readEventRow } from "../src/examples/events.js"; +import { readRows } from "../src/rows.js"; + +/** + * Runs the `events` example end to end: CREATE a table, populate it (here via + * `JSONEachRow` — an INSERT carries its rows in the same HTTP body after the + * FORMAT clause), SELECT it back `FORMAT RowBinary`, and decode with the reader + * imported from `src/examples/events.ts`. ENGINE = Memory + a finally-drop keeps + * re-runs clean; the SELECT's ORDER BY gives a stable order to assert against. + */ +describe("example: events (scalars via JSONEachRow)", () => { + it("creates, populates, and reads back through readEventRow", async () => { + const t = "rb_example_events"; + await query(`DROP TABLE IF EXISTS ${t}`); + await query( + `CREATE TABLE ${t} (id UInt64, name String, ts DateTime('UTC')) ENGINE = Memory`, + ); + try { + const rows = [ + { id: 1, name: "alpha", ts: "2021-01-01 00:00:00" }, + { id: 2, name: "bravo", ts: "2021-06-15 12:30:00" }, + { id: 3, name: "", ts: "1970-01-01 00:00:00" }, + ]; + await query( + `INSERT INTO ${t} FORMAT JSONEachRow\n` + + rows.map((r) => JSON.stringify(r)).join("\n"), + ); + + const r = new Cursor( + await query( + `SELECT id, name, ts FROM ${t} ORDER BY id FORMAT RowBinary`, + ), + ); + const out: EventRow[] = readRows(readEventRow)(r); + expect(out).toEqual([ + { id: 1n, name: "alpha", ts: "2021-01-01T00:00:00.000Z" }, + { id: 2n, name: "bravo", ts: "2021-06-15T12:30:00.000Z" }, + { id: 3n, name: "", ts: "1970-01-01T00:00:00.000Z" }, + ]); + expect(r.pos).toBe(r.buf.length); + } finally { + await query(`DROP TABLE IF EXISTS ${t}`); + } + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/framing-interleaved.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/framing-interleaved.test.ts new file mode 100644 index 000000000..55b1e2ced --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/framing-interleaved.test.ts @@ -0,0 +1,352 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { + readArray, + readMap, + readNullable, + readTuple, + readVariant, +} from "../src/composite.js"; +import { Cursor } from "../src/core.js"; +import { readDynamic } from "../src/dynamic.js"; +import { readInt32, readUInt8 } from "../src/integers.js"; +import { readJSON } from "../src/json.js"; +import { readString } from "../src/strings.js"; + +/** + * Interleaving framing tests: TWO variable-length / self-describing columns are + * placed adjacent between the Int32 sentinels — `i32(LEAD), X, Y, i32(TRAIL)`. + * + * The point is the X→Y boundary: there is NO sentinel between them, so if X's + * reader stops one byte early or late, Y decodes from the wrong offset (wrong + * value) AND the trailing sentinel is wrong too. Pairing two variadic types + * (Array, Map, Tuple, Nullable, Variant, Dynamic, JSON) — especially two + * self-describing ones back-to-back, and 1-byte NULL values next to them — is + * the case most likely to expose an off-by-one in a buggy reader. + */ +const LEAD = 123456789; +const TRAIL = 987654321; + +const SETTINGS = [ + "enable_time_time64_type = 1", + "allow_experimental_variant_type = 1", + "allow_suspicious_variant_types = 1", + "allow_experimental_dynamic_type = 1", + "allow_experimental_json_type = 1", + "enable_json_type = 1", + "allow_experimental_qbit_type = 1", + "allow_suspicious_low_cardinality_types = 1", +].join(", "); + +/** Build `i32(LEAD), X, Y, i32(TRAIL)` and return a reader over the bytes. */ +async function framed(exprX: string, exprY: string): Promise { + const sql = + `SELECT toInt32(${LEAD}) AS a, ${exprX} AS x, ${exprY} AS y,` + + ` toInt32(${TRAIL}) AS b SETTINGS ${SETTINGS} FORMAT RowBinary`; + return new Cursor(await query(sql)); +} + +// Inner-reader shorthands, as in framing-nested.test.ts. +const u8 = readUInt8; +const str = readString; +// Variant(UInt8, String) sorts to [String(0), UInt8(1)]. +const variantU8Str = readVariant([readString, readUInt8]); + +describe("framing (interleaved): i32, X, Y, i32 — the X→Y boundary must be exact", () => { + describe("X = Array(UInt8)", () => { + it("Array, Array", async () => { + const r = await framed("[1, 2, 3]::Array(UInt8)", "[4, 5]::Array(UInt8)"); + expect(readInt32(r)).toBe(LEAD); + expect(readArray(u8)(r)).toEqual([1, 2, 3]); + expect(readArray(u8)(r)).toEqual([4, 5]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Array, Map", async () => { + const r = await framed( + "[1, 2, 3]::Array(UInt8)", + "map('a', 1, 'b', 2)::Map(String, UInt8)", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readArray(u8)(r)).toEqual([1, 2, 3]); + expect(readMap(str, u8)(r)).toEqual( + new Map([ + ["a", 1], + ["b", 2], + ]), + ); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Array, Variant", async () => { + const r = await framed( + "[1, 2, 3]::Array(UInt8)", + "'hi'::Variant(UInt8, String)", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readArray(u8)(r)).toEqual([1, 2, 3]); + expect(readVariant([readString, readUInt8])(r)).toBe("hi"); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Array, Dynamic", async () => { + const r = await framed("[1, 2, 3]::Array(UInt8)", "toInt32(7)::Dynamic"); + expect(readInt32(r)).toBe(LEAD); + expect(readArray(u8)(r)).toEqual([1, 2, 3]); + expect(readDynamic(r)).toBe(7); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Array, JSON", async () => { + const r = await framed("[1, 2, 3]::Array(UInt8)", `'{"a":1}'::JSON`); + expect(readInt32(r)).toBe(LEAD); + expect(readArray(u8)(r)).toEqual([1, 2, 3]); + expect(readJSON(r)).toEqual(new Map([["a", 1n]])); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Array, Nullable (NULL)", async () => { + const r = await framed( + "[1, 2, 3]::Array(UInt8)", + "CAST(NULL AS Nullable(String))", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readArray(u8)(r)).toEqual([1, 2, 3]); + expect(readNullable(str)(r)).toBeNull(); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("empty Array, Array — the empty array is a lone count byte", async () => { + const r = await framed("[]::Array(UInt8)", "[9]::Array(UInt8)"); + expect(readInt32(r)).toBe(LEAD); + expect(readArray(u8)(r)).toEqual([]); + expect(readArray(u8)(r)).toEqual([9]); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("X = Map(String, UInt8)", () => { + it("Map, Array", async () => { + const r = await framed( + "map('a', 1)::Map(String, UInt8)", + "[7, 8]::Array(UInt8)", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readMap(str, u8)(r)).toEqual(new Map([["a", 1]])); + expect(readArray(u8)(r)).toEqual([7, 8]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Map, Variant", async () => { + const r = await framed( + "map('a', 1)::Map(String, UInt8)", + "42::Variant(UInt8, String)", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readMap(str, u8)(r)).toEqual(new Map([["a", 1]])); + expect(variantU8Str(r)).toBe(42); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Map, Dynamic", async () => { + const r = await framed( + "map('a', 1)::Map(String, UInt8)", + "'hi'::Dynamic", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readMap(str, u8)(r)).toEqual(new Map([["a", 1]])); + expect(readDynamic(r)).toBe("hi"); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("X = Tuple", () => { + it("Tuple, Tuple", async () => { + const r = await framed( + "(1, 'x')::Tuple(UInt8, String)", + "(2, 'y')::Tuple(UInt8, String)", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readTuple([u8, str])(r)).toEqual([1, "x"]); + expect(readTuple([u8, str])(r)).toEqual([2, "y"]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Tuple, Variant", async () => { + const r = await framed( + "(1, 'x')::Tuple(UInt8, String)", + "'z'::Variant(UInt8, String)", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readTuple([u8, str])(r)).toEqual([1, "x"]); + expect(variantU8Str(r)).toBe("z"); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Tuple, Dynamic", async () => { + const r = await framed( + "(1, 'x')::Tuple(UInt8, String)", + "toInt32(9)::Dynamic", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readTuple([u8, str])(r)).toEqual([1, "x"]); + expect(readDynamic(r)).toBe(9); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("X = Variant", () => { + it("Variant, Variant", async () => { + const r = await framed( + "42::Variant(UInt8, String)", + "'hi'::Variant(UInt8, String)", + ); + expect(readInt32(r)).toBe(LEAD); + expect(variantU8Str(r)).toBe(42); + expect(variantU8Str(r)).toBe("hi"); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Variant (NULL), Variant — the NULL is a lone discriminant byte", async () => { + const r = await framed( + "NULL::Variant(UInt8, String)", + "7::Variant(UInt8, String)", + ); + expect(readInt32(r)).toBe(LEAD); + expect(variantU8Str(r)).toBeNull(); + expect(variantU8Str(r)).toBe(7); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Variant, Dynamic", async () => { + const r = await framed( + "'hi'::Variant(UInt8, String)", + "toInt32(7)::Dynamic", + ); + expect(readInt32(r)).toBe(LEAD); + expect(variantU8Str(r)).toBe("hi"); + expect(readDynamic(r)).toBe(7); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Variant, Array", async () => { + const r = await framed( + "42::Variant(UInt8, String)", + "[1, 2]::Array(UInt8)", + ); + expect(readInt32(r)).toBe(LEAD); + expect(variantU8Str(r)).toBe(42); + expect(readArray(u8)(r)).toEqual([1, 2]); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("X = Dynamic", () => { + it("Dynamic, Dynamic", async () => { + const r = await framed("toInt32(7)::Dynamic", "'hi'::Dynamic"); + expect(readInt32(r)).toBe(LEAD); + expect(readDynamic(r)).toBe(7); + expect(readDynamic(r)).toBe("hi"); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Dynamic (NULL), Dynamic — the NULL is a lone Nothing tag", async () => { + const r = await framed("NULL::Dynamic", "toInt32(7)::Dynamic"); + expect(readInt32(r)).toBe(LEAD); + expect(readDynamic(r)).toBeNull(); + expect(readDynamic(r)).toBe(7); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Dynamic, Variant", async () => { + const r = await framed( + "toInt32(7)::Dynamic", + "'hi'::Variant(UInt8, String)", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readDynamic(r)).toBe(7); + expect(variantU8Str(r)).toBe("hi"); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Dynamic, JSON", async () => { + const r = await framed("toInt32(7)::Dynamic", `'{"a":1}'::JSON`); + expect(readInt32(r)).toBe(LEAD); + expect(readDynamic(r)).toBe(7); + expect(readJSON(r)).toEqual(new Map([["a", 1n]])); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Dynamic (Array), Array", async () => { + const r = await framed( + "[1, 2, 3]::Array(UInt8)::Dynamic", + "[4, 5]::Array(UInt8)", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readDynamic(r)).toEqual([1, 2, 3]); + expect(readArray(u8)(r)).toEqual([4, 5]); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("X = JSON", () => { + it("JSON, JSON", async () => { + const r = await framed(`'{"a":1}'::JSON`, `'{"b":2}'::JSON`); + expect(readInt32(r)).toBe(LEAD); + expect(readJSON(r)).toEqual(new Map([["a", 1n]])); + expect(readJSON(r)).toEqual(new Map([["b", 2n]])); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("JSON, Dynamic", async () => { + const r = await framed(`'{"a":1}'::JSON`, "toInt32(7)::Dynamic"); + expect(readInt32(r)).toBe(LEAD); + expect(readJSON(r)).toEqual(new Map([["a", 1n]])); + expect(readDynamic(r)).toBe(7); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("JSON, Array", async () => { + const r = await framed(`'{"a":1}'::JSON`, "[1, 2]::Array(UInt8)"); + expect(readInt32(r)).toBe(LEAD); + expect(readJSON(r)).toEqual(new Map([["a", 1n]])); + expect(readArray(u8)(r)).toEqual([1, 2]); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("X = Nullable (1-byte boundaries)", () => { + it("Nullable (NULL), Nullable (value)", async () => { + const r = await framed( + "CAST(NULL AS Nullable(String))", + "CAST('v' AS Nullable(String))", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readNullable(str)(r)).toBeNull(); + expect(readNullable(str)(r)).toBe("v"); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Nullable (value), Variant", async () => { + const r = await framed( + "CAST('v' AS Nullable(String))", + "42::Variant(UInt8, String)", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readNullable(str)(r)).toBe("v"); + expect(variantU8Str(r)).toBe(42); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Nullable (NULL), Dynamic", async () => { + const r = await framed( + "CAST(NULL AS Nullable(String))", + "toInt32(7)::Dynamic", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readNullable(str)(r)).toBeNull(); + expect(readDynamic(r)).toBe(7); + expect(readInt32(r)).toBe(TRAIL); + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/framing-nested.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/framing-nested.test.ts new file mode 100644 index 000000000..f34b27d60 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/framing-nested.test.ts @@ -0,0 +1,344 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { + readArray, + readMap, + readNullable, + readTuple, + readVariant, +} from "../src/composite.js"; +import { Cursor } from "../src/core.js"; +import { readDynamic } from "../src/dynamic.js"; +import { readInt32, readUInt8 } from "../src/integers.js"; +import { readJSON } from "../src/json.js"; +import { readString } from "../src/strings.js"; + +/** + * Framing tests for NESTED self-describing / variable-length types — the place + * where a buggy reader is most likely to desync. Each value combines two types + * that carry an internal type description or a variable length (`Dynamic`, + * `Variant`, `JSON`, `Array`, `Map`, `Tuple`, `Nullable`), so the boundary + * between inner readers is "blurry": a reader that miscounts one element's bytes + * silently shifts everything after it. + * + * Same harness as framing.test.ts: the value sits as the MIDDLE column between + * two distinct Int32 sentinels `i32(LEAD), X, i32(TRAIL)`. Reading TRAIL back + * correctly is only possible if X consumed EXACTLY its bytes. The adjacency + * cases (Tuple of two variable things, NULL/empty inners) are the sharpest. + */ +const LEAD = 123456789; +const TRAIL = 987654321; + +// Every experimental / suspicious flag on, as in framing.test.ts. +const SETTINGS = [ + "enable_time_time64_type = 1", + "allow_experimental_variant_type = 1", + "allow_suspicious_variant_types = 1", + "allow_experimental_dynamic_type = 1", + "allow_experimental_json_type = 1", + "enable_json_type = 1", + "allow_experimental_qbit_type = 1", + "allow_suspicious_low_cardinality_types = 1", +].join(", "); + +async function framed(expr: string): Promise { + const sql = + `SELECT toInt32(${LEAD}) AS a, ${expr} AS x, toInt32(${TRAIL}) AS b` + + ` SETTINGS ${SETTINGS} FORMAT RowBinary`; + return new Cursor(await query(sql)); +} + +// Shared inner readers, written once so the nesting reads cleanly below. +const u8 = readUInt8; +const str = readString; +// Variant(UInt8, String) sorts to [String(0), UInt8(1)]. +const variantU8Str = readVariant([readString, readUInt8]); + +describe("framing (nested): two self-describing / variable types — boundaries stay exact", () => { + describe("Array of a variable / self-describing inner", () => { + it("Array(Variant(UInt8, String)) with a NULL element", async () => { + const r = await framed( + "[42::Variant(UInt8, String), 'hi'::Variant(UInt8, String), NULL::Variant(UInt8, String)]::Array(Variant(UInt8, String))", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readArray(variantU8Str)(r)).toEqual([42, "hi", null]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Array(Dynamic)", async () => { + const r = await framed("['x'::Dynamic, 'y'::Dynamic]::Array(Dynamic)"); + expect(readInt32(r)).toBe(LEAD); + expect(readArray(readDynamic)(r)).toEqual(["x", "y"]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Array(Nullable(String)) with a hole", async () => { + const r = await framed("['a', NULL, 'b']::Array(Nullable(String))"); + expect(readInt32(r)).toBe(LEAD); + expect(readArray(readNullable(str))(r)).toEqual(["a", null, "b"]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Array(Array(String)) — variable-length inner arrays", async () => { + const r = await framed("[['a', 'b'], ['c']]::Array(Array(String))"); + expect(readInt32(r)).toBe(LEAD); + expect(readArray(readArray(str))(r)).toEqual([["a", "b"], ["c"]]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Array(Map(String, UInt8))", async () => { + const r = await framed( + "[map('a', 1), map('b', 2)]::Array(Map(String, UInt8))", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readArray(readMap(str, u8))(r)).toEqual([ + new Map([["a", 1]]), + new Map([["b", 2]]), + ]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Array(Tuple(UInt8, String))", async () => { + const r = await framed( + "[(1, 'x'), (2, 'y')]::Array(Tuple(UInt8, String))", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readArray(readTuple([u8, str]))(r)).toEqual([ + [1, "x"], + [2, "y"], + ]); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Map(String, variable value)", () => { + it("Map(String, Variant(UInt8, String))", async () => { + const r = await framed( + "map('a', 42::Variant(UInt8, String), 'b', 'hi'::Variant(UInt8, String))", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readMap(str, variantU8Str)(r)).toEqual( + new Map([ + ["a", 42], + ["b", "hi"], + ]), + ); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Map(String, Dynamic) with mixed value types", async () => { + const r = await framed( + "map('a', toInt32(7)::Dynamic, 'b', 'hi'::Dynamic)::Map(String, Dynamic)", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readMap(str, readDynamic)(r)).toEqual( + new Map([ + ["a", 7], + ["b", "hi"], + ]), + ); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Map(String, Array(UInt8))", async () => { + const r = await framed( + "map('x', [1, 2], 'y', [3])::Map(String, Array(UInt8))", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readMap(str, readArray(u8))(r)).toEqual( + new Map([ + ["x", [1, 2]], + ["y", [3]], + ]), + ); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Map(String, Nullable(UInt8)) with a NULL value", async () => { + const r = await framed( + "map('a', 1, 'b', NULL)::Map(String, Nullable(UInt8))", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readMap(str, readNullable(u8))(r)).toEqual( + new Map([ + ["a", 1], + ["b", null], + ]), + ); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Tuple adjacency — two variable things back-to-back", () => { + it("Tuple(Dynamic, Dynamic)", async () => { + const r = await framed( + "(toInt32(7)::Dynamic, 'hi'::Dynamic)::Tuple(Dynamic, Dynamic)", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readTuple([readDynamic, readDynamic])(r)).toEqual([7, "hi"]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Tuple(Dynamic, Dynamic) — first is a 1-byte NULL", async () => { + const r = await framed( + "(NULL::Dynamic, toInt32(9)::Dynamic)::Tuple(Dynamic, Dynamic)", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readTuple([readDynamic, readDynamic])(r)).toEqual([null, 9]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Tuple(Variant, Variant)", async () => { + const r = await framed( + "(42::Variant(UInt8, String), 'x'::Variant(UInt8, String))::Tuple(Variant(UInt8, String), Variant(UInt8, String))", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readTuple([variantU8Str, variantU8Str])(r)).toEqual([42, "x"]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Tuple(Array(UInt8), Array(UInt8)) — two adjacent length-prefixed arrays", async () => { + const r = await framed( + "([1, 2], [3, 4])::Tuple(Array(UInt8), Array(UInt8))", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readTuple([readArray(u8), readArray(u8)])(r)).toEqual([ + [1, 2], + [3, 4], + ]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Tuple(Array(UInt8), String)", async () => { + const r = await framed("([1, 2], 'x')::Tuple(Array(UInt8), String)"); + expect(readInt32(r)).toBe(LEAD); + expect(readTuple([readArray(u8), str])(r)).toEqual([[1, 2], "x"]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Tuple(Nullable(String), Nullable(String)) — NULL then value", async () => { + const r = await framed( + "(NULL, 'x')::Tuple(Nullable(String), Nullable(String))", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readTuple([readNullable(str), readNullable(str)])(r)).toEqual([ + null, + "x", + ]); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Variant whose active alternative is variable-length", () => { + it("Variant(Array(UInt8), String) holding the Array (discriminant 0)", async () => { + const r = await framed("[1, 2, 3]::Variant(Array(UInt8), String)"); + expect(readInt32(r)).toBe(LEAD); + // sorted [Array(UInt8)(0), String(1)] + expect(readVariant([readArray(u8), readString])(r)).toEqual([1, 2, 3]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Variant(Array(UInt8), String) holding the String (discriminant 1)", async () => { + const r = await framed("'hi'::Variant(Array(UInt8), String)"); + expect(readInt32(r)).toBe(LEAD); + expect(readVariant([readArray(u8), readString])(r)).toBe("hi"); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Variant(Map(String, UInt8), UInt8) holding the Map (discriminant 0)", async () => { + const r = await framed("map('a', 1)::Variant(Map(String, UInt8), UInt8)"); + expect(readInt32(r)).toBe(LEAD); + // sorted [Map(String, UInt8)(0), UInt8(1)] + expect(readVariant([readMap(str, u8), readUInt8])(r)).toEqual( + new Map([["a", 1]]), + ); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Variant(Tuple(UInt8, String), UInt8) holding the Tuple (discriminant 0)", async () => { + const r = await framed("(1, 'x')::Variant(Tuple(UInt8, String), UInt8)"); + expect(readInt32(r)).toBe(LEAD); + // sorted [Tuple(UInt8, String)(0), UInt8(1)] + expect(readVariant([readTuple([u8, str]), readUInt8])(r)).toEqual([ + 1, + "x", + ]); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Dynamic wrapping a nested self-describing type", () => { + it("Dynamic(Array(Variant(UInt8, String)))", async () => { + const r = await framed( + "[42::Variant(UInt8, String), 'hi'::Variant(UInt8, String)]::Array(Variant(UInt8, String))::Dynamic", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readDynamic(r)).toEqual([42, "hi"]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Dynamic(Map(String, Variant(UInt8, String)))", async () => { + const r = await framed( + "map('a', 42::Variant(UInt8, String))::Map(String, Variant(UInt8, String))::Dynamic", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readDynamic(r)).toEqual(new Map([["a", 42]])); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Dynamic(Array(Dynamic))", async () => { + const r = await framed( + "['x'::Dynamic, 'y'::Dynamic]::Array(Dynamic)::Dynamic", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readDynamic(r)).toEqual(["x", "y"]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Dynamic(JSON)", async () => { + const r = await framed(`'{"a":1}'::JSON::Dynamic`); + expect(readInt32(r)).toBe(LEAD); + expect(readDynamic(r)).toEqual(new Map([["a", 1n]])); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Dynamic(Nested)", async () => { + const r = await framed( + "[(1, 'x'), (2, 'y')]::Nested(a UInt8, b String)::Dynamic", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readDynamic(r)).toEqual([ + { a: 1, b: "x" }, + { a: 2, b: "y" }, + ]); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("JSON with several self-describing paths", () => { + it("mixed value types (Int64, String, Array)", async () => { + const r = await framed(`'{"i":1,"s":"hi","arr":[1,2]}'::JSON`); + expect(readInt32(r)).toBe(LEAD); + expect(readJSON(r)).toEqual( + new Map([ + ["arr", [1n, 2n]], + ["s", "hi"], + ["i", 1n], + ]), + ); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("nested object flattened to dotted paths", async () => { + const r = await framed(`'{"a":{"b":2},"c":3}'::JSON`); + expect(readInt32(r)).toBe(LEAD); + expect(readJSON(r)).toEqual( + new Map([ + ["a.b", 2n], + ["c", 3n], + ]), + ); + expect(readInt32(r)).toBe(TRAIL); + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/framing.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/framing.test.ts new file mode 100644 index 000000000..8d19363a6 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/framing.test.ts @@ -0,0 +1,609 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { readBool } from "../src/bool.js"; +import { + readArray, + readMap, + readNullable, + readQBit, + readTuple, + readTupleNamed, + readVariant, +} from "../src/composite.js"; +import { Cursor } from "../src/core.js"; +import { + readDate, + readDate32, + readDateTime, + readDateTime64, +} from "../src/datetime.js"; +import { + readDecimal128, + readDecimal256, + readDecimal32, + readDecimal64, +} from "../src/decimals.js"; +import { readDynamic } from "../src/dynamic.js"; +import { readEnum16, readEnum8 } from "../src/enums.js"; +import { readBFloat16, readFloat32, readFloat64 } from "../src/floats.js"; +import { + readGeometry, + readLineString, + readMultiLineString, + readMultiPolygon, + readPoint, + readPolygon, + readRing, +} from "../src/geo.js"; +import { + readInt128, + readInt16, + readInt256, + readInt32, + readInt64, + readInt8, + readUInt128, + readUInt16, + readUInt256, + readUInt32, + readUInt64, + readUInt8, +} from "../src/integers.js"; +import { readInterval } from "../src/interval.js"; +import { formatIPv4, formatIPv6, readIPv4, readIPv6 } from "../src/ip.js"; +import { readJSON } from "../src/json.js"; +import { readFixedString, readString } from "../src/strings.js"; +import { readTime, readTime64 } from "../src/time.js"; +import { formatUUID, readUUID } from "../src/uuid.js"; + +/** + * Framing tests: every type is placed as the MIDDLE column between two distinct + * Int32 sentinels — `i32(LEAD), X, i32(TRAIL)`. Reading the trailing sentinel + * correctly only works if X's reader consumed EXACTLY its bytes: one byte short + * or long and the final `readInt32()` returns garbage instead of TRAIL. So each + * test reads LEAD, then X, then asserts TRAIL — a tight check that the middle + * reader stops on the dot. + * + * The NULL / empty cases (Nullable null, Variant null, Dynamic null, empty + * Array) are the sharpest: the value is a single byte, so an over-read is caught + * immediately by the sentinel. + */ +const LEAD = 123456789; +const TRAIL = 987654321; + +/** + * Every experimental / suspicious type flag this skill targets, enabled for all + * queries. It is fine to turn everything on unconditionally here — the skill is + * meant to read whatever a server emits — so each case just names its type and + * shares this one settings string. + */ +const SETTINGS = [ + "enable_time_time64_type = 1", + "allow_experimental_variant_type = 1", + "allow_suspicious_variant_types = 1", + "allow_experimental_dynamic_type = 1", + "allow_experimental_json_type = 1", + "enable_json_type = 1", + "allow_experimental_qbit_type = 1", + "allow_suspicious_low_cardinality_types = 1", +].join(", "); + +/** + * Build a `i32(LEAD), X, i32(TRAIL)` row, run it, and return a fresh reader over + * the bytes. Each test reads the leading sentinel, X, and the trailing sentinel + * itself. + */ +async function framed(expr: string): Promise { + const sql = + `SELECT toInt32(${LEAD}) AS a, ${expr} AS x, toInt32(${TRAIL}) AS b` + + ` SETTINGS ${SETTINGS} FORMAT RowBinary`; + return new Cursor(await query(sql)); +} + +describe("framing: i32, X, i32 — the middle reader must stop at the exact byte", () => { + describe("Integers", () => { + it("Int8", async () => { + const r = await framed("toInt8(-5)"); + expect(readInt32(r)).toBe(LEAD); + expect(readInt8(r)).toBe(-5); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Int16", async () => { + const r = await framed("toInt16(-12345)"); + expect(readInt32(r)).toBe(LEAD); + expect(readInt16(r)).toBe(-12345); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Int32", async () => { + const r = await framed("toInt32(-70000)"); + expect(readInt32(r)).toBe(LEAD); + expect(readInt32(r)).toBe(-70000); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Int64", async () => { + const r = await framed("toInt64(-5)"); + expect(readInt32(r)).toBe(LEAD); + expect(readInt64(r)).toBe(-5n); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Int128", async () => { + const r = await framed("toInt128(-5)"); + expect(readInt32(r)).toBe(LEAD); + expect(readInt128(r)).toBe(-5n); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Int256", async () => { + const r = await framed("toInt256(-5)"); + expect(readInt32(r)).toBe(LEAD); + expect(readInt256(r)).toBe(-5n); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("UInt8", async () => { + const r = await framed("toUInt8(200)"); + expect(readInt32(r)).toBe(LEAD); + expect(readUInt8(r)).toBe(200); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("UInt16", async () => { + const r = await framed("toUInt16(60000)"); + expect(readInt32(r)).toBe(LEAD); + expect(readUInt16(r)).toBe(60000); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("UInt32", async () => { + const r = await framed("toUInt32(4000000000)"); + expect(readInt32(r)).toBe(LEAD); + expect(readUInt32(r)).toBe(4000000000); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("UInt64", async () => { + const r = await framed("toUInt64(5)"); + expect(readInt32(r)).toBe(LEAD); + expect(readUInt64(r)).toBe(5n); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("UInt128", async () => { + const r = await framed("toUInt128(5)"); + expect(readInt32(r)).toBe(LEAD); + expect(readUInt128(r)).toBe(5n); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("UInt256", async () => { + const r = await framed("toUInt256(5)"); + expect(readInt32(r)).toBe(LEAD); + expect(readUInt256(r)).toBe(5n); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Floats", () => { + it("Float32", async () => { + const r = await framed("toFloat32(1.5)"); + expect(readInt32(r)).toBe(LEAD); + expect(readFloat32(r)).toBe(1.5); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Float64", async () => { + const r = await framed("toFloat64(1.5)"); + expect(readInt32(r)).toBe(LEAD); + expect(readFloat64(r)).toBe(1.5); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("BFloat16", async () => { + const r = await framed("toBFloat16(1.5)"); + expect(readInt32(r)).toBe(LEAD); + expect(readBFloat16(r)).toBe(1.5); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Decimals", () => { + it("Decimal32", async () => { + const r = await framed("toDecimal32(1.5, 4)"); + expect(readInt32(r)).toBe(LEAD); + expect(readDecimal32(4)(r)).toEqual([15000n, 4]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Decimal64", async () => { + const r = await framed("toDecimal64(-12.34, 2)"); + expect(readInt32(r)).toBe(LEAD); + expect(readDecimal64(2)(r)).toEqual([-1234n, 2]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Decimal128", async () => { + const r = await framed("toDecimal128(1.5, 4)"); + expect(readInt32(r)).toBe(LEAD); + expect(readDecimal128(4)(r)).toEqual([15000n, 4]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Decimal256", async () => { + const r = await framed("toDecimal256(1.5, 4)"); + expect(readInt32(r)).toBe(LEAD); + expect(readDecimal256(4)(r)).toEqual([15000n, 4]); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Bool and strings", () => { + it("Bool", async () => { + const r = await framed("true"); + expect(readInt32(r)).toBe(LEAD); + expect(readBool(r)).toBe(true); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("String", async () => { + const r = await framed("'hello'"); + expect(readInt32(r)).toBe(LEAD); + expect(readString(r)).toBe("hello"); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("FixedString (NUL padding counts toward the fixed width)", async () => { + const r = await framed("CAST('ab' AS FixedString(5))"); + expect(readInt32(r)).toBe(LEAD); + expect(readFixedString(5)(r)).toBe("ab\x00\x00\x00"); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Dates and times", () => { + it("Date", async () => { + const r = await framed("toDate('2021-03-15')"); + expect(readInt32(r)).toBe(LEAD); + expect(readDate(r).toISOString()).toBe("2021-03-15T00:00:00.000Z"); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Date32", async () => { + const r = await framed("toDate32('1950-01-01')"); + expect(readInt32(r)).toBe(LEAD); + expect(readDate32(r).toISOString()).toBe("1950-01-01T00:00:00.000Z"); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("DateTime", async () => { + const r = await framed("toDateTime('2021-01-01 00:00:00', 'UTC')"); + expect(readInt32(r)).toBe(LEAD); + expect(readDateTime(r).getTime()).toBe(1609459200000); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("DateTime64", async () => { + const r = await framed( + "toDateTime64('2021-01-01 00:00:00.123', 3, 'UTC')", + ); + expect(readInt32(r)).toBe(LEAD); + const [d, n] = readDateTime64(3)(r); + expect(d.toISOString()).toBe("2021-01-01T00:00:00.000Z"); + expect(n).toBe(123_000_000); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Time", async () => { + const r = await framed("CAST('12:34:56' AS Time)"); + expect(readInt32(r)).toBe(LEAD); + expect(readTime(r)).toBe(45296); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Time64", async () => { + const r = await framed("toTime64('12:34:56.123', 3)"); + expect(readInt32(r)).toBe(LEAD); + expect(readTime64(3)(r)).toEqual([45296123n, 3]); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("UUID and networking", () => { + it("UUID", async () => { + const r = await framed("toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')"); + expect(readInt32(r)).toBe(LEAD); + expect(formatUUID(readUUID(r))).toBe( + "61f0c404-5cb3-11e7-907b-a6006ad3dba0", + ); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("IPv4", async () => { + const r = await framed("toIPv4('1.2.3.4')"); + expect(readInt32(r)).toBe(LEAD); + expect(formatIPv4(readIPv4(r))).toBe("1.2.3.4"); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("IPv6", async () => { + const r = await framed("toIPv6('2001:db8::1')"); + expect(readInt32(r)).toBe(LEAD); + expect(formatIPv6(readIPv6(r))).toBe("2001:db8::1"); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Enums", () => { + it("Enum8", async () => { + const r = await framed("CAST('b' AS Enum8('a' = 1, 'b' = 2))"); + expect(readInt32(r)).toBe(LEAD); + expect(readEnum8(r)).toBe(2); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Enum16", async () => { + const r = await framed("CAST('big' AS Enum16('small' = 1, 'big' = 300))"); + expect(readInt32(r)).toBe(LEAD); + expect(readEnum16(r)).toBe(300); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Composite and wrappers", () => { + it("Nullable (value present)", async () => { + const r = await framed("CAST(7 AS Nullable(UInt8))"); + expect(readInt32(r)).toBe(LEAD); + expect(readNullable(readUInt8)(r)).toBe(7); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Nullable (NULL — a single flag byte)", async () => { + const r = await framed("CAST(NULL AS Nullable(UInt8))"); + expect(readInt32(r)).toBe(LEAD); + expect(readNullable(readUInt8)(r)).toBeNull(); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("LowCardinality(String)", async () => { + const r = await framed("CAST('x' AS LowCardinality(String))"); + expect(readInt32(r)).toBe(LEAD); + expect(readString(r)).toBe("x"); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Array(UInt8)", async () => { + const r = await framed("CAST([1, 2, 3] AS Array(UInt8))"); + expect(readInt32(r)).toBe(LEAD); + expect(readArray(readUInt8)(r)).toEqual([1, 2, 3]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Array empty (a single count byte)", async () => { + const r = await framed("CAST([] AS Array(UInt8))"); + expect(readInt32(r)).toBe(LEAD); + expect(readArray(readUInt8)(r)).toEqual([]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Tuple", async () => { + const r = await framed("CAST((1, 'x') AS Tuple(UInt8, String))"); + expect(readInt32(r)).toBe(LEAD); + expect(readTuple([readUInt8, readString])(r)).toEqual([1, "x"]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Tuple named", async () => { + const r = await framed("CAST((1, 'x') AS Tuple(a UInt8, b String))"); + expect(readInt32(r)).toBe(LEAD); + expect(readTupleNamed({ a: readUInt8, b: readString })(r)).toEqual({ + a: 1, + b: "x", + }); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Map", async () => { + const r = await framed("CAST(map('a', 1, 'b', 2) AS Map(String, UInt8))"); + expect(readInt32(r)).toBe(LEAD); + expect(readMap(readString, readUInt8)(r)).toEqual( + new Map([ + ["a", 1], + ["b", 2], + ]), + ); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Nested", async () => { + const r = await framed( + "CAST([(1, 'x'), (2, 'y')] AS Nested(a UInt8, b String))", + ); + expect(readInt32(r)).toBe(LEAD); + expect( + readArray(readTupleNamed({ a: readUInt8, b: readString }))(r), + ).toEqual([ + { a: 1, b: "x" }, + { a: 2, b: "y" }, + ]); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Geo", () => { + it("Point", async () => { + const r = await framed("CAST((1.5, 2.5) AS Point)"); + expect(readInt32(r)).toBe(LEAD); + expect(readPoint(r)).toEqual([1.5, 2.5]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Ring", async () => { + const r = await framed("CAST([(0, 0), (1, 2)] AS Ring)"); + expect(readInt32(r)).toBe(LEAD); + expect(readRing(r)).toEqual([ + [0, 0], + [1, 2], + ]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("LineString", async () => { + const r = await framed("CAST([(3, 4), (5, 6)] AS LineString)"); + expect(readInt32(r)).toBe(LEAD); + expect(readLineString(r)).toEqual([ + [3, 4], + [5, 6], + ]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("MultiLineString", async () => { + const r = await framed("CAST([[(0, 0), (1, 1)]] AS MultiLineString)"); + expect(readInt32(r)).toBe(LEAD); + expect(readMultiLineString(r)).toEqual([ + [ + [0, 0], + [1, 1], + ], + ]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Polygon", async () => { + const r = await framed("CAST([[(0, 0), (1, 0), (1, 1)]] AS Polygon)"); + expect(readInt32(r)).toBe(LEAD); + expect(readPolygon(r)).toEqual([ + [ + [0, 0], + [1, 0], + [1, 1], + ], + ]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("MultiPolygon", async () => { + const r = await framed( + "CAST([[[(0, 0), (1, 0), (1, 1)]]] AS MultiPolygon)", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readMultiPolygon(r)).toEqual([ + [ + [ + [0, 0], + [1, 0], + [1, 1], + ], + ], + ]); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Geometry", async () => { + const r = await framed("CAST(CAST((1.5, 2.5) AS Point) AS Geometry)"); + expect(readInt32(r)).toBe(LEAD); + expect(readGeometry(r)).toEqual([1.5, 2.5]); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Intervals", () => { + it("Interval", async () => { + const r = await framed("toIntervalSecond(5)"); + expect(readInt32(r)).toBe(LEAD); + expect(readInterval(r)).toBe(5n); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Variant", () => { + it("Variant (value)", async () => { + const r = await framed("CAST(42 AS Variant(UInt8, String))"); + expect(readInt32(r)).toBe(LEAD); + expect(readVariant([readString, readUInt8])(r)).toBe(42); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Variant (NULL — a single discriminant byte)", async () => { + const r = await framed("CAST(NULL AS Variant(UInt8, String))"); + expect(readInt32(r)).toBe(LEAD); + expect(readVariant([readString, readUInt8])(r)).toBeNull(); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Dynamic", () => { + it("Dynamic (scalar)", async () => { + const r = await framed("CAST(toUInt64(42) AS Dynamic)"); + expect(readInt32(r)).toBe(LEAD); + expect(readDynamic(r)).toBe(42n); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Dynamic (NULL — a single Nothing tag)", async () => { + const r = await framed("CAST(NULL AS Dynamic)"); + expect(readInt32(r)).toBe(LEAD); + expect(readDynamic(r)).toBeNull(); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("Dynamic (Array)", async () => { + const r = await framed("CAST([1, 2, 3]::Array(UInt8) AS Dynamic)"); + expect(readInt32(r)).toBe(LEAD); + expect(readDynamic(r)).toEqual([1, 2, 3]); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("JSON", () => { + it("JSON", async () => { + const r = await framed(`'{"a":1}'::JSON`); + expect(readInt32(r)).toBe(LEAD); + expect(readJSON(r)).toEqual(new Map([["a", 1n]])); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Aggregate state", () => { + it("SimpleAggregateFunction (transparent → inner UInt64)", async () => { + const r = await framed( + "CAST(42 AS SimpleAggregateFunction(sum, UInt64))", + ); + expect(readInt32(r)).toBe(LEAD); + expect(readUInt64(r)).toBe(42n); + expect(readInt32(r)).toBe(TRAIL); + }); + + it("AggregateFunction (sumState: known UInt64 layout)", async () => { + const r = await framed("sumState(toUInt64(42))"); + expect(readInt32(r)).toBe(LEAD); + expect(readUInt64(r)).toBe(42n); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Vector search", () => { + it("QBit", async () => { + const r = await framed("CAST([1.0, 2.0] AS QBit(Float32, 2))"); + expect(readInt32(r)).toBe(LEAD); + expect(readQBit(readFloat32)(r)).toEqual([1, 2]); + expect(readInt32(r)).toBe(TRAIL); + }); + }); + + describe("Misc", () => { + it("Nothing (empty Array(Nothing) — element reader never runs)", async () => { + const r = await framed("[]"); + expect(readInt32(r)).toBe(LEAD); + expect( + readArray(() => { + throw new Error("unreachable"); + })(r), + ).toEqual([]); + expect(readInt32(r)).toBe(TRAIL); + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/iot.bench.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/iot.bench.ts new file mode 100644 index 000000000..f104170c7 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/iot.bench.ts @@ -0,0 +1,134 @@ +import { bench, describe } from "vitest"; +import { query } from "./clickhouse.js"; +import { type Reader, Cursor } from "../src/core.js"; +import { + type IotRow, + readIotRow, + readIotRowFast, +} from "../src/examples/iot.js"; + +/** + * Benchmark: RowBinary vs JSON for a table of IoT sensor readings — the + * dense-numeric, fixed-width shape RowBinary exists for. The SKILL's + * format-choice guidance says reach for JSON when the result is string-heavy and + * for RowBinary when it is "high-volume fixed-width numeric"; this is the latter, + * so we measure the gap honestly against the JSON formats a knowledgeable user + * would actually choose: + * + * - JSONEachRow — newline-delimited objects (keys repeated every row) + * - JSONCompactEachRow — newline-delimited arrays (no repeated keys; smaller) + * + * For each JSON format we use the fastest idiomatic decode: splice the rows into + * one `[...]` document and hand it to V8's native `JSON.parse` in a single call. + */ +const N = 50_000; + +// Same rows, three formats. Deterministic, dense-numeric IoT readings. +const SELECT = + `SELECT toUInt32(number % 1000) AS sensor_id, ` + + `toDateTime64(1700000000 + number, 3) AS ts, ` + + `20 + (number % 1500) / 100 AS temperature, ` + + `30 + (number % 7000) / 100 AS humidity, ` + + `980 + (number % 6000) / 100 AS pressure, ` + + `toFloat32(3 + (number % 200) / 100) AS battery, ` + + `toUInt8(number % 4) AS status ` + + `FROM numbers(${N})`; + +const RB_BUF = await query(`${SELECT} FORMAT RowBinary`); +const JSON_BUF = await query(`${SELECT} FORMAT JSONEachRow`); +const JSON_COMPACT_BUF = await query(`${SELECT} FORMAT JSONCompactEachRow`); + +// --- decoders --------------------------------------------------------------- + +function decodeRowBinary(read: Reader): IotRow[] { + const s = new Cursor(RB_BUF); + const out: IotRow[] = []; + while (s.pos < s.buf.length) out.push(read(s)); + return out; +} + +// Wrap newline-delimited JSON rows into one array and parse in a single call — +// the fastest way to drive V8's native JSON.parse over a whole response. +function decodeJsonArray(buf: Buffer): unknown[] { + const text = buf.toString("utf8"); + return JSON.parse(`[${text.trimEnd().replaceAll("\n", ",")}]`); +} + +// --- correctness cross-check (runs once at load) ---------------------------- + +const COLS: (keyof IotRow)[] = [ + "sensor_id", + "temperature", + "humidity", + "pressure", + "battery", + "status", +]; +{ + const rb = decodeRowBinary(readIotRowFast); + const api = decodeRowBinary(readIotRow); + const je = decodeJsonArray(JSON_BUF) as Record[]; + const jc = decodeJsonArray(JSON_COMPACT_BUF) as (number | string)[][]; + + if (rb.length !== N) + throw new Error(`RowBinary: ${rb.length} rows, expected ${N}`); + if (je.length !== N) + throw new Error(`JSONEachRow: ${je.length} rows, expected ${N}`); + if (jc.length !== N) + throw new Error(`JSONCompactEachRow: ${jc.length} rows, expected ${N}`); + + // API reader and fast reader must agree exactly. + if (JSON.stringify(api) !== JSON.stringify(rb)) + throw new Error("iot: API vs fast mismatch"); + + // RowBinary (binary float) and JSON (decimal text -> float) must agree to a + // tiny epsilon on every numeric column, spot-checked across the result. + const order: (keyof IotRow)[] = [ + "sensor_id", + "ts", + "temperature", + "humidity", + "pressure", + "battery", + "status", + ]; + for (const i of [0, 1, 123, 4999, N - 1]) { + for (const c of COLS) { + const a = rb[i]![c] as number; + const b = Number(je[i]![c]); + const d = Number(jc[i]![order.indexOf(c)]); + const tol = c === "battery" ? 1e-2 : 1e-9; // Float32 battery has less precision + if (Math.abs(a - b) > tol || Math.abs(a - d) > tol) { + throw new Error( + `iot: ${c}@${i} mismatch rb=${a} json=${b} compact=${d}`, + ); + } + } + } + + const mb = (b: Buffer) => (b.length / 1e6).toFixed(2); + const perRow = (b: Buffer) => (b.length / N).toFixed(1); + console.log( + `\n IoT readings — ${N.toLocaleString()} rows, wire size on the HTTP response:\n` + + ` RowBinary ${mb(RB_BUF)} MB (${perRow(RB_BUF)} B/row)\n` + + ` JSONCompactEachRow ${mb(JSON_COMPACT_BUF)} MB (${perRow(JSON_COMPACT_BUF)} B/row) ${(JSON_COMPACT_BUF.length / RB_BUF.length).toFixed(1)}x\n` + + ` JSONEachRow ${mb(JSON_BUF)} MB (${perRow(JSON_BUF)} B/row) ${(JSON_BUF.length / RB_BUF.length).toFixed(1)}x\n`, + ); +} + +// --- benchmarks ------------------------------------------------------------- + +describe("IoT readings: RowBinary vs JSON decode throughput", () => { + bench("RowBinary — optimized (monomorphized)", () => { + decodeRowBinary(readIotRowFast); + }); + bench("RowBinary — API (combinators)", () => { + decodeRowBinary(readIotRow); + }); + bench("JSONCompactEachRow — JSON.parse", () => { + decodeJsonArray(JSON_COMPACT_BUF); + }); + bench("JSONEachRow — JSON.parse", () => { + decodeJsonArray(JSON_BUF); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/iot.columnar.bench.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/iot.columnar.bench.ts new file mode 100644 index 000000000..6148b7b6f --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/iot.columnar.bench.ts @@ -0,0 +1,57 @@ +import { bench, describe } from "vitest"; +import { query } from "./clickhouse.js"; +import { Cursor } from "../src/core.js"; +import { + type IotRow, + decodeIotColumnar, + iotRowAt, + readIotRowFast, +} from "../src/examples/iot.js"; + +/** + * Benchmark: row-objects (AoS, `readIotRowFast`) vs columnar (SoA, + * `decodeIotColumnar`) over the same fixed-width IoT buffer. Columnar removes + * the per-row object + `Date` allocation that dominates a numeric decode, for a + * ~4x win — the "free 4x, in plain JS" the WASM investigation surfaced (see + * `case-studies/wasm-vs-js.md`). + */ +const N = 50_000; +const SELECT = + `SELECT toUInt32(number % 1000) AS sensor_id, ` + + `toDateTime64(1700000000 + number, 3) AS ts, ` + + `20 + (number % 1500) / 100 AS temperature, ` + + `30 + (number % 7000) / 100 AS humidity, ` + + `980 + (number % 6000) / 100 AS pressure, ` + + `toFloat32(3 + (number % 200) / 100) AS battery, ` + + `toUInt8(number % 4) AS status ` + + `FROM numbers(${N})`; +const BUF = await query(`${SELECT} FORMAT RowBinary`); + +function decodeRows(): IotRow[] { + const s = new Cursor(BUF); + const out: IotRow[] = []; + while (s.pos < s.buf.length) out.push(readIotRowFast(s)); + return out; +} + +// correctness: columnar (via the lazy row accessor) must equal the row decode +{ + const rows = decodeRows(); + const cols = decodeIotColumnar(BUF); + if (rows.length !== N || cols.sensor_id.length !== N) + throw new Error("columnar: row count"); + for (const i of [0, 1, 123, 4999, N - 1]) { + if (JSON.stringify(rows[i]) !== JSON.stringify(iotRowAt(cols, i))) { + throw new Error(`columnar: row ${i} mismatch`); + } + } +} + +describe("IoT decode: row-objects vs columnar", () => { + bench("rows — readIotRowFast (objects + Date)", () => { + decodeRows(); + }); + bench("columnar — decodeIotColumnar (typed arrays)", () => { + decodeIotColumnar(BUF); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/iot.wasm-headroom.bench.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/iot.wasm-headroom.bench.ts new file mode 100644 index 000000000..0a6eb7292 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/iot.wasm-headroom.bench.ts @@ -0,0 +1,132 @@ +import { bench, describe } from "vitest"; +import { query } from "./clickhouse.js"; +import { Cursor } from "../src/core.js"; +import { type IotRow, readIotRowFast } from "../src/examples/iot.js"; + +/** + * WASM headroom probe — NOT a WASM implementation, but the measurement that + * decides whether writing one is worth it. + * + * A WASM parser can read bytes, but it CANNOT allocate JS objects / strings / + * BigInts / Dates — those must be materialized on the JS side whatever decodes + * the bytes. So the maximum a WASM parser could ever shave off our current + * row-object decode is bounded by: + * + * (full row-object decode time) − (unavoidable JS-side materialization) + * + * We bracket that headroom with three decoders over the SAME IoT buffer (the + * best case for RowBinary — every column fixed-width numeric): + * + * 1. rows — the current fast reader: builds {…} objects + Date per row. + * 2. columnar — same reads, written into preallocated typed arrays, NO + * per-row objects. The "different output contract" a WASM + * parser would target. + * 3. parseOnly — same reads, accumulated into a scalar checksum, allocates + * NOTHING. The pure byte-arithmetic floor: a WASM parser + * cannot beat this slice by much (V8 already compiles DataView + * reads to native loads), and still has to pay it. + * + * Read the gaps: rows→parseOnly is the materialization WASM can't remove; + * rows→columnar is the win available in plain JS by changing the output shape. + */ +const N = 50_000; +const SELECT = + `SELECT toUInt32(number % 1000) AS sensor_id, ` + + `toDateTime64(1700000000 + number, 3) AS ts, ` + + `20 + (number % 1500) / 100 AS temperature, ` + + `30 + (number % 7000) / 100 AS humidity, ` + + `980 + (number % 6000) / 100 AS pressure, ` + + `toFloat32(3 + (number % 200) / 100) AS battery, ` + + `toUInt8(number % 4) AS status ` + + `FROM numbers(${N})`; +const BUF = await query(`${SELECT} FORMAT RowBinary`); +const ROW_BYTES = 41; + +// 1. Current output contract: an array of row objects. +function decodeRows(): IotRow[] { + const s = new Cursor(BUF); + const out: IotRow[] = []; + while (s.pos < s.buf.length) out.push(readIotRowFast(s)); + return out; +} + +type Columns = { + sensor_id: Uint32Array; + ts: Float64Array; // epoch ms + temperature: Float64Array; + humidity: Float64Array; + pressure: Float64Array; + battery: Float32Array; + status: Uint8Array; +}; + +// 2. Columnar contract: straight into typed arrays, no per-row objects. +function decodeColumnar(): Columns { + const view = new DataView(BUF.buffer, BUF.byteOffset, BUF.byteLength); + const n = (BUF.length / ROW_BYTES) | 0; + const c: Columns = { + sensor_id: new Uint32Array(n), + ts: new Float64Array(n), + temperature: new Float64Array(n), + humidity: new Float64Array(n), + pressure: new Float64Array(n), + battery: new Float32Array(n), + status: new Uint8Array(n), + }; + let o = 0; + for (let i = 0; i < n; i++) { + c.sensor_id[i] = view.getUint32(o, true); + c.ts[i] = Number(view.getBigInt64(o + 4, true)); + c.temperature[i] = view.getFloat64(o + 12, true); + c.humidity[i] = view.getFloat64(o + 20, true); + c.pressure[i] = view.getFloat64(o + 28, true); + c.battery[i] = view.getFloat32(o + 36, true); + c.status[i] = BUF[o + 40]!; + o += ROW_BYTES; + } + return c; +} + +// 3. Pure parse floor: read everything, allocate nothing, fold into a checksum. +let sink = 0; +function parseOnly(): number { + const view = new DataView(BUF.buffer, BUF.byteOffset, BUF.byteLength); + const n = (BUF.length / ROW_BYTES) | 0; + let acc = 0; + let o = 0; + for (let i = 0; i < n; i++) { + acc += view.getUint32(o, true); + acc += Number(view.getBigInt64(o + 4, true)); + acc += view.getFloat64(o + 12, true); + acc += view.getFloat64(o + 20, true); + acc += view.getFloat64(o + 28, true); + acc += view.getFloat32(o + 36, true); + acc += BUF[o + 40]!; + o += ROW_BYTES; + } + return (sink = acc); // observable, so V8 can't elide the reads +} + +// sanity: all three agree on row count / a sampled value +{ + const rows = decodeRows(); + const cols = decodeColumnar(); + if (rows.length !== N || cols.sensor_id.length !== N) + throw new Error("headroom: row count"); + if (rows[123]!.temperature !== cols.temperature[123]) + throw new Error("headroom: value mismatch"); + parseOnly(); + if (!Number.isFinite(sink)) throw new Error("headroom: checksum"); +} + +describe("WASM headroom on IoT RowBinary (best case for RowBinary)", () => { + bench("rows — current fast reader (objects + Date)", () => { + decodeRows(); + }); + bench("columnar — into typed arrays (no per-row objects)", () => { + decodeColumnar(); + }); + bench("parseOnly — reads only, zero allocation (the WASM floor)", () => { + parseOnly(); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/ledger.bench.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/ledger.bench.ts new file mode 100644 index 000000000..b2bffc00f --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/ledger.bench.ts @@ -0,0 +1,201 @@ +import { bench, describe } from "vitest"; +import { query } from "./clickhouse.js"; +import { type Reader, Cursor } from "../src/core.js"; +import { type DecimalValue, formatDecimal } from "../src/decimals.js"; +import { + type LedgerRow, + readLedgerRow, + readLedgerRowFast, +} from "../src/examples/ledger.js"; + +/** + * Benchmark + correctness proof: RowBinary vs JSON for a financial ledger whose + * every column is WIDER than a JS `number` can hold — `UInt128`, `Int64`, + * `Decimal128(18)`, `UInt256`. The SKILL says RowBinary "clearly wins" on wide + * numerics; here it wins twice over, because for this shape JSON isn't just + * slower, it's WRONG: + * + * - ClickHouse emits these as BARE JSON numbers, so stock `JSON.parse` rounds + * every one to a float64 — silent, lossy corruption (demonstrated below). + * - The only correct JSON path quotes the values server-side + * (`output_format_json_quote_64bit_integers` + `..._quote_decimals`) and + * re-parses each string into a `bigint`/decimal pair by hand — extra work on + * top of a larger wire. + * + * RowBinary reads each value as an exact `bigint` straight off the wire. + */ +const N = 50_000; + +const SELECT = + `SELECT ` + + // UInt128 near the top of the range, varied per row. + `toUInt128('340282366920938463463374607431768200000') + number AS txn_id, ` + + // Int64 starting at 2^53 + 1 — already past exact-double range on row 0. + `toInt64(9007199254740993) + number AS account, ` + + // Decimal128(18): ~14 integer digits + 18 fractional = 32 significant digits. + `CAST(concat(toString(toUInt64(98765432109876 + number)), '.123456789012345678') AS Decimal128(18)) AS amount, ` + + `CAST(concat(toString(toUInt64(12345678901234 + number)), '.111111111111111111') AS Decimal128(18)) AS balance, ` + + `CAST(concat(toString(toUInt64(1000 + number % 9000)), '.5678') AS Decimal64(4)) AS fee, ` + + // UInt256 near the top of the range. + `toUInt256('115792089237316195423570985008687907853269984665640564039457000000000') + number AS volume ` + + `FROM numbers(${N})`; + +const RB_BUF = await query(`${SELECT} FORMAT RowBinary`); +// Naive JSON: bare numbers. Fast to parse, but every wide value is corrupted. +const JSON_BARE_BUF = await query(`${SELECT} FORMAT JSONEachRow`); +// Correct JSON: quote wide ints AND decimals so values arrive as exact strings. +const QUOTE = + "SETTINGS output_format_json_quote_64bit_integers = 1, output_format_json_quote_decimals = 1"; +const JSON_STR_BUF = await query(`${SELECT} ${QUOTE} FORMAT JSONEachRow`); +const JSON_COMPACT_STR_BUF = await query( + `${SELECT} ${QUOTE} FORMAT JSONCompactEachRow`, +); + +// --- decoders --------------------------------------------------------------- + +function decodeRowBinary(read: Reader): LedgerRow[] { + const s = new Cursor(RB_BUF); + const out: LedgerRow[] = []; + while (s.pos < s.buf.length) out.push(read(s)); + return out; +} + +function jsonArray(buf: Buffer): unknown[] { + return JSON.parse( + `[${buf.toString("utf8").trimEnd().replaceAll("\n", ",")}]`, + ); +} + +// Parse a fixed-point decimal string ("123.456") into the exact [unscaled, scale] +// pair RowBinary returns — the per-field work JSON must do to stay lossless. +function parseDecimal(str: string, scale: number): DecimalValue { + const neg = str.charCodeAt(0) === 45; // '-' + const s = neg ? str.slice(1) : str; + const dot = s.indexOf("."); + let digits: string; + let frac: number; + if (dot === -1) { + digits = s; + frac = 0; + } else { + digits = s.slice(0, dot) + s.slice(dot + 1); + frac = s.length - dot - 1; + } + let unscaled = BigInt(digits); + if (frac < scale) unscaled *= 10n ** BigInt(scale - frac); + else if (frac > scale) unscaled /= 10n ** BigInt(frac - scale); + return [neg ? -unscaled : unscaled, scale]; +} + +// Correct decode of the quoted JSON: turn the string fields back into the exact +// bigint / decimal-pair shape RowBinary produces. +function decodeJsonObjectsCorrect(buf: Buffer): LedgerRow[] { + const rows = jsonArray(buf) as Record[]; + const out: LedgerRow[] = new Array(rows.length); + for (let i = 0; i < rows.length; i++) { + const r = rows[i]!; + out[i] = { + txn_id: BigInt(r.txn_id!), + account: BigInt(r.account!), + amount: parseDecimal(r.amount!, 18), + balance: parseDecimal(r.balance!, 18), + fee: parseDecimal(r.fee!, 4), + volume: BigInt(r.volume!), + }; + } + return out; +} + +function decodeJsonCompactCorrect(buf: Buffer): LedgerRow[] { + const rows = jsonArray(buf) as string[][]; + const out: LedgerRow[] = new Array(rows.length); + for (let i = 0; i < rows.length; i++) { + const r = rows[i]!; + out[i] = { + txn_id: BigInt(r[0]!), + account: BigInt(r[1]!), + amount: parseDecimal(r[2]!, 18), + balance: parseDecimal(r[3]!, 18), + fee: parseDecimal(r[4]!, 4), + volume: BigInt(r[5]!), + }; + } + return out; +} + +// --- correctness cross-check + the corruption demonstration (runs at load) --- + +const eqDec = (a: DecimalValue, b: DecimalValue) => + a[0] === b[0] && a[1] === b[1]; +const eqRow = (a: LedgerRow, b: LedgerRow) => + a.txn_id === b.txn_id && + a.account === b.account && + eqDec(a.amount, b.amount) && + eqDec(a.balance, b.balance) && + eqDec(a.fee, b.fee) && + a.volume === b.volume; + +{ + const rb = decodeRowBinary(readLedgerRowFast); + const api = decodeRowBinary(readLedgerRow); + const jObj = decodeJsonObjectsCorrect(JSON_STR_BUF); + const jCompact = decodeJsonCompactCorrect(JSON_COMPACT_STR_BUF); + const bare = jsonArray(JSON_BARE_BUF) as Record[]; // the WRONG path + + if (rb.length !== N) + throw new Error(`RowBinary: ${rb.length} rows, expected ${N}`); + for (let i = 0; i < N; i++) { + if (!eqRow(rb[i]!, api[i]!)) + throw new Error(`ledger: API vs fast mismatch @${i}`); + if (!eqRow(rb[i]!, jObj[i]!)) + throw new Error(`ledger: RowBinary vs quoted-JSON mismatch @${i}`); + if (!eqRow(rb[i]!, jCompact[i]!)) + throw new Error(`ledger: RowBinary vs quoted-compact mismatch @${i}`); + } + + // The corruption: stock JSON.parse over the BARE numbers disagrees with the + // exact RowBinary value on every wide column of row 0. + const r0 = rb[0]!; + const b0 = bare[0]!; + console.log( + `\n Financial ledger — ${N.toLocaleString()} rows. Stock JSON.parse on bare numbers (row 0):\n` + + ` txn_id RowBinary ${r0.txn_id}\n` + + ` JSON.parse ${BigInt(Math.trunc(b0.txn_id as unknown as number)).toString()} ${BigInt(Math.trunc(b0.txn_id as unknown as number)) === r0.txn_id ? "ok" : "✗ CORRUPTED"}\n` + + ` account RowBinary ${r0.account}\n` + + ` JSON.parse ${b0.account} ${BigInt(b0.account!) === r0.account ? "ok" : "✗ CORRUPTED"}\n` + + ` amount RowBinary ${formatDecimal(r0.amount)}\n` + + ` JSON.parse ${b0.amount} ✗ CORRUPTED (only ~16 sig digits survive)\n`, + ); + + const mb = (b: Buffer) => (b.length / 1e6).toFixed(2); + const x = (b: Buffer) => `${(b.length / RB_BUF.length).toFixed(1)}x`; + console.log( + ` Wire size (correct paths quote wide values as strings):\n` + + ` RowBinary ${mb(RB_BUF)} MB\n` + + ` JSONCompactEachRow quoted ${mb(JSON_COMPACT_STR_BUF)} MB ${x(JSON_COMPACT_STR_BUF)}\n` + + ` JSONEachRow quoted ${mb(JSON_STR_BUF)} MB ${x(JSON_STR_BUF)}\n`, + ); +} + +// --- benchmarks ------------------------------------------------------------- + +describe("Financial ledger: RowBinary vs JSON decode throughput", () => { + bench("RowBinary — optimized (monomorphized)", () => { + decodeRowBinary(readLedgerRowFast); + }); + bench("RowBinary — API (combinators)", () => { + decodeRowBinary(readLedgerRow); + }); + bench( + "JSONCompactEachRow quoted — JSON.parse + BigInt/decimal (correct)", + () => { + decodeJsonCompactCorrect(JSON_COMPACT_STR_BUF); + }, + ); + bench("JSONEachRow quoted — JSON.parse + BigInt/decimal (correct)", () => { + decodeJsonObjectsCorrect(JSON_STR_BUF); + }); + bench("JSONEachRow bare — JSON.parse only (FAST BUT WRONG)", () => { + jsonArray(JSON_BARE_BUF); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/logs.bench.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/logs.bench.ts new file mode 100644 index 000000000..89c520524 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/logs.bench.ts @@ -0,0 +1,112 @@ +import { gzipSync, zstdCompressSync } from "node:zlib"; +import { bench, describe } from "vitest"; +import { query } from "./clickhouse.js"; +import { type Reader, Cursor } from "../src/core.js"; +import { + type LogRow, + readLogRow, + readLogRowFast, +} from "../src/examples/logs.js"; + +/** + * Benchmark: RowBinary vs JSON for a STRING-HEAVY application log table — the + * honest counter-case. The SKILL's format-choice guidance says prefer a `JSON*` + * format when the result is mostly strings consumed wholesale, because V8's + * native `JSON.parse` builds JS strings in optimized C++ faster than a JS-level + * RowBinary string decoder, and JSON's repetitive keys compress away on the + * wire. This measures both halves of that claim and is expected to show JSON + * WINNING — the result that makes the skill's "don't use RowBinary here" advice + * trustworthy. + */ +const N = 50_000; + +// Realistic log lines: repeated templates (compress well) with varying values, +// two LowCardinality columns, a high-cardinality hex trace_id. Deterministic. +const SELECT = + `SELECT ` + + `toDateTime(1700000000 + number) AS ts, ` + + `['INFO','INFO','INFO','WARN','ERROR','DEBUG'][number % 6 + 1]::LowCardinality(String) AS level, ` + + `['api','auth','db','cache','worker','scheduler'][number % 6 + 1]::LowCardinality(String) AS service, ` + + `concat('handled ', ['GET','POST','PUT'][number % 3 + 1], ' /api/v1/resource/', toString(number % 200), ` + + `' in ', toString(number % 1000), 'ms status=', toString([200,200,200,404,500][number % 5 + 1])) AS message, ` + + `lower(hex(MD5(toString(number)))) AS trace_id ` + + `FROM numbers(${N})`; + +const RB_BUF = await query(`${SELECT} FORMAT RowBinary`); +const JSON_BUF = await query(`${SELECT} FORMAT JSONEachRow`); +const JSON_COMPACT_BUF = await query(`${SELECT} FORMAT JSONCompactEachRow`); + +// --- decoders --------------------------------------------------------------- + +function decodeRowBinary(read: Reader): LogRow[] { + const s = new Cursor(RB_BUF); + const out: LogRow[] = []; + while (s.pos < s.buf.length) out.push(read(s)); + return out; +} + +function decodeJsonArray(buf: Buffer): unknown[] { + return JSON.parse( + `[${buf.toString("utf8").trimEnd().replaceAll("\n", ",")}]`, + ); +} + +// --- correctness cross-check + wire-size report (runs at load) -------------- + +const COLS: (keyof LogRow)[] = ["level", "service", "message", "trace_id"]; +{ + const rb = decodeRowBinary(readLogRowFast); + const api = decodeRowBinary(readLogRow); + const je = decodeJsonArray(JSON_BUF) as Record[]; + const order: (keyof LogRow)[] = [ + "ts", + "level", + "service", + "message", + "trace_id", + ]; + const jc = decodeJsonArray(JSON_COMPACT_BUF) as string[][]; + + if (rb.length !== N) + throw new Error(`RowBinary: ${rb.length} rows, expected ${N}`); + if (JSON.stringify(api) !== JSON.stringify(rb)) + throw new Error("logs: API vs fast mismatch"); + for (const i of [0, 1, 123, 4999, N - 1]) { + for (const c of COLS) { + if (rb[i]![c] !== je[i]![c]) + throw new Error(`logs: ${c}@${i} RowBinary vs JSON mismatch`); + if (rb[i]![c] !== jc[i]![order.indexOf(c)]) + throw new Error(`logs: ${c}@${i} RowBinary vs compact mismatch`); + } + } + + const mb = (b: Buffer) => (b.length / 1e6).toFixed(2); + // Compressed wire size: what gzip / zstd on the HTTP response would send. + const gz = (b: Buffer) => (gzipSync(b, { level: 6 }).length / 1e6).toFixed(2); + const zs = (b: Buffer) => (zstdCompressSync(b).length / 1e6).toFixed(2); + const row = (name: string, b: Buffer) => + ` ${name.padEnd(18)} raw ${mb(b)} MB gzip ${gz(b)} MB zstd ${zs(b)} MB`; + console.log( + `\n Application logs — ${N.toLocaleString()} rows, wire size (raw + compressed):\n` + + `${row("RowBinary", RB_BUF)}\n` + + `${row("JSONCompactEachRow", JSON_COMPACT_BUF)}\n` + + `${row("JSONEachRow", JSON_BUF)}\n`, + ); +} + +// --- benchmarks ------------------------------------------------------------- + +describe("Application logs (string-heavy): RowBinary vs JSON decode throughput", () => { + bench("JSONEachRow — JSON.parse", () => { + decodeJsonArray(JSON_BUF); + }); + bench("JSONCompactEachRow — JSON.parse", () => { + decodeJsonArray(JSON_COMPACT_BUF); + }); + bench("RowBinary — optimized (monomorphized)", () => { + decodeRowBinary(readLogRowFast); + }); + bench("RowBinary — API (combinators)", () => { + decodeRowBinary(readLogRow); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/lowCardinality.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/lowCardinality.test.ts new file mode 100644 index 000000000..dbc39d66b --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/lowCardinality.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { readNullable } from "../src/composite.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readLowCardinality } from "../src/lowCardinality.js"; +import { readString } from "../src/strings.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +/** + * `LowCardinality(T)` is TRANSPARENT in RowBinary: it is encoded byte-for-byte + * the same as `T`, with NO dictionary/index layer. (The dictionary encoding + * exists only in the Native format — do not look for it here.) So there is no + * dedicated reader: decode the inner `T` directly. + */ +describe("LowCardinality (transparent — decode as the inner type)", () => { + it("LowCardinality(String) decodes exactly like String", async () => { + const r = await reader("CAST('x' AS LowCardinality(String))"); + // readLowCardinality is the identity combinator: it just returns readString. + expect(readLowCardinality(readString)(r)).toBe("x"); // identical bytes to String 'x': 01 78 + expect(r.pos).toBe(2); + }); + + it("LowCardinality(Nullable(String)) is just the inner Nullable(String)", async () => { + const r = await reader("CAST(NULL AS LowCardinality(Nullable(String)))"); + expect(readNullable(readString)(r)).toBeNull(); + expect(r.pos).toBe(1); // lone null flag, no dictionary anything + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST('x' AS LowCardinality(String)) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readString(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/nested.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/nested.test.ts new file mode 100644 index 000000000..7723b5f42 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/nested.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { readArray, readTupleNamed } from "../src/composite.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUInt8 } from "../src/integers.js"; +import { readNested } from "../src/nested.js"; +import { readString } from "../src/strings.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +/** + * `Nested(...)` has no wire format of its own: + * - With the default `flatten_nested=1`, a `Nested(a T1, b T2)` column expands + * into separate columns `a Array(T1)`, `b Array(T2)` — decode each with + * readArray. + * - With `flatten_nested=0`, the column is `Array(Tuple(a T1, b T2))` — decode + * with readArray + readTupleNamed (verified byte-identical to a real Nested + * column). + * + * Either way it reuses existing readers; there is no dedicated Nested reader. + */ +describe("Nested (decode as Array(Tuple(...)))", () => { + it("decodes a Nested column as an array of named rows", async () => { + // Byte-identical to `Nested(x UInt8, y String)` under flatten_nested=0. + const r = await reader( + "CAST([(1, 'a'), (2, 'b')] AS Array(Tuple(x UInt8, y String)))", + ); + // readNested is the thin alias readArray(readTupleNamed(...)). + const rows = readNested({ x: readUInt8, y: readString })(r); + expect(rows).toEqual([ + { x: 1, y: "a" }, + { x: 2, y: "b" }, + ]); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST([(1, 'a'), (2, 'b')] AS Array(Tuple(x UInt8, y String))) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readArray(readTupleNamed({ x: readUInt8, y: readString }))(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/nothing.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/nothing.test.ts new file mode 100644 index 000000000..9e26c4414 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/nothing.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { readArray, readNullable } from "../src/composite.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readNothing } from "../src/nothing.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +/** + * `Nothing` is the empty type: it has NO values and occupies ZERO bytes. It is + * never a column on its own (you cannot materialize a value of it) — it only + * shows up wrapped, as the inferred element of a literal with no type: + * + * - `[]` -> `Array(Nothing)` -> always the empty array (varint len 0) + * - `NULL` -> `Nullable(Nothing)` -> always NULL (lone flag byte 0x01) + * + * So there is no dedicated reader, and no "read a Nothing" ever happens: the + * Array is empty (the element reader is not called) and the Nullable is NULL + * (the inner reader is not called). The throwing readers below assert exactly + * that — wrap with readArray / readNullable and the inner fn is unreachable. + * + * In practice, CAST a bare `[]`/`NULL` to a concrete type before SELECTing if + * you want real elements; `Nothing` only appears for untyped literals. + */ +describe("Nothing (zero-width — only appears as Array(Nothing) / Nullable(Nothing))", () => { + it("Array(Nothing) is the empty array; the element reader is never called", async () => { + const r = await reader("[]"); + // readNothing throws if ever called; the empty array means it never is. + expect(readArray(readNothing)(r)).toEqual([]); + expect(r.pos).toBe(1); // just the varint length 0x00 + }); + + it("Nullable(Nothing) is NULL; the inner reader is never called", async () => { + const r = await reader("NULL"); + // readNothing throws if ever called; the NULL flag means it never is. + expect(readNullable(readNothing)(r)).toBeNull(); + expect(r.pos).toBe(1); // lone NULL flag 0x01 + }); + + describe("advance() edge cases", () => { + it("Array(Nothing): throws NeedMoreData for every incomplete prefix", async () => { + const full = await query("SELECT [] FORMAT RowBinary"); // single 0x00 count byte + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readArray(() => { + throw new Error("element reader must not run"); + })(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len}`).toBe(NeedMoreData); + } + }); + + it("Nullable(Nothing): throws NeedMoreData for every incomplete prefix", async () => { + const full = await query("SELECT NULL FORMAT RowBinary"); // single 0x01 flag byte + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readNullable(() => { + throw new Error("inner reader must not run"); + })(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len}`).toBe(NeedMoreData); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/observability.bench.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/observability.bench.ts new file mode 100644 index 000000000..631b7f969 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/observability.bench.ts @@ -0,0 +1,64 @@ +import { bench, describe } from "vitest"; +import { query } from "./clickhouse.js"; +import { type Reader, Cursor } from "../src/core.js"; +import { + type ObsRow, + readObsRow, + readObsRowFast, +} from "../src/examples/observability.js"; + +/** + * API-combinator `readObsRow` vs flattened `readObsRowFast` over the same large + * buffer. The gotcha-heavy schema is also the most composite-heavy (Variant + + * Map + nested Tuple array + Nullable array), so it's where the flatten tier — + * coalesced `advance()` over the 33-byte fixed head, inlined reads, pre-sized + * arrays, `formatUUIDTable` — should pay the most. + */ +const N = 20_000; +const BUF = await query( + `SELECT + toUInt64(number) AS id, + toDateTime64('2021-01-01 00:00:00', 3, 'UTC') + number AS ts, + CAST(number % 4 + 1 AS Enum8('debug'=1,'info'=2,'warn'=3,'error'=4)) AS level, + generateUUIDv4() AS trace_id, + multiIf( + number % 3 = 0, CAST(toInt64(number) AS Variant(String, Int64, Float64)), + number % 3 = 1, CAST(concat('s', toString(number)) AS Variant(String, Int64, Float64)), + CAST(toFloat64(number) / 2 AS Variant(String, Int64, Float64)) + ) AS payload, + CAST(map('env','prod','az',toString(number % 3)) AS Map(LowCardinality(String), String)) AS tags, + arrayMap(x -> CAST(tuple(concat('m', toString(x)), toFloat64(x)/10) AS Tuple(name LowCardinality(String), value Float64)), range(number % 3)) AS metrics, + arrayMap(x -> CAST(if(x % 2 = 0, toInt64(x)*1000000000, NULL) AS Nullable(Int64)), range(number % 4)) AS attrs + FROM numbers(${N}) + SETTINGS allow_experimental_variant_type=1, allow_suspicious_variant_types=1, allow_suspicious_low_cardinality_types=1 + FORMAT RowBinary`, +); + +function decodeAll(read: Reader): ObsRow[] { + const s = new Cursor(BUF); + const out: ObsRow[] = []; + while (s.pos < s.buf.length) out.push(read(s)); + return out; +} + +const norm = (rows: ObsRow[]): string => + JSON.stringify(rows, (_k, v) => + typeof v === "bigint" ? `${v}n` : v instanceof Map ? [...v] : v, + ); +{ + const a = decodeAll(readObsRow); + const b = decodeAll(readObsRowFast); + if (a.length !== N) + throw new Error(`observability: decoded ${a.length} rows, expected ${N}`); + if (norm(a) !== norm(b)) + throw new Error("observability: API vs fast mismatch"); +} + +describe("example observability: API vs optimized", () => { + bench("API (combinators)", () => { + decodeAll(readObsRow); + }); + bench("optimized (flattened)", () => { + decodeAll(readObsRowFast); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/observability.example.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/observability.example.test.ts new file mode 100644 index 000000000..22e4c4570 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/observability.example.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { Cursor } from "../src/core.js"; +import { + type ObsRow, + readObsRow, + readObsRowFast, +} from "../src/examples/observability.js"; +import { readRows } from "../src/rows.js"; + +/** + * The gotcha-heavy example end to end: a single SELECT (no table needed) builds + * rows with `Variant`, `DateTime64(3)`, `Map(LowCardinality(String), String)`, + * `Array(Tuple(LowCardinality(String), Float64))`, `Array(Nullable(Int64))`, + * `Enum8`, and `UUID`, then both readers decode it. The experimental types need + * their `SETTINGS` flags on the query. + */ +const SQL = (n: number): string => + `SELECT + toUInt64(number) AS id, + toDateTime64('2021-01-01 00:00:00', 3, 'UTC') + number AS ts, + CAST(number % 4 + 1 AS Enum8('debug'=1,'info'=2,'warn'=3,'error'=4)) AS level, + generateUUIDv4() AS trace_id, + multiIf( + number % 3 = 0, CAST(toInt64(number) AS Variant(String, Int64, Float64)), + number % 3 = 1, CAST(concat('s', toString(number)) AS Variant(String, Int64, Float64)), + CAST(toFloat64(number) / 2 AS Variant(String, Int64, Float64)) + ) AS payload, + CAST(map('env','prod','az',toString(number % 3)) AS Map(LowCardinality(String), String)) AS tags, + arrayMap(x -> CAST(tuple(concat('m', toString(x)), toFloat64(x)/10) AS Tuple(name LowCardinality(String), value Float64)), range(number % 3)) AS metrics, + arrayMap(x -> CAST(if(x % 2 = 0, toInt64(x)*1000000000, NULL) AS Nullable(Int64)), range(number % 4)) AS attrs + FROM numbers(${n}) + SETTINGS allow_experimental_variant_type=1, allow_suspicious_variant_types=1, allow_suspicious_low_cardinality_types=1 + FORMAT RowBinary`; + +describe("example: observability (Variant / DateTime64 / LowCardinality / nested)", () => { + it("API and optimized readers agree, consume exactly, and decode the gotchas", async () => { + const buf = await query(SQL(64)); + + const a = new Cursor(buf); + const viaApi: ObsRow[] = readRows(readObsRow)(a); + expect(a.pos, "API reader consumes the whole buffer").toBe(a.buf.length); + + const b = new Cursor(buf); + const viaFast: ObsRow[] = readRows(readObsRowFast)(b); + expect(b.pos, "fast reader consumes the whole buffer").toBe(b.buf.length); + + // The optimized reader must produce byte-identical results to the API one. + expect(viaFast).toEqual(viaApi); + expect(viaApi).toHaveLength(64); + + // Spot-check the gotchas on the first rows. + expect(viaApi[0]!.id).toBe(0n); // UInt64 -> bigint + expect(viaApi[0]!.ts).toBe("2021-01-01T00:00:00.000Z"); // DateTime64(3) + expect(viaApi[0]!.level).toBe(1); // Enum8 underlying int ('debug') + expect(viaApi[0]!.tags).toEqual( + new Map([ + ["env", "prod"], + ["az", "0"], + ]), + ); + expect(viaApi[0]!.metrics).toEqual([]); + expect(viaApi[0]!.attrs).toEqual([]); + + // Variant active type rotates by row — proves the sort-by-type-name + // discriminant mapping (0=Float64, 1=Int64, 2=String) is right. + expect(viaApi[0]!.payload).toBe(0n); // number%3==0 -> Int64 + expect(viaApi[1]!.payload).toBe("s1"); // number%3==1 -> String + expect(viaApi[2]!.payload).toBe(1); // number==2 -> Float64 1.0 + + // Nested + Nullable + wide int. + expect(viaApi[2]!.metrics).toEqual([ + { name: "m0", value: 0 }, + { name: "m1", value: 0.1 }, + ]); + expect(viaApi[1]!.attrs).toEqual([0n]); + expect(viaApi[2]!.attrs).toEqual([0n, null]); + + // trace_id is a canonical UUID string. + expect(viaApi[0]!.traceId).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/, + ); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/orders.bench.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/orders.bench.ts new file mode 100644 index 000000000..3a8ffc8cf --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/orders.bench.ts @@ -0,0 +1,48 @@ +import { bench, describe } from "vitest"; +import { query } from "./clickhouse.js"; +import { type Reader, Cursor } from "../src/core.js"; +import { + type OrderRow, + readOrderRow, + readOrderRowFast, +} from "../src/examples/orders.js"; + +/** + * Benchmark: API-combinator `readOrderRow` (BigInt `formatUUID`) vs + * `readOrderRowFast` (lookup-table `formatUUIDTable` + inlined reads) over the + * same large buffer. Since every row stringifies a UUID, the formatter swap is + * expected to dominate the win. + */ +const N = 20_000; +const BUF = await query( + `SELECT toUInt8(number % 251) AS id, generateUUIDv4() AS uid, ` + + `toDecimal64(number / 100, 2) AS price, ` + + `CAST(toInt8(number % 3 + 1) AS Enum8('new' = 1, 'shipped' = 2, 'done' = 3)) AS status ` + + `FROM numbers(${N}) FORMAT RowBinary`, +); + +function decodeAll(read: Reader): OrderRow[] { + const s = new Cursor(BUF); + const out: OrderRow[] = []; + while (s.pos < s.buf.length) out.push(read(s)); + return out; +} + +const norm = (rows: OrderRow[]): string => + JSON.stringify(rows, (_k, v) => (typeof v === "bigint" ? `${v}n` : v)); +{ + const a = decodeAll(readOrderRow); + const b = decodeAll(readOrderRowFast); + if (a.length !== N) + throw new Error(`orders: decoded ${a.length} rows, expected ${N}`); + if (norm(a) !== norm(b)) throw new Error("orders: API vs fast mismatch"); +} + +describe("example orders: API vs optimized", () => { + bench("API (formatUUID + combinators)", () => { + decodeAll(readOrderRow); + }); + bench("optimized (formatUUIDTable + inlined)", () => { + decodeAll(readOrderRowFast); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/orders.example.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/orders.example.test.ts new file mode 100644 index 000000000..1e4bef328 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/orders.example.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { Cursor } from "../src/core.js"; +import { type OrderRow, readOrderRow } from "../src/examples/orders.js"; +import { readRows } from "../src/rows.js"; + +/** + * Runs the `orders` example end to end (UUID / Decimal / Enum). These types are + * awkward or lossy as JSON — UUID and Enum as names, Decimal as a float that + * can't represent every value exactly — so the rows go in as raw SQL `VALUES` + * instead of `JSONEachRow`. + */ +describe("example: orders (UUID / Decimal / Enum via raw VALUES)", () => { + it("creates, populates, and reads back through readOrderRow", async () => { + const t = "rb_example_orders"; + await query(`DROP TABLE IF EXISTS ${t}`); + await query( + `CREATE TABLE ${t} (` + + `id UInt8, ` + + `uid UUID, ` + + `price Decimal64(2), ` + + `status Enum8('new' = 1, 'shipped' = 2, 'done' = 3)` + + `) ENGINE = Memory`, + ); + try { + await query( + `INSERT INTO ${t} VALUES ` + + `(1, '61f0c404-5cb3-11e7-907b-a6006ad3dba0', 12.34, 'new'), ` + + `(2, '00000000-0000-0000-0000-000000000000', 0.00, 'shipped'), ` + + `(3, 'ffffffff-ffff-ffff-ffff-ffffffffffff', -9.99, 'done')`, + ); + + const r = new Cursor( + await query( + `SELECT id, uid, price, status FROM ${t} ORDER BY id FORMAT RowBinary`, + ), + ); + const out: OrderRow[] = readRows(readOrderRow)(r); + expect(out).toEqual([ + { + id: 1, + uid: "61f0c404-5cb3-11e7-907b-a6006ad3dba0", + price: [1234n, 2], + status: 1, + }, + { + id: 2, + uid: "00000000-0000-0000-0000-000000000000", + price: [0n, 2], + status: 2, + }, + { + id: 3, + uid: "ffffffff-ffff-ffff-ffff-ffffffffffff", + price: [-999n, 2], + status: 3, + }, + ]); + expect(r.pos).toBe(r.buf.length); + } finally { + await query(`DROP TABLE IF EXISTS ${t}`); + } + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/profiles.bench.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/profiles.bench.ts new file mode 100644 index 000000000..7276bc15f --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/profiles.bench.ts @@ -0,0 +1,47 @@ +import { bench, describe } from "vitest"; +import { query } from "./clickhouse.js"; +import { type Reader, Cursor } from "../src/core.js"; +import { + type ProfileRow, + readProfileRow, + readProfileRowFast, +} from "../src/examples/profiles.js"; + +/** + * Benchmark: API-combinator `readProfileRow` vs monomorphized + * `readProfileRowFast` over the same large buffer. This is the first case where + * the API version allocates a combinator closure per row (`readArray(readString)` + * and `readNullable(readInt32)`), so the inlined version should pull ahead. + */ +const N = 20_000; +const BUF = await query( + `SELECT toUInt32(number) AS id, ` + + `arrayMap(x -> concat('t', toString(x)), range(number % 4)) AS tags, ` + + `CAST(if(number % 3 = 0, NULL, toInt32(number) - 50) AS Nullable(Int32)) AS score ` + + `FROM numbers(${N}) FORMAT RowBinary`, +); + +function decodeAll(read: Reader): ProfileRow[] { + const s = new Cursor(BUF); + const out: ProfileRow[] = []; + while (s.pos < s.buf.length) out.push(read(s)); + return out; +} + +const norm = (rows: ProfileRow[]): string => JSON.stringify(rows); +{ + const a = decodeAll(readProfileRow); + const b = decodeAll(readProfileRowFast); + if (a.length !== N) + throw new Error(`profiles: decoded ${a.length} rows, expected ${N}`); + if (norm(a) !== norm(b)) throw new Error("profiles: API vs fast mismatch"); +} + +describe("example profiles: API vs optimized", () => { + bench("API (combinators)", () => { + decodeAll(readProfileRow); + }); + bench("optimized (monomorphized)", () => { + decodeAll(readProfileRowFast); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/profiles.example.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/profiles.example.test.ts new file mode 100644 index 000000000..c42a54d15 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/profiles.example.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { Cursor } from "../src/core.js"; +import { type ProfileRow, readProfileRow } from "../src/examples/profiles.js"; +import { readRows } from "../src/rows.js"; + +/** + * Runs the `profiles` example end to end (Array + Nullable). Populated via + * `JSONEachRow`; the empty array and the NULL score are the single-byte edge + * cases the reader has to get right. + */ +describe("example: profiles (Array + Nullable via JSONEachRow)", () => { + it("creates, populates, and reads back through readProfileRow", async () => { + const t = "rb_example_profiles"; + await query(`DROP TABLE IF EXISTS ${t}`); + await query( + `CREATE TABLE ${t} (id UInt32, tags Array(String), score Nullable(Int32)) ENGINE = Memory`, + ); + try { + const rows = [ + { id: 1, tags: ["a", "b"], score: 10 }, + { id: 2, tags: [], score: null }, + { id: 3, tags: ["solo"], score: -5 }, + ]; + await query( + `INSERT INTO ${t} FORMAT JSONEachRow\n` + + rows.map((r) => JSON.stringify(r)).join("\n"), + ); + + const r = new Cursor( + await query( + `SELECT id, tags, score FROM ${t} ORDER BY id FORMAT RowBinary`, + ), + ); + const out: ProfileRow[] = readRows(readProfileRow)(r); + expect(out).toEqual([ + { id: 1, tags: ["a", "b"], score: 10 }, + { id: 2, tags: [], score: null }, + { id: 3, tags: ["solo"], score: -5 }, + ]); + expect(r.pos).toBe(r.buf.length); + } finally { + await query(`DROP TABLE IF EXISTS ${t}`); + } + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/qbit.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/qbit.test.ts new file mode 100644 index 000000000..d5637c7ba --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/qbit.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { readQBit } from "../src/composite.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readBFloat16, readFloat32, readFloat64 } from "../src/floats.js"; +import { readUInt8 } from "../src/integers.js"; + +// QBit is experimental; the type needs allow_experimental_qbit_type. +async function reader(expr: string): Promise { + return new Cursor( + await query( + `SELECT ${expr} SETTINGS allow_experimental_qbit_type = 1 FORMAT RowBinary`, + ), + ); +} + +/** + * `QBit(element_type, dimension)` is a vector-search type whose storage keeps + * the vector bit-transposed for quantized distance math. But that layout is a + * STORAGE / Native-format concern: in RowBinary a `QBit` is materialized as the + * plain vector — encoded byte-for-byte like `Array(element_type)` (a LEB128 + * length, then `dimension` element values). So `readQBit` is transparent: it + * just reads it as an array of the element type. + * + * The element type is one of the quantizable floats: `BFloat16`, `Float32`, + * `Float64` — read each with the matching float reader. + */ +describe("QBit (transparent in RowBinary — decode as Array(element_type))", () => { + it("QBit(Float32, N) decodes exactly like Array(Float32)", async () => { + const r = await reader("[1.0, 2.0, 3.0, 4.0]::QBit(Float32, 4)"); + expect(readQBit(readFloat32)(r)).toEqual([1, 2, 3, 4]); + expect(r.pos).toBe(1 + 4 * 4); // length byte + 4 Float32s + }); + + it("QBit(Float64, N) decodes exactly like Array(Float64)", async () => { + const r = await reader("[1.5, 2.5]::QBit(Float64, 2)"); + expect(readQBit(readFloat64)(r)).toEqual([1.5, 2.5]); + }); + + it("QBit(BFloat16, N) decodes exactly like Array(BFloat16)", async () => { + const r = await reader("[1.0, 2.0, 3.0, 4.0]::QBit(BFloat16, 4)"); + // BFloat16 of small integers is exact (they fit the float32 high half). + expect(readQBit(readBFloat16)(r)).toEqual([1, 2, 3, 4]); + }); + + it("is length-prefixed like Array — the next column starts right after", async () => { + const r = await reader( + "[1.0, 2.0]::QBit(Float32, 2) AS q, toUInt8(255) AS m", + ); + expect(readQBit(readFloat32)(r)).toEqual([1, 2]); + expect(r.pos).toBe(1 + 2 * 4); // exact: no quantization padding + expect(readUInt8(r)).toBe(255); // proves the framing matched + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT [1.0, 2.0]::QBit(Float32, 2) SETTINGS allow_experimental_qbit_type = 1 FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readQBit(readFloat32)(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/readUUID.bench.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/readUUID.bench.ts new file mode 100644 index 000000000..5f894b5ca --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/readUUID.bench.ts @@ -0,0 +1,38 @@ +import { bench, describe } from "vitest"; +import { formatUUID, formatUUIDTable } from "../src/reader.js"; + +/** + * Benchmark: the BigInt-based formatUUID vs the lookup-table formatUUIDTable + * (byte -> packed-hex table written into a preallocated buffer, no BigInt, no + * intermediate slices). Both format the same raw 16 bytes. + * + * This benchmark is reader-independent: each case formats a static 16-byte + * buffer directly, so it measures only the formatting cost — `readUUID` just + * returns a zero-copy 16-byte subarray, which would otherwise add noise. + */ + +// Wire bytes for 61f0c404-5cb3-11e7-907b-a6006ad3dba0 (two LE UInt64 halves). +const WIRE = Buffer.from([ + 0xe7, 0x11, 0xb3, 0x5c, 0x04, 0xc4, 0xf0, 0x61, 0xa0, 0xdb, 0xd3, 0x6a, 0x00, + 0xa6, 0x7b, 0x90, +]); +const EXPECTED = "61f0c404-5cb3-11e7-907b-a6006ad3dba0"; + +// Equivalence guard: a faster wrong answer is worthless. Validate before timing. +const viaFormat = formatUUID(WIRE); +const viaTable = formatUUIDTable(WIRE); +if (viaFormat !== EXPECTED || viaTable !== EXPECTED) { + throw new Error( + `UUID mismatch: format=${viaFormat} table=${viaTable} expected=${EXPECTED}`, + ); +} + +describe("UUID formatting", () => { + bench("formatUUID (BigInt + toString)", () => { + formatUUID(WIRE); + }); + + bench("formatUUIDTable (lookup table + preallocated buffer)", () => { + formatUUIDTable(WIRE); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/rows.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/rows.test.ts new file mode 100644 index 000000000..ca45f3bca --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/rows.test.ts @@ -0,0 +1,213 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { Cursor } from "../src/core.js"; +import { readInt32, readUInt64 } from "../src/integers.js"; +import { readRows } from "../src/rows.js"; +import { readString } from "../src/strings.js"; + +/** + * Multi-row tests: plain `RowBinary` concatenates rows back-to-back with no row + * count, length prefix, or delimiter between them — a row is just its columns, + * and the next row's bytes begin immediately after the previous row's last + * column. So decoding N rows is the same per-row read run N times against one + * shared buffer/cursor. + * + * Each test asserts two things: (a) every row decodes to the expected values, + * and (b) the cursor lands EXACTLY on `buf.length` after the final row. The + * end-position check is the tight one — if any row's reader stopped one byte + * short or long, the misalignment compounds across rows and the cursor misses + * the end, so a wrong row boundary can't slip through unnoticed. + * + * Variable-width columns (String) are the sharp case: there is no fixed row + * stride to resync on, so the only thing keeping rows aligned is each reader + * consuming exactly its bytes. + */ +describe("multiple rows from one buffer", () => { + it("fixed-width single column: read a known row count in a loop", async () => { + const r = new Cursor( + await query("SELECT toInt32(number) FROM numbers(5) FORMAT RowBinary"), + ); + const out: number[] = []; + for (let i = 0; i < 5; i++) out.push(readInt32(r)); + expect(out).toEqual([0, 1, 2, 3, 4]); + expect(r.pos).toBe(r.buf.length); + }); + + it("two fixed-width columns per row", async () => { + const r = new Cursor( + await query( + "SELECT toUInt64(number), toInt32(-number) FROM numbers(4) FORMAT RowBinary", + ), + ); + const out: Array<[bigint, number]> = []; + for (let i = 0; i < 4; i++) out.push([readUInt64(r), readInt32(r)]); + expect(out).toEqual([ + [0n, 0], + [1n, -1], + [2n, -2], + [3n, -3], + ]); + expect(r.pos).toBe(r.buf.length); + }); + + it("variable-width column: rows of differing String length stay aligned", async () => { + // repeat('x', number) yields strings of length 0,1,2,3,4 — every row has a + // different byte width, so alignment depends entirely on readString + // consuming exactly its varint length + bytes. + const r = new Cursor( + await query( + "SELECT repeat('x', number) FROM numbers(5) FORMAT RowBinary", + ), + ); + const out: string[] = []; + for (let i = 0; i < 5; i++) out.push(readString(r)); + expect(out).toEqual(["", "x", "xx", "xxx", "xxxx"]); + expect(r.pos).toBe(r.buf.length); + }); + + it("mixed fixed + variable columns per row", async () => { + const r = new Cursor( + await query( + "SELECT number AS n, repeat('ab', number) AS s FROM numbers(3) FORMAT RowBinary", + ), + ); + const out: Array<[bigint, string]> = []; + for (let i = 0; i < 3; i++) out.push([readUInt64(r), readString(r)]); + expect(out).toEqual([ + [0n, ""], + [1n, "ab"], + [2n, "abab"], + ]); + expect(r.pos).toBe(r.buf.length); + }); + + it("drives the loop on cursor position, without knowing the row count", async () => { + // With no row count on the wire, a reader that doesn't know N up front + // loops until the cursor reaches the buffer end. This works precisely + // because each row consumes exactly its bytes — the end is a row boundary. + const r = new Cursor( + await query("SELECT toInt32(number) FROM numbers(10) FORMAT RowBinary"), + ); + const out: number[] = []; + while (r.pos < r.buf.length) out.push(readInt32(r)); + expect(out).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + expect(r.pos).toBe(r.buf.length); + }); + + it("zero rows: an empty result is an empty buffer", async () => { + const r = new Cursor( + await query("SELECT toInt32(1) WHERE 0 FORMAT RowBinary"), + ); + expect(r.buf.length).toBe(0); + const out: number[] = []; + while (r.pos < r.buf.length) out.push(readInt32(r)); + expect(out).toEqual([]); + expect(r.pos).toBe(r.buf.length); + }); + + describe("readRows() helper: the position-bounded loop as a method", () => { + it("reads every row via a per-row callback", async () => { + const r = new Cursor( + await query( + "SELECT number AS id, repeat('ab', number) AS name FROM numbers(3) FORMAT RowBinary", + ), + ); + const out = readRows((s) => ({ + id: readUInt64(s), + name: readString(s), + }))(r); + expect(out).toEqual([ + { id: 0n, name: "" }, + { id: 1n, name: "ab" }, + { id: 2n, name: "abab" }, + ]); + expect(r.pos).toBe(r.buf.length); + }); + + it("returns [] for an empty result", async () => { + const r = new Cursor( + await query("SELECT toInt32(1) WHERE 0 FORMAT RowBinary"), + ); + expect(readRows(readInt32)(r)).toEqual([]); + expect(r.pos).toBe(r.buf.length); + }); + }); + + describe("readRows() with NeedMoreData (partial trailing row)", () => { + it("returns the complete rows and rewinds pos to the last row boundary", async () => { + // 4 rows of (UInt64, String) — fixed 8 bytes + a varint-prefixed string. + const full = await query( + "SELECT number AS id, repeat('ab', number) AS s FROM numbers(4) FORMAT RowBinary", + ); + + // Find the byte offset where row 2 (0-indexed) ends, by decoding the full + // buffer once and committing per row. + const probe = new Cursor(full); + const ends: number[] = []; + readRows((s) => { + readUInt64(s); + readString(s); + ends.push(s.pos); + return null; + })(probe); + // Cut the buffer one byte before the end so the LAST row is truncated. + const r = new Cursor(full.subarray(0, full.length - 1)); + const rows = readRows((s) => ({ + id: readUInt64(s), + s: readString(s), + }))(r); + // Only the 3 complete rows come back; the 4th (straddling) is dropped. + expect(rows).toEqual([ + { id: 0n, s: "" }, + { id: 1n, s: "ab" }, + { id: 2n, s: "abab" }, + ]); + // pos is rewound to the start of the incomplete 4th row, NOT left mid-row. + expect(r.pos).toBe(ends[2]); + expect(r.pos).toBeLessThan(r.buf.length); + }); + + it("drives a chunked stream to completion via the commit point", async () => { + const full = await query( + "SELECT number AS id, repeat('x', number % 7) AS s FROM numbers(50) FORMAT RowBinary", + ); + const expected = Array.from({ length: 50 }, (_, i) => ({ + id: BigInt(i), + s: "x".repeat(i % 7), + })); + + // Reveal the buffer in fixed chunks; each step decodes whatever complete + // rows are now visible and carries the commit point forward. Tiny chunk + // sizes guarantee rows straddle boundaries. + for (const chunk of [1, 5, 13, 4096]) { + const rows: Array<{ id: bigint; s: string }> = []; + let committed = 0; + let avail = 0; + while (committed < full.length) { + avail = Math.min(full.length, avail + chunk); + const r = new Cursor(full.subarray(0, avail)); + r.pos = committed; + rows.push( + ...readRows((s) => ({ id: readUInt64(s), s: readString(s) }))(r), + ); + // Guard against a stall: a non-final chunk that completed no new row + // just means we need more bytes — the loop reveals more next pass. + committed = r.pos; + } + expect(rows, `chunk size ${chunk}`).toEqual(expected); + } + }); + + it("still propagates a non-NeedMoreData error from the row reader", async () => { + const r = new Cursor( + await query("SELECT toInt32(1) FROM numbers(3) FORMAT RowBinary"), + ); + const boom = new Error("decode fault"); + expect(() => + readRows(() => { + throw boom; + })(r), + ).toThrow(boom); + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/simpleAggregateFunction.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/simpleAggregateFunction.test.ts new file mode 100644 index 000000000..817ef55e3 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/simpleAggregateFunction.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { readArray } from "../src/composite.js"; +import { NeedMoreData, Cursor } from "../src/core.js"; +import { readUInt64, readUInt8 } from "../src/integers.js"; +import { readSimpleAggregateFunction } from "../src/simpleAggregateFunction.js"; + +async function reader(expr: string): Promise { + return new Cursor(await query(`SELECT ${expr} FORMAT RowBinary`)); +} + +/** + * `SimpleAggregateFunction(func, T)` is TRANSPARENT in RowBinary: the column + * already holds a finished value of the underlying type `T` (the partial + * aggregate of a "simple" function — sum, min, max, groupArrayArray, ... — is + * just a value of `T`), so it is encoded byte-for-byte the same as `T`. There + * is no dedicated reader: decode the inner `T` directly. + * + * Do NOT confuse it with `AggregateFunction(func, T)`, whose value is an opaque + * serialized aggregation STATE with a function-specific binary layout. + */ +describe("SimpleAggregateFunction (transparent — decode as the inner type)", () => { + it("SimpleAggregateFunction(sum, UInt64) decodes exactly like UInt64", async () => { + const r = await reader("CAST(42 AS SimpleAggregateFunction(sum, UInt64))"); + // readSimpleAggregateFunction is the identity combinator: just readUInt64. + expect(readSimpleAggregateFunction(readUInt64)(r)).toBe(42n); // identical bytes to UInt64 42: 2a 00 00 00 00 00 00 00 + expect(r.pos).toBe(8); + }); + + it("SimpleAggregateFunction(groupArrayArray, Array(UInt8)) is just Array(UInt8)", async () => { + const r = await reader( + "CAST([1, 2, 3] AS SimpleAggregateFunction(groupArrayArray, Array(UInt8)))", + ); + expect(readArray(readUInt8)(r)).toEqual([1, 2, 3]); // varint len 03, then bytes + expect(r.pos).toBe(4); + }); + + describe("advance() edge cases", () => { + it("throws NeedMoreData for every incomplete prefix (0 .. full.length-1)", async () => { + const full = await query( + "SELECT CAST(42 AS SimpleAggregateFunction(sum, UInt64)) FORMAT RowBinary", + ); + for (let len = 0; len < full.length; len++) { + const r = new Cursor(full.subarray(0, len)); + let thrown: unknown; + try { + readUInt64(r); + } catch (e) { + thrown = e; + } + expect(thrown, `prefix length ${len} of ${full.length}`).toBe( + NeedMoreData, + ); + } + }); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/streamRowBatches.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/streamRowBatches.test.ts new file mode 100644 index 000000000..0239bfc4e --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/streamRowBatches.test.ts @@ -0,0 +1,169 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { Cursor } from "../src/core.js"; +import { readUInt64 } from "../src/integers.js"; +import { type SmallChunkStats, streamRowBatches } from "../src/stream.js"; +import { readString } from "../src/strings.js"; + +/** + * `streamRowBatches` is the async front door over `readRows`: an async iterable + * of byte chunks in, an async generator of row batches out. A real HTTP response + * arrives as such a chunk stream; here we take the full `FORMAT RowBinary` bytes + * ClickHouse returns and re-slice them into fixed-size chunks, which is exactly + * what the function consumes — and lets us drive every chunk-boundary case + * (mid-field, mid-row, aligned) deterministically. + */ +async function* chunked(buf: Buffer, size: number): AsyncGenerator { + for (let i = 0; i < buf.length; i += size) { + yield buf.subarray(i, Math.min(i + size, buf.length)); + } +} + +type Row = { id: bigint; s: string }; +const readRow = (s: Cursor): Row => ({ + id: readUInt64(s), + s: readString(s), +}); + +describe("streamRowBatches (async, chunked stream -> row batches)", () => { + it("reassembles every row across chunk sizes, with no empty batches", async () => { + const full = await query( + "SELECT number AS id, repeat('x', number % 9) AS s FROM numbers(60) FORMAT RowBinary", + ); + const expected: Row[] = Array.from({ length: 60 }, (_, i) => ({ + id: BigInt(i), + s: "x".repeat(i % 9), + })); + + // Tiny sizes force rows to straddle boundaries; the large one delivers + // everything in a single batch. + for (const size of [1, 3, 13, 64, full.length, full.length * 2]) { + const batches: Row[][] = []; + for await (const batch of streamRowBatches( + chunked(full, size), + readRow, + )) { + batches.push(batch); + } + expect( + batches.every((b) => b.length > 0), + `chunk size ${size}: no empty batches`, + ).toBe(true); + expect(batches.flat(), `chunk size ${size}`).toEqual(expected); + } + }); + + it("delivers everything in one batch when the whole buffer is one chunk", async () => { + const full = await query( + "SELECT number AS id, repeat('y', number) AS s FROM numbers(5) FORMAT RowBinary", + ); + const batches: Row[][] = []; + for await (const batch of streamRowBatches( + chunked(full, full.length), + readRow, + )) { + batches.push(batch); + } + expect(batches).toHaveLength(1); + expect(batches[0]).toHaveLength(5); + }); + + it("yields nothing for an empty result", async () => { + const full = await query("SELECT toInt32(1) WHERE 0 FORMAT RowBinary"); + expect(full.length).toBe(0); + const batches: Row[][] = []; + for await (const batch of streamRowBatches(chunked(full, 8), readRow)) { + batches.push(batch); + } + expect(batches).toEqual([]); + }); + + it("throws when the stream ends mid-row (truncated response)", async () => { + const full = await query( + "SELECT number AS id, repeat('z', number + 1) AS s FROM numbers(4) FORMAT RowBinary", + ); + const truncated = full.subarray(0, full.length - 1); // cut the last row short + const consume = async () => { + const out: Row[] = []; + for await (const batch of streamRowBatches( + chunked(truncated, 5), + readRow, + )) { + out.push(...batch); + } + return out; + }; + await expect(consume()).rejects.toThrow(/ended mid-row/); + }); + + it("warns once when chunks are pathologically small (rows straddle them)", async () => { + const full = await query( + "SELECT number AS id, repeat('w', number % 5) AS s FROM numbers(80) FORMAT RowBinary", + ); + const warnings: SmallChunkStats[] = []; + const out: Row[] = []; + // 1-byte chunks: every row spans many chunks, so rows/chunk is far below 1. + for await (const batch of streamRowBatches(chunked(full, 1), readRow, { + warnOnSmallChunks: { warn: (_msg, stats) => warnings.push(stats) }, + })) { + out.push(...batch); + } + expect(out).toHaveLength(80); // still decodes correctly + expect(warnings).toHaveLength(1); // fires exactly once, not per chunk + expect(warnings[0]!.rowsPerChunk).toBeLessThan(2); + expect(warnings[0]!.chunks).toBeGreaterThanOrEqual(16); // past the default warmup + }); + + it("does not warn on a healthy stream (many rows per chunk)", async () => { + const full = await query( + "SELECT number AS id, '' AS s FROM numbers(2000) FORMAT RowBinary", + ); + let warned = false; + // ~50-byte rows in 4 KB chunks → ~450 rows/chunk, well above the threshold, + // and a low warmup so the average is actually evaluated. + for await (const batch of streamRowBatches(chunked(full, 4096), readRow, { + warnOnSmallChunks: { warmupChunks: 2, warn: () => (warned = true) }, + })) { + void batch; + } + expect(warned).toBe(false); + }); + + it("respects warnOnSmallChunks: false (disabled)", async () => { + const full = await query( + "SELECT number AS id, '' AS s FROM numbers(80) FORMAT RowBinary", + ); + let warned = false; + const orig = console.warn; + console.warn = () => (warned = true); // would catch the default sink too + try { + for await (const batch of streamRowBatches(chunked(full, 1), readRow, { + warnOnSmallChunks: false, + })) { + void batch; + } + } finally { + console.warn = orig; + } + expect(warned).toBe(false); + }); + + it("accepts plain Uint8Array chunks (not just Buffer)", async () => { + const full = await query( + "SELECT number AS id, '' AS s FROM numbers(3) FORMAT RowBinary", + ); + async function* asU8(): AsyncGenerator { + // Hand back a non-Buffer view to exercise the normalization path. + yield new Uint8Array(full.buffer, full.byteOffset, full.byteLength); + } + const batches: Row[][] = []; + for await (const batch of streamRowBatches(asU8(), readRow)) { + batches.push(batch); + } + expect(batches.flat()).toEqual([ + { id: 0n, s: "" }, + { id: 1n, s: "" }, + { id: 2n, s: "" }, + ]); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/streamingRow.bench.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/streamingRow.bench.ts new file mode 100644 index 000000000..b619b0860 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/streamingRow.bench.ts @@ -0,0 +1,227 @@ +import { bench, describe } from "vitest"; + +/** + * Streaming "need more bytes" mechanism benchmark — throw vs generator-yield. + * + * A streaming parser must, when it runs out of bytes mid-result, suspend and + * resume once more arrive. Two ways to signal "give me more bytes": + * + * 1. THROW a sentinel and let the driver re-enter the parser (this file's + * `parseThrow`). A plain function; the cost is throw/catch stack unwinding. + * 2. YIELD a request from a generator and resume it via `.next()` once bytes + * arrive. Ergonomic — the parser reads as if synchronous — but every read + * runs inside a generator state machine. + * + * To measure ONLY that mechanism, the model is deliberately stripped down: + * + * - The stream is ONE contiguous buffer whose AVAILABLE length grows. "More + * bytes arrived" = bump `avail` by a chunk. So no contender pays for + * stitching separate chunk buffers — that cost is identical for both in + * reality and would only add noise here. + * - The row shape is known (5 × little-endian UInt32 = 20 bytes), as it always + * is for a bespoke generated parser. So each parser checks whether the WHOLE + * row is available before reading any field. Consequence: neither parser + * ever re-reads a field on resume — `parseThrow` restarts from a clean row + * boundary, the generators suspend between rows. The "generators avoid + * re-parsing" argument therefore does NOT apply here; what's left is purely + * throw/catch unwinding vs generator resume. + * + * Contenders: + * - throw + restart — plain function, `throw MORE` when starved. + * - generator (yield* reader) — combinator style: `const a = yield* r.u32()`. + * Two levels of generator delegation per field; + * the "great on paper" form. + * - generator (inline yield) — one generator, row-level `while(...) yield`, + * fields read inline. The lean generator, shown + * so the combinator's overhead isn't mistaken + * for "all generators". + * + * Two chunk regimes are timed: a realistic large chunk (suspends rarely; steady + * state dominates) and a tiny sub-row chunk (suspends constantly; the mechanism + * cost dominates). Read the numbers on your own machine — that's the point. + */ + +const ROWS = 50_000; +const FIELDS_PER_ROW = 5; +const ROW_BYTES = FIELDS_PER_ROW * 4; // 5 × UInt32 +const N = ROWS * FIELDS_PER_ROW; // total field count +const TOTAL = ROWS * ROW_BYTES; + +// Build the payload once: field k (global index) holds the value k, so the +// expected checksum is the exact triangular sum and stays < 2^53 (no masking). +const PAYLOAD = new Uint8Array(TOTAL); +{ + const dv = new DataView(PAYLOAD.buffer); + for (let k = 0; k < N; k++) dv.setUint32(k * 4, k, true); +} +const EXPECTED = { rows: ROWS, sum: (N * (N - 1)) / 2 }; + +type Result = { rows: number; sum: number }; + +/** Singleton sentinel thrown on starvation — a bare value, so no Error stack is + * captured (that capture, not the unwind, is what makes throwing Errors slow). */ +const MORE = Symbol("need-more-bytes"); + +/** + * THROW approach. `avail` grows by `chunkSize` each time the parser starves. + * Reads restart from `committed` (the last completed row); because the whole-row + * check precedes any field read, a restart re-reads nothing. + */ +function parseThrow(bytes: Uint8Array, chunkSize: number): Result { + const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + let avail = 0; + let committed = 0; + let rows = 0; + let sum = 0; + for (;;) { + try { + let pos = committed; + while (pos < TOTAL) { + if (pos + ROW_BYTES > avail) throw MORE; + sum += + dv.getUint32(pos, true) + + dv.getUint32(pos + 4, true) + + dv.getUint32(pos + 8, true) + + dv.getUint32(pos + 12, true) + + dv.getUint32(pos + 16, true); + pos += ROW_BYTES; + rows++; + committed = pos; + } + return { rows, sum }; + } catch (err) { + if (err !== MORE) throw err; + avail = Math.min(TOTAL, avail + chunkSize); + } + } +} + +/** + * Combinator generator reader. Each `u32()` is itself a generator that suspends + * until its 4 bytes are available, so the parse body reads as if synchronous: + * `const a = yield* r.u32()`. `avail` is mutated on the reader between resumes. + */ +function makeGenReader(bytes: Uint8Array) { + const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + return { + pos: 0, + avail: 0, + *u32(): Generator { + while (this.pos + 4 > this.avail) yield; + const v = dv.getUint32(this.pos, true); + this.pos += 4; + return v; + }, + }; +} + +function* parseGenCombinator( + r: ReturnType, +): Generator { + let rows = 0; + let sum = 0; + while (r.pos < TOTAL) { + const a = yield* r.u32(); + const b = yield* r.u32(); + const c = yield* r.u32(); + const d = yield* r.u32(); + const e = yield* r.u32(); + sum += a + b + c + d + e; + rows++; + } + return { rows, sum }; +} + +/** + * Lean generator: a single generator, whole-row availability checked with an + * inline `while (...) yield`, fields read inline. No per-field delegation, so it + * runs at near-normal speed between the (rare) suspensions. + */ +function* parseGenInline( + bytes: Uint8Array, + box: { avail: number }, +): Generator { + const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + let pos = 0; + let rows = 0; + let sum = 0; + while (pos < TOTAL) { + while (pos + ROW_BYTES > box.avail) yield; + sum += + dv.getUint32(pos, true) + + dv.getUint32(pos + 4, true) + + dv.getUint32(pos + 8, true) + + dv.getUint32(pos + 12, true) + + dv.getUint32(pos + 16, true); + pos += ROW_BYTES; + rows++; + } + return { rows, sum }; +} + +/** Drive a generator to completion, revealing `chunkSize` more bytes per + * suspension via the shared `holder.avail`. Returns the generator's `return`. */ +function driveGen( + gen: Generator, + holder: { avail: number }, + chunkSize: number, +): Result { + let step = gen.next(); + while (!step.done) { + holder.avail = Math.min(TOTAL, holder.avail + chunkSize); + step = gen.next(); + } + return step.value; +} + +function runGenCombinator(chunkSize: number): Result { + const r = makeGenReader(PAYLOAD); + return driveGen(parseGenCombinator(r), r, chunkSize); +} + +function runGenInline(chunkSize: number): Result { + const box = { avail: 0 }; + return driveGen(parseGenInline(PAYLOAD, box), box, chunkSize); +} + +// Equivalence guard: a faster wrong answer is worthless. Validate every +// contender at both chunk regimes before any timing runs. +function assertCorrect(label: string, got: Result): void { + if (got.rows !== EXPECTED.rows || got.sum !== EXPECTED.sum) { + throw new Error( + `${label} mismatch: got rows=${got.rows} sum=${got.sum}, ` + + `expected rows=${EXPECTED.rows} sum=${EXPECTED.sum}`, + ); + } +} +for (const cs of [64 * 1024, 8]) { + assertCorrect(`throw cs=${cs}`, parseThrow(PAYLOAD, cs)); + assertCorrect(`gen-combinator cs=${cs}`, runGenCombinator(cs)); + assertCorrect(`gen-inline cs=${cs}`, runGenInline(cs)); +} + +describe("streaming need-more-bytes: 64 KB chunks (suspends rarely)", () => { + const cs = 64 * 1024; + bench("throw + restart", () => { + parseThrow(PAYLOAD, cs); + }); + bench("generator (yield* reader)", () => { + runGenCombinator(cs); + }); + bench("generator (inline yield)", () => { + runGenInline(cs); + }); +}); + +describe("streaming need-more-bytes: 8-byte chunks (suspends constantly)", () => { + const cs = 8; + bench("throw + restart", () => { + parseThrow(PAYLOAD, cs); + }); + bench("generator (yield* reader)", () => { + runGenCombinator(cs); + }); + bench("generator (inline yield)", () => { + runGenInline(cs); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/telemetry.bench.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/telemetry.bench.ts new file mode 100644 index 000000000..499d23272 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/telemetry.bench.ts @@ -0,0 +1,50 @@ +import { bench, describe } from "vitest"; +import { query } from "./clickhouse.js"; +import { type Reader, Cursor } from "../src/core.js"; +import { + type TelemetryRow, + readTelemetryRow, + readTelemetryRowFast, +} from "../src/examples/telemetry.js"; + +/** + * Benchmark: API-combinator `readTelemetryRow` vs fully monomorphized + * `readTelemetryRowFast` over the same large buffer. The most composite-heavy + * example — Map + Array + Nullable + named Tuple — so the API version builds four + * closures per row plus a keyed object build; the biggest expected win. + */ +const N = 20_000; +const BUF = await query( + `SELECT concat('h', toString(number)) AS host, ` + + `map('env', 'prod', 'az', toString(number % 3)) AS tags, ` + + `arrayMap(x -> toFloat64(x) / 10, range(number % 5)) AS cpu, ` + + `CAST(if(number % 2 = 0, 'us', NULL) AS Nullable(String)) AS region, ` + + `CAST(tuple(toUInt32(number), toUInt16(number % 100)) AS Tuple(start UInt32, count UInt16)) AS window ` + + `FROM numbers(${N}) FORMAT RowBinary`, +); + +function decodeAll(read: Reader): TelemetryRow[] { + const s = new Cursor(BUF); + const out: TelemetryRow[] = []; + while (s.pos < s.buf.length) out.push(read(s)); + return out; +} + +const norm = (rows: TelemetryRow[]): string => + JSON.stringify(rows, (_k, v) => (v instanceof Map ? [...v] : v)); +{ + const a = decodeAll(readTelemetryRow); + const b = decodeAll(readTelemetryRowFast); + if (a.length !== N) + throw new Error(`telemetry: decoded ${a.length} rows, expected ${N}`); + if (norm(a) !== norm(b)) throw new Error("telemetry: API vs fast mismatch"); +} + +describe("example telemetry: API vs optimized", () => { + bench("API (combinators)", () => { + decodeAll(readTelemetryRow); + }); + bench("optimized (monomorphized)", () => { + decodeAll(readTelemetryRowFast); + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/telemetry.example.test.ts b/skills/clickhouse-js-node-rowbinary-parser/tests/telemetry.example.test.ts new file mode 100644 index 000000000..9e5ab1fff --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/telemetry.example.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import { query } from "./clickhouse.js"; +import { Cursor } from "../src/core.js"; +import { + type TelemetryRow, + readTelemetryRow, +} from "../src/examples/telemetry.js"; +import { readRows } from "../src/rows.js"; + +/** + * Runs the `telemetry` example end to end (Map / Array / Nullable / named + * Tuple). Populated via `JSONEachRow`; the second row exercises every empty / + * NULL branch at once (empty Map, empty Array, NULL region). + */ +describe("example: telemetry (composite columns via JSONEachRow)", () => { + it("creates, populates, and reads back through readTelemetryRow", async () => { + const t = "rb_example_telemetry"; + await query(`DROP TABLE IF EXISTS ${t}`); + await query( + `CREATE TABLE ${t} (` + + `host String, ` + + `tags Map(String, String), ` + + `cpu Array(Float64), ` + + `region Nullable(String), ` + + `window Tuple(start UInt32, count UInt16)` + + `) ENGINE = Memory`, + ); + try { + const rows = [ + { + host: "a", + tags: { env: "prod", az: "1" }, + cpu: [0.5, 0.25], + region: "us", + window: { start: 1000, count: 3 }, + }, + { + host: "b", + tags: {}, + cpu: [], + region: null, + window: { start: 2000, count: 0 }, + }, + ]; + await query( + `INSERT INTO ${t} FORMAT JSONEachRow\n` + + rows.map((r) => JSON.stringify(r)).join("\n"), + ); + + const r = new Cursor( + await query( + `SELECT host, tags, cpu, region, window FROM ${t} ORDER BY host FORMAT RowBinary`, + ), + ); + const out: TelemetryRow[] = readRows(readTelemetryRow)(r); + expect(out).toEqual([ + { + host: "a", + tags: new Map([ + ["env", "prod"], + ["az", "1"], + ]), + cpu: [0.5, 0.25], + region: "us", + window: { start: 1000, count: 3 }, + }, + { + host: "b", + tags: new Map(), + cpu: [], + region: null, + window: { start: 2000, count: 0 }, + }, + ]); + expect(r.pos).toBe(r.buf.length); + } finally { + await query(`DROP TABLE IF EXISTS ${t}`); + } + }); +}); diff --git a/skills/clickhouse-js-node-rowbinary-parser/tests/wasm-int128.experiment.mjs b/skills/clickhouse-js-node-rowbinary-parser/tests/wasm-int128.experiment.mjs new file mode 100644 index 000000000..9f30b82dc --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tests/wasm-int128.experiment.mjs @@ -0,0 +1,246 @@ +/** + * WASM proof kernel — the one experiment from the "why JS, not WASM" case study. + * + * Sums an `Int128` column three ways over the SAME 32 MB RowBinary buffer: + * 1. JS BigInt — what JS MUST do to add 128-bit integers (heap bigints). + * 2. JS f64 fold — the raw read floor (reads the same bytes, wrong math) to + * show how fast V8 streams the memory: "JITed JS at mem speed". + * 3. WASM — a hand-emitted kernel doing native i64 add-with-carry, + * the one place WASM structurally beats JS. + * + * Also measures the boundary tax: copying the buffer into WASM linear memory. + * + * Run: node tests/wasm-int128.experiment.mjs + * Needs a live ClickHouse at $CLICKHOUSE_URL (default http://localhost:8123). + */ +const URL_BASE = process.env.CLICKHOUSE_URL ?? "http://localhost:8123"; +const N = 2_000_000; + +// --- 1. hand-emit a tiny WASM module: void sum128(i32 ptr, i32 lenBytes) ---- +// It folds 16-byte little-endian Int128s into a 128-bit accumulator (two i64s +// with carry) and writes [lo @ mem0, hi @ mem8]. The whole point: the per-row +// add never touches the JS heap. +const leb = (n) => { + const out = []; + do { + let b = n & 0x7f; + n >>>= 7; + if (n) b |= 0x80; + out.push(b); + } while (n); + return out; +}; +const str = (s) => [s.length, ...[...Buffer.from(s)]]; +const section = (id, content) => [id, ...leb(content.length), ...content]; + +// locals after params (0=ptr i32, 1=len i32): +// 2=p i32, 3=end i32, 4=accLo i64, 5=accHi i64, 6=lo i64, 7=hi i64, 8=newLo i64 +const body = [ + 0x02, + 0x02, + 0x7f, + 0x05, + 0x7e, // locals: 2×i32, 5×i64 + 0x20, + 0x00, + 0x21, + 0x02, // p = ptr + 0x20, + 0x00, + 0x20, + 0x01, + 0x6a, + 0x21, + 0x03, // end = ptr + len + 0x02, + 0x40, // block + 0x03, + 0x40, // loop + 0x20, + 0x02, + 0x20, + 0x03, + 0x4f, + 0x0d, + 0x01, // if p >=u end: break + 0x20, + 0x02, + 0x29, + 0x03, + 0x00, + 0x21, + 0x06, // lo = i64.load[p] + 0x20, + 0x02, + 0x29, + 0x03, + 0x08, + 0x21, + 0x07, // hi = i64.load[p+8] + 0x20, + 0x04, + 0x20, + 0x06, + 0x7c, + 0x21, + 0x08, // newLo = accLo + lo + 0x20, + 0x05, + 0x20, + 0x07, + 0x7c, // accHi + hi + 0x20, + 0x08, + 0x20, + 0x04, + 0x54, + 0xad, + 0x7c, // + (newLo () + ...section(3, [0x01, 0x00]), // func 0 : type 0 + ...section(5, [0x01, 0x00, 0x02]), // memory: min 2 pages + ...section(7, [ + 0x02, + ...str("memory"), + 0x02, + 0x00, + ...str("sum128"), + 0x00, + 0x00, + ]), + ...section(10, [0x01, ...leb(body.length), ...body]), // code +]); + +const mod = await WebAssembly.compile(moduleBytes); // throws if malformed — a free validator +const inst = await WebAssembly.instantiate(mod, {}); +const { memory, sum128 } = inst.exports; + +// --- fetch the Int128 column ------------------------------------------------ +const sql = `SELECT toInt128(number) * 123456789012345 AS x FROM numbers(${N}) FORMAT RowBinary`; +const res = await fetch(URL_BASE, { method: "POST", body: sql }); +if (!res.ok) throw new Error(`ClickHouse ${res.status}: ${await res.text()}`); +const buf = Buffer.from(await res.arrayBuffer()); +const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength); +const MB = buf.length / 1e6; +const reps = 20; +const ms = (t) => Number(t) / 1e6 / reps; +const gbs = (msPass) => MB / 1e3 / (msPass / 1e3); + +// --- 1. JS BigInt 128-bit sum (correct; what you must do today) ------------- +let acc = 0n; +let t = process.hrtime.bigint(); +for (let r = 0; r < reps; r++) { + acc = 0n; + for (let o = 0; o < buf.length; o += 16) { + const lo = view.getBigUint64(o, true); + const hi = view.getBigInt64(o + 8, true); + acc += (hi << 64n) + lo; + } +} +const bigintMs = ms(process.hrtime.bigint() - t); + +// --- 2. JS f64 fold: the raw read floor (V8 at memory speed) ---------------- +let sink = 0; +t = process.hrtime.bigint(); +for (let r = 0; r < reps; r++) { + let a = 0; + for (let o = 0; o < buf.length; o += 16) + a += view.getFloat64(o, true) + view.getFloat64(o + 8, true); + sink = a; +} +const floorMs = ms(process.hrtime.bigint() - t); + +// --- 3. WASM kernel --------------------------------------------------------- +const INPUT_OFF = 64; // results live in mem[0..16); input clear of them +const needPages = Math.ceil((INPUT_OFF + buf.length) / 65536); +const havePages = memory.buffer.byteLength / 65536; +if (needPages > havePages) memory.grow(needPages - havePages); + +// boundary tax: copy the network buffer into linear memory +t = process.hrtime.bigint(); +for (let r = 0; r < reps; r++) + new Uint8Array(memory.buffer, INPUT_OFF, buf.length).set(buf); +const copyMs = ms(process.hrtime.bigint() - t); + +// kernel only (buffer already resident) +t = process.hrtime.bigint(); +for (let r = 0; r < reps; r++) sum128(INPUT_OFF, buf.length); +const kernelMs = ms(process.hrtime.bigint() - t); + +const mview = new DataView(memory.buffer); +const wasmSum = + (mview.getBigUint64(8, true) << 64n) + mview.getBigUint64(0, true); +if (wasmSum !== acc) throw new Error(`WASM sum ${wasmSum} != BigInt ${acc}`); + +console.log( + `\nInt128 column sum — ${N.toLocaleString()} rows, ${MB.toFixed(0)} MB, ${reps} reps (Node ${process.version})`, +); +console.log(` correctness: WASM == BigInt == ${acc} ✓\n`); +const row = (name, msPass, note = "") => + console.log( + ` ${name.padEnd(34)} ${msPass.toFixed(2).padStart(7)} ms ${gbs(msPass).toFixed(1).padStart(5)} GB/s ${note}`, + ); +row("1. JS BigInt-128 sum (correct)", bigintMs, "← what JS must do"); +row("2. JS f64 fold (read floor)", floorMs, "← V8 at memory speed"); +row("3. WASM i64 add-carry, kernel only", kernelMs); +row( + " WASM + copy-in boundary tax", + kernelMs + copyMs, + `(copy ${copyMs.toFixed(2)} ms)`, +); +console.log( + `\n BigInt tax vs read floor : ${(bigintMs / floorMs).toFixed(1)}x`, +); +console.log( + ` WASM kernel vs BigInt : ${(bigintMs / kernelMs).toFixed(1)}x faster`, +); +console.log( + ` WASM+copy vs BigInt : ${(bigintMs / (kernelMs + copyMs)).toFixed(1)}x faster`, +); +console.log( + ` WASM kernel vs read floor: ${(kernelMs / floorMs).toFixed(2)}x (1.0 = at memory speed)`, +); +if (sink === undefined) throw new Error("floor sink"); // keep `sink` observable so V8 can't elide the fold diff --git a/skills/clickhouse-js-node-rowbinary-parser/tsconfig.build.json b/skills/clickhouse-js-node-rowbinary-parser/tsconfig.build.json new file mode 100644 index 000000000..29ed1e716 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tsconfig.build.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "noEmitOnError": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "dist", + "rootDir": "src", + "types": ["node"] + }, + "include": ["src"], + "exclude": ["src/examples"] +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/tsconfig.json b/skills/clickhouse-js-node-rowbinary-parser/tsconfig.json new file mode 100644 index 000000000..2a7a84367 --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "types": ["vitest/globals", "node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "erasableSyntaxOnly": true, + "noEmit": true, + "skipLibCheck": true, + "verbatimModuleSyntax": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src", "tests", "vitest.config.ts"] +} diff --git a/skills/clickhouse-js-node-rowbinary-parser/vitest.config.ts b/skills/clickhouse-js-node-rowbinary-parser/vitest.config.ts new file mode 100644 index 000000000..8a3bbd8be --- /dev/null +++ b/skills/clickhouse-js-node-rowbinary-parser/vitest.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + globals: true, + include: ["tests/**/*.test.ts"], + benchmark: { + include: ["tests/**/*.bench.ts"], + }, + }, +}); diff --git a/tests/clickhouse-test-runner/package.json b/tests/clickhouse-test-runner/package.json index 996fe280a..c1e24caea 100644 --- a/tests/clickhouse-test-runner/package.json +++ b/tests/clickhouse-test-runner/package.json @@ -1,7 +1,7 @@ { "name": "@clickhouse/clickhouse-test-runner", "private": true, - "version": "1.22.0", + "version": "1.23.0", "description": "Node.js port of ClickHouse/clickhouse-java tests/clickhouse-client harness", "engines": { "node": ">=20.19.0"