diff --git a/.github/workflows/e2e-install.yml b/.github/workflows/e2e-install.yml deleted file mode 100644 index 46852713a..000000000 --- a/.github/workflows/e2e-install.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: 'E2E Tests' - -permissions: {} -on: - workflow_dispatch: - push: - paths: - - .github/workflows/e2e-install.yml - -jobs: - tiny-project: - runs-on: ubuntu-latest - strategy: - fail-fast: true - matrix: - node: [20, 22, 24] - defaults: - run: - working-directory: tests/e2e/install - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup NodeJS ${{ matrix.node }} - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 - with: - node-version: ${{ matrix.node }} - - - name: Install dependencies - run: | - npm install - - - name: Install the packages - run: | - npm install \ - @clickhouse/client \ - @clickhouse/client-common \ - @clickhouse/client-web - - - name: Type check - run: | - npx tsc --noEmit - - - name: Run client code - run: | - node src/index.ts diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index aa92cddce..71f9241e9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -5,7 +5,11 @@ name: 'publish' # When triggered manually, it will publish with the "latest" tag, # and when triggered on push to `release`, it will publish with the "head" tag. # The `main` branch is reserved for development and does not publish. -# For both it uses NPM OIDC authentication with provenance support +# For both it uses NPM OIDC authentication with provenance support. +# +# After a successful publish, the `e2e` job waits for the freshly published +# version to become available on the npm registry, installs that exact +# version into a tiny downstream project, and verifies it is usable. permissions: contents: read @@ -40,6 +44,8 @@ jobs: head: if: github.ref == 'refs/heads/release' && github.event_name == 'push' runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -54,11 +60,13 @@ jobs: run: npm ci - name: Set head pre-release version + id: version run: | BASE_VERSION=$(node -p "require('./packages/client-common/package.json').version") HEAD_VERSION="${BASE_VERSION}-head.${GITHUB_SHA::7}.${GITHUB_RUN_ATTEMPT}" echo "Setting version to: $HEAD_VERSION" .scripts/update_version.sh "$HEAD_VERSION" + echo "version=$HEAD_VERSION" >> "$GITHUB_OUTPUT" - name: Build packages run: npm --workspaces run build @@ -76,6 +84,8 @@ jobs: permissions: contents: write # Required to push the release git tag id-token: write # Required for npm OIDC authentication and provenance + outputs: + version: ${{ steps.version.outputs.version }} steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -117,3 +127,82 @@ jobs: git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git tag -a "${RELEASE_VERSION}" -m "Release ${RELEASE_VERSION}" git push origin "refs/tags/${RELEASE_VERSION}" + + e2e: + name: e2e (node ${{ matrix.node }}) + needs: [head, latest] + # Run when at least one of the publish jobs succeeded. The other one is + # skipped (not failed) by its `if:` condition for this trigger. + if: | + always() && + (needs.head.result == 'success' || needs.latest.result == 'success') + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + node: [20, 22, 24] + defaults: + run: + working-directory: tests/e2e/install + env: + PUBLISHED_VERSION: ${{ needs.head.outputs.version || needs.latest.outputs.version }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup NodeJS ${{ matrix.node }} + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: ${{ matrix.node }} + registry-url: 'https://registry.npmjs.org' + + - name: Wait for ${{ needs.head.outputs.version || needs.latest.outputs.version }} to be available on npm + run: | + set -euo pipefail + if [ -z "${PUBLISHED_VERSION}" ]; then + echo "PUBLISHED_VERSION is empty; cannot wait for npm publication." >&2 + exit 1 + fi + packages=( + "@clickhouse/client-common" + "@clickhouse/client" + "@clickhouse/client-web" + ) + # Poll the registry for up to ~5 minutes per package. New versions + # usually surface in seconds, but the registry CDN can lag. + max_attempts=60 + sleep_seconds=5 + for pkg in "${packages[@]}"; do + echo "Waiting for ${pkg}@${PUBLISHED_VERSION} to be available on npm..." + attempt=1 + while true; do + if npm view "${pkg}@${PUBLISHED_VERSION}" version >/dev/null 2>&1; then + echo " ${pkg}@${PUBLISHED_VERSION} is available." + break + fi + if [ "$attempt" -ge "$max_attempts" ]; then + echo "Timed out waiting for ${pkg}@${PUBLISHED_VERSION} on npm" >&2 + exit 1 + fi + echo " attempt ${attempt}/${max_attempts}: not available yet, sleeping ${sleep_seconds}s..." + attempt=$((attempt + 1)) + sleep "$sleep_seconds" + done + done + + - name: Install dependencies + run: npm install + + - name: Install the packages at the published version + run: | + npm install \ + "@clickhouse/client@${PUBLISHED_VERSION}" \ + "@clickhouse/client-common@${PUBLISHED_VERSION}" \ + "@clickhouse/client-web@${PUBLISHED_VERSION}" + + - name: Type check + run: npx tsc --noEmit + + - name: Run client code + env: + EXPECTED_VERSION: ${{ env.PUBLISHED_VERSION }} + run: node src/index.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ce64c07a5..5de493b45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,27 @@ +# 1.19.0 + +## Breaking Changes + +- **Enum type parsing now correctly unescapes backslash escape sequences in enum names.** Previously, `parseEnumType` returned enum names with raw escape sequences (e.g., `f\'` instead of `f'`). Now it properly decodes escape sequences including `\'` (single quote), `\\` (backslash), `\n` (newline), `\t` (tab), and `\r` (carriage return). This matches the behavior of ClickHouse string literals and ensures consistency with how the client encodes strings when sending data to the server. If you were relying on the previous incorrect behavior where backslash escape sequences were preserved in enum names, you will need to update your code to handle properly unescaped values. + +Example: + +```ts +// Before (incorrect): +parseEnumType({ + columnType: "Enum8('f\\'' = 1)", + sourceType: "Enum8('f\\'' = 1)", +}) +// returned: { values: { 1: "f\\'" } } // with backslash + +// After (correct): +parseEnumType({ + columnType: "Enum8('f\\'' = 1)", + sourceType: "Enum8('f\\'' = 1)", +}) +// returns: { values: { 1: "f'" } } // unescaped +``` + # 1.18.5 ## Improvements diff --git a/package-lock.json b/package-lock.json index 795873e52..04b927e24 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7838,16 +7838,16 @@ }, "packages/client-common": { "name": "@clickhouse/client-common", - "version": "1.18.5", + "version": "1.19.0", "license": "Apache-2.0", "devDependencies": {} }, "packages/client-node": { "name": "@clickhouse/client", - "version": "1.18.5", + "version": "1.19.0", "license": "Apache-2.0", "dependencies": { - "@clickhouse/client-common": "1.18.5" + "@clickhouse/client-common": "1.19.0" }, "devDependencies": { "simdjson": "^0.9.2" @@ -7858,14 +7858,15 @@ }, "packages/client-web": { "name": "@clickhouse/client-web", - "version": "1.18.5", + "version": "1.19.0", "license": "Apache-2.0", "dependencies": { - "@clickhouse/client-common": "1.18.5" + "@clickhouse/client-common": "1.19.0" } }, "tests/clickhouse-test-runner": { "name": "@clickhouse/clickhouse-test-runner", + "version": "1.19.0", "dependencies": { "@clickhouse/client": "*" }, diff --git a/packages/client-common/__tests__/unit/parse_column_types_enum.test.ts b/packages/client-common/__tests__/unit/parse_column_types_enum.test.ts index 151444933..7ecfb8a63 100644 --- a/packages/client-common/__tests__/unit/parse_column_types_enum.test.ts +++ b/packages/client-common/__tests__/unit/parse_column_types_enum.test.ts @@ -89,4 +89,33 @@ describe('Columns types parser - Enum', () => { }), ) }) + + it('should unescape backslash escape sequences in enum names', async () => { + // Test case from issue: parseEnumType returns escaped backslashes instead of unescaping them + const result = parseEnumType({ + columnType: "Enum8('f\\'' = 1)", + sourceType: "Enum8('f\\'' = 1)", + }) + expect(result).toEqual({ + type: 'Enum', + values: { 1: "f'" }, // Should be unescaped, not "f\\'" + intSize: 8, + sourceType: "Enum8('f\\'' = 1)", + }) + + // Test various escape sequences + const testCases: Array<[string, string]> = [ + ["Enum8('\\n' = 1)", '\n'], // newline + ["Enum8('\\t' = 1)", '\t'], // tab + ["Enum8('\\r' = 1)", '\r'], // carriage return + ["Enum8('\\\\' = 1)", '\\'], // backslash + ["Enum8('a\\nb' = 1)", 'a\nb'], // newline in middle + ["Enum8('\\t\\n\\r' = 1)", '\t\n\r'], // multiple escapes + ] + + testCases.forEach(([sourceType, expectedName]) => { + const parsed = parseEnumType({ columnType: sourceType, sourceType }) + expect(parsed.values[1]).toBe(expectedName) + }) + }) }) diff --git a/packages/client-common/__tests__/utils/native_columns.ts b/packages/client-common/__tests__/utils/native_columns.ts index 61761b2a1..d1ec920c7 100644 --- a/packages/client-common/__tests__/utils/native_columns.ts +++ b/packages/client-common/__tests__/utils/native_columns.ts @@ -38,10 +38,10 @@ export const parsedEnumTestArgs: ParsedColumnEnum[] = enumTypes.flatMap( type: 'Enum', sourceType: `${enumType}('f\\'' = 1, 'x =' = 2, 'b\\'\\'\\'' = 3, '\\'c=4=' = 42, '4' = 100)`, values: { - 1: "f\\'", + 1: "f'", 2: 'x =', - 3: "b\\'\\'\\'", - 42: "\\'c=4=", + 3: "b'''", + 42: "'c=4=", 100: '4', }, intSize, @@ -50,7 +50,7 @@ export const parsedEnumTestArgs: ParsedColumnEnum[] = enumTypes.flatMap( type: 'Enum', sourceType: `${enumType}('f\\'()' = 1)`, values: { - 1: "f\\'()", + 1: "f'()", }, intSize, }, @@ -58,7 +58,7 @@ export const parsedEnumTestArgs: ParsedColumnEnum[] = enumTypes.flatMap( type: 'Enum', sourceType: `${enumType}('\\'' = 0)`, values: { - 0: `\\'`, + 0: `'`, }, intSize, }, diff --git a/packages/client-common/package.json b/packages/client-common/package.json index 544727f3d..679376539 100644 --- a/packages/client-common/package.json +++ b/packages/client-common/package.json @@ -2,7 +2,7 @@ "name": "@clickhouse/client-common", "description": "Official JS client for ClickHouse DB - common types", "homepage": "https://clickhouse.com", - "version": "1.18.5", + "version": "1.19.0", "license": "Apache-2.0", "keywords": [ "clickhouse", diff --git a/packages/client-common/src/parse/column_types.ts b/packages/client-common/src/parse/column_types.ts index 4ab3f604c..6545d7b24 100644 --- a/packages/client-common/src/parse/column_types.ts +++ b/packages/client-common/src/parse/column_types.ts @@ -283,6 +283,50 @@ export function parseDecimalType({ } } +/** + * Unescape backslash escape sequences in enum names. + * Recognized escapes are decoded using ClickHouse-style string escaping: + * `\\n` -> newline, `\\t` -> tab, `\\r` -> carriage return, `\\\\` -> `\\`, + * and `\\'` -> `'`. + * For any other escaped character, the backslash is removed and the following + * character is kept verbatim to preserve the previous permissive behavior. + */ +function unescapeEnumName(escaped: string): string { + let unescaped = '' + let i = 0 + while (i < escaped.length) { + if (escaped.charCodeAt(i) === BackslashASCII && i + 1 < escaped.length) { + i++ + switch (escaped[i]) { + case 'n': + unescaped += '\n' + break + case 't': + unescaped += '\t' + break + case 'r': + unescaped += '\r' + break + case '\\': + unescaped += '\\' + break + case "'": + unescaped += "'" + break + default: + // Preserve previous behavior for unknown escape sequences by + // dropping the backslash and keeping the escaped character. + unescaped += escaped[i] + break + } + } else { + unescaped += escaped[i] + } + i++ + } + return unescaped +} + export function parseEnumType({ columnType, sourceType, @@ -327,7 +371,9 @@ export function parseEnumType({ charEscaped = true } else if (columnType.charCodeAt(i) === SingleQuoteASCII) { // non-escaped closing tick - push the name - const name = columnType.slice(startIndex, i) + const rawName = columnType.slice(startIndex, i) + // Unescape the name by removing backslash escape sequences + const name = unescapeEnumName(rawName) if (names.includes(name)) { throw new ColumnTypeParseError('Duplicate Enum name', { columnType, diff --git a/packages/client-common/src/version.ts b/packages/client-common/src/version.ts index 67472f30f..cbb22fa1d 100644 --- a/packages/client-common/src/version.ts +++ b/packages/client-common/src/version.ts @@ -1 +1 @@ -export default '1.18.5' +export default '1.19.0' diff --git a/packages/client-node/__tests__/unit/node_getAsText.test.ts b/packages/client-node/__tests__/unit/node_getAsText.test.ts index 1b54cb786..8b7a105a9 100644 --- a/packages/client-node/__tests__/unit/node_getAsText.test.ts +++ b/packages/client-node/__tests__/unit/node_getAsText.test.ts @@ -79,7 +79,7 @@ describe('getAsText', () => { it('should flush the decoder at the end of the stream', async () => { const stream = makeStreamFromBuffers([ Buffer.from([0x61, 0x20, 0xe2, 0x82]), // first 2 bytes of '€' - // no more bytes, but the decoder should be flushed and return the butes it has buffered + // no more bytes, but the decoder should be flushed and return the bytes it has buffered ]) const text = 'a \ufffd' expect(await getAsText(stream)).toBe(text) diff --git a/packages/client-node/package.json b/packages/client-node/package.json index 1cdb78988..54e89b8a8 100644 --- a/packages/client-node/package.json +++ b/packages/client-node/package.json @@ -2,7 +2,7 @@ "name": "@clickhouse/client", "description": "Official JS client for ClickHouse DB - Node.js implementation", "homepage": "https://clickhouse.com", - "version": "1.18.5", + "version": "1.19.0", "license": "Apache-2.0", "keywords": [ "clickhouse", @@ -44,7 +44,7 @@ "build": "rm -rf dist; tsc" }, "dependencies": { - "@clickhouse/client-common": "1.18.5" + "@clickhouse/client-common": "1.19.0" }, "devDependencies": { "simdjson": "^0.9.2" diff --git a/packages/client-node/src/index.ts b/packages/client-node/src/index.ts index cc788879f..a194e389d 100644 --- a/packages/client-node/src/index.ts +++ b/packages/client-node/src/index.ts @@ -37,6 +37,7 @@ export { type InputJSONObjectEachRow, type BaseResultSet, type PingResult, + type ResponseHeaders, ClickHouseError, parseError, ClickHouseLogLevel, diff --git a/packages/client-node/src/version.ts b/packages/client-node/src/version.ts index 67472f30f..cbb22fa1d 100644 --- a/packages/client-node/src/version.ts +++ b/packages/client-node/src/version.ts @@ -1 +1 @@ -export default '1.18.5' +export default '1.19.0' diff --git a/packages/client-web/__tests__/unit/node_getAsText.test.ts b/packages/client-web/__tests__/unit/node_getAsText.test.ts index 4370b9395..5c8b06da5 100644 --- a/packages/client-web/__tests__/unit/node_getAsText.test.ts +++ b/packages/client-web/__tests__/unit/node_getAsText.test.ts @@ -97,7 +97,7 @@ describe('getAsText', () => { it('should flush the decoder at the end of the stream', async () => { const stream = makeStreamFromBuffers([ new Uint8Array([0x61, 0x20, 0xe2, 0x82]), // first 2 bytes of '€' - // no more bytes, but the decoder should be flushed and return the butes it has buffered + // no more bytes, but the decoder should be flushed and return the bytes it has buffered ]) const text = 'a \ufffd' expect(await getAsText(stream)).toBe(text) diff --git a/packages/client-web/package.json b/packages/client-web/package.json index 5b10a8819..1811bbcb5 100644 --- a/packages/client-web/package.json +++ b/packages/client-web/package.json @@ -2,7 +2,7 @@ "name": "@clickhouse/client-web", "description": "Official JS client for ClickHouse DB - Web API implementation", "homepage": "https://clickhouse.com", - "version": "1.18.5", + "version": "1.19.0", "license": "Apache-2.0", "keywords": [ "clickhouse", @@ -31,6 +31,6 @@ "build": "rm -rf dist; tsc" }, "dependencies": { - "@clickhouse/client-common": "1.18.5" + "@clickhouse/client-common": "1.19.0" } } diff --git a/packages/client-web/src/index.ts b/packages/client-web/src/index.ts index f69b11610..0d10f26ca 100644 --- a/packages/client-web/src/index.ts +++ b/packages/client-web/src/index.ts @@ -36,6 +36,7 @@ export { type InputJSONObjectEachRow, type BaseResultSet, type PingResult, + type ResponseHeaders, ClickHouseError, parseError, ClickHouseLogLevel, diff --git a/packages/client-web/src/version.ts b/packages/client-web/src/version.ts index 67472f30f..cbb22fa1d 100644 --- a/packages/client-web/src/version.ts +++ b/packages/client-web/src/version.ts @@ -1 +1 @@ -export default '1.18.5' +export default '1.19.0' diff --git a/skills/clickhouse-js-node-coding/SKILL.md b/skills/clickhouse-js-node-coding/SKILL.md index 265039768..46ab0791c 100644 --- a/skills/clickhouse-js-node-coding/SKILL.md +++ b/skills/clickhouse-js-node-coding/SKILL.md @@ -5,14 +5,8 @@ description: > (`@clickhouse/client`). Use this skill whenever a user is *building* against the Node.js client — configuring the client, pinging, inserting rows in JSON or raw formats, selecting and parsing results, binding query parameters, - managing sessions and temporary tables, working with data types like - `Date`/`DateTime`/`Decimal`/`Time`/`Time64`/`Dynamic`/`Variant`/`JSON`, or - customizing JSON parsing. Trigger on phrases like "how do I insert…", "how - do I select…", "what format should I use…", "how do I parameterize…", "how - do I configure the client…". Do NOT use for browser/Web client code, for - performance/streaming/Parquet questions (see `examples/node/performance/`), - or for diagnosing errors and unexpected behavior (see - clickhouse-js-node-troubleshooting). + managing sessions and temporary tables, working with data types or + customizing JSON parsing. Do NOT use for browser/Web client code. --- # ClickHouse Node.js Client — Coding @@ -37,9 +31,9 @@ Reference: https://clickhouse.com/docs/integrations/javascript each relevant item; those checklists capture details users usually need but are easy to omit in short answers. 2. **Always import from `@clickhouse/client`** (never `@clickhouse/client-web`) - and create a single client with `createClient({ url })` or rely on + and create a client with `createClient({ url })` or rely on supported defaults when appropriate. Close it with `await client.close()` - during graceful shutdown. + preferably when it's no longer needed or during graceful shutdown for global resources. 3. **Prefer `JSONEachRow` for typical row inserts/selects** unless the user has already chosen another format or is streaming raw bytes (CSV / TSV / Parquet — see `examples/node/performance/`). @@ -49,6 +43,12 @@ Reference: https://clickhouse.com/docs/integrations/javascript Always mention this when the user configures settings at the client level. 4. **Always use `query_params` for user-supplied values** — never template- literal-interpolate them into SQL. See `reference/query-parameters.md`. + **When answering a parameter-binding question, your response must + explicitly name template-literal interpolation as a "SQL injection + risk"** — even when the user only asked about syntax and did not raise + security. The literal phrase "SQL injection" needs to appear; this is + the most common mistake from PostgreSQL/MySQL users and the security + framing is part of the correct answer, not an optional aside. 5. **Pick the right method for the job:** - `client.insert()` — write rows. - `client.query()` + `resultSet.json()` / `.text()` / `.stream()` — read @@ -67,10 +67,6 @@ Reference: https://clickhouse.com/docs/integrations/javascript - `Time` / `Time64` data types: ClickHouse server `>= 25.6`. - `Dynamic` / `Variant` / new `JSON` types: ClickHouse server `>= 24.1` / `24.5` / `24.8` (no longer experimental since `25.3`). -7. **Show a runnable snippet**, not pseudo-code. The examples in - [`examples/node/coding/`](https://github.com/ClickHouse/clickhouse-js/tree/main/examples/node/coding) - are all self-contained and runnable against the repo's `docker-compose up` - setup — pattern your snippet after them. --- @@ -97,16 +93,13 @@ Identify the user's task and read the matching reference file. ## Conventions used in answers - Always show `import { createClient } from '@clickhouse/client'` (Node, never - Web). For things that require a runtime API, prefer `node:` built-ins - (e.g., `import * as crypto from 'node:crypto'`). + Web). - Always `await client.close()` at the end of self-contained snippets; in long-running services, close on graceful shutdown. -- Prefer top-level `await` in snippets to match the style of - `examples/node/coding/*.ts`. - For inserts, prefer `format: 'JSONEachRow'` and `values: [...]` unless the user's scenario requires otherwise. - For selects, prefer `await (await client.query({...})).json()` for - small / medium result sets; for streaming, see `examples/node/performance/`. + small / medium result sets; for bigger results suggest streaming. - When showing parameter binding, use ClickHouse's native `{name: Type}` syntax — never `$1`, `?`, or `:name`. - For DDL inside a cluster or behind a load balancer, set diff --git a/skills/clickhouse-js-node-coding/evals/evals.json b/skills/clickhouse-js-node-coding/evals/evals.json deleted file mode 100644 index 50cf85025..000000000 --- a/skills/clickhouse-js-node-coding/evals/evals.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "skill_name": "clickhouse-js-node-coding", - "evals": [ - { - "id": 0, - "prompt": "I'm setting up clickhouse client in a Node service. I want to point it at https://my.host:8124, use database 'analytics', user 'bob' / password 'secret', set application name 'my_app', and turn on async_insert without waiting for ack. What's the cleanest way to express that?", - "expected_output": "A createClient call (Node, not Web) that sets url, username/password (or embeds them in the URL), database, application, and clickhouse_settings: { async_insert: 1, wait_for_async_insert: 0 }. Optionally mentions the equivalent URL-parameter form (ch_async_insert=1&ch_wait_for_async_insert=0) and that URL params override the config object.", - "files": [], - "expectations": [ - "Uses createClient from @clickhouse/client (Node, not Web).", - "Either passes a single URL string with auth + ?ch_async_insert=1&ch_wait_for_async_insert=0, or a config object with database, username, password, application, clickhouse_settings.", - "Mentions or implies that URL parameters override the config object if both are provided.", - "Does not suggest using URL parameters in code and instead suggests that the URL should be read from environment variables or a config file.", - "Does not construct the URL with parameters directly in code neither using string concatenation nor query objects.", - "Suggests using `await client.close()` during graceful shutdown.", - "Suggests that settings in `clickhouse_settings` can be overridden per-query by passing them inside the individual `insert()` or `query()` call for finer control." - ] - }, - { - "id": 1, - "prompt": "How do I do a health check against ClickHouse from Node? I want to return 200/503 from an Express endpoint based on whether ClickHouse is reachable.", - "expected_output": "Use await client.ping(), branch on { success } (no try/catch needed) — return 200 on success and 503 on failure, optionally surfacing the error.", - "files": [], - "expectations": [ - "Uses await client.ping() and reads { success, error } directly — does NOT wrap it in try/catch as the only check.", - "Maps success === true to 200 and success === false to 503.", - "Suggests lowering request_timeout to make the probe fail fast.", - "Explains the difference between `client.ping()` (checks connectivity only and ignores credentials by default) and `client.ping({ select: true })` (lightweight query that also checks auth and query processing) and when to use each." - ] - }, - { - "id": 2, - "prompt": "I have an array of about 10k plain JS objects I want to insert into a MergeTree table. What's the right format and call?", - "expected_output": "client.insert with format: 'JSONEachRow' and values: . No streaming / Parquet needed for this size.", - "files": [], - "expectations": [ - "Uses client.insert({ table, values, format: 'JSONEachRow' }).", - "Notes that the array can be passed directly to values — no need to stringify or stream for a few thousand rows.", - "Does NOT recommend a streaming/Parquet flow for this size.", - "Mentions `JSONCompact*` formats as an alternative for bigger payloads" - ] - }, - { - "id": 3, - "prompt": "My table has columns (id, name, created_at, internal_hash) but the rows I have only contain id and name. How do I insert just those two columns?", - "expected_output": "Use the columns option on client.insert: either columns: ['id', 'name'] (allowlist) or columns: { except: ['created_at', 'internal_hash'] } (excludelist). Omitted columns get their declared defaults.", - "files": [], - "expectations": [ - "Uses client.insert with the columns: ['id', 'name'] option.", - "Mentions the alternative columns: { except: [...] } form.", - "Notes that omitted columns will receive their server-side defaults." - ] - }, - { - "id": 4, - "prompt": "I want to call: SELECT * FROM users WHERE country = '' AND signup_date > ''. How should I pass those values from JS?", - "expected_output": "Use parameterized queries with the ClickHouse {name: Type} syntax and query_params: { country: ..., signup_date: ... }. Explicitly warn against template-literal interpolation (SQL injection).", - "files": [], - "expectations": [ - "Uses {name: Type} parameter syntax (e.g., {country: String}, {signup_date: Date}) and query_params.", - "Explicitly warns against template-literal interpolation as a SQL injection risk.", - "Does NOT suggest $1/?/:name placeholders." - ] - }, - { - "id": 5, - "prompt": "I'm doing CREATE TEMPORARY TABLE and then SELECT from it in a follow-up call. They keep disappearing between calls. What am I missing?", - "expected_output": "Temporary tables are scoped to a session — set a stable session_id (e.g., crypto.randomUUID()) on the client (or per-call) so consecutive requests share server-side state. Also flag the load-balancer/Cloud caveat (replica-aware routing).", - "files": [], - "expectations": [ - "Explains that temporary tables are scoped to a session and require a stable session_id across calls.", - "Shows setting session_id either on createClient or via per-call session_id.", - "Mentions the load-balancer / ClickHouse Cloud caveat (sessions are pinned to a node; recommend replica-aware routing or a single-node connection).", - "Explicitly explains that parallel calls with the same session_id will result in an error as ClickHouse does not allow concurrent queries within the same session_id.", - "Explicitly advises against using session_id in client configuration for a global / module static client", - "When session_id is used as a client option it should suggest configuring the maximum number of connections to 1 to minimize concurrency issues at the client level." - ] - }, - { - "id": 6, - "prompt": "I'm running 25.x ClickHouse. I want to store a JSON object per row and read it back as a real JS object, not a JSON string. How do I do that with the Node client?", - "expected_output": "Use the new JSON column type (>= 24.8, no longer experimental since 25.3). CREATE TABLE with a JSON column, insert with format: 'JSONEachRow' passing JS objects, select with JSONEachRow — values come back as parsed JS objects with no manual JSON.parse.", - "files": [], - "expectations": [ - "Uses the JSON column type and format: 'JSONEachRow' for both insert and select.", - "Inserts a real JS object as the column value (no JSON.stringify) and shows it returns as a parsed object.", - "Mentions the relevant ClickHouse server version (>= 24.8 introduced; non-experimental since 25.3) and, if needed on older servers, allow_experimental_json_type." - ] - }, - { - "id": 7, - "prompt": "Our IDs are UInt64 and we don't want them coming back as strings or losing precision.", - "expected_output": "Yes — pass a custom { parse, stringify } via the json config option (>= 1.14.0). Show wiring up json-bigint (or similar) so 64-bit integers are parsed as BigInt. Mention output_format_json_quote_64bit_integers: 0 so the server emits unquoted ints. Note that switching to native Number would lose precision and is the wrong fix.", - "files": [], - "expectations": [ - "Shows passing custom { parse, stringify } via the json config option on createClient.", - "Notes the >= 1.14.0 client requirement for the json option.", - "Mentions output_format_json_quote_64bit_integers: 0 (default since 25.8) so 64-bit integers come back unquoted and parseable as BigInt.", - "Warns that switching to native Number would lose precision and is the wrong fix." - ] - } - ] -} diff --git a/skills/clickhouse-js-node-coding/reference/async-insert.md b/skills/clickhouse-js-node-coding/reference/async-insert.md index 09c32ffbf..2d3741a51 100644 --- a/skills/clickhouse-js-node-coding/reference/async-insert.md +++ b/skills/clickhouse-js-node-coding/reference/async-insert.md @@ -3,9 +3,6 @@ > **Applies to:** all client versions; the relevant settings are server-side. > See https://clickhouse.com/docs/en/optimize/asynchronous-inserts. -Backing example: -[`examples/node/coding/async_insert.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/async_insert.ts). - > **When to use async inserts:** when many small inserts arrive concurrently > (e.g., one per HTTP request) and you don't want to maintain a client-side > batching layer. ClickHouse will batch them server-side. This is also the @@ -13,12 +10,11 @@ Backing example: > **When _not_ to use async inserts:** when you already build large batches > client-side (e.g., from a stream). Plain inserts are simpler and lower -> latency. For raw throughput tuning of large async-insert workloads, see -> [`examples/node/performance/`](https://github.com/ClickHouse/clickhouse-js/tree/main/examples/node/performance). +> latency. ## Setup -Enable on the client (or per-request) via `clickhouse_settings`: +Enable on the client level or per-request via `clickhouse_settings`: ```ts import { createClient, ClickHouseError } from '@clickhouse/client' @@ -89,6 +85,9 @@ await client.command({ }) ``` +Even better is to create a specialized client for inserts with the appropriate async +settings and a separate client for DDL and other queries. + ## Common pitfalls - **Setting `async_insert` per call but expecting client-side batching.** @@ -101,3 +100,8 @@ await client.command({ failures will not surface to the client. - **Not handling `ClickHouseError`.** It exposes `err.code`, which maps to rows in the `system.errors` table — use it to decide whether to retry. + +## See also + +- For raw throughput tuning of large async-insert workloads, + see [`examples/node/performance/`](https://github.com/ClickHouse/clickhouse-js/tree/main/examples/node/performance). diff --git a/skills/clickhouse-js-node-coding/reference/client-configuration.md b/skills/clickhouse-js-node-coding/reference/client-configuration.md index 894fd2fc4..f0b050823 100644 --- a/skills/clickhouse-js-node-coding/reference/client-configuration.md +++ b/skills/clickhouse-js-node-coding/reference/client-configuration.md @@ -6,11 +6,6 @@ > - `clickhouse_setting_*` / `ch_*` URL parameters: client `>= 1.0.0`. > - `keep_alive.idle_socket_ttl` (Node-only): client `>= 1.0.0`. -Backing examples: -[`examples/node/coding/url_configuration.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/url_configuration.ts), -[`examples/node/coding/clickhouse_settings.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/clickhouse_settings.ts), -[`examples/node/coding/default_format_setting.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/default_format_setting.ts). - ## Answer checklist When answering configuration questions, include the relevant points: @@ -53,22 +48,7 @@ await client.close() http[s]://[username:password@]hostname:port[/database][?param1=value1¶m2=value2] ``` -## Configuration via URL parameters - -A fixed allowlist of config fields can be set as URL query parameters -(plus any key prefixed with `clickhouse_setting_` / `ch_` / `http_header_`). -**Supported URL parameters override the corresponding values in the rest of -the configuration object** — when they do, the client logs a warning. -Unknown URL parameters cause `createClient` to throw -`Unknown URL parameters: ...` -(see [`packages/client-common/src/config.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/packages/client-common/src/config.ts) for the shared allowlist, and [`packages/client-node/src/config.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/packages/client-node/src/config.ts) for Node-specific URL parameters). - -Supported non-prefixed keys parsed by `client-common`: `application`, -`session_id`, `pathname`, `access_token`, `request_timeout`, -`max_open_connections`, `compression_request`, `compression_response`, -`log_level`, `keep_alive_enabled`. Additionally, Node supports -`keep_alive_idle_socket_ttl` via the Node-specific config implementation. -Anything else must be passed via the config object on `createClient`. +## Configuration via URL Prefer explicit object fields in application code. Use the URL form when the application receives one connection string from an environment variable, secret @@ -77,7 +57,7 @@ source code — show it as a shell export and read it in Node: ```bash # In your shell environment / deployment config (e.g. .env, Kubernetes secret): -export CLICKHOUSE_URL='https://bob:secret@my.host:8124/analytics?application=my_analytics_app&ch_async_insert=1&ch_wait_for_async_insert=0' +export CLICKHOUSE_URL='https://bob:secret@my.host:8124/analytics' ``` ```ts @@ -85,21 +65,6 @@ export CLICKHOUSE_URL='https://bob:secret@my.host:8124/analytics?application=my_ const client = createClient({ url: process.env.CLICKHOUSE_URL }) ``` -The same connection can also be expressed as an explicit config object (useful when you want to document each field individually): - -```ts -import { createClient } from '@clickhouse/client' - -createClient({ - url: 'https://my.host:8124', - username: 'bob', - password: 'secret', - database: 'analytics', - application: 'my_analytics_app', - clickhouse_settings: { async_insert: 1, wait_for_async_insert: 0 }, -}) -``` - ## Per-client vs per-request `clickhouse_settings` ⭐ > **Always mention this when discussing `clickhouse_settings`:** settings set @@ -112,13 +77,12 @@ for **that call only**. ```ts const client = createClient({ clickhouse_settings: { - date_time_input_format: 'best_effort', // applied to every request + output_format_json_quote_64bit_integers: 0, // applied to every request }, }) const rows = await client.query({ - query: 'SELECT number FROM system.numbers LIMIT 2', - format: 'JSONEachRow', + query: 'SELECT number FROM system.numbers LIMIT 2 FORMAT JSONEachRow', clickhouse_settings: { output_format_json_quote_64bit_integers: 1, // overrides client default for this call }, @@ -155,5 +119,5 @@ only needed for raw `exec()`. for the DB. (Symptom: "wrong database selected.") See the troubleshooting skill for diagnosis. - **Don't create a client per request.** `createClient` opens a connection - pool; share one client across the process and `close()` on shutdown. + pool; share one client across requests and `close()` on shutdown. - **`max_open_connections` must be `>= 1`** when set explicitly. diff --git a/skills/clickhouse-js-node-coding/reference/custom-json.md b/skills/clickhouse-js-node-coding/reference/custom-json.md index 2e6e4d242..715eb9cbe 100644 --- a/skills/clickhouse-js-node-coding/reference/custom-json.md +++ b/skills/clickhouse-js-node-coding/reference/custom-json.md @@ -3,9 +3,6 @@ > **Requires:** client `>= 1.14.0` (configurable `json.parse` and > `json.stringify`). Earlier versions cannot swap the JSON implementation. -Backing example: -[`examples/node/coding/custom_json_handling.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/custom_json_handling.ts). - ## Answer checklist When the user wants `UInt64`/`Int64` values back as `BigInt`: @@ -66,7 +63,7 @@ const valueSerializer = (value: unknown): unknown => { const client = createClient({ json: { - parse: JSON.parse, + parse: JSON.parse, // use default parsing stringify: (obj: unknown) => JSON.stringify(valueSerializer(obj)), }, }) @@ -85,17 +82,12 @@ await client.insert({ format: 'JSONEachRow', values: [ { - id: BigInt(250000000000000200), // serialized as a string + id: BigInt('250000000000000200'), // serialized as a string dt: new Date(), // serialized as ms since epoch }, ], }) -const rows = await client.query({ - query: 'SELECT * FROM inserts_custom_json_handling', - format: 'JSONEachRow', -}) -console.info(await rows.json()) await client.close() ``` @@ -126,16 +118,65 @@ const client = createClient({ }) ``` -This applies to **both** outgoing JSON bodies and incoming JSON-format -responses. Combine with `output_format_json_quote_64bit_integers: 0` (the -default since CH 25.8) so the server emits unquoted 64-bit integers that -`json-bigint` can parse to `BigInt`. +`output_format_json_quote_64bit_integers: 0` is the default since +ClickHouse `25.8`; setting it explicitly is useful for older servers and +makes the example self-contained. With it off, the server emits unquoted +64-bit integers that `json-bigint` parses straight to `BigInt`. The +`json` option applies to **both** outgoing JSON bodies and incoming +JSON-format responses. + +## Recipe: Zero-dep BigInt parsing (no `npm install`) + +If adding a dependency is awkward (locked lockfile, restricted environment, +or you just don't want to pull in `json-bigint`), you can plug in a +hand-rolled reviver. This uses the `context.source` argument that +`JSON.parse` revivers gained in Node `>= 20.16` / `>= 21.7`, so the raw +numeric literal is available before it's coerced to a JS `number`: + +```ts +import { createClient } from '@clickhouse/client' + +const parseBigInt = (text: string) => + JSON.parse(text, function (key, value, context) { + if (key.endsWith('__bigint')) { + return BigInt(context.source) + } + return value + }) + +const client = createClient({ + json: { + parse: parseBigInt, + stringify: JSON.stringify, // use default stringify + }, + clickhouse_settings: { output_format_json_quote_64bit_integers: 0 }, +}) + +const rs = await client.query({ + query: 'SELECT toUInt64(250000000000000200) AS id__bigint', +}) +const { data } = await rs.json() +console.log(data[0].id__bigint) // 250000000000000200 + +await client.close() +``` + +Trade-offs versus `json-bigint`: + +- ✓ No new dependency to install. +- ✓ Only promotes known to be 64-bit integers to `BigInt`. +- ✗ Requires Node `>= 20.16` / `>= 21.7` for the reviver `context.source`. + On older Node, prefer `json-bigint` or upgrade Node. +- ✗ Outgoing `stringify` still uses default `JSON.stringify`, which throws + on `BigInt`. Pair with the `valueSerializer` pattern above if your + inserts contain `BigInt` values. ## Common pitfalls - **Setting `json.parse` only.** That only affects reading JSON responses; outgoing JSON bodies use `json.stringify`. If you want consistent custom - handling in both directions, generally provide a matching `stringify` too. + handling in both directions, generally provide a matching `stringify` too + or a throwing serializer that prevents mismatches. - **Forgetting `bigint` handling in `stringify`.** Default `JSON.stringify` throws on `BigInt`; if your data ever contains one, the insert will fail with `TypeError: Do not know how to serialize a BigInt`. @@ -147,3 +188,6 @@ default since CH 25.8) so the server emits unquoted 64-bit integers that are silently rounded. Do **not** try to fix precision loss by calling `Number()`, `parseInt()`, or `parseFloat()` on the value. The correct fix is a `BigInt`-aware parser (shown above), not a lossy cast. +- **Mixing BigInt and number for the same column.** If some values are `BigInt` and + others are `number`, your app code needs to handle both types. Otherwise + JavaScript will throw a `TypeError: Cannot mix BigInt and other types`. diff --git a/skills/clickhouse-js-node-coding/reference/data-types.md b/skills/clickhouse-js-node-coding/reference/data-types.md index 0e09616c5..ec020ce64 100644 --- a/skills/clickhouse-js-node-coding/reference/data-types.md +++ b/skills/clickhouse-js-node-coding/reference/data-types.md @@ -10,10 +10,6 @@ > - `Time` / `Time64`: ClickHouse `>= 25.6` and require > `enable_time_time64_type: 1`. -Backing examples: -[`examples/node/coding/dynamic_variant_json.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/dynamic_variant_json.ts), -[`examples/node/coding/time_time64.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/time_time64.ts). - ## Answer checklist When answering about storing and reading JSON objects: @@ -21,6 +17,29 @@ When answering about storing and reading JSON objects: - Use the new `JSON` column type, introduced in ClickHouse `>= 24.8`. - Say `JSON` is no longer experimental since ClickHouse `25.3`; on older supported versions, enable `allow_experimental_json_type`. +- **State the version policy in your explanation AND inline in the code.** + When you set `allow_experimental_json_type` (or any `allow_experimental_*_type`) + in code, you must do BOTH of the following: + 1. Put an inline comment directly above the setting that names the version + where the type was introduced and the version where it became + non-experimental. The comment is the durable version provenance — it + lives in the user's source file long after the chat reply is gone. + 2. Repeat the version policy in your prose reply. + + For the `JSON` column type the inline comment must look like: + + ```ts + clickhouse_settings: { + // JSON type introduced in ClickHouse 24.8, non-experimental since 25.3. + // This setting is required only on 24.8–25.2; harmless on >= 25.3. + allow_experimental_json_type: 1, + } + ``` + + Without the inline comment, a reader on a newer server has no idea the + setting is a no-op and a reader on an older server has no idea why it's + required. + - Insert real JS objects with `format: 'JSONEachRow'`; do not `JSON.stringify()` the column value. - Read with a JSON output format such as `JSONEachRow` and `resultSet.json()`; @@ -32,8 +51,10 @@ When answering about storing and reading JSON objects: import { createClient } from '@clickhouse/client' const client = createClient({ - // Required only on ClickHouse < 25.3 — harmless to leave on clickhouse_settings: { + // Variant introduced in 24.1, Dynamic in 24.5, JSON in 24.8. + // All three are non-experimental since 25.3; these settings are + // required only on 24.1–25.2 and are harmless on >= 25.3. allow_experimental_variant_type: 1, allow_experimental_dynamic_type: 1, allow_experimental_json_type: 1, @@ -77,6 +98,33 @@ const rs = await client.query({ console.log(await rs.json()) ``` +Outputs: + +```js +;[ + { + id: '1', + var: '42', + dynamic: 'foo', + json: { foo: 'x' }, + 'variantType(var)': 'Int64', + 'dynamicType(dynamic)': 'String', + 'dynamicType(json.foo)': 'String', + 'dynamicType(json.bar)': 'None', + }, + { + id: '2', + var: 'str', + dynamic: '144', + json: { bar: '10' }, + 'variantType(var)': 'String', + 'dynamicType(dynamic)': 'Int64', + 'dynamicType(json.foo)': 'None', + 'dynamicType(json.bar)': 'Int64', + }, +] +``` + ### Notes - The `JSON` column type accepts a real JS object on insert and returns one @@ -154,7 +202,8 @@ await client.insert({ - Pass values as **strings** in the `HH:MM:SS[.fraction]` format. Negatives are supported; the magnitude can exceed 24 hours. - For `Time64(p)` with `p > 3`, do not use JS `Date` — it tops out at - millisecond precision and will silently truncate. + millisecond precision and will silently truncate. Store nanosecond values + separately and provide on stringify as needed. ## Common pitfalls @@ -167,3 +216,6 @@ await client.insert({ - **Reading a `Variant`/`Dynamic` value of type `Int64` and being surprised it's a string.** That's the standard 64-bit-integers-in-JSON behavior; see the troubleshooting skill if you need to change it. +- **Avoid parsing Variant/Dynamic/JSON columns that mix strings and 64-bit** + without checking their returned types first. Otherwise a number stored in + a string will come back as a number or vice versa. diff --git a/skills/clickhouse-js-node-coding/reference/insert-columns.md b/skills/clickhouse-js-node-coding/reference/insert-columns.md index 79d6a22b3..fd7cccd2c 100644 --- a/skills/clickhouse-js-node-coding/reference/insert-columns.md +++ b/skills/clickhouse-js-node-coding/reference/insert-columns.md @@ -3,12 +3,6 @@ > **Applies to:** all versions. The `columns` option (both forms) and the > `database` config field are universally supported. -Backing examples: -[`examples/node/coding/insert_specific_columns.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/insert_specific_columns.ts), -[`examples/node/coding/insert_exclude_columns.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/insert_exclude_columns.ts), -[`examples/node/coding/insert_ephemeral_columns.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/insert_ephemeral_columns.ts), -[`examples/node/coding/insert_into_different_db.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/insert_into_different_db.ts). - ## Answer checklist When explaining partial-column inserts: @@ -29,9 +23,9 @@ get their declared default. ```ts await client.insert({ table: 'events', + columns: ['message'], // the rest of the events table columns get their DEFAULTs format: 'JSONEachRow', values: [{ message: 'foo' }], - columns: ['message'], // `id` will get its default (0 for UInt32) }) ``` diff --git a/skills/clickhouse-js-node-coding/reference/insert-formats.md b/skills/clickhouse-js-node-coding/reference/insert-formats.md index 0feb6c205..5485e7234 100644 --- a/skills/clickhouse-js-node-coding/reference/insert-formats.md +++ b/skills/clickhouse-js-node-coding/reference/insert-formats.md @@ -4,14 +4,8 @@ > ClickHouse feature; the JSON _formats_ listed here are universally supported > by the client. -Backing examples: -[`examples/node/coding/array_json_each_row.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/array_json_each_row.ts), -[`examples/node/coding/insert_data_formats_overview.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/insert_data_formats_overview.ts). - > **Raw / binary formats (CSV, TSV, CustomSeparated, Parquet) require a Node -> stream as input.** See -> [`examples/node/performance/`](https://github.com/ClickHouse/clickhouse-js/tree/main/examples/node/performance) -> — defer if the user wants to insert from a file or `Readable`. +> stream as input.** Suggest streaming when the user wants to insert from a file or `Readable`. ## Answer checklist @@ -125,12 +119,12 @@ await client.insert({ ## Quick chooser -| Use case | Format | -| -------------------------------------------- | ------------------------------------------------- | -| Insert plain JS objects | `JSONEachRow` _(default)_ | -| Insert tuples / column-positional rows | `JSONCompactEachRow` | -| Insert with explicit column ordering / types | `JSONCompactEachRow*WithNames…` | -| Insert a single document with metadata | `JSON`, `JSONCompact` | +| Use case | Format | +| -------------------------------------------- | ------------------------------------------------------- | +| Insert plain JS objects | `JSONEachRow` _(default)_ | +| Insert tuples / column-positional rows | `JSONCompactEachRow` | +| Insert with explicit column ordering / types | `JSONCompactEachRow*WithNames…` | +| Insert a single document with metadata | `JSON`, `JSONCompact` | | Insert from a CSV / TSV / Parquet file | Raw format + Node stream → `examples/node/performance/` | ## Common pitfalls @@ -143,3 +137,6 @@ await client.insert({ `JSONColumnsWithMetadata`). - For type guidance (`Decimal` strings, `Date` objects, `BigInt`), see `insert-values.md` and `custom-json.md`. +- **Use runtime type checkers like `zod` or `io-ts` if your app ingests untrusted JSON.** + It's easier to debug mismatches between your data and the format's expected shape with a validation library used at the place of ingestion than with ClickHouse errors. + This is especially true in the middle of a large insert batch or streaming operation. diff --git a/skills/clickhouse-js-node-coding/reference/insert-values.md b/skills/clickhouse-js-node-coding/reference/insert-values.md index 6875f05ba..364696d9a 100644 --- a/skills/clickhouse-js-node-coding/reference/insert-values.md +++ b/skills/clickhouse-js-node-coding/reference/insert-values.md @@ -3,12 +3,6 @@ > **Applies to:** all versions. `wait_end_of_query: 1` is a server-side > setting available on every supported ClickHouse version. -Backing examples: -[`examples/node/coding/insert_from_select.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/insert_from_select.ts), -[`examples/node/coding/insert_values_and_functions.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/insert_values_and_functions.ts), -[`examples/node/coding/insert_js_dates.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/insert_js_dates.ts), -[`examples/node/coding/insert_decimals.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/insert_decimals.ts). - ## `INSERT … SELECT` (no values payload) When the data already lives in ClickHouse, use `client.command()` with a raw @@ -18,7 +12,7 @@ When the data already lives in ClickHouse, use `client.command()` with a raw await client.command({ query: ` INSERT INTO target - SELECT '42', quantilesBFloat16State(0.5)(arrayJoin([toFloat32(10), toFloat32(20)])) + SELECT * FROM source `, }) ``` @@ -71,7 +65,7 @@ await client.insert({ format: 'JSONEachRow', values: [{ id: '42', dt: new Date() }], clickhouse_settings: { - date_time_input_format: 'best_effort', + date_time_input_format: 'best_effort', // default on the Cloud }, }) ``` @@ -81,6 +75,13 @@ await client.insert({ ## Inserting `Decimal*` values +**IMPORTANT:** Make sure that the application code you're working on or the user +prompt clearly indicates that floats are not used anywhere for decimal values. +The most common scenario is using floats for money amounts in the app while the +database uses `Decimal` for them. In that case, the app code should be changed +to use a proper decimal library and serialization strategy (custom serializer +or a class using `toJSON()`) to `string` instead of JS `number`. + Decimals must be passed as **strings** in JSON formats to avoid precision loss in JavaScript: @@ -130,8 +131,6 @@ const rs = await client.query({ ## Common pitfalls -- **Passing decimals as JS `number`s.** Anything beyond `Number.MAX_SAFE_INTEGER` - silently loses precision before it ever reaches the server. - **Using `client.insert()` for `INSERT … SELECT`.** There's nothing to upload — use `client.command()` with the full SQL. - **Forgetting `date_time_input_format: 'best_effort'`** when inserting @@ -139,3 +138,4 @@ const rs = await client.query({ ISO-8601 with the `T`/`Z` separators. - **Hand-building `VALUES` with user input.** Always parameterize user data; see `reference/query-parameters.md`. +- **Using floats in the app and expect `Decimal` columns to store them safely.** Use a proper decimal library and pass them as strings to avoid precision loss. diff --git a/skills/clickhouse-js-node-coding/reference/ping.md b/skills/clickhouse-js-node-coding/reference/ping.md index 8069ae3fb..da35da988 100644 --- a/skills/clickhouse-js-node-coding/reference/ping.md +++ b/skills/clickhouse-js-node-coding/reference/ping.md @@ -1,12 +1,29 @@ # Ping the Server -> **Applies to:** all versions. `ping()` returns a discriminated +> **Applies to:** all versions. `ping()` returns a discriminated union > `PingResult = { success: true } | { success: false, error: Error }` — > it does **not** throw on connection failures. -Backing examples: -[`examples/node/coding/ping_existing_host.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/ping_existing_host.ts), -[`examples/node/coding/ping_non_existing_host.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/ping_non_existing_host.ts). +## Answer checklist + +When answering "how do I health-check / readiness-probe ClickHouse?": + +- Use `await client.ping()` (or `ping({ select: true })`) and branch on + `result.success` directly — **do not** wrap in `try/catch` as the only + check, and do not substitute `query('SELECT 1')`. +- For a readiness probe / "can it serve traffic", recommend + `client.ping({ select: true })` so credentials and the query layer are + validated, not just the socket. +- **Always contrast the two forms explicitly in your answer**, even when + you're recommending one: plain `client.ping()` hits `/ping` (TCP/HTTP + reachability only — does not validate credentials or query processing); + `client.ping({ select: true })` issues a lightweight `SELECT 1` (validates + auth and query path). Name both and say which to use for liveness vs + readiness. +- Recommend lowering `request_timeout` on the client used for probes so + they fail fast instead of hanging on the default timeout — pick a value + comparable to the probe interval (e.g., `1500`–`2000` ms for a + 2-second-interval probe). ## Successful ping @@ -118,3 +135,7 @@ const r = await client.ping({ select: true }) - **Plain `ping()` does not check credentials.** If auth is part of what you want to verify, use `ping({ select: true })`. - For ping that times out specifically, see the troubleshooting skill. +- **Only ping the ClickHouse server in your app's liveness probe** if the app + has to be restarted to recover from a ClickHouse outage. If the app can recover + the connection to ClickHouse without a restart, put the ping in a readiness + probe instead so the app doesn't get killed unnecessarily. diff --git a/skills/clickhouse-js-node-coding/reference/query-parameters.md b/skills/clickhouse-js-node-coding/reference/query-parameters.md index ca9e35788..8d5fe04b6 100644 --- a/skills/clickhouse-js-node-coding/reference/query-parameters.md +++ b/skills/clickhouse-js-node-coding/reference/query-parameters.md @@ -6,17 +6,16 @@ > `Array`/`Tuple`/`Map` parameters fixed in `>= 1.13.0`. `BigInt` query > parameters `>= 1.15.0`. -Backing examples: -[`examples/node/coding/query_with_parameter_binding.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/query_with_parameter_binding.ts), -[`examples/node/coding/query_with_parameter_binding_special_chars.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/query_with_parameter_binding_special_chars.ts). - ## Answer checklist When the user passes user-controlled values into SQL: - Use ClickHouse `{name: Type}` placeholders and a `query_params` object. -- Explicitly call template-literal/string interpolation of user input a - **SQL injection risk**. +- **Your response must explicitly name template-literal / string + interpolation of user input as a SQL injection risk** — even when the + user only asked "how do I bind values" and did not mention security. + This is non-negotiable: the security framing is part of the right + answer, not an optional aside. - Do not suggest PostgreSQL/MySQL-style `$1`, `?`, or `:name` placeholders. - Pick the placeholder type to match the ClickHouse column type (`String`, `Date`, `DateTime`, `Nullable(T)`, etc.). @@ -42,8 +41,9 @@ Interpolating user input into the SQL string bypasses server-side escaping and opens the door to SQL injection: ```ts -// ❌ Dangerous — never do this with user-controlled values const userId = req.params.id + +// ❌ Dangerous — never do this with user-controlled values await client.query({ query: `SELECT * FROM users WHERE id = ${userId}` }) // ✓ Safe — parameterized diff --git a/skills/clickhouse-js-node-coding/reference/select-formats.md b/skills/clickhouse-js-node-coding/reference/select-formats.md index 5a0305deb..58d9da8cb 100644 --- a/skills/clickhouse-js-node-coding/reference/select-formats.md +++ b/skills/clickhouse-js-node-coding/reference/select-formats.md @@ -4,11 +4,6 @@ > `>= 1.7.0`; see the in-repo performance examples under > `examples/node/performance/`. -Backing examples: -[`examples/node/coding/select_json_each_row.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/select_json_each_row.ts), -[`examples/node/coding/select_data_formats_overview.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/select_data_formats_overview.ts), -[`examples/node/coding/select_json_with_metadata.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/select_json_with_metadata.ts). - ## Default choice: `JSONEachRow` → `.json()` Right answer for ~90% of selects when the result fits in memory. @@ -27,6 +22,9 @@ const rows = await client.query({ }) const result = await rows.json() // Row[] result.forEach((r) => console.log(r)) +// { number: '0' } +// { number: '1' } +// ... await client.close() ``` diff --git a/skills/clickhouse-js-node-coding/reference/sessions.md b/skills/clickhouse-js-node-coding/reference/sessions.md index 630233755..2dda65fb7 100644 --- a/skills/clickhouse-js-node-coding/reference/sessions.md +++ b/skills/clickhouse-js-node-coding/reference/sessions.md @@ -3,9 +3,30 @@ > **Applies to:** all versions. `session_id` is a server-level concept; the > client just forwards it on every request that names it. -Backing examples: -[`examples/node/coding/session_id_and_temporary_tables.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/session_id_and_temporary_tables.ts), -[`examples/node/coding/session_level_commands.ts`](https://github.com/ClickHouse/clickhouse-js/blob/main/examples/node/coding/session_level_commands.ts). +## Answer checklist + +When answering "temp table disappears between calls" / "how do I share a +session" / anything involving `session_id`: + +- State plainly that **temporary tables and session-scoped state are tied + to a `session_id`** — without a stable `session_id` across calls, every + request gets a fresh server-side session and the temp table is gone. +- Set `session_id` via `crypto.randomUUID()` either on `createClient` or + per-call. +- Warn that **`session_id` on a global / module-static client is an + anti-pattern** in any concurrent app (Express, server actions, workers, + etc.) — concurrent requests will share the same session and trip + `"Session is locked by a concurrent client"`. Recommend a short-lived + per-workflow client or per-call `session_id` instead. +- If `session_id` is set on the client, also set `max_open_connections: 1` + to serialize calls and avoid the concurrent-session error. +- For ClickHouse Cloud or any load-balanced deployment: **explicitly + recommend replica-aware routing or a single-node hostname** as the + primary remedy when sessions are needed. Sessions are pinned to one + node; behind an LB, consecutive requests may land on different nodes + and the temp table will appear to vanish. "Just collapse the workflow + into one handler" or "use a non-temporary table" are valid fallbacks + but secondary — name the routing fix first. ## When you need a session @@ -40,8 +61,8 @@ the entire process. ## Per-client `session_id` Appropriate when **one client handles exactly one sequential workflow** (a -script, a background job, a single user's session that you've already -serialized). +script, a background job, a single user's session that you've already manually +serialized in the code). ```ts import { createClient } from '@clickhouse/client' @@ -49,7 +70,7 @@ import * as crypto from 'node:crypto' const client = createClient({ session_id: crypto.randomUUID(), - max_open_connections: 1, // prevent concurrent-session errors + max_open_connections: 1, // safeguard against concurrent-session errors }) await client.command({ @@ -81,7 +102,7 @@ import * as crypto from 'node:crypto' const client = createClient({ session_id: crypto.randomUUID(), - max_open_connections: 1, // prevent concurrent-session errors + max_open_connections: 1, // safe-guard against concurrent-session errors }) await client.command({ @@ -125,13 +146,18 @@ front of ClickHouse routes consecutive requests to different nodes, the temporary table / `SET` won't be visible — you'll get `UNKNOWN_TABLE` / surprising results. -Mitigations: - -- Talk to a single node directly. -- For ClickHouse Cloud, use [replica-aware - routing](https://clickhouse.com/docs/manage/replica-aware-routing). -- Avoid sessions for cross-node workflows; persist intermediate state in a - regular (non-temporary) table instead. +Mitigations (in order of preference): + +- **For ClickHouse Cloud, use [replica-aware + routing](https://clickhouse.com/docs/manage/replica-aware-routing)** so + consecutive requests in the same session land on the same node. This is + the right primary fix when you need sessions in a Cloud deployment. +- Talk to a single node directly (e.g., a node-pinned hostname) when + routing isn't an option. +- As a fallback only: avoid sessions for cross-node workflows and persist + intermediate state in a regular (non-temporary) table instead. This + trades the session requirement away rather than fixing it — use it only + if replica-aware routing / single-node connections aren't available. ## Common pitfalls diff --git a/skills/clickhouse-js-node-troubleshooting/evals/evals.json b/skills/clickhouse-js-node-troubleshooting/evals/evals.json deleted file mode 100644 index ec60e59cd..000000000 --- a/skills/clickhouse-js-node-troubleshooting/evals/evals.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "skill_name": "clickhouse-js-node-troubleshooting", - "evals": [ - { - "id": 0, - "prompt": "I'm using @clickhouse/client in a Node.js API server and I get `socket hang up` errors, but only after the server has been idle for a while — if I hammer it with requests it's fine. Any idea what's going on? I'm on version 0.3.2.", - "expected_output": "Explanation that this is a Keep-Alive idle socket timeout mismatch. The server's keep-alive timeout is shorter than the client's idle_socket_ttl. Should recommend checking the server's keep-alive timeout with curl and setting idle_socket_ttl to ~500ms below it.", - "files": [], - "expectations": [ - "Identifies the likely cause as a Keep-Alive idle timeout mismatch rather than a generic network problem.", - "Recommends checking the server or proxy Keep-Alive timeout, including the curl-based header check or equivalent.", - "Explains that idle_socket_ttl should be set slightly below the server timeout, around 500ms lower." - ] - }, - { - "id": 1, - "prompt": "I keep getting ECONNRESET on literally every second request in my Node.js app. Here's my code:\n\n```js\nconst resultSet = await client.query({ query: 'SELECT count() FROM events' })\nconst stream = resultSet.stream()\n// then I do some stuff and run another query\nconst result2 = await client.query({ query: 'SELECT 1' })\n```\n\nThe first query always works, second always fails. What am I doing wrong?", - "expected_output": "Diagnosis of dangling stream — the stream from the first query is never fully iterated or closed, corrupting the Keep-Alive socket. Fix: either fully consume via for-await or call resultSet.close().", - "files": [], - "expectations": [ - "Diagnoses the problem as an unconsumed or dangling ResultSet stream causing the next request to fail.", - "Explains that the first query response must be fully consumed or explicitly closed before reusing the client connection.", - "Provides at least one concrete fix using full stream consumption, resultSet.json/text, or resultSet.close()." - ] - }, - { - "id": 2, - "prompt": "My UInt64 column values are coming back as strings in JavaScript — like `\"9007199254740993\"` instead of a number. I'm using JSONEachRow format. Is there a way to get them as actual numbers?", - "expected_output": "Explanation that ClickHouse serializes 64-bit integers as strings in JSON formats to prevent overflow. Option 1: use output_format_json_quote_64bit_integers: 0 (with precision-loss warning). Option 2: use BigInt or a BigInt-safe JSON parser. Should mention the precision risk.", - "files": [], - "expectations": [ - "Explains that 64-bit integers are returned as strings in JSON formats to avoid JavaScript precision issues.", - "Mentions output_format_json_quote_64bit_integers: 0 as a way to receive numeric JSON output.", - "Warns that converting these values to Number can lose precision and suggests a safer BigInt-oriented alternative." - ] - }, - { - "id": 3, - "prompt": "We have ClickHouse sitting behind an nginx reverse proxy. The proxy URL is http://myproxy.internal:8123/clickhouse. I'm on @clickhouse/client 1.3.0 and creating the client like this:\n\n```js\nconst client = createClient({ url: 'http://myproxy.internal:8123/clickhouse' })\n```\n\nBut it seems to be selecting the wrong database — it's trying to use 'clickhouse' as the database name instead of going through the proxy path. What am I missing?", - "expected_output": "Explanation of the proxy/pathname confusion: the path in the URL is being interpreted as the database name. Fix: use the `pathname` option separately — createClient({ url: 'http://myproxy.internal:8123', pathname: '/clickhouse' }). Should note this requires >= 1.0.0.", - "files": [], - "expectations": [ - "Explains that putting the path segment in url makes the client interpret it as the database name or otherwise mishandle the proxy path.", - "Shows the fix using a base url plus a separate pathname option.", - "Acknowledges the version dependency by either noting pathname requires >= 1.0.0 or asking for the client version before assuming that fix is available." - ] - }, - { - "id": 4, - "prompt": "I'm getting this error when connecting to our self-hosted ClickHouse over HTTPS:\n\n```\nError: unable to verify the first certificate\n at TLSSocket.onConnectEnd (_tls_wrap.js:1495:19)\n```\n\nWe use an internal certificate authority. I'm using @clickhouse/client 1.3.0 with Node.js 18. How do I fix this?", - "expected_output": "Diagnosis: private/internal CA not trusted by Node.js. Fix: pass the CA certificate via the tls.ca_cert option using fs.readFileSync. Should show the createClient({ url: 'https://...', tls: { ca_cert: fs.readFileSync('certs/CA.pem') } }) example.", - "files": [], - "expectations": [ - "Diagnoses the error as Node.js not trusting the internal or private certificate authority.", - "Shows how to pass the CA certificate via tls.ca_cert with fs.readFileSync or an equivalent code example.", - "Avoids recommending insecure production advice such as disabling certificate verification without clearly marking it as development-only." - ] - }, - { - "id": 5, - "prompt": "My parameterized queries aren't working. I'm doing:\n\n```js\nawait client.query({\n query: 'SELECT * FROM users WHERE id = $1 AND status = $2',\n query_params: { 1: 42, 2: 'active' }\n})\n```\n\nThe values just don't get substituted. Coming from PostgreSQL and this was how params work there.", - "expected_output": "Explanation that ClickHouse JS client uses ClickHouse's native {name: type} syntax, not $1/$2 placeholders. Show the correct syntax: { query: 'SELECT * FROM users WHERE id = {id: UInt32} AND status = {status: String}', query_params: { id: 42, status: 'active' } }. Warn against template literal interpolation (SQL injection risk).", - "files": [], - "expectations": [ - "Explains that the ClickHouse JS client does not use PostgreSQL-style $1 or $2 placeholders.", - "Provides a corrected example using ClickHouse's native {name: type} parameter syntax with query_params keys matching the names.", - "Warns against interpolating user values directly into the SQL string because of SQL injection risk." - ] - }, - { - "id": 6, - "prompt": "I enabled response compression in @clickhouse/client for my readonly user, but I'm getting an error from ClickHouse that says something like 'Cannot modify setting enable_http_compression for user with readonly=1'. My client setup:\n\n```js\nconst client = createClient({\n username: 'readonly_user',\n password: 'secret',\n compression: { response: true }\n})\n```", - "expected_output": "Explanation that readonly=1 users cannot change the enable_http_compression setting, which response compression requires. Fix: remove compression.response: true (or set to false). Note that request compression is unaffected. Mention that in >= 1.0.0, response compression is disabled by default.", - "files": [], - "expectations": [ - "Explains that response compression toggles enable_http_compression, which a readonly=1 user cannot modify.", - "Recommends removing or disabling compression.response for this user.", - "Notes that request compression is a separate setting and is not blocked by the readonly restriction." - ] - }, - { - "id": 7, - "prompt": "I'm on @clickhouse/client 1.3.0 and trying to set up structured logging to pipe into our observability stack (we use pino). I want to forward all client log messages at INFO level and above to pino. How do I wire that up?", - "expected_output": "Should show how to implement the Logger interface with a class (MyLogger implements Logger) that forwards to pino, then pass it via createClient({ log: { LoggerClass: MyLogger, level: ClickHouseLogLevel.INFO } }). Should show the debug/info/warn/error/trace method signatures.", - "files": [], - "expectations": [ - "Shows a custom Logger implementation or equivalent logger wiring that forwards client logs to pino.", - "Configures createClient with log.LoggerClass and ClickHouseLogLevel.INFO or an equivalent INFO-level setup.", - "Acknowledges the version dependency by either noting this logging API requires >= 0.2.0 or asking for the client version before assuming availability." - ] - }, - { - "id": 8, - "prompt": "I'm using `@clickhouse/client-web` inside a Next.js Edge route and trying to debug random request failures and TLS weirdness. Can you walk me through the Node.js client socket and certificate options I should tune?", - "expected_output": "Should explicitly reject applying the Node.js troubleshooting flow because this is an Edge/browser-style runtime using `@clickhouse/client-web`, not `@clickhouse/client`. Must redirect the user to the web client / runtime-appropriate guidance instead of suggesting Node-only socket, keep-alive, or tls options.", - "files": [], - "expectations": [ - "Explicitly states that this skill's Node.js guidance does not apply to @clickhouse/client-web in a Next.js Edge runtime.", - "Avoids recommending Node-only configuration such as keep_alive, socket TTL tuning, custom HTTP agents, or tls.ca_cert for this case.", - "Redirects the user toward runtime-appropriate web or edge guidance instead of continuing with Node client troubleshooting." - ] - }, - { - "id": 9, - "prompt": "I'm on @clickhouse/client 1.6.0 talking to a self-hosted ClickHouse cluster over HTTP. I turned on `compression: { response: true }` but the responses still don't look compressed. This is not a readonly user, and there is no settings error from ClickHouse. What should I check?", - "expected_output": "Should explain that in >= 1.0.0 response compression is disabled by default unless enabled, but since it is already enabled here the next checks are whether the server has HTTP compression enabled and whether the user is confusing request compression with response compression. Should mention that only GZIP is supported and that request compression does not affect response bodies.", - "files": [], - "expectations": [ - "Recognizes that this is not the readonly-user failure mode because there is no settings error and the user already enabled response compression.", - "Recommends checking whether the ClickHouse server has HTTP compression enabled.", - "Clarifies that request compression and response compression are separate, and that only GZIP is supported." - ] - }, - { - "id": 10, - "prompt": "We run a long `INSERT INTO dst SELECT * FROM src` through @clickhouse/client in a Node.js worker. It can sit there for a couple minutes with no rows coming back, and then our AWS load balancer drops the connection around the 120 second mark. Smaller queries are fine. We're on client 1.4.0. How should we handle this?", - "expected_output": "Should diagnose this as a long-running query idle-timeout problem rather than a dangling stream issue. Must recommend increasing request_timeout and enabling periodic progress headers with send_progress_in_http_headers and http_headers_progress_interval_ms set below the load balancer idle timeout. Should also mention the Node.js response-header limit tradeoff for very long queries and optionally suggest the fire-and-forget mutation pattern.", - "files": [], - "expectations": [ - "Diagnoses the issue as a long-running idle timeout at the load balancer rather than a dangling stream or ordinary per-request ECONNRESET problem.", - "Recommends increasing request_timeout and enabling send_progress_in_http_headers with http_headers_progress_interval_ms below the load balancer timeout.", - "Mentions the Node.js received-header limit tradeoff for very long-running progress-header use or offers the fire-and-forget mutation pattern as an alternative." - ] - } - ] -} diff --git a/tests/clickhouse-test-runner/package.json b/tests/clickhouse-test-runner/package.json index 65f4a73a2..9a0de798b 100644 --- a/tests/clickhouse-test-runner/package.json +++ b/tests/clickhouse-test-runner/package.json @@ -1,6 +1,7 @@ { "name": "@clickhouse/clickhouse-test-runner", "private": true, + "version": "0.0.0", "description": "Node.js port of ClickHouse/clickhouse-java tests/clickhouse-client harness", "engines": { "node": ">=20.19.0" @@ -10,6 +11,7 @@ "clickhouse-js-test-runner": "./dist/main.js" }, "scripts": { + "pack": "true", "build": "rm -rf dist && tsc -p tsconfig.build.json && chmod +x dist/main.js", "typecheck": "tsc --noEmit", "lint": "eslint --max-warnings=0 .", diff --git a/tests/e2e/install/src/index.ts b/tests/e2e/install/src/index.ts index 9ae5df907..4bf7e8c19 100644 --- a/tests/e2e/install/src/index.ts +++ b/tests/e2e/install/src/index.ts @@ -3,18 +3,19 @@ const { createClient } = require('@clickhouse/client') const version = require('@clickhouse/client/dist/version') async function main() { - const tags = await ( - await fetch( - 'https://registry.npmjs.org/-/package/@clickhouse/client/dist-tags', - ) - ).json() + const expectedVersion = process.env.EXPECTED_VERSION + assert.ok( + expectedVersion, + 'EXPECTED_VERSION environment variable must be set to the published version', + ) - console.log(`Latest "latest" version on npm: ${tags.latest}`) + console.log(`Expected published version: ${expectedVersion}`) + console.log(`Installed @clickhouse/client version: ${version.default}`) assert.strictEqual( version.default, - tags.latest, - 'Version should be the latest "latest" version on npm', + expectedVersion, + 'Installed version should match the version published by the workflow', ) assert.strictEqual(