diff --git a/packages/client-common/__tests__/integration/select_query_binding.test.ts b/packages/client-common/__tests__/integration/select_query_binding.test.ts index b8aff351c..adc07fba2 100644 --- a/packages/client-common/__tests__/integration/select_query_binding.test.ts +++ b/packages/client-common/__tests__/integration/select_query_binding.test.ts @@ -217,6 +217,28 @@ describe("select with query binding", () => { expect(response).toBe('"2022-05-02"\n'); }); + it("binds a JS Date to a scalar Date/Date32 parameter", async () => { + const rs = await client.query({ + query: "SELECT {d: Date} AS d, {d32: Date32} AS d32", + format: "JSONEachRow", + query_params: { + d: new Date(Date.UTC(2022, 4, 2, 13, 25, 55)), + d32: new Date(Date.UTC(2022, 4, 2, 13, 25, 55)), + }, + }); + + expect(await rs.json()).toEqual([{ d: "2022-05-02", d32: "2022-05-02" }]); + }); + + it("binds a JS Date to a scalar Date parameter on the command path", async () => { + await client.command({ + query: "SELECT {d: Date} AS d FORMAT Null", + query_params: { + d: new Date(Date.UTC(2022, 4, 2, 13, 25, 55)), + }, + }); + }); + it("handles DateTime in a parameterized query", async () => { const rs = await client.query({ query: "SELECT toDateTime({min_time: DateTime})", diff --git a/packages/client-common/__tests__/unit/format_query_params.test.ts b/packages/client-common/__tests__/unit/format_query_params.test.ts index 67d45205b..de96a66aa 100644 --- a/packages/client-common/__tests__/unit/format_query_params.test.ts +++ b/packages/client-common/__tests__/unit/format_query_params.test.ts @@ -91,7 +91,7 @@ describe("formatQueryParams", () => { expect(formatQueryParams({ value: [] })).toBe("[]"); }); - it("formats a date without timezone", () => { + it("formats a scalar date as a Unix timestamp when the type is unknown", () => { const date = new Date(Date.UTC(2022, 6, 29, 7, 52, 14)); expect( @@ -129,6 +129,33 @@ describe("formatQueryParams", () => { ).toBe("1659081134.005"); }); + it("formats a Date/Date32 scalar parameter as a UTC date string", () => { + const date = new Date(Date.UTC(2022, 6, 29, 7, 52, 14)); + + for (const columnType of ["Date", "Date32", "Nullable(Date)"]) { + expect(formatQueryParams({ value: date, columnType })).toBe("2022-07-29"); + } + }); + + it("formats a pre-epoch Date scalar parameter without corruption", () => { + const date = new Date(Date.UTC(1969, 11, 31, 23, 59, 59)); + + expect(formatQueryParams({ value: date, columnType: "Date" })).toBe( + "1969-12-31", + ); + }); + + it("keeps the Unix timestamp for a DateTime/DateTime64 scalar parameter", () => { + const date = new Date(Date.UTC(2022, 6, 29, 7, 52, 14, 123)); + + expect(formatQueryParams({ value: date, columnType: "DateTime" })).toBe( + "1659081134.123", + ); + expect( + formatQueryParams({ value: date, columnType: "Nullable(DateTime64(3))" }), + ).toBe("1659081134.123"); + }); + it("does not wrap a string in quotes", () => { expect( formatQueryParams({ diff --git a/packages/client-common/src/data_formatter/format_query_params.ts b/packages/client-common/src/data_formatter/format_query_params.ts index d82c49ec0..cbea242c2 100644 --- a/packages/client-common/src/data_formatter/format_query_params.ts +++ b/packages/client-common/src/data_formatter/format_query_params.ts @@ -5,15 +5,35 @@ export class TupleParam { } } +// Matches a `Date` or `Date32` server type (optionally wrapped, e.g. `Nullable(Date)`), +// but not `DateTime`/`DateTime64`: the `\b` after `Date` fails on the `T` of `DateTime`. +const DATE_OR_DATE32_PARAM_RE = /\bDate(32)?\b/; + +// Extracts the ClickHouse type of a bound parameter from its `{name:Type}` placeholder in the +// query, so its value can be formatted for that specific type. Returns undefined when unknown. +export function extractQueryParamType( + query: string | undefined, + key: string, +): string | undefined { + if (query === undefined) return undefined; + const escapedKey = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const match = query.match( + new RegExp(`\\{\\s*${escapedKey}\\s*:\\s*([^}]+?)\\s*\\}`), + ); + return match?.[1]; +} + export function formatQueryParams({ value, wrapStringInQuotes, printNullAsKeyword, + columnType, }: FormatQueryParamsOptions): string { return formatQueryParamsInternal({ value, wrapStringInQuotes, printNullAsKeyword, + columnType, isInArrayOrTuple: false, }); } @@ -22,6 +42,7 @@ function formatQueryParamsInternal({ value, wrapStringInQuotes, printNullAsKeyword, + columnType, isInArrayOrTuple, }: FormatQueryParamsOptions & { isInArrayOrTuple: boolean }): string { if (value === null || value === undefined) { @@ -80,15 +101,17 @@ function formatQueryParamsInternal({ } if (value instanceof Date) { + // Array(Date)/Array(Date32) reject a bare Unix timestamp, so container elements are + // always a quoted 'YYYY-MM-DD' string. A scalar is type-directed: Date/Date32 only accept + // a date string (a Unix timestamp is rejected with BAD_QUERY_PARAMETER), while + // DateTime/DateTime64 keep the timezone-agnostic Unix timestamp so the time of day is + // preserved. if (isInArrayOrTuple) { - // Inside a container each element is parsed by the element type's own - // parser: Array(Date)/Array(Date32) reject a bare Unix timestamp and only - // accept a quoted date string. A quoted UTC 'YYYY-MM-DD' is the single - // representation accepted by every temporal element type (Date, Date32, - // DateTime, DateTime64). return `'${value.toISOString().slice(0, 10)}'`; } - // The ClickHouse server parses numbers as time-zone-agnostic Unix timestamps + if (columnType !== undefined && DATE_OR_DATE32_PARAM_RE.test(columnType)) { + return value.toISOString().slice(0, 10); + } const unixTimestamp = Math.floor(value.getTime() / 1000) .toString() .padStart(10, "0"); @@ -152,6 +175,9 @@ interface FormatQueryParamsOptions { wrapStringInQuotes?: boolean; // For tuples/arrays, it is required to print NULL instead of \N printNullAsKeyword?: boolean; + // The ClickHouse type of the bound parameter (from `{name:Type}` in the query), used to + // pick a type-correct representation for a scalar `Date` value. Undefined when unknown. + columnType?: string; } const TabASCII = 9; diff --git a/packages/client-common/src/data_formatter/index.ts b/packages/client-common/src/data_formatter/index.ts index 01b003879..89d248030 100644 --- a/packages/client-common/src/data_formatter/index.ts +++ b/packages/client-common/src/data_formatter/index.ts @@ -1,3 +1,7 @@ export * from "./formatter"; -export { TupleParam, formatQueryParams } from "./format_query_params"; +export { + TupleParam, + formatQueryParams, + extractQueryParamType, +} from "./format_query_params"; export { formatQuerySettings } from "./format_query_settings"; diff --git a/packages/client-common/src/index.ts b/packages/client-common/src/index.ts index 2a751ae7c..9df719f9b 100644 --- a/packages/client-common/src/index.ts +++ b/packages/client-common/src/index.ts @@ -141,6 +141,7 @@ export { export { formatQuerySettings, formatQueryParams, + extractQueryParamType, encodeJSON, isSupportedRawFormat, isStreamableJSONFamily, diff --git a/packages/client-common/src/utils/multipart.ts b/packages/client-common/src/utils/multipart.ts index ec62f5e03..2c901fba9 100644 --- a/packages/client-common/src/utils/multipart.ts +++ b/packages/client-common/src/utils/multipart.ts @@ -1,4 +1,4 @@ -import { formatQueryParams } from "../data_formatter"; +import { extractQueryParamType, formatQueryParams } from "../data_formatter"; const SAFE_PART_NAME = /^[A-Za-z0-9_.-]+$/; @@ -22,6 +22,7 @@ export const MAX_URL_BIND_PARAM_LENGTH = 4096; */ export function serializeQueryParamsForUrl( query_params: Record, + query?: string, ): [string, string][] | null { const entries: [string, string][] = []; // Raw length is a lower bound on the encoded length, so large payloads @@ -29,7 +30,8 @@ export function serializeQueryParamsForUrl( let rawLength = 0; for (const [key, value] of Object.entries(query_params)) { const name = `param_${key}`; - const formatted = formatQueryParams({ value }); + const columnType = extractQueryParamType(query, key); + const formatted = formatQueryParams({ value, columnType }); rawLength += name.length + formatted.length; if (rawLength > MAX_URL_BIND_PARAM_LENGTH) { return null; diff --git a/packages/client-common/src/utils/url.ts b/packages/client-common/src/utils/url.ts index 6798adcc4..acf9ccf8c 100644 --- a/packages/client-common/src/utils/url.ts +++ b/packages/client-common/src/utils/url.ts @@ -1,4 +1,8 @@ -import { formatQueryParams, formatQuerySettings } from "../data_formatter"; +import { + extractQueryParamType, + formatQueryParams, + formatQuerySettings, +} from "../data_formatter"; import type { ClickHouseSettings } from "../settings"; export function transformUrl({ @@ -38,6 +42,9 @@ interface ToSearchParamsOptions { * used as-is instead of serializing {@link query_params} again. */ param_entries?: [string, string][]; query?: string; + /** The query text used only to read `{name:Type}` param types when serializing + * {@link query_params}; unlike {@link query} it is never added to the URL. */ + param_types_query?: string; session_id?: string; query_id: string; role?: string | Array; @@ -48,6 +55,7 @@ export function toSearchParams({ query, query_params, param_entries, + param_types_query, clickhouse_settings, session_id, query_id, @@ -61,7 +69,11 @@ export function toSearchParams({ entries = [["query_id", query_id]]; if (query_params !== undefined) { for (const [key, value] of Object.entries(query_params)) { - const formattedParam = formatQueryParams({ value }); + const columnType = extractQueryParamType( + param_types_query ?? query, + key, + ); + const formattedParam = formatQueryParams({ value, columnType }); entries.push([`param_${key}`, formattedParam]); } } diff --git a/packages/client-node/CHANGELOG.md b/packages/client-node/CHANGELOG.md index 435f9b73f..22375709f 100644 --- a/packages/client-node/CHANGELOG.md +++ b/packages/client-node/CHANGELOG.md @@ -15,8 +15,10 @@ ## Bug fixes - Fixed `Array(Date)` / `Array(Date32)` query-parameter binding (and other temporal element types nested in arrays, tuples, and maps). A JS `Date` inside a container was serialized as a bare Unix timestamp (e.g. `[1683244800]`), which the server's `Array(Date)` element parser rejects (`CANNOT_PARSE_INPUT_ASSERTION_FAILED`). Container-nested `Date` values are now emitted as a quoted UTC date string (e.g. `['2023-05-05']`), the one encoding every temporal element type accepts. Note: a `Date` used inside `Array(DateTime)` / `Array(DateTime64)` is now bound at day precision (the time-of-day is dropped), since date-only is the only form `Array(Date)` accepts; scalar `Date` / `DateTime` binding is unchanged. ([#947]) +- Fixed scalar `Date` / `Date32` query-parameter binding. A JS `Date` bound to a scalar `{name:Date}` / `{name:Date32}` parameter was serialized as a bare Unix timestamp, which the server rejects with `BAD_QUERY_PARAMETER` (only `DateTime` / `DateTime64` accept a numeric timestamp). The bound parameter's type is now read from the query, so a `Date` value is formatted as a UTC date string for `Date` / `Date32` while `DateTime` / `DateTime64` keep the Unix timestamp and preserve the time of day. ([#968]) [#947]: https://github.com/ClickHouse/clickhouse-js/pull/947 +[#968]: https://github.com/ClickHouse/clickhouse-js/pull/968 # 1.23.1 diff --git a/packages/client-node/src/connection/node_base_connection.ts b/packages/client-node/src/connection/node_base_connection.ts index 7ab67c7b9..fa6c4f803 100644 --- a/packages/client-node/src/connection/node_base_connection.ts +++ b/packages/client-node/src/connection/node_base_connection.ts @@ -17,6 +17,7 @@ import { buildMultipartBody, serializeQueryParamsForUrl, formatQueryParams, + extractQueryParamType, isCredentialsAuth, isJWTAuth, toSearchParams, @@ -204,7 +205,7 @@ export abstract class NodeBaseConnection implements Connection (params.use_multipart_params_auto ?? this.params.use_multipart_params_auto) ) { - const entries = serializeQueryParamsForUrl(queryParams); + const entries = serializeQueryParamsForUrl(queryParams, params.query); if (entries === null) { useMultipart = true; } else { @@ -217,6 +218,7 @@ export abstract class NodeBaseConnection implements Connection // When using multipart, query_params are sent in the multipart body query_params: useMultipart ? undefined : params.query_params, param_entries: urlParamEntries, + param_types_query: params.query, session_id: params.session_id, clickhouse_settings, query_id, @@ -237,7 +239,8 @@ export abstract class NodeBaseConnection implements Connection const boundary = `----clickhouse-js-${crypto.randomUUID()}`; const parts: Record = { query: params.query }; for (const [key, value] of Object.entries(params.query_params)) { - parts[`param_${key}`] = formatQueryParams({ value }); + const columnType = extractQueryParamType(params.query, key); + parts[`param_${key}`] = formatQueryParams({ value, columnType }); } body = buildMultipartBody(parts, boundary); headers["Content-Type"] = `multipart/form-data; boundary=${boundary}`; @@ -553,6 +556,7 @@ export abstract class NodeBaseConnection implements Connection ); const toSearchParamsOptions = { query: sendQueryInParams ? params.query : undefined, + param_types_query: params.query, database: this.params.database, query_params: params.query_params, session_id: params.session_id, diff --git a/packages/client-web/CHANGELOG.md b/packages/client-web/CHANGELOG.md index 12fb839c6..3c7df4b8f 100644 --- a/packages/client-web/CHANGELOG.md +++ b/packages/client-web/CHANGELOG.md @@ -15,8 +15,10 @@ ## Bug fixes - Fixed `Array(Date)` / `Array(Date32)` query-parameter binding (and other temporal element types nested in arrays, tuples, and maps). A JS `Date` inside a container was serialized as a bare Unix timestamp (e.g. `[1683244800]`), which the server's `Array(Date)` element parser rejects (`CANNOT_PARSE_INPUT_ASSERTION_FAILED`). Container-nested `Date` values are now emitted as a quoted UTC date string (e.g. `['2023-05-05']`), the one encoding every temporal element type accepts. Note: a `Date` used inside `Array(DateTime)` / `Array(DateTime64)` is now bound at day precision (the time-of-day is dropped), since date-only is the only form `Array(Date)` accepts; scalar `Date` / `DateTime` binding is unchanged. ([#947]) +- Fixed scalar `Date` / `Date32` query-parameter binding. A JS `Date` bound to a scalar `{name:Date}` / `{name:Date32}` parameter was serialized as a bare Unix timestamp, which the server rejects with `BAD_QUERY_PARAMETER` (only `DateTime` / `DateTime64` accept a numeric timestamp). The bound parameter's type is now read from the query, so a `Date` value is formatted as a UTC date string for `Date` / `Date32` while `DateTime` / `DateTime64` keep the Unix timestamp and preserve the time of day. ([#968]) [#947]: https://github.com/ClickHouse/clickhouse-js/pull/947 +[#968]: https://github.com/ClickHouse/clickhouse-js/pull/968 # 1.23.1 diff --git a/packages/client-web/src/connection/web_connection.ts b/packages/client-web/src/connection/web_connection.ts index 32ddb08d9..8e196dd3b 100644 --- a/packages/client-web/src/connection/web_connection.ts +++ b/packages/client-web/src/connection/web_connection.ts @@ -14,6 +14,7 @@ import { buildMultipartBody, serializeQueryParamsForUrl, formatQueryParams, + extractQueryParamType, isCredentialsAuth, isJWTAuth, isSuccessfulResponse, @@ -75,7 +76,7 @@ export class WebConnection implements Connection { (params.use_multipart_params_auto ?? this.params.use_multipart_params_auto) ) { - const entries = serializeQueryParamsForUrl(queryParams); + const entries = serializeQueryParamsForUrl(queryParams, params.query); if (entries === null) { useMultipart = true; } else { @@ -88,6 +89,7 @@ export class WebConnection implements Connection { clickhouse_settings, query_params: useMultipart ? undefined : params.query_params, param_entries: urlParamEntries, + param_types_query: params.query, session_id: params.session_id, role: params.role, query_id, @@ -99,7 +101,11 @@ export class WebConnection implements Connection { const boundary = `----clickhouse-js-${crypto.randomUUID()}`; const multipartParts: Record = { query: params.query }; for (const [key, value] of Object.entries(params.query_params)) { - multipartParts[`param_${key}`] = formatQueryParams({ value }); + const columnType = extractQueryParamType(params.query, key); + multipartParts[`param_${key}`] = formatQueryParams({ + value, + columnType, + }); } body = buildMultipartBody(multipartParts, boundary); headers["Content-Type"] = `multipart/form-data; boundary=${boundary}`; @@ -294,6 +300,7 @@ export class WebConnection implements Connection { database: this.params.database, clickhouse_settings: params.clickhouse_settings, query_params: params.query_params, + param_types_query: params.query, session_id: params.session_id, role: params.role, query_id,