Standalone ClickHouse data-type parser (C++ + TypeScript port)

type-parser/mini-parser-extracted/.gitignore

@@ -0,0 +1 @@
+build/

type-parser/mini-parser-extracted/CMakeLists.txt

@@ -0,0 +1,32 @@
+cmake_minimum_required(VERSION 3.16)
+project(chdt_datatype_parser CXX)
+
+# A self-contained library: no dependency on the ClickHouse source tree.
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+if(NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE Release)
+endif()
+
+# Warning flags are GCC/Clang-specific; guard them so MSVC builds still work.
+if(MSVC)
+    add_compile_options(/W4)
+else()
+    add_compile_options(-Wall -Wextra)
+endif()
+
+add_library(chdt_datatype_parser
+    src/lexer.cpp
+    src/parser.cpp
+    src/json.cpp
+)
+target_include_directories(chdt_datatype_parser PUBLIC include)
+
+# CLI: read a data-type string, print its JSON AST. Exit non-zero on error.
+add_executable(chdt-parse tool/main.cpp)
+target_link_libraries(chdt-parse PRIVATE chdt_datatype_parser)
+
+enable_testing()
+add_subdirectory(test)

type-parser/mini-parser-extracted/README.md

@@ -0,0 +1,363 @@
+# chdt — standalone ClickHouse data-type parser
+
+## Examples
+
+```bash
+./build/chdt-parse "Tuple(id UInt64, name LowCardinality(String), price Decimal(18, 4), ts DateTime64(9, 'UTC'), tags
+  Array(LowCardinality(Nullable(String))), attrs Map(String, Array(Nullable(Int32))), status Enum8('active' = 1, 'closed' = -2), coords
+  Array(Tuple(Float64, Float64)), meta Nested(k String, v UInt32), fixed FixedString(16), dyn Dynamic(max_types = 8), variant Variant(UInt64, String,
+  Array(UInt8)), raw Object('json'))"
+```
+
+```json
+{
+  "type": "TupleDataType",
+  "name": "Tuple",
+  "arguments": [
+    {
+      "type": "DataType",
+      "name": "UInt64"
+    },
+    {
+      "type": "DataType",
+      "name": "LowCardinality",
+      "arguments": [
+        {
+          "type": "DataType",
+          "name": "String"
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "Decimal",
+      "arguments": [
+        {
+          "type": "Literal",
+          "value_type": "UInt64",
+          "value": "18"
+        },
+        {
+          "type": "Literal",
+          "value_type": "UInt64",
+          "value": "4"
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "DateTime64",
+      "arguments": [
+        {
+          "type": "Literal",
+          "value_type": "UInt64",
+          "value": "9"
+        },
+        {
+          "type": "Literal",
+          "value_type": "String",
+          "value": "UTC"
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "Array",
+      "arguments": [
+        {
+          "type": "DataType",
+          "name": "LowCardinality",
+          "arguments": [
+            {
+              "type": "DataType",
+              "name": "Nullable",
+              "arguments": [
+                {
+                  "type": "DataType",
+                  "name": "String"
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "Map",
+      "arguments": [
+        {
+          "type": "DataType",
+          "name": "String"
+        },
+        {
+          "type": "DataType",
+          "name": "Array",
+          "arguments": [
+            {
+              "type": "DataType",
+              "name": "Nullable",
+              "arguments": [
+                {
+                  "type": "DataType",
+                  "name": "Int32"
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "type": "EnumDataType",
+      "name": "Enum8",
+      "values": [
+        {
+          "name": "active",
+          "value": 1
+        },
+        {
+          "name": "closed",
+          "value": -2
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "Array",
+      "arguments": [
+        {
+          "type": "TupleDataType",
+          "name": "Tuple",
+          "arguments": [
+            {
+              "type": "DataType",
+              "name": "Float64"
+            },
+            {
+              "type": "DataType",
+              "name": "Float64"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "Nested",
+      "arguments": [
+        {
+          "type": "NameTypePair",
+          "name": "k",
+          "data_type": {
+            "type": "DataType",
+            "name": "String"
+          }
+        },
+        {
+          "type": "NameTypePair",
+          "name": "v",
+          "data_type": {
+            "type": "DataType",
+            "name": "UInt32"
+          }
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "FixedString",
+      "arguments": [
+        {
+          "type": "Literal",
+          "value_type": "UInt64",
+          "value": "16"
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "Dynamic",
+      "arguments": [
+        {
+          "type": "Function",
+          "name": "equals",
+          "is_operator": true,
+          "arguments": [
+            {
+              "type": "Identifier",
+              "name": "max_types"
+            },
+            {
+              "type": "Literal",
+              "value_type": "UInt64",
+              "value": "8"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "Variant",
+      "arguments": [
+        {
+          "type": "DataType",
+          "name": "UInt64"
+        },
+        {
+          "type": "DataType",
+          "name": "String"
+        },
+        {
+          "type": "DataType",
+          "name": "Array",
+          "arguments": [
+            {
+              "type": "DataType",
+              "name": "UInt8"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "type": "DataType",
+      "name": "Object",
+      "arguments": [
+        {
+          "type": "Literal",
+          "value_type": "String",
+          "value": "json"
+        }
+      ]
+    }
+  ],
+  "element_names": [
+    "id",
+    "name",
+    "price",
+    "ts",
+    "tags",
+    "attrs",
+    "status",
+    "coords",
+    "meta",
+    "fixed",
+    "dyn",
+    "variant",
+    "raw"
+  ]
+}
+```
+
+## About
+
+A small, self-contained C++ library that parses a ClickHouse **data-type
+string** (the kind sent in the types row of `RowBinaryWithNamesAndTypes`, e.g.
+`Array(Nullable(UInt64))`, `Tuple(a UInt8, b String)`, `Enum8('a' = 1)`,
+`Decimal(10, 2)`) into a JSON AST.
+
+It is extracted from the server's `ParserDataType`
+(`src/Parsers/ParserDataType.cpp`) but has **no dependency on the ClickHouse
+source tree** — only the C++20 standard library. The JSON it emits mirrors the
+data-type subtree of the frozen `EXPLAIN AST json = 1` document (format
+**version 2**; see `AST.md` in the ClickHouse repo), so its output is a drop-in
+match for what the server produces.
+
+## Why this exists
+
+The server's type parser is entangled with the lexer, the expression parsers,
+and the `IAST` / `Field` machinery. Vendoring all of that verbatim would pull
+in ~9–10k lines (`Field`, `ReadHelpers`, `Exception`, the formatting/hashing
+layer). Instead, this reimplements just the type grammar on a minimal AST:
+
+- a purpose-built tokenizer (`src/lexer.*`) covering the slice of SQL that type
+  strings use, in place of the full `Lexer` and its `UTF8Helpers` /
+  `find_symbols` dependencies;
+- a faithful port of `ParserDataType::parseImpl` (`src/parser.cpp`) — same
+  control flow: identifier + SQL-standard multi-word aliases, the Enum and
+  Tuple special cases, then the generic parametric-argument loop;
+- plain structs for the AST (`include/chdt/ast.h`) instead of `IAST` + `Field`.
+
+## Build and test
+
+```bash
+rm -rf build # clean the build dir for a fresh run
+cmake -S . -B build -DCMAKE_BUILD_TYPE=Release
+cmake --build build
+```
+
+## Usage
+
+Library:
+
+```cpp
+#include "chdt/parser.h"
+
+chdt::ParseResult r = chdt::parseDataType("Tuple(a UInt8, b String)");
+if (r.ok())
+    std::string json = chdt::toJSON(*r.ast);
+else
+    /* r.error->message, r.error->position */;
+```
+
+CLI:
+
+```bash
+./build/chdt-parse "Array(Nullable(UInt64))"
+echo "Enum8('a' = 1, 'b' = 2)" | ./build/chdt-parse
+```
+
+## Output shape
+
+Node types and slots match the server (format v2):
+
+| `type`          | slots                                                    |
+| --------------- | -------------------------------------------------------- |
+| `DataType`      | `name`, `arguments?` (present iff the type had `(...)`)  |
+| `EnumDataType`  | `name`, `values` (array of `{ name, value }`)            |
+| `TupleDataType` | `name`, `arguments?`, `element_names?` (named tuples)    |
+| `NameTypePair`  | `name`, `data_type` (a `Nested(...)` element)            |
+| `Literal`       | `value_type`, `value` (64-bit ints as JSON strings)      |
+| `Function`      | `name`, `is_operator?`, `arguments` (e.g. `max_types=5`) |
+| `Identifier`    | `name`, `name_parts?`                                    |
+
+`EnumDataType.values` and `TupleDataType.element_names` are carried here exactly
+as the server emits them since format v2.
+
+## Coverage
+
+Supported: scalars, parametric types with literal args (`Decimal`,
+`FixedString`, `DateTime64`, …), nested type args (`Array`, `Map`, `Nullable`,
+`LowCardinality`, `Variant`, …), enums (explicit → `EnumDataType`;
+auto-assigned → generic `DataType`), named/unnamed/mixed tuples, `Nested`,
+`Dynamic(max_types = N)`, the legacy `Object('json')`, and the SQL-standard
+multi-word aliases (`DOUBLE PRECISION`, `CHAR VARYING`, `INT SIGNED`, …).
+
+**Deliberately not supported yet** (the parser returns a clear error):
+
+- `AggregateFunction` / `SimpleAggregateFunction` — needs the function-expression
+  parser the server reaches for here.
+- the new `JSON(...)` object-argument syntax (`JSON(a.b UInt32, SKIP x)`). Bare
+  `JSON` and legacy `Object('json')` parse fine.
+
+## Tests
+
+`ctest` runs two suites against the parser:
+
+- **oracle** (`test/oracle_compare.py`) — for each type in `test/cases.txt`,
+  compares the parser's JSON against the `data_type` subtree the real server
+  emits for `CREATE TABLE t (c <TYPE>) ENGINE = Null`. Needs a `clickhouse`
+  binary (default: `../build/programs/clickhouse`; override with
+  `-DCLICKHOUSE_BINARY=...`).
+- **unsupported** (`test/check_unsupported.py`) — asserts the deferred types in
+  `test/cases_unsupported.txt` are rejected.
+
+```bash
+cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DCLICKHOUSE_BINARY=/work/ClickHouse/build/programs/clickhouse
+cmake --build build
+ctest --test-dir build --output-on-failure
+```
+
+When the AST format changes get merged the special build `/work/ClickHouse/build/programs/clickhouse` wouldn't be needed.