diff --git a/Cargo.lock b/Cargo.lock index 9b1556b..5d25989 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,7 +82,7 @@ dependencies = [ [[package]] name = "clickhouse-query-ext" -version = "1.0.0" +version = "1.0.1" dependencies = [ "anyhow", "futures", diff --git a/Cargo.toml b/Cargo.toml index d35d01b..904d8b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clickhouse-query-ext" -version = "1.0.0" +version = "1.0.1" edition = "2024" authors = ["Querya Community"] description = "High-performance ClickHouse driver for Querya Desktop" diff --git a/README.md b/README.md index b56c296..a4c729a 100644 --- a/README.md +++ b/README.md @@ -1,148 +1,65 @@ -# πŸš€ clickhouse-query-ext (Querya ClickHouse Database Extension) +# clickhouse-query-ext [![Rust CI](https://github.com/QueryaHub/clickhouse-query-ext/actions/workflows/rust.yml/badge.svg)](https://github.com/QueryaHub/clickhouse-query-ext/actions/workflows/rust.yml) -[![Rust Edition](https://img.shields.io/badge/Edition-2024-brightgreen.svg)](https://doc.rust-lang.org/edition-guide/rust-2024/) -[![Protocol](https://img.shields.io/badge/Protocol-JSON--RPC%202.0%20over%20NDJSON%20%2F%20stdio-blue.svg)](#architecture) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -**`clickhouse-query-ext`** β€” это Π²Ρ‹ΡΠΎΠΊΠΎΠΏΡ€ΠΎΠΈΠ·Π²ΠΎΠ΄ΠΈΡ‚Π΅Π»ΡŒΠ½Ρ‹ΠΉ, отказоустойчивый асинхронный Π΄Ρ€Π°ΠΉΠ²Π΅Ρ€ ΠΈ Ρ€Π°ΡΡˆΠΈΡ€Π΅Π½ΠΈΠ΅ Π‘Π£Π‘Π” [ClickHouse](https://clickhouse.com/) для ΠΏΠ»Π°Ρ‚Ρ„ΠΎΡ€ΠΌΡ‹ **Querya Desktop (Analyst Edition)**. -Π”Ρ€Π°ΠΉΠ²Π΅Ρ€ построСн Π½Π° **Rust (Edition 2024)** ΠΈ Ρ€Π°Π±ΠΎΡ‚Π°Π΅Ρ‚ ΠΊΠ°ΠΊ ΠΈΠ·ΠΎΠ»ΠΈΡ€ΠΎΠ²Π°Π½Π½Ρ‹ΠΉ подпроцСсс (Zero-Trust Sandbox), взаимодСйствуя с хостом Querya Ρ‡Π΅Ρ€Π΅Π· ΠΏΡ€ΠΎΡ‚ΠΎΠΊΠΎΠ» **JSON-RPC 2.0 (NDJSON over `stdin` / `stdout`)**. - ---- - -## πŸ“ АрхитСктурная схСма Ρ€Π°ΡΡˆΠΈΡ€Π΅Π½ΠΈΡ - -```mermaid -graph LR - subgraph Querya [Querya Desktop Host] - UI[πŸ’» Generative SDUI] - Bridge[πŸ”Œ Bridge Process Manager] - end - - subgraph RustSandbox [clickhouse-query-ext (Rust Subprocess)] - Reader[πŸ“₯ LineStream stdin] --> Router[πŸ”„ JSON-RPC 2.0 Router] - Router --> Sys[βš™οΈ system.* Handshake / Ping / Secrets] - Router --> Conn[πŸ”Œ db.connect / disconnect] - Router --> Query[πŸ“Š db.query / execute / cancel] - Router --> Sdui[🎨 SDUI Tree & Form Schemas] - - Sys --> Pool[πŸ”’ ConnectionSecretsPool zeroize] - Query --> Safe[πŸ›‘οΈ Safe Mode AST Filter & Limits] - Sdui --> Parser[🌳 SYSTEM.* Introspection] - - Router --> Writer[πŸ“€ NDJSON stdout Mutex] - Writer --> Bridge - end - - Query -->|HTTP/HTTPS ClickHouse Client| CH[(🌐 ClickHouse Server)] -``` +Π Π°ΡΡˆΠΈΡ€Π΅Π½ΠΈΠ΅ Π±Π°Π·Ρ‹ Π΄Π°Π½Π½Ρ‹Ρ… [ClickHouse](https://clickhouse.com/) для [Querya Desktop](https://github.com/QueryaHub). УстанавливаСтся ΠΊΠ°ΠΊ `.qext`-ΠΏΠ°ΠΊΠ΅Ρ‚ ΠΈ добавляСт ClickHouse Π² список ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡Π΅Π½ΠΈΠΉ: Ρ„ΠΎΡ€ΠΌΠ° настройки, Π΄Π΅Ρ€Π΅Π²ΠΎ схСмы, просмотр Ρ‚Π°Π±Π»ΠΈΡ† ΠΈ SQL-Ρ€Π΅Π΄Π°ΠΊΡ‚ΠΎΡ€. ---- +## ВозмоТности -## ✨ ΠšΠ»ΡŽΡ‡Π΅Π²Ρ‹Π΅ возмоТности ΠΈ этапы Ρ€Π΅Π°Π»ΠΈΠ·Π°Ρ†ΠΈΠΈ +- ΠŸΠΎΠ΄ΠΊΠ»ΡŽΡ‡Π΅Π½ΠΈΠ΅ ΠΊ ClickHouse ΠΏΠΎ HTTP/HTTPS +- Π”Π΅Ρ€Π΅Π²ΠΎ ΠΎΠ±ΡŠΠ΅ΠΊΡ‚ΠΎΠ²: Π±Π°Π·Ρ‹, Ρ‚Π°Π±Π»ΠΈΡ†Ρ‹, прСдставлСния, ΠΊΠΎΠ»ΠΎΠ½ΠΊΠΈ, ΠΏΠ°Ρ€Ρ‚ΠΈΡ†ΠΈΠΈ +- Π’Ρ‹ΠΏΠΎΠ»Π½Π΅Π½ΠΈΠ΅ SQL-запросов ΠΈ просмотр Ρ€Π΅Π·ΡƒΠ»ΡŒΡ‚Π°Ρ‚ΠΎΠ² +- АналитичСский Ρ€Π΅ΠΆΠΈΠΌ (Safe Mode): ΠΎΠ³Ρ€Π°Π½ΠΈΡ‡Π΅Π½ΠΈΠ΅ опасных ΠΎΠΏΠ΅Ρ€Π°Ρ†ΠΈΠΉ ΠΈ Π»ΠΈΠΌΠΈΡ‚Ρ‹ Π½Π° сСссию +- ΠšΠΎΠ½Ρ‚Π΅ΠΊΡΡ‚Π½Ρ‹Π΅ дСйствия для Ρ‚Π°Π±Π»ΠΈΡ† ΠΈ ΠΏΠ°Ρ€Ρ‚ΠΈΡ†ΠΈΠΉ (статистика, DDL, optimize ΠΈ Π΄Ρ€.) -ВсС 6 этапов тСхничСского задания ΠΏΠΎΠ»Π½ΠΎΡΡ‚ΡŒΡŽ Ρ€Π΅Π°Π»ΠΈΠ·ΠΎΠ²Π°Π½Ρ‹, ΠΏΡ€ΠΎΠ²Π΅Ρ€Π΅Π½Ρ‹ ΠΈ ΠΏΠΎΠΊΡ€Ρ‹Ρ‚Ρ‹ автоматичСскими тСстами: +## ВрСбования -### 1️⃣ ΠšΠ°Ρ€ΠΊΠ°Ρ, асинхронный I/O ΠΈ NDJSON-транспорт ([Stage 1/6]) -- Π§Ρ‚Π΅Π½ΠΈΠ΅ ΠΏΠΎΡ‚ΠΎΠΊΠ° `stdin` Ρ‡Π΅Ρ€Π΅Π· асинхронный `tokio_util::codec::LinesCodec` Π±Π΅Π· Π±Π»ΠΎΠΊΠΈΡ€ΠΎΠ²ΠΊΠΈ Π³Π»Π°Π²Π½ΠΎΠ³ΠΎ ΠΏΠΎΡ‚ΠΎΠΊΠ°. -- Π—Π°ΠΏΠΈΡΡŒ ΠΎΡ‚Π²Π΅Ρ‚ΠΎΠ² Π² `stdout` Ρ‡Π΅Ρ€Π΅Π· потокобСзопасный `Mutex` с автоматичСской очисткой символов ΠΏΠ΅Ρ€Π΅Π²ΠΎΠ΄Π° строки (`\n`, `\r\n`), Π³Π°Ρ€Π°Π½Ρ‚ΠΈΡ€ΡƒΡŽΡ‰Π°Ρ 100% Π²Π°Π»ΠΈΠ΄Π½Ρ‹ΠΉ **NDJSON (Newline Delimited JSON)**. -- Π Π°Π·Π΄Π΅Π»Π΅Π½ΠΈΠ΅ быстрых ΠΌΠ΅Ρ‚ΠΎΠ΄ΠΎΠ² (`system.*` β€” Π·Π°Π΄Π΅Ρ€ΠΆΠΊΠ° `< 5ms`) ΠΈ тяТёлых SQL-Π²Ρ‹Π±ΠΎΡ€ΠΎΠΊ (Π²Ρ‹ΠΏΠΎΠ»Π½ΡΡŽΡ‚ΡΡ Π² ΠΏΡƒΠ»Π΅ Π·Π°Π΄Π°Ρ‡ `tokio::spawn`). +- Querya Desktop 2.0+ +- Для сборки ΠΈΠ· исходников: Rust stable (1.85+) -### 2️⃣ Π–ΠΈΠ·Π½Π΅Π½Π½Ρ‹ΠΉ Ρ†ΠΈΠΊΠ», ΡƒΠΏΡ€Π°Π²Π»Π΅Π½ΠΈΠ΅ сСкрСтами ΠΈ Π»ΠΎΠ³ΠΈΡ€ΠΎΠ²Π°Π½ΠΈΠ΅ ([Stage 2/6]) -- **`system.handshake`**: ΠΎΠ±ΠΌΠ΅Π½ вСрсиями ΠΈ рСгистрация возмоТностСй Π΄Ρ€Π°ΠΉΠ²Π΅Ρ€Π° (`db.connect`, `db.query`, `db.getSchemaTree`, `sdui.contextActions` ΠΈ Π΄Ρ€.). -- **`system.ping`**: Watchdog-Ρ‚Π°ΠΉΠΌΠ΅Ρ€ ΠΌΠ³Π½ΠΎΠ²Π΅Π½Π½ΠΎΠ³ΠΎ ΠΎΡ‚Π²Π΅Ρ‚Π° для прСдотвращСния зависаний (`result: "pong"`). -- **`system.injectCredentials`**: ΠΏΠ΅Ρ€Π΅Π΄Π°Ρ‡Π° ΠΏΠ°Ρ€ΠΎΠ»Π΅ΠΉ ΠΈ JWT Π² ΠΈΠ·ΠΎΠ»ΠΈΡ€ΠΎΠ²Π°Π½Π½Ρ‹ΠΉ `ConnectionSecretsPool`. -- **Π—Π°Ρ‰ΠΈΡ‚Π° памяти (`zeroize` & `secrecy`)**: ΠΏΠ°Ρ€ΠΎΠ»ΠΈ ΠΈ Ρ‚ΠΎΠΊΠ΅Π½Ρ‹ хранятся Π² Π·Π°Ρ‰ΠΈΡ‰Ρ‘Π½Π½ΠΎΠΉ памяти ΠΈ Π·Π°Π½ΡƒΠ»ΡΡŽΡ‚ΡΡ ΠΏΡ€ΠΈ ΡƒΠ΄Π°Π»Π΅Π½ΠΈΠΈ соСдинСния ΠΈΠ»ΠΈ Π°Π²Π°Ρ€ΠΈΠΉΠ½ΠΎΠΌ Π·Π°Π²Π΅Ρ€ΡˆΠ΅Π½ΠΈΠΈ (`clear_all`). -- **Π‘Π°Π½ΠΈΡ‚Π°Π·ΠΈΡ€ΠΎΠ²Π°Π½Π½Ρ‹ΠΉ Π»ΠΎΠ³Π³Π΅Ρ€ ([src/utils/logger.rs](src/utils/logger.rs))**: всС Π»ΠΎΠ³ΠΈ Π½Π°ΠΏΡ€Π°Π²Π»ΡΡŽΡ‚ΡΡ ΠΈΡΠΊΠ»ΡŽΡ‡ΠΈΡ‚Π΅Π»ΡŒΠ½ΠΎ Π² `stderr` с автоматичСским маскированиСм ΠΏΠ°Ρ€ΠΎΠ»Π΅ΠΉ ΠΈ HTTP-Π·Π°Π³ΠΎΠ»ΠΎΠ²ΠΊΠΎΠ² Π°Π²Ρ‚ΠΎΡ€ΠΈΠ·Π°Ρ†ΠΈΠΈ. +## Π‘Π±ΠΎΡ€ΠΊΠ° -### 3️⃣ ИсполнСниС SQL, конвСртация Ρ‚ΠΈΠΏΠΎΠ² ΠΈ Safe Mode ([Stage 3/6]) -- **`db.query` / `db.execute` / `db.cancelQuery`**: Π²Ρ‹ΠΏΠΎΠ»Π½Π΅Π½ΠΈΠ΅ SQL-запросов Ρ‡Π΅Ρ€Π΅Π· HTTP API ClickHouse с ΠΏΠΎΠ΄Π΄Π΅Ρ€ΠΆΠΊΠΎΠΉ стримингового парсинга Ρ„ΠΎΡ€ΠΌΠ°Ρ‚Π° `FORMAT JSONCompactEachRowWithNamesAndTypes`. -- **ΠšΠΎΠ½Π²Π΅Ρ€Ρ‚Π΅Ρ€ Ρ‚ΠΈΠΏΠΎΠ² ([src/mapper/types.rs](src/mapper/types.rs))**: полная ΠΏΠΎΠ΄Π΄Π΅Ρ€ΠΆΠΊΠ° `Int64/UInt64/Int128/UInt256`, `Decimal(P, S)`, `DateTime64`, `Array(T)`, `Tuple(...)`, `Map(K, V)`, `Nullable(T)` ΠΈ `LowCardinality(T)`. Π‘ΠΎΠ»ΡŒΡˆΠΈΠ΅ числа автоматичСски ΡΠ΅Ρ€ΠΈΠ°Π»ΠΈΠ·ΡƒΡŽΡ‚ΡΡ Π² строки (`"18446744073709551615"`), прСдотвращая ΠΏΠΎΡ‚Π΅Ρ€ΡŽ точности Π² JS. -- **πŸ›‘οΈ АналитичСский Safe Mode (Read-Only)**: - - ΠŸΡ€Π΅-Ρ„ΠΈΠ»ΡŒΡ‚Ρ€Π°Ρ†ΠΈΡ AST Π½Π° сторонС Rust (мгновСнная Π±Π»ΠΎΠΊΠΈΡ€ΠΎΠ²ΠΊΠ° `DROP DATABASE`, `TRUNCATE TABLE`, `ALTER ... DROP COLUMN` Π΄ΠΎ ΠΎΡ‚ΠΏΡ€Π°Π²ΠΊΠΈ Π½Π° сСрвСр). - - Установка сСссионных ΠΊΠ²ΠΎΡ‚ Π½Π° сСрвСрС ClickHouse (`readonly=1`, `max_execution_time=300`, `max_memory_usage=10000000000`). +```bash +cargo fmt --all -- --check +cargo clippy --all-targets --all-features -- -D warnings +cargo test +cargo build --release +./scripts/package_qext.sh +``` -### 4️⃣ Π˜Π½Ρ‚Ρ€ΠΎΡΠΏΠ΅ΠΊΡ†ΠΈΡ схСмы ΠΈ Π»Π΅Π½ΠΈΠ²ΠΎΠ΅ Π΄Π΅Ρ€Π΅Π²ΠΎ ([Stage 4/6]) -- **`db.getSchemaTree` & `db.expandTreeNode`**: иСрархичСская навигация ΠΏΠΎ ΠΎΠ±ΡŠΠ΅ΠΊΡ‚Π°ΠΌ Π‘Π£Π‘Π” с ΠΏΠΎΠ΄Π΄Π΅Ρ€ΠΆΠΊΠΎΠΉ Π»Π΅Π½ΠΈΠ²ΠΎΠΉ Π΄ΠΎΠ·Π°Π³Ρ€ΡƒΠ·ΠΊΠΈ. -- ΠžΡ‚ΠΎΠ±Ρ€Π°ΠΆΠ΅Π½ΠΈΠ΅ Π±Π°Π· Π΄Π°Π½Π½Ρ‹Ρ… (`SYSTEM.databases`), Ρ‚Π°Π±Π»ΠΈΡ† ΠΈ прСдставлСний (`SYSTEM.tables`), словарСй (`SYSTEM.dictionaries` с ΠΌΠ΅Ρ‚Ρ€ΠΈΠΊΠ°ΠΌΠΈ `HitRate`), ΠΊΠΎΠ»ΠΎΠ½ΠΎΠΊ (`SYSTEM.columns`) ΠΈ ΠΏΠ°Ρ€Ρ‚ΠΈΡ†ΠΈΠΉ (`SYSTEM.parts` с расчётом количСства строк ΠΈ сТатого Ρ€Π°Π·ΠΌΠ΅Ρ€Π° Π½Π° дискС). +АртСфакты: -### 5️⃣ Π“Π΅Π½Π΅Ρ€Π°Ρ‚ΠΎΡ€ SDUI-Ρ„ΠΎΡ€ΠΌ ΠΈ контСкстныС дСйствия ([Stage 5/6]) -- **`db.getConnectionFormSchema`**: гСнСрация Ρ„ΠΎΡ€ΠΌΡ‹ настройки ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡Π΅Π½ΠΈΡ (`assets/connection_form.json`). -- **`sdui.contextActions` ([src/sdui/actions.rs](src/sdui/actions.rs))**: контСкстноС мСню для Π°Π½Π°Π»ΠΈΡ‚ΠΈΠΊΠΎΠ² Π±Π΅Π· написания DDL: - - **Для Ρ‚Π°Π±Π»ΠΈΡ† (`table`)**: Π²Ρ‹Π±ΠΎΡ€ΠΊΠ° *Top 100 Rows*, *Column Statistics* (быстрый ΠΏΡ€ΠΎΡ„Π°ΠΉΠ»Π΅Ρ€), *Optimize Table (FINAL)*, *Deduplicate*, *Show DDL*. - - **Для ΠΏΠ°Ρ€Ρ‚ΠΈΡ†ΠΈΠΉ (`partition`)**: *Drop Partition*, *Freeze Partition* (созданиС бэкапа/hardlink), *Detach Partition*. - - **Для Π±Π°Π· Π΄Π°Π½Π½Ρ‹Ρ… (`database`, `view`)**: ΠΌΠΎΠ½ΠΈΡ‚ΠΎΡ€ΠΈΠ½Π³ Π°ΠΊΡ‚ΠΈΠ²Π½Ρ‹Ρ… ΠΌΡƒΡ‚Π°Ρ†ΠΈΠΉ (`SYSTEM.mutations`) ΠΈ процСссов (`SYSTEM.processes`). +| Π€Π°ΠΉΠ» | НазначСниС | +|------|------------| +| `target/release/clickhouse-query-ext` | Π‘ΠΈΠ½Π°Ρ€Π½ΠΈΠΊ Π΄Ρ€Π°ΠΉΠ²Π΅Ρ€Π° | +| `dist/clickhouse-query-ext-1.0.1.qext` | ΠŸΠ°ΠΊΠ΅Ρ‚ для установки Π² Querya Desktop | +| `dist/clickhouse-query-ext-1.0.1.qext.sha256` | ΠšΠΎΠ½Ρ‚Ρ€ΠΎΠ»ΡŒΠ½Π°Ρ сумма | -### 6️⃣ ΠžΡ‚ΠΊΠ°Π·ΠΎΡƒΡΡ‚ΠΎΠΉΡ‡ΠΈΠ²ΠΎΡΡ‚ΡŒ, автовосстановлСниС ΠΈ Panic Hook ([Stage 6/6]) -- **`std::panic::set_hook`**: ΠΏΠ΅Ρ€Π΅Ρ…Π²Π°Ρ‚ Π»ΡŽΠ±Ρ‹Ρ… ΠΏΠ°Π½ΠΈΠΊ Rust, Π²Ρ‹Π²ΠΎΠ΄ диагностичСского ΠΎΡ‚Ρ‡Π΅Ρ‚Π° Π² `stderr`, Π·Π°Π½ΡƒΠ»Π΅Π½ΠΈΠ΅ всСх сСкрСтов Π² памяти (`zeroize`) ΠΈ Π·Π°Π²Π΅Ρ€ΡˆΠ΅Π½ΠΈΠ΅ с ΠΊΠΎΠ΄ΠΎΠΌ **`101`** для ΠΊΠΎΡ€Ρ€Π΅ΠΊΡ‚Π½ΠΎΠ³ΠΎ запуска ΡΠΊΡΠΏΠΎΠ½Π΅Π½Ρ†ΠΈΠ°Π»ΡŒΠ½ΠΎΠ³ΠΎ backoff-пСрСзапуска (`SandboxAutoRecovery`). -- ΠŸΡ€ΠΎΠ²Π΅Ρ€ΠΊΠ° ΠΈ восстановлСниС структуры scratch-Π΄ΠΈΡ€Π΅ΠΊΡ‚ΠΎΡ€ΠΈΠΉ (`ensure_scratch_directories`) ΠΏΡ€ΠΈ ΠΊΠ°ΠΆΠ΄ΠΎΠΌ запускС Π΄Ρ€Π°ΠΉΠ²Π΅Ρ€Π°. +Установка: **Querya Desktop β†’ Extensions β†’ Install from file** β†’ Π²Ρ‹Π±Ρ€Π°Ρ‚ΡŒ `.qext`. ---- +## ΠšΡ€ΠΎΡΡ-компиляция -## πŸ› οΈ Π‘Π±ΠΎΡ€ΠΊΠ° ΠΈ тСстированиС +```bash +./scripts/build_cross.sh x86_64-unknown-linux-gnu +./scripts/package_qext.sh --target x86_64-unknown-linux-gnu +``` -### ВрСбования -- **Rust toolchain:** `stable` (edition 2024, Rust 1.85+) -- **OS:** Linux / macOS / Windows +ΠŸΠΎΠ΄Π΄Π΅Ρ€ΠΆΠΈΠ²Π°Π΅ΠΌΡ‹Π΅ Ρ†Π΅Π»ΠΈ: `x86_64-unknown-linux-gnu`, `aarch64-apple-darwin`, `x86_64-pc-windows-msvc`. -### ΠšΠΎΠΌΠ°Π½Π΄Ρ‹ сборки ΠΈ ΠΏΡ€ΠΎΠ²Π΅Ρ€ΠΊΠΈ +## Π Π΅Π»ΠΈΠ· + +Π’Π΅Π³ `v*` запускаСт GitHub Actions: сборка для всСх ΠΏΠ»Π°Ρ‚Ρ„ΠΎΡ€ΠΌ ΠΈ публикация `.qext` Π² Releases. ```bash -# ΠŸΡ€ΠΎΠ²Π΅Ρ€ΠΊΠ° форматирования -cargo fmt --all -- --check +git tag v1.0.1 +git push origin v1.0.1 +``` -# Запуск строгого Π»ΠΈΠ½Ρ‚Π΅Ρ€Π° -cargo clippy --all-targets --all-features -- -D warnings +## ДокумСнтация -# Запуск ΠΏΠΎΠ»Π½ΠΎΠ³ΠΎ ΠΊΠΎΠΌΠΏΠ»Π΅ΠΊΡ‚Π° unit- ΠΈ ΠΈΠ½Ρ‚Π΅Π³Ρ€Π°Ρ†ΠΈΠΎΠ½Π½Ρ‹Ρ… тСстов (48+ тСстов) -cargo test --verbose --all +ΠŸΠΎΠ΄Ρ€ΠΎΠ±Π½Π°Ρ спСцификация RPC, SDUI ΠΈ аналитичСских Ρ„ΡƒΠ½ΠΊΡ†ΠΈΠΉ β€” Π² ΠΊΠ°Ρ‚Π°Π»ΠΎΠ³Π΅ [`docs/`](docs/). -# Π‘Π±ΠΎΡ€ΠΊΠ° Ρ€Π΅Π»ΠΈΠ·Π½ΠΎΠ³ΠΎ Π±ΠΈΠ½Π°Ρ€Π½ΠΎΠ³ΠΎ Ρ„Π°ΠΉΠ»Π° Π΄Ρ€Π°ΠΉΠ²Π΅Ρ€Π° -cargo build --release -``` +## ЛицСнзия -ПослС ΡƒΡΠΏΠ΅ΡˆΠ½ΠΎΠΉ сборки исполняСмый Ρ„Π°ΠΉΠ» Π±ΡƒΠ΄Π΅Ρ‚ доступСн ΠΏΠΎ ΠΏΡƒΡ‚ΠΈ `target/release/clickhouse-query-ext`. - ---- - -## πŸ“‹ БпСцификация JSON-RPC ΠΌΠ΅Ρ‚ΠΎΠ΄ΠΎΠ² - -| ΠœΠ΅Ρ‚ΠΎΠ΄ | ОписаниС | НазначСниС | -| :--- | :--- | :--- | -| `system.handshake` | ОбмСн вСрсиями ΠΈ `capabilities` | Π˜Π½ΠΈΡ†ΠΈΠ°Π»ΠΈΠ·Π°Ρ†ΠΈΡ сСссии Querya Host ↔ Rust | -| `system.ping` | Watchdog heartbeat | Быстрая ΠΏΡ€ΠΎΠ²Π΅Ρ€ΠΊΠ° ТизнСспособности (`< 5ms`) | -| `system.injectCredentials` | ΠŸΠ΅Ρ€Π΅Π΄Π°Ρ‡Π° пароля/JWT | БСзопасноС сохранСниС Π² In-Memory Pool | -| `system.shutdown` | Π—Π°Π²Π΅Ρ€ΡˆΠ΅Π½ΠΈΠ΅ Ρ€Π°Π±ΠΎΡ‚Ρ‹ | ΠžΡ‡ΠΈΡΡ‚ΠΊΠ° памяти ΠΈ Π²Ρ‹Ρ…ΠΎΠ΄ с ΠΊΠΎΠ΄ΠΎΠΌ `0` | -| `db.connect` | Π‘ΠΎΠ·Π΄Π°Π½ΠΈΠ΅ HTTP-ΠΊΠ»ΠΈΠ΅Π½Ρ‚Π° | Π˜Π½ΠΈΡ†ΠΈΠ°Π»ΠΈΠ·Π°Ρ†ΠΈΡ TLS ΠΈ ΠΏΡ€ΠΎΠ²Π΅Ρ€ΠΊΠ° соСдинСния | -| `db.disconnect` | Π—Π°ΠΊΡ€Ρ‹Ρ‚ΠΈΠ΅ сСссии | Π£Π΄Π°Π»Π΅Π½ΠΈΠ΅ ΠΊΠ»ΠΈΠ΅Π½Ρ‚Π° ΠΈΠ· глобального ΠΏΡƒΠ»Π° | -| `db.query` | Π’Ρ‹Π±ΠΎΡ€ΠΊΠ° строк (`SELECT`) | Π’ΠΎΠ·Π²Ρ€Π°Ρ‚ `RowCompact` с ΠΌΠ°ΠΏΠΏΠΈΠ½Π³ΠΎΠΌ Ρ‚ΠΈΠΏΠΎΠ² | -| `db.execute` | Π’Ρ‹ΠΏΠΎΠ»Π½Π΅Π½ΠΈΠ΅ DDL/DML | Π’ΠΎΠ·Π²Ρ€Π°Ρ‚ количСства Π·Π°Ρ‚Ρ€ΠΎΠ½ΡƒΡ‚Ρ‹Ρ… строк (`affectedRows`) | -| `db.cancelQuery` | ΠžΡ‚ΠΌΠ΅Π½Π° запроса (`KILL QUERY`) | ΠžΡΡ‚Π°Π½ΠΎΠ²ΠΊΠ° Π΄ΠΎΠ»Π³ΠΈΡ… вычислСний ΠΏΠΎ `query_id` | -| `db.getSchemaTree` | Бписок Π±Π°Π· Π΄Π°Π½Π½Ρ‹Ρ… | ΠšΠΎΡ€Π½Π΅Π²ΠΎΠΉ ΡƒΡ€ΠΎΠ²Π΅Π½ΡŒ SDUI-Π΄Π΅Ρ€Π΅Π²Π° | -| `db.expandTreeNode` | Π Π°Π·Π²ΠΎΡ€Π°Ρ‡ΠΈΠ²Π°Π½ΠΈΠ΅ ΡƒΠ·Π»Π° | ΠŸΠΎΠ΄Π³Ρ€ΡƒΠ·ΠΊΠ° Ρ‚Π°Π±Π»ΠΈΡ†, Π²ΡŒΡŽΡ…, ΠΊΠΎΠ»ΠΎΠ½ΠΎΠΊ ΠΈ ΠΏΠ°Ρ€Ρ‚ΠΈΡ†ΠΈΠΉ | -| `db.getConnectionFormSchema`| Π€ΠΎΡ€ΠΌΠ° ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡Π΅Π½ΠΈΡ | ΠžΡ‚Π΄Π°Ρ‡Π° JSON-схСмы настроСк Π‘Π£Π‘Π” | -| `sdui.contextActions` | ΠšΠΎΠ½Ρ‚Π΅ΠΊΡΡ‚Π½ΠΎΠ΅ мСню | ГСнСрация аналитичСских ΠΊΠΎΠΌΠ°Π½Π΄ для UI | - ---- - -## πŸ‘₯ Π‘Ρ‚Ρ€ΡƒΠΊΡ‚ΡƒΡ€Π° рСпозитория - -```text -clickhouse-query-ext/ -β”œβ”€β”€ assets/ -β”‚ β”œβ”€β”€ connection_form.json # JSON-схСма Ρ„ΠΎΡ€ΠΌΡ‹ ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡Π΅Π½ΠΈΡ -β”‚ └── icon.svg # Иконка Ρ€Π°ΡΡˆΠΈΡ€Π΅Π½ΠΈΡ -β”œβ”€β”€ docs/ -β”‚ β”œβ”€β”€ 01_TZ_RUST_ARCHITECTURE.md -β”‚ └── 02_CLICKHOUSE_ANALYST_FEATURES.md -β”œβ”€β”€ src/ -β”‚ β”œβ”€β”€ main.rs # Π’ΠΎΡ‡ΠΊΠ° Π²Ρ…ΠΎΠ΄Π°, инициализация Sandbox ΠΈ асинхронный Ρ†ΠΈΠΊΠ» -β”‚ β”œβ”€β”€ error.rs # Π”ΠΎΠΌΠ΅Π½Π½Ρ‹Π΅ ошибки DriverError ΠΈ ΠΌΠ°ΠΏΠΏΠΈΠ½Π³ Π² ΠΊΠΎΠ΄Ρ‹ JSON-RPC (-3260x) -β”‚ β”œβ”€β”€ transport/ # Асинхронный NDJSON-транспорт (stdio.rs, framing.rs) -β”‚ β”œβ”€β”€ rpc/ # Π ΠΎΡƒΡ‚Π΅Ρ€ ΠΈ ΠΎΠ±Ρ€Π°Π±ΠΎΡ‚Ρ‡ΠΈΠΊΠΈ JSON-RPC 2.0 -β”‚ β”œβ”€β”€ driver/ # HTTP/TLS ΠΊΠ»ΠΈΠ΅Π½Ρ‚, ΠΏΡƒΠ» соСдинСний, настройки сСссий -β”‚ β”œβ”€β”€ mapper/ # ΠŸΠ°Ρ€ΡΠ΅Ρ€ Ρ‚ΠΈΠΏΠΎΠ² ClickHouse ΠΈ Row Format -β”‚ β”œβ”€β”€ sdui/ # Generative SDUI: Π΄Π΅Ρ€Π΅Π²ΠΎ, Ρ„ΠΎΡ€ΠΌΡ‹ ΠΈ контСкстныС дСйствия -β”‚ └── utils/ # Panic hook, recovery, zeroize сСкрСты, санитазированный Π»ΠΎΠ³Π³Π΅Ρ€ -└── manifest.json # ΠœΠ°Π½ΠΈΡ„Π΅ΡΡ‚ Ρ€Π°ΡΡˆΠΈΡ€Π΅Π½ΠΈΡ для Querya Desktop -``` +MIT diff --git a/assets/connection_form.json b/assets/connection_form.json index 7cec2c8..61bcb6a 100644 --- a/assets/connection_form.json +++ b/assets/connection_form.json @@ -23,6 +23,14 @@ "required": true, "defaultValue": "default" }, + { + "key": "database", + "label": "Π‘Π°Π·Π° Π΄Π°Π½Π½Ρ‹Ρ… ΠΏΠΎ ΡƒΠΌΠΎΠ»Ρ‡Π°Π½ΠΈΡŽ", + "type": "text", + "required": false, + "defaultValue": "default", + "helperText": "Имя Π±Π°Π·Ρ‹ ClickHouse для ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡Π΅Π½ΠΈΡ ΠΈ Π΄Π΅Ρ€Π΅Π²Π° схСмы" + }, { "key": "password", "label": "ΠŸΠ°Ρ€ΠΎΠ»ΡŒ", diff --git a/manifest.json b/manifest.json index 6eb143b..32c86eb 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "queryahub.clickhouse-driver", "name": "ClickHouse Database Driver (Analyst Edition)", - "version": "1.0.0", + "version": "1.0.1", "publisher": "Querya Community", "description": "Π˜Π·ΠΎΠ»ΠΈΡ€ΠΎΠ²Π°Π½Π½Ρ‹ΠΉ Π½Π°Ρ‚ΠΈΠ²Π½Ρ‹ΠΉ Rust-Π΄Ρ€Π°ΠΉΠ²Π΅Ρ€ для аналитичСской Π‘Π£Π‘Π” ClickHouse с ΠΏΠΎΠ»Π½ΠΎΠΉ ΠΏΠΎΠ΄Π΄Π΅Ρ€ΠΆΠΊΠΎΠΉ MergeTree, словарСй, ΠΏΠ°Ρ€Ρ‚ΠΈΡ†ΠΈΠΉ ΠΈ SDUI-интроспСкции.", "type": "database_driver", diff --git a/scripts/package_qext.sh b/scripts/package_qext.sh index 24f9689..37461d5 100755 --- a/scripts/package_qext.sh +++ b/scripts/package_qext.sh @@ -69,6 +69,10 @@ cp "${BIN_PATH}" "${STAGING_DIR}/bin/${DEST_BIN_NAME}" if [ "${DEST_BIN_NAME}" != "${BIN_NAME}" ] && [ ! -f "${STAGING_DIR}/bin/${BIN_NAME}" ]; then cp "${BIN_PATH}" "${STAGING_DIR}/bin/${BIN_NAME}" fi +chmod +x "${STAGING_DIR}/bin/${DEST_BIN_NAME}" +if [ -f "${STAGING_DIR}/bin/${BIN_NAME}" ]; then + chmod +x "${STAGING_DIR}/bin/${BIN_NAME}" +fi # 3. Create .qext (.zip) archive using python zipfile module or zip CLI ARCHIVE_NAME="clickhouse-query-ext-${VERSION}-${TARGET}.qext" diff --git a/src/driver/client.rs b/src/driver/client.rs index ca2f637..ce605cf 100644 --- a/src/driver/client.rs +++ b/src/driver/client.rs @@ -1,7 +1,9 @@ use crate::error::DriverError; use crate::utils::secret_guard::ConnectionSecretsPool; -use reqwest::{Client, ClientBuilder}; +use reqwest::{Client, ClientBuilder, RequestBuilder}; use serde::Deserialize; +use std::future::Future; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use url::Url; @@ -18,13 +20,16 @@ pub struct ConnectParams { pub readonly: Option, } -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct ClickHouseClient { pub connection_id: u64, pub base_url: String, pub user: String, pub database: String, pub readonly: bool, + /// When true, omit `readonly=1` on HTTP requests because the server/user + /// profile already enforces read-only mode. + server_enforces_readonly: AtomicBool, pub http_client: Client, } @@ -52,8 +57,7 @@ impl ClickHouseClient { } else { params.database.unwrap_or_else(|| "default".to_string()) }; - // Safe Mode: default to readonly = true unless explicitly disabled - let readonly = params.readonly.unwrap_or(true); + let readonly = params.readonly.unwrap_or(false); let base = format!("{}://{}:{}", scheme, host, port); (base, user, database, readonly) } @@ -62,7 +66,7 @@ impl ClickHouseClient { let port = params.port.unwrap_or(8123); let user = params.user.unwrap_or_else(|| "default".to_string()); let database = params.database.unwrap_or_else(|| "default".to_string()); - let readonly = params.readonly.unwrap_or(true); + let readonly = params.readonly.unwrap_or(false); ( format!("http://{}:{}", host, port), user, @@ -82,31 +86,50 @@ impl ClickHouseClient { user, database, readonly, + server_enforces_readonly: AtomicBool::new(false), http_client, }) } - /// Check connection health by executing `SELECT version()` against ClickHouse. - /// Utilizes zero-trust `ConnectionSecretsPool` for authentication headers without persisting secrets in struct memory. - pub async fn ping_connection(&self) -> Result { - if self.base_url.starts_with("mock://") || self.base_url.starts_with("test://") { - return Ok("mock-clickhouse-23.8.1.1".to_string()); - } + /// Appends Safe Mode session settings to a ClickHouse HTTP URL. + pub fn append_safe_mode_settings(&self, url: &mut Url) { + self.append_safe_mode_settings_with(url, self.omit_readonly_setting()); + } - let mut url = Url::parse(&self.base_url)?; - url.query_pairs_mut() - .append_pair("query", "SELECT version()") - .append_pair("database", &self.database); - if self.readonly { - url.query_pairs_mut() - .append_pair("readonly", "1") - .append_pair("max_execution_time", "300") - .append_pair("max_memory_usage", "10000000000"); + fn omit_readonly_setting(&self) -> bool { + self.server_enforces_readonly.load(Ordering::Relaxed) + } + + fn append_safe_mode_settings_with(&self, url: &mut Url, omit_readonly_setting: bool) { + if !self.readonly { + return; } + if !omit_readonly_setting { + url.query_pairs_mut().append_pair("readonly", "1"); + } + url.query_pairs_mut() + .append_pair("max_execution_time", "300") + .append_pair("max_memory_usage", "10000000000"); + } - let mut req = self.http_client.get(url); + fn mark_server_readonly_enforced(&self) { + self.server_enforces_readonly.store(true, Ordering::Relaxed); + } - // Retrieve secret securely just-in-time from ConnectionSecretsPool + /// Returns true when ClickHouse rejects `readonly=1` because the session is + /// already read-only at server/profile level. + pub fn is_readonly_setting_conflict(err: &DriverError) -> bool { + match err { + DriverError::Client(msg) => { + msg.contains("Cannot modify 'readonly' setting in readonly mode") + || msg.contains("Code: 164") + || msg.contains("(READONLY)") + } + _ => false, + } + } + + fn apply_auth(&self, mut req: RequestBuilder) -> RequestBuilder { if let Some(secrets) = ConnectionSecretsPool::global().get(self.connection_id) { if let Some(jwt) = secrets.expose_jwt_token() { req = req.header("Authorization", format!("Bearer {}", jwt)); @@ -120,8 +143,10 @@ impl ClickHouseClient { } else { req = req.header("X-ClickHouse-User", &self.user); } + req + } - let resp = req.send().await?; + async fn read_response(resp: reqwest::Response) -> Result { if !resp.status().is_success() { let status = resp.status(); let text = resp.text().await.unwrap_or_default(); @@ -130,9 +155,84 @@ impl ClickHouseClient { status, text ))); } + Ok(resp.text().await?) + } - let version = resp.text().await?.trim().to_string(); - Ok(version) + pub async fn execute_with_readonly_retry( + &self, + mut run: F, + ) -> Result + where + F: FnMut(bool) -> Fut, + Fut: Future>, + { + let mut omit = self.omit_readonly_setting(); + loop { + match run(omit).await { + Ok(text) => return Ok(text), + Err(err) if self.readonly && !omit && Self::is_readonly_setting_conflict(&err) => { + self.mark_server_readonly_enforced(); + omit = true; + } + Err(err) => return Err(err), + } + } + } + + /// Executes a GET request with optional SQL in the `query` URL parameter. + pub async fn get_with_query(&self, sql: &str) -> Result { + if self.base_url.starts_with("mock://") || self.base_url.starts_with("test://") { + return Ok("mock-clickhouse-23.8.1.1".to_string()); + } + + self.execute_with_readonly_retry(|omit_readonly| async move { + let mut url = Url::parse(&self.base_url)?; + url.query_pairs_mut() + .append_pair("query", sql) + .append_pair("database", &self.database); + self.append_safe_mode_settings_with(&mut url, omit_readonly); + let req = self.apply_auth(self.http_client.get(url)); + Self::read_response(req.send().await?).await + }) + .await + } + + /// Executes a POST request with SQL in the request body. + pub async fn post_sql( + &self, + sql: &str, + mut extra_params: impl FnMut(&mut Url), + ) -> Result { + if self.base_url.starts_with("mock://") || self.base_url.starts_with("test://") { + return Ok(String::new()); + } + + let sql = sql.to_string(); + let mut omit = self.omit_readonly_setting(); + loop { + let mut url = Url::parse(&self.base_url)?; + url.query_pairs_mut() + .append_pair("database", &self.database); + extra_params(&mut url); + self.append_safe_mode_settings_with(&mut url, omit); + let req = self + .apply_auth(self.http_client.post(url)) + .body(sql.clone()); + match Self::read_response(req.send().await?).await { + Ok(text) => return Ok(text), + Err(err) if self.readonly && !omit && Self::is_readonly_setting_conflict(&err) => { + self.mark_server_readonly_enforced(); + omit = true; + } + Err(err) => return Err(err), + } + } + } + + /// Check connection health by executing `SELECT version()` against ClickHouse. + pub async fn ping_connection(&self) -> Result { + let version = self.get_with_query("SELECT version()").await?; + Ok(version.trim().to_string()) } } @@ -145,6 +245,7 @@ mod tests { let params = ConnectParams { connection_id: 1, connection_string: Some("http://admin@localhost:8123/analytics?readonly=1".to_string()), + readonly: Some(true), ..Default::default() }; let client = ClickHouseClient::from_params(params).unwrap(); @@ -169,7 +270,6 @@ mod tests { #[test] fn test_from_params_safe_mode_aliases() { - // Test parsing safe_mode alias via serde let json_val = serde_json::json!({ "connectionId": 10, "host": "localhost", @@ -188,4 +288,30 @@ mod tests { let client_off = ClickHouseClient::from_params(params_off).unwrap(); assert!(!client_off.readonly); } + + #[test] + fn test_append_safe_mode_settings_omits_readonly_when_server_enforces() { + let client = ClickHouseClient::from_params(ConnectParams { + connection_id: 3, + host: Some("localhost".to_string()), + readonly: Some(true), + ..Default::default() + }) + .unwrap(); + client.mark_server_readonly_enforced(); + + let mut url = Url::parse("http://localhost:8123").unwrap(); + client.append_safe_mode_settings(&mut url); + let query = url.query().unwrap_or_default(); + assert!(!query.contains("readonly=1")); + assert!(query.contains("max_execution_time=300")); + } + + #[test] + fn test_is_readonly_setting_conflict() { + let err = DriverError::Client( + "ClickHouse HTTP error 500 Internal Server Error: Code: 164. DB::Exception: Cannot modify 'readonly' setting in readonly mode. (READONLY)".to_string(), + ); + assert!(ClickHouseClient::is_readonly_setting_conflict(&err)); + } } diff --git a/src/rpc/handlers/query.rs b/src/rpc/handlers/query.rs index fc941e8..a233daa 100644 --- a/src/rpc/handlers/query.rs +++ b/src/rpc/handlers/query.rs @@ -231,44 +231,13 @@ pub async fn handle_query(params: Option) -> Result { } // 3. Real ClickHouse HTTP request - let mut url = Url::parse(&client.base_url)?; - url.query_pairs_mut() - .append_pair("database", &client.database) - .append_pair("query_id", &actual_query_id); - if client.readonly { - url.query_pairs_mut() - .append_pair("readonly", "1") - .append_pair("max_execution_time", "300") - .append_pair("max_memory_usage", "10000000000"); - } - - let mut req = client.http_client.post(url).body(sql_to_run); - - if let Some(secrets) = ConnectionSecretsPool::global().get(client.connection_id) { - if let Some(jwt) = secrets.expose_jwt_token() { - req = req.header("Authorization", format!("Bearer {}", jwt)); - } else if let Some(pass) = secrets.expose_password() { - req = req - .header("X-ClickHouse-User", &client.user) - .header("X-ClickHouse-Key", pass); - } else { - req = req.header("X-ClickHouse-User", &client.user); - } - } else { - req = req.header("X-ClickHouse-User", &client.user); - } - - let resp = req.send().await?; - if !resp.status().is_success() { - let status = resp.status(); - let text = resp.text().await.unwrap_or_default(); - return Err(DriverError::Client(format!( - "ClickHouse SQL error {}: {}", - status, text - ))); - } - - let text = resp.text().await?; + let actual_query_id_for_url = actual_query_id.clone(); + let text = client + .post_sql(&sql_to_run, |url| { + url.query_pairs_mut() + .append_pair("query_id", &actual_query_id_for_url); + }) + .await?; let elapsed = start_time.elapsed().as_millis() as u64; if is_tabular_query { diff --git a/src/rpc/handlers/schema.rs b/src/rpc/handlers/schema.rs index 4c6404a..9457340 100644 --- a/src/rpc/handlers/schema.rs +++ b/src/rpc/handlers/schema.rs @@ -1,11 +1,9 @@ use crate::driver::pool::ConnectionPool; use crate::error::DriverError; use crate::sdui::tree::*; -use crate::utils::secret_guard::ConnectionSecretsPool; use serde::Deserialize; use serde_json::{Value, json}; use tracing::info; -use url::Url; #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] @@ -34,44 +32,7 @@ async fn run_introspection_query(connection_id: u64, sql: &str) -> Result