questdb · kafka1991 · May 24, 2026 · May 24, 2026 · May 24, 2026 · May 24, 2026
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -71,6 +71,11 @@ option(
     "Build the C/C++ tests with -fsanitize=address,undefined."
     OFF)
 
+option(
+    QUESTDB_ENABLE_ARROW
+    "Build with Apache Arrow C Data Interface exports. Opt-in: pulls arrow-rs."
+    OFF)
+
 # Build static and dynamic lib written in Rust by invoking `cargo`.
 # Imports `questdb_client` target.
 add_subdirectory(corrosion)
@@ -81,6 +86,13 @@ endif()
 if(QUESTDB_ENABLE_INSECURE_SKIP_VERIFY)
     list(APPEND QUESTDB_CARGO_FEATURES insecure-skip-verify)
 endif()
+if(QUESTDB_TESTS_AND_EXAMPLES AND NOT QUESTDB_ENABLE_ARROW)
+    message(STATUS "QUESTDB_TESTS_AND_EXAMPLES=ON: enabling QUESTDB_ENABLE_ARROW")
+    set(QUESTDB_ENABLE_ARROW ON)
+endif()
+if(QUESTDB_ENABLE_ARROW)
+    list(APPEND QUESTDB_CARGO_FEATURES arrow)
+endif()
 if(QUESTDB_CARGO_FEATURES)
     corrosion_import_crate(
         MANIFEST_PATH questdb-rs-ffi/Cargo.toml
@@ -94,6 +106,9 @@ endif()
 target_include_directories(
     questdb_client INTERFACE
     ${CMAKE_CURRENT_SOURCE_DIR}/include)
+if(QUESTDB_ENABLE_ARROW)
+    target_compile_definitions(questdb_client INTERFACE QUESTDB_CLIENT_ENABLE_ARROW)
+endif()
 if(WIN32)
     set_target_properties(
         questdb_client-shared
@@ -280,6 +295,26 @@ if (QUESTDB_TESTS_AND_EXAMPLES)
     compile_example(
         line_reader_c_example_columns
         examples/line_reader_c_example_columns.c)
+    compile_example(
+        line_reader_c_example_arrow
+        examples/line_reader_c_example_arrow.c)
+
+    find_package(Arrow QUIET)
+    if(Arrow_FOUND)
+        compile_example(
+            line_sender_cpp_example_arrow
+            examples/line_sender_cpp_example_arrow.cpp)
+        target_link_libraries(
+            line_sender_cpp_example_arrow Arrow::arrow_shared)
+        compile_example(
+            line_reader_cpp_example_arrow
+            examples/line_reader_cpp_example_arrow.cpp)
+        target_link_libraries(
+            line_reader_cpp_example_arrow Arrow::arrow_shared)
+    else()
+        message(STATUS
+            "arrow-cpp not found; skipping line_{sender,reader}_cpp_example_arrow.")
+    endif()
 
     # Include Rust tests as part of the tests run
     add_test(
@@ -358,6 +393,28 @@ if (QUESTDB_TESTS_AND_EXAMPLES)
         cpp_test/qwp_mock_server.cpp
         cpp_test/test_line_reader_mock.cpp)
 
+    # Apache Arrow C Data Interface tests. The fatal_error gate above
+    # forces QUESTDB_ENABLE_ARROW=ON when tests are enabled, so these
+    # always build alongside the rest of the suite.
+    compile_test(
+        test_arrow_c
+        cpp_test/qwp_mock_server.cpp
+        cpp_test/qwp_mock_c.cpp
+        cpp_test/test_arrow_c.c)
+    compile_test(
+        test_arrow_egress
+        cpp_test/qwp_mock_server.cpp
+        cpp_test/test_arrow_egress.cpp)
+    compile_test(
+        test_arrow_ingress
+        cpp_test/qwp_mock_server.cpp
+        cpp_test/test_arrow_ingress.cpp)
+
+    compile_test(
+        test_column_sender
+        cpp_test/qwp_mock_server.cpp
+        cpp_test/test_column_sender.cpp)
+
     # System testing Python3 script.
     # This will download the latest QuestDB instance from Github,
     # thus will also require a Java 11 installation to run the tests.

diff --git a/ci/compile.yaml b/ci/compile.yaml
@@ -1,19 +1,29 @@
 steps:
+  - bash: |
+      df -h /
+      sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
+                  /opt/hostedtoolcache/CodeQL /usr/local/share/boost || true
+      sudo docker image prune --all --force >/dev/null 2>&1 || true
+      df -h /
+    condition: eq(variables['imageName'], 'ubuntu-latest')
+    displayName: "Free disk space (Microsoft-hosted ubuntu)"
+  - bash: |
+      echo "##vso[task.setvariable variable=CARGO_INCREMENTAL]0"
+    condition: eq(variables['imageName'], 'ubuntu-latest')
+    displayName: "Disable cargo incremental on Linux (saves ~30-50% target/ size)"
   - script: |
       rustup update $(toolchain)
       rustup default $(toolchain)
     condition: ne(variables['toolchain'], '')
     displayName: "Update and set Rust toolchain"
   - script: |
       brew install numpy
+      python3 -m pip install --break-system-packages pyarrow polars
     condition: eq(variables['imageName'], 'macos-latest')
-    displayName: "Install numpy via brew on macOS"
+    displayName: "Install numpy + pyarrow + polars on macOS"
   - script: |
       python -m pip install --upgrade pip
-      pip install numpy
-    # hetzner-incus provisions numpy via apt (python3-numpy) before this
-    # template runs because Ubuntu 24.04+ enforces PEP 668 and rejects
-    # pip into the system interpreter.
+      pip install numpy pyarrow polars tzdata
     condition: |
       and(
         ne(variables['imageName'], 'macos-latest'),

diff --git a/ci/run_all_tests.py b/ci/run_all_tests.py
@@ -37,6 +37,10 @@ def main():
         'test_line_reader_mock',
         'line_reader_c_smoke',
         'test_line_reader',  # live-broker; skips per-test when no broker reachable
+        'test_arrow_c',
+        'test_arrow_egress',
+        'test_arrow_ingress',
+        'test_column_sender',
     ]
     test_paths = [
         (d, find_binary(d, name, exe_suffix))
@@ -45,7 +49,7 @@ def main():
     ]
 
     system_test_path = pathlib.Path('system_test') / 'test.py'
-    qdb_v = '9.2.0'  # The version of QuestDB we'll test against.
+    qdb_v = '9.4.1'  # The version of QuestDB we'll test against.
 
     run_cmd('cargo', 'test',
             '--', '--nocapture', cwd='questdb-rs')
@@ -64,7 +68,14 @@ def main():
             '--', '--nocapture', cwd='questdb-rs')
     run_cmd('cargo', 'test', '--features=almost-all-features',
             '--', '--nocapture', cwd='questdb-rs')
+    run_cmd('cargo', 'test',
+            '--features=almost-all-features,arrow,polars',
+            '--', '--nocapture', cwd='questdb-rs')
+    run_cmd('cargo', 'test', '--no-default-features',
+            '--features=ring-crypto,tls-webpki-certs,sync-sender-qwp-ws,sync-reader-ws,arrow',
+            '--', '--nocapture', cwd='questdb-rs')
     run_cmd('cargo', 'test', cwd='questdb-rs-ffi')
+    run_cmd('cargo', 'test', '--features=arrow', cwd='questdb-rs-ffi')
     for _, path in test_paths:
         run_cmd(str(path))
     run_cmd('python3', str(system_test_path), 'run', '--versions', qdb_v, '-v')

diff --git a/ci/run_fuzz_pipeline.yaml b/ci/run_fuzz_pipeline.yaml
@@ -137,7 +137,8 @@ stages:
           - bash: |
               set -eux
               sudo apt-get update
-              sudo apt-get install -y --no-install-recommends cmake python3-numpy
+              sudo apt-get install -y --no-install-recommends cmake python3-numpy python3-pip
+              sudo python3 -m pip install --break-system-packages pyarrow polars
               # Image-provided JDK paths (see provision.sh's
               # `apt-get install -y openjdk-17-jdk openjdk-25-jdk maven`).
               JAVA_PATH_17="/usr/lib/jvm/java-17-openjdk-amd64"
@@ -200,6 +201,9 @@ stages:
           - script: |
               python3 system_test/test.py run --repo ./questdb TestQwpWsFuzz -v
             displayName: "TestQwpWsFuzz"
+          - script: |
+              python3 system_test/test.py run --repo ./questdb TestArrowEgressFuzz TestArrowEgressPerKind TestArrowEgressEmpty TestArrowIngressFuzz TestArrowIngressPerKind TestArrowIngressDesignatedTs TestArrowIngressErrors TestArrowIngressMultiBatch TestArrowIngressExtraTypes TestArrowIngressUnsupportedTypes TestArrowRoundTripFuzz TestArrowRoundTripPerKind TestArrowAlignment TestArrowPolarsFuzz TestArrowPolarsRoundTripPerKind TestArrowPolarsPerDtype -v
+            displayName: "TestArrowFuzz"
           - task: ArchiveFiles@2
             displayName: "Compress QuestDB server log on failure"
             condition: failed()
@@ -277,6 +281,8 @@ stages:
         pool:
           vmImage: "ubuntu-latest"
         timeoutInMinutes: 30
+        variables:
+          imageName: ubuntu-latest
         steps:
           - checkout: self
             fetchDepth: 1

diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml
@@ -43,7 +43,7 @@ stages:
         pool:
           name: $(poolName)
           vmImage: $(imageName)
-        timeoutInMinutes: 60
+        timeoutInMinutes: 90
         steps:
           - checkout: self
             fetchDepth: 1
@@ -67,7 +67,7 @@ stages:
           ############################# temp for test end #####################
           - script: python3 ci/run_all_tests.py
             env:
-              JAVA_HOME: $(JAVA_HOME_17_X64)
+              JAVA_HOME: $(JAVA_HOME_25_X64)
             displayName: "Tests"
           # - task: PublishBuildArtifacts@1
           #   inputs:
@@ -181,7 +181,7 @@ stages:
               # debian-installed packages because the wheel RECORD file is
               # missing). --break-system-packages overrides PEP 668.
               sudo apt-get install -y --no-install-recommends cmake python3-pip
-              sudo python3 -m pip install --break-system-packages 'numpy>=2'
+              sudo python3 -m pip install --break-system-packages 'numpy>=2' pyarrow polars
               JAVA_PATH_17="/usr/lib/jvm/java-17-openjdk-amd64"
               JAVA_PATH_25="/usr/lib/jvm/java-25-openjdk-amd64"
               for p in "$JAVA_PATH_17" "$JAVA_PATH_25"; do
@@ -313,6 +313,9 @@ stages:
           - script: |
               python3 system_test/test.py run --repo ./questdb TestQwpWsFuzz -v
             displayName: "TestQwpWsFuzz"
+          - script: |
+              python3 system_test/test.py run --repo ./questdb TestArrowEgressFuzz TestArrowEgressPerKind TestArrowEgressEmpty TestArrowIngressFuzz TestArrowIngressPerKind TestArrowIngressDesignatedTs TestArrowIngressErrors TestArrowIngressMultiBatch TestArrowIngressExtraTypes TestArrowIngressUnsupportedTypes TestArrowRoundTripFuzz TestArrowRoundTripPerKind TestArrowAlignment TestArrowPolarsFuzz TestArrowPolarsRoundTripPerKind TestArrowPolarsPerDtype -v
+            displayName: "TestArrowWsFuzz"
           # Mirrors ci/run_fuzz_pipeline.yaml: on failure, archive and
           # publish the QuestDB server log so PR reviewers don't have to
           # repro locally. Path comes from system_test/fixture.py:_log_path.
@@ -360,7 +363,8 @@ stages:
           - bash: |
               set -eux
               sudo apt-get update
-              sudo apt-get install -y --no-install-recommends cmake python3-numpy
+              sudo apt-get install -y --no-install-recommends cmake python3-numpy python3-pip
+              sudo python3 -m pip install --break-system-packages pyarrow polars
               JAVA_PATH_17="/usr/lib/jvm/java-17-openjdk-amd64"
               JAVA_PATH_25="/usr/lib/jvm/java-25-openjdk-amd64"
               for p in "$JAVA_PATH_17" "$JAVA_PATH_25"; do
@@ -414,6 +418,9 @@ stages:
           - script: |
               python3 system_test/test.py run --repo ./questdb TestQwpWsFuzz -v
             displayName: "TestQwpWsFuzz"
+          - script: |
+              python3 system_test/test.py run --repo ./questdb TestArrowEgressFuzz TestArrowEgressPerKind TestArrowEgressEmpty TestArrowIngressFuzz TestArrowIngressPerKind TestArrowIngressDesignatedTs TestArrowIngressErrors TestArrowIngressMultiBatch TestArrowIngressExtraTypes TestArrowIngressUnsupportedTypes TestArrowRoundTripFuzz TestArrowRoundTripPerKind TestArrowAlignment TestArrowPolarsFuzz TestArrowPolarsRoundTripPerKind TestArrowPolarsPerDtype -v
+            displayName: "TestArrowWsFuzz"
           - task: ArchiveFiles@2
             displayName: "Compress QuestDB server log on failure"
             condition: failed()
@@ -444,6 +451,8 @@ stages:
         pool:
           vmImage: "ubuntu-latest"
         timeoutInMinutes: 30
+        variables:
+          imageName: ubuntu-latest
         steps:
           - checkout: self
             fetchDepth: 1

diff --git a/cpp_test/qwp_mock_c.cpp b/cpp_test/qwp_mock_c.cpp
@@ -0,0 +1,56 @@
+#include "qwp_mock_c.h"
+#include "qwp_mock_server.hpp"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace qm = qwp_mock;
+
+struct qwp_mock_c
+{
+    std::unique_ptr<qm::MockServer> server;
+    std::string addr_cached;
+};
+
+extern "C" qwp_mock_c* qwp_mock_c_start(int slot_count)
+{
+    if (slot_count < 1)
+        slot_count = 1;
+    // Per-connection script: wait for one client binary frame whose
+    // first byte is 'Q' (the QWP1 magic byte that every column-sender
+    // publish frame starts with). This blocks the worker from
+    // `graceful_close`ing before the client has finished writing.
+    qm::Script accept_one_frame = {
+        qm::ActionAwaitClientFrame{0x51},
+    };
+    std::vector<qm::Script> scripts;
+    scripts.reserve(static_cast<size_t>(slot_count));
+    for (int i = 0; i < slot_count; ++i)
+        scripts.push_back(accept_one_frame);
+
+    auto holder = new qwp_mock_c{};
+    try
+    {
+        holder->server = std::make_unique<qm::MockServer>(std::move(scripts));
+        holder->addr_cached = holder->server->addr();
+    }
+    catch (...)
+    {
+        delete holder;
+        return nullptr;
+    }
+    return holder;
+}
+
+extern "C" const char* qwp_mock_c_addr(qwp_mock_c* mock)
+{
+    if (mock == nullptr)
+        return nullptr;
+    return mock->addr_cached.c_str();
+}
+
+extern "C" void qwp_mock_c_stop(qwp_mock_c* mock)
+{
+    delete mock;
+}
diff --git a/cpp_test/qwp_mock_c.h b/cpp_test/qwp_mock_c.h
@@ -0,0 +1,47 @@
+/* C-friendly shim around `qwp_mock::MockServer` for the pure-C
+ * test_arrow_c.c suite.
+ *
+ * Spins up an in-process mock that accepts one WS-Upgrade per slot and
+ * silently swallows the first inbound QWP binary frame on each
+ * connection — enough to drive `column_sender_flush_arrow_batch`
+ * end-to-end without hitting a live QuestDB instance.
+ *
+ * CMake note: when wiring this into the build, add
+ *   `cpp_test/qwp_mock_c.cpp` to the `c-questdb-client-test`
+ * executable's source list (alongside `qwp_mock_server.cpp`). The
+ * shim itself has no extra link deps beyond what
+ * `qwp_mock_server.cpp` already pulls in.
+ */
+
+#ifndef QWP_MOCK_C_H
+#define QWP_MOCK_C_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+typedef struct qwp_mock_c qwp_mock_c;
+
+/* Start a mock server bound to 127.0.0.1:0. The mock accepts up to
+ * `slot_count` WS upgrades and, on each, waits for one inbound QWP
+ * binary frame (first payload byte == 'Q', i.e. the QWP1 magic) before
+ * cleanly closing the connection. `slot_count` must be >= 1 — pass 1
+ * when using the default `pool_size=1` connect string.
+ *
+ * Returns NULL on failure (e.g. OS-level bind failure). */
+qwp_mock_c* qwp_mock_c_start(int slot_count);
+
+/* Return the mock's listening address as "127.0.0.1:NNNN", suitable for
+ * splicing into a `qwpws::addr=...` connect string. Pointer is valid
+ * until `qwp_mock_c_stop`. */
+const char* qwp_mock_c_addr(qwp_mock_c* mock);
+
+/* Shut down the mock and free its resources. Safe to pass NULL. */
+void qwp_mock_c_stop(qwp_mock_c* mock);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* QWP_MOCK_C_H */