feat: Apache Arrow / Polars integration for QWP-WS

CMakeLists.txt

@@ -71,6 +71,11 @@ option(
     "Build the C/C++ tests with -fsanitize=address,undefined."
     OFF)
 
+option(
+    QUESTDB_ENABLE_ARROW
+    "Build with Apache Arrow C Data Interface exports. Opt-in: pulls arrow-rs."
+    OFF)
+
 # Build static and dynamic lib written in Rust by invoking `cargo`.
 # Imports `questdb_client` target.
 add_subdirectory(corrosion)
@@ -81,6 +86,13 @@ endif()
 if(QUESTDB_ENABLE_INSECURE_SKIP_VERIFY)
     list(APPEND QUESTDB_CARGO_FEATURES insecure-skip-verify)
 endif()
+if(QUESTDB_TESTS_AND_EXAMPLES AND NOT QUESTDB_ENABLE_ARROW)
+    message(STATUS "QUESTDB_TESTS_AND_EXAMPLES=ON: enabling QUESTDB_ENABLE_ARROW")
+    set(QUESTDB_ENABLE_ARROW ON)
+endif()
+if(QUESTDB_ENABLE_ARROW)
+    list(APPEND QUESTDB_CARGO_FEATURES arrow)
+endif()
 if(QUESTDB_CARGO_FEATURES)
     corrosion_import_crate(
         MANIFEST_PATH questdb-rs-ffi/Cargo.toml
@@ -94,6 +106,9 @@ endif()
 target_include_directories(
     questdb_client INTERFACE
     ${CMAKE_CURRENT_SOURCE_DIR}/include)
+if(QUESTDB_ENABLE_ARROW)
+    target_compile_definitions(questdb_client INTERFACE QUESTDB_CLIENT_ENABLE_ARROW)
+endif()
 if(WIN32)
     set_target_properties(
         questdb_client-shared
@@ -280,6 +295,26 @@ if (QUESTDB_TESTS_AND_EXAMPLES)
     compile_example(
         line_reader_c_example_columns
         examples/line_reader_c_example_columns.c)
+    compile_example(
+        line_reader_c_example_arrow
+        examples/line_reader_c_example_arrow.c)
+
+    find_package(Arrow QUIET)
+    if(Arrow_FOUND)
+        compile_example(
+            line_sender_cpp_example_arrow
+            examples/line_sender_cpp_example_arrow.cpp)
+        target_link_libraries(
+            line_sender_cpp_example_arrow Arrow::arrow_shared)
+        compile_example(
+            line_reader_cpp_example_arrow
+            examples/line_reader_cpp_example_arrow.cpp)
+        target_link_libraries(
+            line_reader_cpp_example_arrow Arrow::arrow_shared)
+    else()
+        message(STATUS
+            "arrow-cpp not found; skipping line_{sender,reader}_cpp_example_arrow.")
+    endif()
 
     # Include Rust tests as part of the tests run
     add_test(
@@ -358,6 +393,20 @@ if (QUESTDB_TESTS_AND_EXAMPLES)
         cpp_test/qwp_mock_server.cpp
         cpp_test/test_line_reader_mock.cpp)
 
+    # Apache Arrow C Data Interface tests. The fatal_error gate above
+    # forces QUESTDB_ENABLE_ARROW=ON when tests are enabled, so these
+    # always build alongside the rest of the suite.
+    compile_test(
+        test_arrow_c
+        cpp_test/test_arrow_c.c)
+    compile_test(
+        test_arrow_egress
+        cpp_test/qwp_mock_server.cpp
+        cpp_test/test_arrow_egress.cpp)
+    compile_test(
+        test_arrow_ingress
+        cpp_test/test_arrow_ingress.cpp)
+
     # System testing Python3 script.
     # This will download the latest QuestDB instance from Github,
     # thus will also require a Java 11 installation to run the tests.

ci/compile.yaml

@@ -1,19 +1,25 @@
 steps:
+  - bash: |
+      df -h /
+      sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
+                  /opt/hostedtoolcache/CodeQL /usr/local/share/boost || true
+      sudo docker image prune --all --force >/dev/null 2>&1 || true
+      df -h /
+    condition: eq(variables['imageName'], 'ubuntu-latest')
+    displayName: "Free disk space (Microsoft-hosted ubuntu)"
   - script: |
       rustup update $(toolchain)
       rustup default $(toolchain)
     condition: ne(variables['toolchain'], '')
     displayName: "Update and set Rust toolchain"
   - script: |
       brew install numpy
+      python3 -m pip install --break-system-packages pyarrow polars
     condition: eq(variables['imageName'], 'macos-latest')
-    displayName: "Install numpy via brew on macOS"
+    displayName: "Install numpy + pyarrow + polars on macOS"
   - script: |
       python -m pip install --upgrade pip
-      pip install numpy
-    # hetzner-incus provisions numpy via apt (python3-numpy) before this
-    # template runs because Ubuntu 24.04+ enforces PEP 668 and rejects
-    # pip into the system interpreter.
+      pip install numpy pyarrow polars
     condition: |
       and(
         ne(variables['imageName'], 'macos-latest'),

ci/run_all_tests.py

@@ -37,6 +37,9 @@ def main():
         'test_line_reader_mock',
         'line_reader_c_smoke',
         'test_line_reader',  # live-broker; skips per-test when no broker reachable
+        'test_arrow_c',
+        'test_arrow_egress',
+        'test_arrow_ingress',
     ]
     test_paths = [
         (d, find_binary(d, name, exe_suffix))
@@ -64,7 +67,14 @@ def main():
             '--', '--nocapture', cwd='questdb-rs')
     run_cmd('cargo', 'test', '--features=almost-all-features',
             '--', '--nocapture', cwd='questdb-rs')
+    run_cmd('cargo', 'test',
+            '--features=almost-all-features,arrow,polars',
+            '--', '--nocapture', cwd='questdb-rs')
+    run_cmd('cargo', 'test', '--no-default-features',
+            '--features=ring-crypto,tls-webpki-certs,sync-sender-qwp-ws,sync-reader-ws,arrow',
+            '--', '--nocapture', cwd='questdb-rs')
     run_cmd('cargo', 'test', cwd='questdb-rs-ffi')
+    run_cmd('cargo', 'test', '--features=arrow', cwd='questdb-rs-ffi')
     for _, path in test_paths:
         run_cmd(str(path))
     run_cmd('python3', str(system_test_path), 'run', '--versions', qdb_v, '-v')

ci/run_fuzz_pipeline.yaml

@@ -137,7 +137,8 @@ stages:
           - bash: |
               set -eux
               sudo apt-get update
-              sudo apt-get install -y --no-install-recommends cmake python3-numpy
+              sudo apt-get install -y --no-install-recommends cmake python3-numpy python3-pip
+              sudo python3 -m pip install --break-system-packages pyarrow polars
               # Image-provided JDK paths (see provision.sh's
               # `apt-get install -y openjdk-17-jdk openjdk-25-jdk maven`).
               JAVA_PATH_17="/usr/lib/jvm/java-17-openjdk-amd64"
@@ -200,6 +201,13 @@ stages:
           - script: |
               python3 system_test/test.py run --repo ./questdb TestQwpWsFuzz -v
             displayName: "TestQwpWsFuzz"
+          - script: |
+              python3 system_test/test.py run --repo ./questdb \
+                TestArrowEgressFuzz TestArrowIngressFuzz \
+                TestArrowIngressExtraTypes TestArrowIngressUnsupportedTypes \
+                TestArrowRoundTripFuzz TestArrowAlignment \
+                TestArrowPolarsFuzz TestArrowPolarsPerDtype -v
+            displayName: "TestArrowFuzz"
           - task: ArchiveFiles@2
             displayName: "Compress QuestDB server log on failure"
             condition: failed()
@@ -277,6 +285,8 @@ stages:
         pool:
           vmImage: "ubuntu-latest"
         timeoutInMinutes: 30
+        variables:
+          imageName: ubuntu-latest
         steps:
           - checkout: self
             fetchDepth: 1

ci/run_tests_pipeline.yaml

@@ -43,7 +43,7 @@ stages:
         pool:
           name: $(poolName)
           vmImage: $(imageName)
-        timeoutInMinutes: 60
+        timeoutInMinutes: 90
         steps:
           - checkout: self
             fetchDepth: 1
@@ -181,7 +181,7 @@ stages:
               # debian-installed packages because the wheel RECORD file is
               # missing). --break-system-packages overrides PEP 668.
               sudo apt-get install -y --no-install-recommends cmake python3-pip
-              sudo python3 -m pip install --break-system-packages 'numpy>=2'
+              sudo python3 -m pip install --break-system-packages 'numpy>=2' pyarrow polars
               JAVA_PATH_17="/usr/lib/jvm/java-17-openjdk-amd64"
               JAVA_PATH_25="/usr/lib/jvm/java-25-openjdk-amd64"
               for p in "$JAVA_PATH_17" "$JAVA_PATH_25"; do
@@ -360,7 +360,8 @@ stages:
           - bash: |
               set -eux
               sudo apt-get update
-              sudo apt-get install -y --no-install-recommends cmake python3-numpy
+              sudo apt-get install -y --no-install-recommends cmake python3-numpy python3-pip
+              sudo python3 -m pip install --break-system-packages pyarrow polars
               JAVA_PATH_17="/usr/lib/jvm/java-17-openjdk-amd64"
               JAVA_PATH_25="/usr/lib/jvm/java-25-openjdk-amd64"
               for p in "$JAVA_PATH_17" "$JAVA_PATH_25"; do
@@ -414,6 +415,13 @@ stages:
           - script: |
               python3 system_test/test.py run --repo ./questdb TestQwpWsFuzz -v
             displayName: "TestQwpWsFuzz"
+          - script: |
+              python3 system_test/test.py run --repo ./questdb \
+                TestArrowEgressFuzz TestArrowIngressFuzz \
+                TestArrowIngressExtraTypes TestArrowIngressUnsupportedTypes \
+                TestArrowRoundTripFuzz TestArrowAlignment \
+                TestArrowPolarsFuzz TestArrowPolarsPerDtype -v
+            displayName: "TestArrowWsFuzz"
           - task: ArchiveFiles@2
             displayName: "Compress QuestDB server log on failure"
             condition: failed()
@@ -444,6 +452,8 @@ stages:
         pool:
           vmImage: "ubuntu-latest"
         timeoutInMinutes: 30
+        variables:
+          imageName: ubuntu-latest
         steps:
           - checkout: self
             fetchDepth: 1