From 6ac52013ecd8a67ef30b32af2dbbda38bec7dc48 Mon Sep 17 00:00:00 2001 From: dentiny Date: Tue, 10 Mar 2026 09:35:55 +0000 Subject: [PATCH] [WIP] Fix link --- CMakeLists.txt | 24 +++++++++++++++---- extension_config.cmake | 16 +++++++++++++ src/client/distributed_alter_table.cpp | 4 ++-- src/client/distributed_create_index.cpp | 4 ++-- src/client/distributed_delete.cpp | 4 ++-- src/client/distributed_insert.cpp | 4 ++-- src/duckherder_extension.cpp | 3 ++- .../client/distributed_alter_table.hpp | 3 ++- .../client/distributed_create_index.hpp | 3 ++- src/include/client/distributed_delete.hpp | 3 ++- src/include/client/distributed_insert.hpp | 3 ++- .../duckherder_extension_instance_state.hpp | 4 ++++ src/server/driver/distributed_executor.cpp | 2 +- .../driver/distributed_flight_server.cpp | 3 ++- test/unittest/CMakeLists.txt | 3 +++ 15 files changed, 63 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0975419..85c018b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.5) # Set extension name here set(TARGET_NAME duckherder) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_FLAGS"${CMAKE_CXX_FLAGS} -Wunused-variable -Wunused-function -Wunused-but-set-variable" ) @@ -28,8 +28,11 @@ if(${BUILD_UNITTESTS}) include_directories(duckdb/third_party/catch) add_executable(test_distributed_flight unit/test_distributed_flight.cpp) target_link_libraries( - test_distributed_flight ${EXTENSION_NAME} Arrow::arrow_static + test_distributed_flight ${EXTENSION_NAME} dummy_static_extension_loader Arrow::arrow_static ArrowFlight::arrow_flight_static protobuf::libprotobuf gRPC::grpc++) + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_link_libraries(test_distributed_flight -Wl,--allow-multiple-definition) + endif() endif() # Generate protobuf code. @@ -102,6 +105,11 @@ set(EXTENSION_SOURCES build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) +# Allow duplicate TableCatalogEntry::Name (see extension_config.cmake). Loadable extension gets it via DUCKDB_EXTRA_LINK_FLAGS. +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_link_libraries(${LOADABLE_EXTENSION_NAME} -Wl,--allow-multiple-definition) +endif() + # Note: gRPC and Protobuf are transitive dependencies of ArrowFlight. target_link_libraries( ${EXTENSION_NAME} Arrow::arrow_static ArrowFlight::arrow_flight_static @@ -117,14 +125,20 @@ install( LIBRARY DESTINATION "${INSTALL_LIB_DIR}" ARCHIVE DESTINATION "${INSTALL_LIB_DIR}") -# Add distributed server executable. +# Add distributed server executable. Link all static extensions so LoadAllExtensions resolves. add_executable(distributed_server src/server/driver/distributed_server_main.cpp) target_link_libraries( - distributed_server ${EXTENSION_NAME} Arrow::arrow_static + distributed_server ${EXTENSION_NAME} dummy_static_extension_loader Arrow::arrow_static ArrowFlight::arrow_flight_static protobuf::libprotobuf gRPC::grpc++) +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_link_libraries(distributed_server -Wl,--allow-multiple-definition) +endif() # Add distributed worker executable. add_executable(distributed_worker src/server/worker/distributed_worker_main.cpp) target_link_libraries( - distributed_worker ${EXTENSION_NAME} Arrow::arrow_static + distributed_worker ${EXTENSION_NAME} dummy_static_extension_loader Arrow::arrow_static ArrowFlight::arrow_flight_static protobuf::libprotobuf gRPC::grpc++) +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_link_libraries(distributed_worker -Wl,--allow-multiple-definition) +endif() diff --git a/extension_config.cmake b/extension_config.cmake index f1a62fd..e59c983 100644 --- a/extension_config.cmake +++ b/extension_config.cmake @@ -1,5 +1,21 @@ # This file is included by DuckDB's build system. It specifies which extension to load +# Allow duplicate TableCatalogEntry::Name when linking extension with libduckdb_static (C++17 inline constexpr). +# No leading/trailing space (CMP0004). Set both per-target (DUCKDB_EXTRA_LINK_FLAGS) and global so libduckdb.so gets it. +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + set(DUCKDB_EXTRA_LINK_FLAGS "-Wl,--allow-multiple-definition" CACHE STRING "" FORCE) + if(CMAKE_EXE_LINKER_FLAGS) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--allow-multiple-definition" CACHE STRING "" FORCE) + else() + set(CMAKE_EXE_LINKER_FLAGS "-Wl,--allow-multiple-definition" CACHE STRING "" FORCE) + endif() + if(CMAKE_SHARED_LINKER_FLAGS) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--allow-multiple-definition" CACHE STRING "" FORCE) + else() + set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--allow-multiple-definition" CACHE STRING "" FORCE) + endif() +endif() + # Extension from this repo duckdb_extension_load(duckherder SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} diff --git a/src/client/distributed_alter_table.cpp b/src/client/distributed_alter_table.cpp index de99497..49d7aae 100644 --- a/src/client/distributed_alter_table.cpp +++ b/src/client/distributed_alter_table.cpp @@ -45,8 +45,8 @@ unique_ptr PhysicalRemoteAlterTableOperator::GetGlobalSourceS return make_uniq(); } -SourceResultType PhysicalRemoteAlterTableOperator::GetData(ExecutionContext &context, DataChunk &chunk, - OperatorSourceInput &input) const { +SourceResultType PhysicalRemoteAlterTableOperator::GetDataInternal(ExecutionContext &context, DataChunk &chunk, + OperatorSourceInput &input) const { auto &gstate = input.global_state.Cast(); auto &db_instance = DatabaseInstance::GetDatabase(context.client); diff --git a/src/client/distributed_create_index.cpp b/src/client/distributed_create_index.cpp index 019550f..8cd453c 100644 --- a/src/client/distributed_create_index.cpp +++ b/src/client/distributed_create_index.cpp @@ -44,8 +44,8 @@ unique_ptr PhysicalRemoteCreateIndexOperator::GetGlobalSource return make_uniq(); } -SourceResultType PhysicalRemoteCreateIndexOperator::GetData(ExecutionContext &context, DataChunk &chunk, - OperatorSourceInput &input) const { +SourceResultType PhysicalRemoteCreateIndexOperator::GetDataInternal(ExecutionContext &context, DataChunk &chunk, + OperatorSourceInput &input) const { auto &gstate = input.global_state.Cast(); auto &db_instance = DatabaseInstance::GetDatabase(context.client); diff --git a/src/client/distributed_delete.cpp b/src/client/distributed_delete.cpp index e421187..412ddf0 100644 --- a/src/client/distributed_delete.cpp +++ b/src/client/distributed_delete.cpp @@ -118,8 +118,8 @@ SinkFinalizeType PhysicalDistributedDelete::Finalize(Pipeline &pipeline, Event & return SinkFinalizeType::READY; } -SourceResultType PhysicalDistributedDelete::GetData(ExecutionContext &context, DataChunk &chunk, - OperatorSourceInput &input) const { +SourceResultType PhysicalDistributedDelete::GetDataInternal(ExecutionContext &context, DataChunk &chunk, + OperatorSourceInput &input) const { // TODO(hjiang): Implement return chunk. chunk.SetCardinality(0); return SourceResultType::FINISHED; diff --git a/src/client/distributed_insert.cpp b/src/client/distributed_insert.cpp index 7f12569..da06fc8 100644 --- a/src/client/distributed_insert.cpp +++ b/src/client/distributed_insert.cpp @@ -93,8 +93,8 @@ SinkFinalizeType PhysicalDistributedInsert::Finalize(Pipeline &pipeline, Event & return SinkFinalizeType::READY; } -SourceResultType PhysicalDistributedInsert::GetData(ExecutionContext &context, DataChunk &chunk, - OperatorSourceInput &input) const { +SourceResultType PhysicalDistributedInsert::GetDataInternal(ExecutionContext &context, DataChunk &chunk, + OperatorSourceInput &input) const { chunk.SetCardinality(0); return SourceResultType::FINISHED; } diff --git a/src/duckherder_extension.cpp b/src/duckherder_extension.cpp index 5f65e26..e657f24 100644 --- a/src/duckherder_extension.cpp +++ b/src/duckherder_extension.cpp @@ -1,6 +1,7 @@ #define DUCKDB_EXTENSION_MAIN #include "duckdb.hpp" +#include "duckdb/storage/storage_extension.hpp" #include "duckherder_extension.hpp" #include "duckherder_extension_instance_state.hpp" #include "duckherder_pragmas.hpp" @@ -35,7 +36,7 @@ void ClearQueryRecorderStats(const DataChunk &args, ExpressionState &state, Vect void LoadInternal(ExtensionLoader &loader) { auto &db = loader.GetDatabaseInstance(); auto &config = DBConfig::GetConfig(db); - config.storage_extensions["duckherder"] = make_uniq(); + StorageExtension::Register(config, "duckherder", make_shared_ptr()); // Set extension state. SetInstanceState(db, make_shared_ptr()); diff --git a/src/include/client/distributed_alter_table.hpp b/src/include/client/distributed_alter_table.hpp index 7089d7f..da2d7c4 100644 --- a/src/include/client/distributed_alter_table.hpp +++ b/src/include/client/distributed_alter_table.hpp @@ -22,7 +22,8 @@ class PhysicalRemoteAlterTableOperator : public PhysicalOperator { public: // Source interface. unique_ptr GetGlobalSourceState(ClientContext &context) const override; - SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override; + SourceResultType GetDataInternal(ExecutionContext &context, DataChunk &chunk, + OperatorSourceInput &input) const override; bool IsSource() const override { return true; diff --git a/src/include/client/distributed_create_index.hpp b/src/include/client/distributed_create_index.hpp index 8c1d29d..a3c3691 100644 --- a/src/include/client/distributed_create_index.hpp +++ b/src/include/client/distributed_create_index.hpp @@ -23,7 +23,8 @@ class PhysicalRemoteCreateIndexOperator : public PhysicalOperator { public: // Source interface. unique_ptr GetGlobalSourceState(ClientContext &context) const override; - SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override; + SourceResultType GetDataInternal(ExecutionContext &context, DataChunk &chunk, + OperatorSourceInput &input) const override; bool IsSource() const override { return true; diff --git a/src/include/client/distributed_delete.hpp b/src/include/client/distributed_delete.hpp index fe041be..49a1e3e 100644 --- a/src/include/client/distributed_delete.hpp +++ b/src/include/client/distributed_delete.hpp @@ -35,7 +35,8 @@ class PhysicalDistributedDelete : public PhysicalOperator { } // Source interface. - SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override; + SourceResultType GetDataInternal(ExecutionContext &context, DataChunk &chunk, + OperatorSourceInput &input) const override; bool IsSource() const override { return true; diff --git a/src/include/client/distributed_insert.hpp b/src/include/client/distributed_insert.hpp index 6a13812..18cc13d 100644 --- a/src/include/client/distributed_insert.hpp +++ b/src/include/client/distributed_insert.hpp @@ -32,7 +32,8 @@ class PhysicalDistributedInsert : public PhysicalOperator { } // Source interface. - SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override; + SourceResultType GetDataInternal(ExecutionContext &context, DataChunk &chunk, + OperatorSourceInput &input) const override; bool IsSource() const override { return true; diff --git a/src/include/duckherder_extension_instance_state.hpp b/src/include/duckherder_extension_instance_state.hpp index 23b6a53..82f81c9 100644 --- a/src/include/duckherder_extension_instance_state.hpp +++ b/src/include/duckherder_extension_instance_state.hpp @@ -38,6 +38,10 @@ struct DuckherderInstanceState : public ObjectCacheEntry { return OBJECT_TYPE; } + optional_idx GetEstimatedCacheMemory() const override { + return optional_idx {}; + } + private: mutable std::mutex mu; shared_ptr query_recorder; diff --git a/src/server/driver/distributed_executor.cpp b/src/server/driver/distributed_executor.cpp index 8e26f2f..6262a33 100644 --- a/src/server/driver/distributed_executor.cpp +++ b/src/server/driver/distributed_executor.cpp @@ -189,7 +189,7 @@ DistributedExecutionResult DistributedExecutor::ExecuteDistributed(const string std::unique_ptr stream; auto status = worker->client->ExecutePartition(req, stream); if (!status.ok()) { - DUCKDB_LOG_WARN(db_instance, + DUCKDB_LOG_WARNING(db_instance, StringUtil::Format("Worker %s failed executing task %llu: %s", worker->worker_id, static_cast(task.task_id), status.ToString())); continue; diff --git a/src/server/driver/distributed_flight_server.cpp b/src/server/driver/distributed_flight_server.cpp index c23622d..d744144 100644 --- a/src/server/driver/distributed_flight_server.cpp +++ b/src/server/driver/distributed_flight_server.cpp @@ -6,6 +6,7 @@ #include "duckdb/common/string_util.hpp" #include "duckdb/logging/logger.hpp" #include "duckdb/main/config.hpp" +#include "duckdb/storage/storage_extension.hpp" #include "query_common.hpp" #include "server/driver/duckling_storage.hpp" @@ -74,7 +75,7 @@ void DistributedFlightServer::Initialize() { // Register the Duckling storage extension. DBConfig config; - config.storage_extensions["duckling"] = make_uniq(); + StorageExtension::Register(config, "duckling", make_shared_ptr()); db = make_uniq(nullptr, &config); conn = make_uniq(*db); diff --git a/test/unittest/CMakeLists.txt b/test/unittest/CMakeLists.txt index ff85495..e06f77d 100644 --- a/test/unittest/CMakeLists.txt +++ b/test/unittest/CMakeLists.txt @@ -19,3 +19,6 @@ endif() target_link_libraries( unittest_duckherder Arrow::arrow_static ArrowFlight::arrow_flight_static protobuf::libprotobuf gRPC::grpc++) +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_link_libraries(unittest_duckherder -Wl,--allow-multiple-definition) +endif()