diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 746d6ba35..92caf59ca 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -209,6 +209,20 @@ jobs: name: neuron-build path: ./build + - name: debug kafka plugin deps + run: | + echo "=== libs in staging prefix ===" + ls -la /home/neuron/main/libs/x86_64-linux-gnu/lib/librdkafka* || echo "no librdkafka" + ls -la /home/neuron/main/libs/x86_64-linux-gnu/lib/libzstd* || echo "no libzstd" + ls -la /home/neuron/main/libs/x86_64-linux-gnu/lib/libsasl2* || echo "no libsasl2" + echo "=== libs in /usr/local/lib ===" + ls -la /usr/local/lib/librdkafka* /usr/local/lib/libzstd* /usr/local/lib/libsasl2* 2>/dev/null || echo "none yet" + echo "=== ldd librdkafka from staging ===" + ldd /home/neuron/main/libs/x86_64-linux-gnu/lib/librdkafka.so.1 || true + echo "=== ldd plugin-kafka from build ===" + chmod -R +x ./build/ 2>/dev/null || true + ldd ./build/plugins/libplugin-kafka.so || true + - name: function test env: KAFKA_BROKER: kafka:9092 @@ -218,6 +232,11 @@ jobs: cmake -DUSE_GCOV=1 .. -DCMAKE_TOOLCHAIN_FILE=../cmake/x86_64-linux-gnu.cmake cd .. ldconfig + echo "=== post-ldconfig ldd check ===" + ldd ./build/plugins/libplugin-kafka.so || true + ldd /usr/local/lib/librdkafka.so.1 || true + echo "=== try manual dlopen ===" + python3 -c "import ctypes; ctypes.CDLL('./build/plugins/libplugin-kafka.so')" 2>&1 || true pytest -s -v tests/ft/app/test_kafka.py - name: create cov report diff --git a/Install-dependencies.md b/Install-dependencies.md index 875e54893..0d2375f53 100644 --- a/Install-dependencies.md +++ b/Install-dependencies.md @@ -74,11 +74,31 @@ $ ./configure --disable-protoc --enable-shared=no CFLAGS=-fPIC CXXFLAGS=-fPIC $ make && sudo make install ``` +[cyrus-sasl](https://github.com/cyrusimap/cyrus-sasl.git) +```shell +$ git clone -b cyrus-sasl-2.1.28 https://github.com/cyrusimap/cyrus-sasl.git +$ cd cyrus-sasl +$ autoreconf -fi +$ ./configure --enable-shared --disable-static --disable-sample \ + --disable-checkapop --disable-cram --disable-digest --disable-otp \ + --disable-srp --disable-gssapi --disable-sql \ + --enable-plain --enable-scram --enable-login \ + --with-dblib=none --without-saslauthd --without-authdaemond +$ make && sudo make install +``` + +[zstd](https://github.com/facebook/zstd.git) +```shell +$ git clone -b v1.5.6 https://github.com/facebook/zstd.git +$ cd zstd/build/cmake && mkdir builddir && cd builddir +$ cmake -DZSTD_BUILD_PROGRAMS=OFF -DZSTD_BUILD_TESTS=OFF -DZSTD_BUILD_SHARED=ON -DZSTD_BUILD_STATIC=OFF -DCMAKE_BUILD_TYPE=Release .. && make && sudo make install +``` + [librdkafka](https://github.com/confluentinc/librdkafka.git) ```shell $ git clone -b v2.6.1 https://github.com/confluentinc/librdkafka.git $ cd librdkafka && mkdir build && cd build -$ cmake -DRDKAFKA_BUILD_STATIC=OFF -DRDKAFKA_BUILD_EXAMPLES=OFF -DRDKAFKA_BUILD_TESTS=OFF -DWITH_SASL=OFF -DWITH_ZSTD=OFF -DWITH_CURL=OFF -DCMAKE_BUILD_TYPE=Release .. && make && sudo make install +$ cmake -DRDKAFKA_BUILD_STATIC=OFF -DRDKAFKA_BUILD_EXAMPLES=OFF -DRDKAFKA_BUILD_TESTS=OFF -DWITH_SSL=ON -DWITH_ZLIB=ON -DWITH_SASL=ON -DWITH_ZSTD=ON -DWITH_CURL=OFF -DCMAKE_BUILD_TYPE=Release .. && make && sudo make install ``` [libxml2](https://github.com/GNOME/libxml2.git) diff --git a/cmake/aarch64-linux-gnu.cmake b/cmake/aarch64-linux-gnu.cmake index eee8e77b0..cafc723a6 100644 --- a/cmake/aarch64-linux-gnu.cmake +++ b/cmake/aarch64-linux-gnu.cmake @@ -22,4 +22,8 @@ set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) link_directories(${CMAKE_STAGING_PREFIX}) file(COPY ${CMAKE_STAGING_PREFIX}/lib/libzlog.so.1.2 DESTINATION /usr/local/lib) -file(COPY ${CMAKE_STAGING_PREFIX}/lib/librdkafka.so.1 DESTINATION /usr/local/lib) \ No newline at end of file +file(COPY ${CMAKE_STAGING_PREFIX}/lib/librdkafka.so.1 DESTINATION /usr/local/lib) +file(GLOB ZSTD_LIBS "${CMAKE_STAGING_PREFIX}/lib/libzstd.so*") +file(COPY ${ZSTD_LIBS} DESTINATION /usr/local/lib) +file(GLOB SASL_LIBS "${CMAKE_STAGING_PREFIX}/lib/libsasl2.so*") +file(COPY ${SASL_LIBS} DESTINATION /usr/local/lib) \ No newline at end of file diff --git a/cmake/arm-linux-gnueabi.cmake b/cmake/arm-linux-gnueabi.cmake index b64f61f17..4fc7c43c0 100644 --- a/cmake/arm-linux-gnueabi.cmake +++ b/cmake/arm-linux-gnueabi.cmake @@ -23,3 +23,7 @@ link_directories(${CMAKE_STAGING_PREFIX}) file(COPY ${CMAKE_STAGING_PREFIX}/lib/libzlog.so.1.2 DESTINATION /usr/local/lib) file(COPY ${CMAKE_STAGING_PREFIX}/lib/librdkafka.so.1 DESTINATION /usr/local/lib) +file(GLOB ZSTD_LIBS "${CMAKE_STAGING_PREFIX}/lib/libzstd.so*") +file(COPY ${ZSTD_LIBS} DESTINATION /usr/local/lib) +file(GLOB SASL_LIBS "${CMAKE_STAGING_PREFIX}/lib/libsasl2.so*") +file(COPY ${SASL_LIBS} DESTINATION /usr/local/lib) diff --git a/cmake/arm-linux-gnueabihf.cmake b/cmake/arm-linux-gnueabihf.cmake index 122046a51..d088767c4 100644 --- a/cmake/arm-linux-gnueabihf.cmake +++ b/cmake/arm-linux-gnueabihf.cmake @@ -22,4 +22,8 @@ set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) link_directories(${CMAKE_STAGING_PREFIX}) file(COPY ${CMAKE_STAGING_PREFIX}/lib/libzlog.so.1.2 DESTINATION /usr/local/lib) -file(COPY ${CMAKE_STAGING_PREFIX}/lib/librdkafka.so.1 DESTINATION /usr/local/lib) \ No newline at end of file +file(COPY ${CMAKE_STAGING_PREFIX}/lib/librdkafka.so.1 DESTINATION /usr/local/lib) +file(GLOB ZSTD_LIBS "${CMAKE_STAGING_PREFIX}/lib/libzstd.so*") +file(COPY ${ZSTD_LIBS} DESTINATION /usr/local/lib) +file(GLOB SASL_LIBS "${CMAKE_STAGING_PREFIX}/lib/libsasl2.so*") +file(COPY ${SASL_LIBS} DESTINATION /usr/local/lib) \ No newline at end of file diff --git a/cmake/x86_64-linux-gnu.cmake b/cmake/x86_64-linux-gnu.cmake index 39ce518fd..4c8ed6cf7 100644 --- a/cmake/x86_64-linux-gnu.cmake +++ b/cmake/x86_64-linux-gnu.cmake @@ -28,3 +28,7 @@ link_directories(${CMAKE_STAGING_PREFIX}) file(COPY ${CMAKE_STAGING_PREFIX}/lib/libzlog.so.1.2 DESTINATION /usr/local/lib) file(COPY ${CMAKE_STAGING_PREFIX}/lib/librdkafka.so.1 DESTINATION /usr/local/lib) +file(GLOB ZSTD_LIBS "${CMAKE_STAGING_PREFIX}/lib/libzstd.so*") +file(COPY ${ZSTD_LIBS} DESTINATION /usr/local/lib) +file(GLOB SASL_LIBS "${CMAKE_STAGING_PREFIX}/lib/libsasl2.so*") +file(COPY ${SASL_LIBS} DESTINATION /usr/local/lib) diff --git a/install_dependencies.sh b/install_dependencies.sh index c1b0c0a90..d8316417f 100755 --- a/install_dependencies.sh +++ b/install_dependencies.sh @@ -73,6 +73,35 @@ cd protobuf-c make && sudo make install cd .. +# Install cyrus-sasl +echo "Installing cyrus-sasl..." +git clone -b cyrus-sasl-2.1.28 https://github.com/cyrusimap/cyrus-sasl.git +cd cyrus-sasl +autoreconf -fi +./configure --enable-shared --disable-static --disable-sample \ + --disable-checkapop --disable-cram --disable-digest --disable-otp \ + --disable-srp --disable-gssapi --disable-sql \ + --enable-plain --enable-scram --enable-login \ + --with-dblib=none --without-saslauthd --without-authdaemond +make && sudo make install +cd .. + +# Install zstd +echo "Installing zstd..." +git clone -b v1.5.6 https://github.com/facebook/zstd.git +cd zstd/build/cmake && mkdir builddir && cd builddir +cmake -DZSTD_BUILD_PROGRAMS=OFF -DZSTD_BUILD_TESTS=OFF -DZSTD_BUILD_SHARED=ON -DZSTD_BUILD_STATIC=OFF -DCMAKE_BUILD_TYPE=Release .. +make && sudo make install +cd ../../../.. + +# Install librdkafka +echo "Installing librdkafka..." +git clone -b v2.6.1 https://github.com/confluentinc/librdkafka.git +cd librdkafka && mkdir build && cd build +cmake -DRDKAFKA_BUILD_STATIC=OFF -DRDKAFKA_BUILD_EXAMPLES=OFF -DRDKAFKA_BUILD_TESTS=OFF -DWITH_SSL=ON -DWITH_ZLIB=ON -DWITH_SASL=ON -DWITH_ZSTD=ON -DWITH_CURL=OFF -DCMAKE_BUILD_TYPE=Release .. +make && sudo make install +cd ../.. + # Install libxml2 echo "Installing libxml2..." git clone -b v2.9.14 https://github.com/GNOME/libxml2.git diff --git a/package-sdk.sh b/package-sdk.sh index d8e8c7363..d9a4c74af 100755 --- a/package-sdk.sh +++ b/package-sdk.sh @@ -71,6 +71,8 @@ cp build/plugins/libplugin-ekuiper.so \ ${package_name}/plugins/ cp /usr/local/lib/librdkafka.so.1 ${package_name}/lib +cp /usr/local/lib/libzstd.so.1 ${package_name}/lib +cp /usr/local/lib/libsasl2.so.3 ${package_name}/lib tar czf ${package_name}-${arch}.tar.gz ${package_name}/ ls ${package_name}