kafka:add libzstd & libsasl2

.github/workflows/check.yml

@@ -209,6 +209,20 @@ jobs:
           name: neuron-build
           path: ./build
 
+      - name: debug kafka plugin deps
+        run: |
+          echo "=== libs in staging prefix ==="
+          ls -la /home/neuron/main/libs/x86_64-linux-gnu/lib/librdkafka* || echo "no librdkafka"
+          ls -la /home/neuron/main/libs/x86_64-linux-gnu/lib/libzstd* || echo "no libzstd"
+          ls -la /home/neuron/main/libs/x86_64-linux-gnu/lib/libsasl2* || echo "no libsasl2"
+          echo "=== libs in /usr/local/lib ==="
+          ls -la /usr/local/lib/librdkafka* /usr/local/lib/libzstd* /usr/local/lib/libsasl2* 2>/dev/null || echo "none yet"
+          echo "=== ldd librdkafka from staging ==="
+          ldd /home/neuron/main/libs/x86_64-linux-gnu/lib/librdkafka.so.1 || true
+          echo "=== ldd plugin-kafka from build ==="
+          chmod -R +x ./build/ 2>/dev/null || true
+          ldd ./build/plugins/libplugin-kafka.so || true
+
       - name: function test
         env:
           KAFKA_BROKER: kafka:9092
@@ -218,6 +232,11 @@ jobs:
           cmake -DUSE_GCOV=1 .. -DCMAKE_TOOLCHAIN_FILE=../cmake/x86_64-linux-gnu.cmake
           cd ..
           ldconfig
+          echo "=== post-ldconfig ldd check ==="
+          ldd ./build/plugins/libplugin-kafka.so || true
+          ldd /usr/local/lib/librdkafka.so.1 || true
+          echo "=== try manual dlopen ==="
+          python3 -c "import ctypes; ctypes.CDLL('./build/plugins/libplugin-kafka.so')" 2>&1 || true
           pytest -s -v tests/ft/app/test_kafka.py
 
       - name: create cov report

Install-dependencies.md

@@ -74,11 +74,31 @@ $ ./configure  --disable-protoc --enable-shared=no CFLAGS=-fPIC CXXFLAGS=-fPIC
 $ make && sudo make install
 ```
 
+[cyrus-sasl](https://github.com/cyrusimap/cyrus-sasl.git)
+```shell
+$ git clone -b cyrus-sasl-2.1.28 https://github.com/cyrusimap/cyrus-sasl.git
+$ cd cyrus-sasl
+$ autoreconf -fi
+$ ./configure --enable-shared --disable-static --disable-sample \
+    --disable-checkapop --disable-cram --disable-digest --disable-otp \
+    --disable-srp --disable-gssapi --disable-sql \
+    --enable-plain --enable-scram --enable-login \
+    --with-dblib=none --without-saslauthd --without-authdaemond
+$ make && sudo make install
+```
+
+[zstd](https://github.com/facebook/zstd.git)
+```shell
+$ git clone -b v1.5.6 https://github.com/facebook/zstd.git
+$ cd zstd/build/cmake && mkdir builddir && cd builddir
+$ cmake -DZSTD_BUILD_PROGRAMS=OFF -DZSTD_BUILD_TESTS=OFF -DZSTD_BUILD_SHARED=ON -DZSTD_BUILD_STATIC=OFF -DCMAKE_BUILD_TYPE=Release .. && make && sudo make install
+```
+
 [librdkafka](https://github.com/confluentinc/librdkafka.git)
 ```shell
 $ git clone -b v2.6.1 https://github.com/confluentinc/librdkafka.git
 $ cd librdkafka && mkdir build && cd build
-$ cmake -DRDKAFKA_BUILD_STATIC=OFF -DRDKAFKA_BUILD_EXAMPLES=OFF -DRDKAFKA_BUILD_TESTS=OFF -DWITH_SASL=OFF -DWITH_ZSTD=OFF -DWITH_CURL=OFF -DCMAKE_BUILD_TYPE=Release .. && make && sudo make install
+$ cmake -DRDKAFKA_BUILD_STATIC=OFF -DRDKAFKA_BUILD_EXAMPLES=OFF -DRDKAFKA_BUILD_TESTS=OFF -DWITH_SSL=ON -DWITH_ZLIB=ON -DWITH_SASL=ON -DWITH_ZSTD=ON -DWITH_CURL=OFF -DCMAKE_BUILD_TYPE=Release .. && make && sudo make install
 ```
 
 [libxml2](https://github.com/GNOME/libxml2.git)

cmake/aarch64-linux-gnu.cmake

@@ -22,4 +22,8 @@ set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
 link_directories(${CMAKE_STAGING_PREFIX})
 
 file(COPY ${CMAKE_STAGING_PREFIX}/lib/libzlog.so.1.2 DESTINATION /usr/local/lib)
-file(COPY ${CMAKE_STAGING_PREFIX}/lib/librdkafka.so.1 DESTINATION /usr/local/lib)
+file(COPY ${CMAKE_STAGING_PREFIX}/lib/librdkafka.so.1 DESTINATION /usr/local/lib)
+file(GLOB ZSTD_LIBS "${CMAKE_STAGING_PREFIX}/lib/libzstd.so*")
+file(COPY ${ZSTD_LIBS} DESTINATION /usr/local/lib)
+file(GLOB SASL_LIBS "${CMAKE_STAGING_PREFIX}/lib/libsasl2.so*")
+file(COPY ${SASL_LIBS} DESTINATION /usr/local/lib)

cmake/arm-linux-gnueabi.cmake

@@ -23,3 +23,7 @@ link_directories(${CMAKE_STAGING_PREFIX})
 
 file(COPY ${CMAKE_STAGING_PREFIX}/lib/libzlog.so.1.2 DESTINATION /usr/local/lib)
 file(COPY ${CMAKE_STAGING_PREFIX}/lib/librdkafka.so.1 DESTINATION /usr/local/lib)
+file(GLOB ZSTD_LIBS "${CMAKE_STAGING_PREFIX}/lib/libzstd.so*")
+file(COPY ${ZSTD_LIBS} DESTINATION /usr/local/lib)
+file(GLOB SASL_LIBS "${CMAKE_STAGING_PREFIX}/lib/libsasl2.so*")
+file(COPY ${SASL_LIBS} DESTINATION /usr/local/lib)

cmake/arm-linux-gnueabihf.cmake

@@ -22,4 +22,8 @@ set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
 link_directories(${CMAKE_STAGING_PREFIX})
 
 file(COPY ${CMAKE_STAGING_PREFIX}/lib/libzlog.so.1.2 DESTINATION /usr/local/lib)
-file(COPY ${CMAKE_STAGING_PREFIX}/lib/librdkafka.so.1 DESTINATION /usr/local/lib)
+file(COPY ${CMAKE_STAGING_PREFIX}/lib/librdkafka.so.1 DESTINATION /usr/local/lib)
+file(GLOB ZSTD_LIBS "${CMAKE_STAGING_PREFIX}/lib/libzstd.so*")
+file(COPY ${ZSTD_LIBS} DESTINATION /usr/local/lib)
+file(GLOB SASL_LIBS "${CMAKE_STAGING_PREFIX}/lib/libsasl2.so*")
+file(COPY ${SASL_LIBS} DESTINATION /usr/local/lib)

cmake/x86_64-linux-gnu.cmake

@@ -28,3 +28,7 @@ link_directories(${CMAKE_STAGING_PREFIX})
 
 file(COPY ${CMAKE_STAGING_PREFIX}/lib/libzlog.so.1.2 DESTINATION /usr/local/lib)
 file(COPY ${CMAKE_STAGING_PREFIX}/lib/librdkafka.so.1 DESTINATION /usr/local/lib)
+file(GLOB ZSTD_LIBS "${CMAKE_STAGING_PREFIX}/lib/libzstd.so*")
+file(COPY ${ZSTD_LIBS} DESTINATION /usr/local/lib)
+file(GLOB SASL_LIBS "${CMAKE_STAGING_PREFIX}/lib/libsasl2.so*")
+file(COPY ${SASL_LIBS} DESTINATION /usr/local/lib)

install_dependencies.sh

@@ -73,6 +73,35 @@ cd protobuf-c
 make && sudo make install
 cd ..
 
+# Install cyrus-sasl
+echo "Installing cyrus-sasl..."
+git clone -b cyrus-sasl-2.1.28 https://github.com/cyrusimap/cyrus-sasl.git
+cd cyrus-sasl
+autoreconf -fi
+./configure --enable-shared --disable-static --disable-sample \
+    --disable-checkapop --disable-cram --disable-digest --disable-otp \
+    --disable-srp --disable-gssapi --disable-sql \
+    --enable-plain --enable-scram --enable-login \
+    --with-dblib=none --without-saslauthd --without-authdaemond
+make && sudo make install
+cd ..
+
+# Install zstd
+echo "Installing zstd..."
+git clone -b v1.5.6 https://github.com/facebook/zstd.git
+cd zstd/build/cmake && mkdir builddir && cd builddir
+cmake -DZSTD_BUILD_PROGRAMS=OFF -DZSTD_BUILD_TESTS=OFF -DZSTD_BUILD_SHARED=ON -DZSTD_BUILD_STATIC=OFF -DCMAKE_BUILD_TYPE=Release ..
+make && sudo make install
+cd ../../../..
+
+# Install librdkafka
+echo "Installing librdkafka..."
+git clone -b v2.6.1 https://github.com/confluentinc/librdkafka.git
+cd librdkafka && mkdir build && cd build
+cmake -DRDKAFKA_BUILD_STATIC=OFF -DRDKAFKA_BUILD_EXAMPLES=OFF -DRDKAFKA_BUILD_TESTS=OFF -DWITH_SSL=ON -DWITH_ZLIB=ON -DWITH_SASL=ON -DWITH_ZSTD=ON -DWITH_CURL=OFF -DCMAKE_BUILD_TYPE=Release ..
+make && sudo make install
+cd ../..
+
 # Install libxml2
 echo "Installing libxml2..."
 git clone -b v2.9.14 https://github.com/GNOME/libxml2.git

package-sdk.sh

@@ -71,6 +71,8 @@ cp build/plugins/libplugin-ekuiper.so \
     ${package_name}/plugins/
 
 cp /usr/local/lib/librdkafka.so.1 ${package_name}/lib
+cp /usr/local/lib/libzstd.so.1 ${package_name}/lib
+cp /usr/local/lib/libsasl2.so.3 ${package_name}/lib
 
 tar czf ${package_name}-${arch}.tar.gz ${package_name}/
 ls ${package_name}