Refactor Kafka Pipeline and Agent Integration

.dockerignore

@@ -0,0 +1,3 @@
+target/
+.git/
+.github/

.github/workflows/ci.yml

@@ -19,8 +19,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install protoc 
-        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+      - name: Install protoc and dependencies
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libcurl4-openssl-dev pkg-config
 
       - name: Install Rust stable
         uses: dtolnay/rust-toolchain@stable
@@ -41,8 +41,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install protoc 
-        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+      - name: Install protoc and dependencies
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libcurl4-openssl-dev pkg-config
 
       - name: Install Rust stable + clippy
         uses: dtolnay/rust-toolchain@stable
@@ -79,8 +79,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install protoc 
-        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+      - name: Install protoc and dependencies
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libcurl4-openssl-dev pkg-config
 
       - name: Install Rust stable
         uses: dtolnay/rust-toolchain@stable

.github/workflows/kafka-pipeline.yml

@@ -0,0 +1,57 @@
+name: Kafka Pipeline
+
+on:
+  push:
+    paths:
+      - 'kafka-pipeline/**'
+      - 'sdk/**'
+  pull_request:
+    paths:
+      - 'kafka-pipeline/**'
+      - 'sdk/**'
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libcurl4-openssl-dev pkg-config
+      - uses: dtolnay/rust-toolchain@stable
+      - run: cargo check --manifest-path kafka-pipeline/Cargo.toml
+      - run: cargo clippy --manifest-path kafka-pipeline/Cargo.toml -- -D warnings
+      - run: cargo fmt --manifest-path kafka-pipeline/Cargo.toml -- --check
+
+  test:
+    runs-on: ubuntu-latest
+    services:
+      kafka:
+        image: apache/kafka:latest
+        env:
+          KAFKA_NODE_ID: 1
+          KAFKA_PROCESS_ROLES: broker,controller
+          KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
+          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,CONTROLLER://kafka:9093
+          KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+          KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
+          CLUSTER_ID: MkQkQzE4NTJjYTEyODQ4MTcwMw
+        ports:
+          - 9092:9092
+    steps:
+      - uses: actions/checkout@v3
+      - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libcurl4-openssl-dev pkg-config
+      - uses: dtolnay/rust-toolchain@stable
+      - run: cargo test --manifest-path kafka-pipeline/Cargo.toml
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libcurl4-openssl-dev pkg-config
+      - uses: dtolnay/rust-toolchain@stable
+      - run: cargo build --release --manifest-path kafka-pipeline/Cargo.toml
+
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: docker build -t aigis-kafka-pipeline -f kafka-pipeline/Dockerfile .

Cargo.toml

@@ -38,20 +38,23 @@ tokio-util = { version = "0.7", features = ["codec"] }
 tokio-tungstenite = "0.29"
 
 tonic = { version = "0.14" }
-tonic-reflection = "0.14"
-tonic-build = "0.14"
-tonic-prost-build = "0.14"
-prost = "0.14"
-tonic-prost = "0.14"
+tonic-prost = { version = "0.14" }
+prost = { version = "0.14" }
+tonic-prost-build = { version = "0.14" }
 
 axum = { version = "0.8", features = ["ws", "macros"] }
 tower = "0.5"
 tower-http = { version = "0.6", features = ["cors", "trace", "compression-gzip"] }
 
-# rdkafka = { version = "0.39", features = ["cmake-build"] }
+hyper = { version = "1.0", features = ["full"] }
+http = "1.1"
+http-body = "1.0"
 
+rdkafka = { version = "0.39", features = ["cmake-build"] }
 sqlx = { version = "0.8", default-features = false, features = ["postgres", "runtime-tokio-native-tls", "uuid", "chrono", "migrate", "macros"] }
 
+
+
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 
@@ -81,7 +84,7 @@ fleet-client = { path = "agent/crates/fleet-client" }
 isolation = { path = "agent/crates/isolation" }
 agent-bin = { path = "agent/crates/agent-bin" }
 agent-tracing = { path = "agent/crates/agent-tracing" }
-rusqlite = { version = "0.31", features = ["bundled"] }
+rusqlite = { version = "0.32", features = ["bundled"] }
 toml = "0.8"
 thrift = "0.17"
 

agent/.env.example

@@ -0,0 +1,7 @@
+# EDR Agent Environment Variables
+
+# Config file path
+EDR_AGENT_CONFIG=/etc/aigis-zero/config.toml
+
+# Logging
+RUST_LOG=info

agent/crates/agent-bin/Cargo.toml

@@ -17,3 +17,13 @@ clap = { version = "4", features = ["derive", "env"] }
 sd-notify = "0.4"
 libc = "0.2"
 toml = { workspace = true }
+hostname = "0.4"
+fleet-client = { workspace = true }
+event-buffer = { workspace = true }
+osquery-client = { workspace = true }
+edr-sdk = { workspace = true }
+serde_json = { workspace = true }
+uuid = { workspace = true }
+agent-tracing = { workspace = true }
+isolation = { workspace = true }
+tokio-util = { workspace = true }