feat(edr): stabilize agent orchestration, complete event ingestion pipeline, and optimize dev tooling

.dev.env.example

@@ -0,0 +1,41 @@
+# =============================================================================
+# EDR Local Development Environment Template (.dev.env)
+# Usage:
+#   cp .dev.env.example .env
+#   # Adjust values below if you have a custom setup
+# =============================================================================
+
+# ── Fleet Server Settings ──────────────────────────────────────────────────────
+# Host/Port the gRPC service will listen on (0.0.0.0 to bind on all interfaces)
+HOST=0.0.0.0
+PORT=50051
+
+# Secret key used to sign and verify node authorization JWT tokens.
+# Change this in staging/production!
+JWT_SECRET=dev-jwt-secret-key-do-not-use-in-production-12345
+
+# Logging configuration for fleet-server
+RUST_LOG=debug
+LOG_FORMAT=human
+
+# ── Database Connections ─────────────────────────────────────────────────────
+# PostgreSQL instance for fleet node registrations (managed by infra/docker-compose.yml)
+# Default port is 5433 for nodes DB
+DATABASE_URL=postgres://edr:changeme@127.0.0.1:5433/edr_nodes
+
+# ── Kafka Configuration ──────────────────────────────────────────────────────
+# Kafka brokers list (using 127.0.0.1 explicitly to bypass IPv6 resolution delays/bugs on local Docker setups)
+KAFKA_BROKERS=127.0.0.1:9092
+
+# Raw telemetry ingestion topic
+KAFKA_TOPIC_AGENTS_EVENTS=edr.events.raw
+
+# Note: The Kafka Web UI is accessible in your browser at http://localhost:8090
+
+# ── Agent Settings ───────────────────────────────────────────────────────────
+# Config path override for running the EDR agent locally without sudo installation.
+# By setting this, 'cargo run --bin edr-agent' will load this config file automatically.
+EDR_AGENT_CONFIG=agent/agent.toml
+
+# Logging configuration for the agent
+AGENT_RUST_LOG=debug

.dockerignore

@@ -0,0 +1,3 @@
+target/
+.git/
+.github/

.env.example

@@ -0,0 +1,25 @@
+# EDR Workspace Environment Variables Template
+# Copy this file to .env at the workspace root and adjust as necessary for your local setup.
+
+# Fleet Server Database Connection
+# Format: postgres://<user>:<password>@<host>:<port>/<database>
+DATABASE_URL=postgres://edr:changeme@localhost:5433/edr_nodes
+
+# Kafka Brokers configuration (using 127.0.0.1 to avoid IPv6 localhost resolution bugs on host)
+KAFKA_BROKERS=127.0.0.1:9092
+KAFKA_TOPIC_AGENTS_EVENTS=edr.events.raw
+
+# Auth JWT Secret (Replace with a real 256-bit secret in production)
+JWT_SECRET=change-me-in-production
+
+# Server bind host and port
+HOST=0.0.0.0
+PORT=50051
+
+# Logging level and output format (human | json)
+RUST_LOG=info
+LOG_FORMAT=human
+
+# EDR Agent configuration path (optional, overrides default /etc/aigis-zero/config.toml)
+# EDR_AGENT_CONFIG=agent/agent.toml
+

.github/workflows/ci.yml

@@ -19,8 +19,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install protoc 
-        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+      - name: Install protoc and dependencies
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libcurl4-openssl-dev pkg-config
 
       - name: Install Rust stable
         uses: dtolnay/rust-toolchain@stable
@@ -41,8 +41,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install protoc 
-        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+      - name: Install protoc and dependencies
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libcurl4-openssl-dev pkg-config
 
       - name: Install Rust stable + clippy
         uses: dtolnay/rust-toolchain@stable
@@ -79,8 +79,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install protoc 
-        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+      - name: Install protoc and dependencies
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libcurl4-openssl-dev pkg-config
 
       - name: Install Rust stable
         uses: dtolnay/rust-toolchain@stable

.github/workflows/kafka-pipeline.yml

@@ -0,0 +1,57 @@
+name: Kafka Pipeline
+
+on:
+  push:
+    paths:
+      - 'kafka-pipeline/**'
+      - 'sdk/**'
+  pull_request:
+    paths:
+      - 'kafka-pipeline/**'
+      - 'sdk/**'
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libcurl4-openssl-dev pkg-config
+      - uses: dtolnay/rust-toolchain@stable
+      - run: cargo check --manifest-path kafka-pipeline/Cargo.toml
+      - run: cargo clippy --manifest-path kafka-pipeline/Cargo.toml -- -D warnings
+      - run: cargo fmt --manifest-path kafka-pipeline/Cargo.toml -- --check
+
+  test:
+    runs-on: ubuntu-latest
+    services:
+      kafka:
+        image: apache/kafka:latest
+        env:
+          KAFKA_NODE_ID: 1
+          KAFKA_PROCESS_ROLES: broker,controller
+          KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
+          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,CONTROLLER://kafka:9093
+          KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+          KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
+          CLUSTER_ID: MkQkQzE4NTJjYTEyODQ4MTcwMw
+        ports:
+          - 9092:9092
+    steps:
+      - uses: actions/checkout@v3
+      - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libcurl4-openssl-dev pkg-config
+      - uses: dtolnay/rust-toolchain@stable
+      - run: cargo test --manifest-path kafka-pipeline/Cargo.toml
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libcurl4-openssl-dev pkg-config
+      - uses: dtolnay/rust-toolchain@stable
+      - run: cargo build --release --manifest-path kafka-pipeline/Cargo.toml
+
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: docker build -t aigis-kafka-pipeline -f kafka-pipeline/Dockerfile .

.gitignore

@@ -52,4 +52,7 @@ dist-ssr/
 *.sw?
 
 */frontendprompt.md
-.agent/*
+.agent/*
+dev-testing-skill.md
+agent-analysis.md
+TEST_GUIDE.md

Cargo.toml

@@ -38,20 +38,23 @@ tokio-util = { version = "0.7", features = ["codec"] }
 tokio-tungstenite = "0.29"
 
 tonic = { version = "0.14" }
-tonic-reflection = "0.14"
-tonic-build = "0.14"
-tonic-prost-build = "0.14"
-prost = "0.14"
-tonic-prost = "0.14"
+tonic-prost = { version = "0.14" }
+prost = { version = "0.14" }
+tonic-prost-build = { version = "0.14" }
 
 axum = { version = "0.8", features = ["ws", "macros"] }
 tower = "0.5"
 tower-http = { version = "0.6", features = ["cors", "trace", "compression-gzip"] }
 
-# rdkafka = { version = "0.39", features = ["cmake-build"] }
+hyper = { version = "1.0", features = ["full"] }
+http = "1.1"
+http-body = "1.0"
 
+rdkafka = { version = "0.39", features = ["cmake-build"] }
 sqlx = { version = "0.8", default-features = false, features = ["postgres", "runtime-tokio-native-tls", "uuid", "chrono", "migrate", "macros"] }
 
+
+
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 
@@ -81,7 +84,7 @@ fleet-client = { path = "agent/crates/fleet-client" }
 isolation = { path = "agent/crates/isolation" }
 agent-bin = { path = "agent/crates/agent-bin" }
 agent-tracing = { path = "agent/crates/agent-tracing" }
-rusqlite = { version = "0.31", features = ["bundled"] }
+rusqlite = { version = "0.32", features = ["bundled"] }
 toml = "0.8"
 thrift = "0.17"