From 6cf386b2f9c8ee6f121a01860a18916050ad60b9 Mon Sep 17 00:00:00 2001 From: Ernst Leierzopf Date: Mon, 2 Mar 2026 23:05:03 +0100 Subject: [PATCH 1/2] update demo setup to work with current components in development. Add possibility to set the used branch for DetectMateLibrary in Dockerfile and docker compose file. --- demo/Dockerfile | 6 ++++ demo/config/detector_config.yaml | 27 ++++++++-------- demo/docker-compose.yml | 34 ++++---------------- demo/manual_demo_run_ipc.py | 52 ++++++++++++++++-------------- demo/manual_demo_run_tcp.py | 54 ++++++++++++++++++-------------- 5 files changed, 84 insertions(+), 89 deletions(-) diff --git a/demo/Dockerfile b/demo/Dockerfile index e63255e..76c371a 100644 --- a/demo/Dockerfile +++ b/demo/Dockerfile @@ -1,6 +1,7 @@ FROM python:3.12-slim WORKDIR /app +ARG DETECTMATELIBRARY_BRANCH=main # Install system dependencies RUN apt-get update && \ @@ -18,4 +19,9 @@ COPY ./tests ./tests RUN uv pip install --system -e . +RUN if [ "$DETECTMATELIBRARY_BRANCH" != "main" ]; then \ + uv pip uninstall --system detectmatelibrary && \ + uv pip install --system "detectmatelibrary @ git+https://github.com/ait-detectmate/DetectMateLibrary.git@$DETECTMATELIBRARY_BRANCH" ; \ + fi + CMD ["detectmate", "--help"] diff --git a/demo/config/detector_config.yaml b/demo/config/detector_config.yaml index a0b35b4..c92685b 100644 --- a/demo/config/detector_config.yaml +++ b/demo/config/detector_config.yaml @@ -2,16 +2,17 @@ detectors: NewValueDetector: method_type: new_value_detector auto_config: False - params: - log_variables: - - id: test - event: 1 - template: adsdas - variables: - - pos: 0 - name: var1 - params: - threshold: 0.5 - header_variables: - - pos: level - params: {} + params: {} + events: + 1: + test: + params: {} + template: adsdas + variables: + - pos: 0 + name: var1 + params: + threshold: 0.5 + header_variables: + - pos: level + params: {} diff --git a/demo/docker-compose.yml b/demo/docker-compose.yml index d59767b..c7f5353 100644 --- a/demo/docker-compose.yml +++ b/demo/docker-compose.yml @@ -1,31 +1,12 @@ -services: - reader: - build: - context: .. - dockerfile: demo/Dockerfile - container_name: detectmate_reader - command: > - detectmate - --settings demo/config/reader_settings.yaml - --config demo/config/reader_config.yaml - volumes: - - ../demo/config:/app/demo/config - - ../logs:/app/logs - environment: - - PYTHONUNBUFFERED=1 - healthcheck: - test: [ "CMD", "python3", "-m", "service.client", "--url", "http://127.0.0.1:8000", "status" ] - interval: 60s - timeout: 3s - retries: 5 - start_period: 5s - networks: - - detectmate_net +x-default-args: &default-args + DETECTMATELIBRARY_BRANCH: main +services: parser: build: context: .. dockerfile: demo/Dockerfile + args: *default-args container_name: detectmate_parser command: > detectmate @@ -35,9 +16,6 @@ services: - ../demo/config:/app/demo/config - ../logs:/app/logs - ../demo/data:/app/demo/data # template file is here - depends_on: - reader: - condition: service_healthy environment: - PYTHONUNBUFFERED=1 healthcheck: @@ -53,6 +31,7 @@ services: build: context: .. dockerfile: demo/Dockerfile + args: *default-args container_name: detectmate_detector command: > detectmate @@ -79,6 +58,7 @@ services: build: context: .. dockerfile: demo/Dockerfile + args: *default-args container_name: detectmate_demo command: python demo/manual_demo_run_tcp.py volumes: @@ -86,8 +66,6 @@ services: - ../demo/data:/app/demo/data - ../logs:/app/logs depends_on: - reader: - condition: service_healthy parser: condition: service_healthy detector: diff --git a/demo/manual_demo_run_ipc.py b/demo/manual_demo_run_ipc.py index f9ee366..9df45cf 100644 --- a/demo/manual_demo_run_ipc.py +++ b/demo/manual_demo_run_ipc.py @@ -1,4 +1,6 @@ import pynng +from detectmatelibrary.helper.from_to import From +from detectmatelibrary.parsers.dummy_parser import DummyParser LOG_PATH = "/app/demo/data/audit.log" @@ -10,30 +12,32 @@ def process_logs() -> None: with open(LOG_PATH, "r") as f: total = sum(1 for _ in f) print(f"Processing {total} log lines...") - with open(LOG_PATH, "rb") as f: - for i, line in enumerate(f, start=1): - line = line.rstrip(b"\n") - print(f"\n--- Processing line {i}/{total} ---") - try: - # Step 1: Reader - with pynng.Pair0(dial="ipc:///tmp/test_reader_engine.ipc") as reader: - reader.send(line) - log_response1 = reader.recv() - # Step 2: Parser - with pynng.Pair0(dial="ipc:///tmp/test_parser_engine.ipc") as parser: - parser.send(log_response1) - log_response2 = parser.recv() - # Step 3: Detector - with pynng.Pair0(dial="ipc:///tmp/test_nvd_engine.ipc", recv_timeout=10) as detector: - detector.send(log_response2) - try: - log_response3 = detector.recv() - print(f"Anomaly detected: {log_response3}") - except pynng.Timeout: - # No anomaly, just continue - pass - except Exception as e: - print(f"Error on line {i}: {e}") + parser = DummyParser() + gen = From.log(parser, LOG_PATH, do_process=False) + i = 1 + while True: + try: + # Step 1: Reader + line = next(gen) + except StopIteration: + break + print(f"\n--- Processing line {i}/{total} ---") + try: + # Step 2: Parser + with pynng.Pair0(dial="ipc:///tmp/test_parser_engine.ipc") as parser: + parser.send(line.serialize) + log_response2 = parser.recv() + # Step 3: Detector + with pynng.Pair0(dial="ipc:///tmp/test_nvd_engine.ipc", recv_timeout=10) as detector: + detector.send(log_response2) + try: + log_response3 = detector.recv() + print(f"Anomaly detected: {log_response3}") + except pynng.Timeout: + # No anomaly, just continue + pass + except Exception as e: + print(f"Error on line {i}: {e}") if __name__ == "__main__": diff --git a/demo/manual_demo_run_tcp.py b/demo/manual_demo_run_tcp.py index 21ba11e..a309c25 100644 --- a/demo/manual_demo_run_tcp.py +++ b/demo/manual_demo_run_tcp.py @@ -1,7 +1,10 @@ import pynng +from detectmatelibrary.helper.from_to import From +from detectmatelibrary.parsers.dummy_parser import DummyParser LOG_PATH = "/app/demo/data/audit.log" +DETECTOR_OUT = "/app/demo/data/detector_out.json" def process_logs() -> None: @@ -10,30 +13,33 @@ def process_logs() -> None: with open(LOG_PATH, "r") as f: total = sum(1 for _ in f) print(f"Processing {total} log lines...") - with open(LOG_PATH, "rb") as f: - for i, line in enumerate(f, start=1): - line = line.rstrip(b"\n") - print(f"\n--- Processing line {i}/{total} ---") - try: - # Step 1: Reader - with pynng.Pair0(dial="tcp://reader:8001") as reader: - reader.send(line) - log_response1 = reader.recv() - # Step 2: Parser - with pynng.Pair0(dial="tcp://parser:8011") as parser: - parser.send(log_response1) - log_response2 = parser.recv() - # Step 3: Detector - with pynng.Pair0(dial="tcp://detector:8021", recv_timeout=10) as detector: - detector.send(log_response2) - try: - log_response3 = detector.recv() - print(f"Anomaly detected: {log_response3}") - except pynng.Timeout: - # No anomaly, just continue - pass - except Exception as e: - print(f"Error on line {i}: {e}") + parser = DummyParser() + gen = From.log(parser, LOG_PATH, do_process=False) + i = 1 + while True: + try: + # Step 1: Reader + line = next(gen) + except StopIteration: + break + print(f"\n--- Processing line {i}/{total} ---") + i += 1 + try: + # Step 2: Parser + with pynng.Pair0(dial="tcp://parser:8011") as parser: + parser.send(line.serialize()) + log_response2 = parser.recv() + # Step 3: Detector + with pynng.Pair0(dial="tcp://detector:8021", recv_timeout=10) as detector: + detector.send(log_response2) + try: + log_response3 = detector.recv() + print(f"Anomaly detected: {log_response3}") + except pynng.Timeout: + # No anomaly, just continue + pass + except Exception as e: + print(f"Error on line {i}: {e}") if __name__ == "__main__": From 1009899e70f060619b4846e86abcc483ddacac7c Mon Sep 17 00:00:00 2001 From: Ernst Leierzopf Date: Sun, 8 Mar 2026 21:19:02 +0100 Subject: [PATCH 2/2] fix log_format in test. --- tests/library_integration/test_pipe_filereader_matcher_nvd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/library_integration/test_pipe_filereader_matcher_nvd.py b/tests/library_integration/test_pipe_filereader_matcher_nvd.py index c14560a..98ac759 100644 --- a/tests/library_integration/test_pipe_filereader_matcher_nvd.py +++ b/tests/library_integration/test_pipe_filereader_matcher_nvd.py @@ -76,7 +76,7 @@ def running_pipeline_services( "MatcherParser": { "method_type": "matcher_parser", "auto_config": False, - "log_format": "type= msg=audit(