update demo setup to work with current components in development.

demo/Dockerfile

@@ -1,6 +1,7 @@
 FROM python:3.12-slim
 
 WORKDIR /app
+ARG DETECTMATELIBRARY_BRANCH=main
 
 # Install system dependencies
 RUN apt-get update && \
@@ -18,4 +19,9 @@ COPY ./tests ./tests
 
 RUN uv pip install --system -e .
 
+RUN if [ "$DETECTMATELIBRARY_BRANCH" != "main" ]; then \
+        uv pip uninstall --system detectmatelibrary && \
+        uv pip install --system "detectmatelibrary @ git+https://github.com/ait-detectmate/DetectMateLibrary.git@$DETECTMATELIBRARY_BRANCH" ; \
+    fi
+
 CMD ["detectmate", "--help"]

demo/config/detector_config.yaml

@@ -2,16 +2,17 @@ detectors:
     NewValueDetector:
         method_type: new_value_detector
         auto_config: False
-        params:
-            log_variables:
-                - id: test
-                  event: 1
-                  template: adsdas
-                  variables:
-                      - pos: 0
-                        name: var1
-                        params:
-                            threshold: 0.5
-                  header_variables:
-                      - pos: level
-                        params: {}
+        params: {}
+        events:
+            1:
+                test:
+                    params: {}
+                    template: adsdas
+                    variables:
+                        - pos: 0
+                          name: var1
+                          params:
+                              threshold: 0.5
+                    header_variables:
+                        - pos: level
+                          params: {}

demo/docker-compose.yml

@@ -1,31 +1,12 @@
-services:
-  reader:
-    build:
-      context: ..
-      dockerfile: demo/Dockerfile
-    container_name: detectmate_reader
-    command: >
-      detectmate
-      --settings demo/config/reader_settings.yaml
-      --config demo/config/reader_config.yaml
-    volumes:
-      - ../demo/config:/app/demo/config
-      - ../logs:/app/logs
-    environment:
-      - PYTHONUNBUFFERED=1
-    healthcheck:
-      test: [ "CMD", "python3", "-m", "service.client", "--url", "http://127.0.0.1:8000", "status" ]
-      interval: 60s
-      timeout: 3s
-      retries: 5
-      start_period: 5s
-    networks:
-      - detectmate_net
+x-default-args: &default-args
+  DETECTMATELIBRARY_BRANCH: main
 
+services:
   parser:
     build:
       context: ..
       dockerfile: demo/Dockerfile
+      args: *default-args
     container_name: detectmate_parser
     command: >
       detectmate
@@ -35,9 +16,6 @@ services:
       - ../demo/config:/app/demo/config
       - ../logs:/app/logs
       - ../demo/data:/app/demo/data  # template file is here
-    depends_on:
-      reader:
-        condition: service_healthy
     environment:
       - PYTHONUNBUFFERED=1
     healthcheck:
@@ -53,6 +31,7 @@ services:
     build:
       context: ..
       dockerfile: demo/Dockerfile
+      args: *default-args
     container_name: detectmate_detector
     command: >
       detectmate
@@ -79,15 +58,14 @@ services:
     build:
       context: ..
       dockerfile: demo/Dockerfile
+      args: *default-args
     container_name: detectmate_demo
     command: python demo/manual_demo_run_tcp.py
     volumes:
       - ../demo:/app/demo
       - ../demo/data:/app/demo/data
       - ../logs:/app/logs
     depends_on:
-      reader:
-        condition: service_healthy
       parser:
         condition: service_healthy
       detector:

demo/manual_demo_run_ipc.py

@@ -1,4 +1,6 @@
 import pynng
+from detectmatelibrary.helper.from_to import From
+from detectmatelibrary.parsers.dummy_parser import DummyParser
 
 
 LOG_PATH = "/app/demo/data/audit.log"
@@ -10,30 +12,32 @@ def process_logs() -> None:
     with open(LOG_PATH, "r") as f:
         total = sum(1 for _ in f)
     print(f"Processing {total} log lines...")
-    with open(LOG_PATH, "rb") as f:
-        for i, line in enumerate(f, start=1):
-            line = line.rstrip(b"\n")
-            print(f"\n--- Processing line {i}/{total} ---")
-            try:
-                # Step 1: Reader
-                with pynng.Pair0(dial="ipc:///tmp/test_reader_engine.ipc") as reader:
-                    reader.send(line)
-                    log_response1 = reader.recv()
-                # Step 2: Parser
-                with pynng.Pair0(dial="ipc:///tmp/test_parser_engine.ipc") as parser:
-                    parser.send(log_response1)
-                    log_response2 = parser.recv()
-                # Step 3: Detector
-                with pynng.Pair0(dial="ipc:///tmp/test_nvd_engine.ipc", recv_timeout=10) as detector:
-                    detector.send(log_response2)
-                    try:
-                        log_response3 = detector.recv()
-                        print(f"Anomaly detected: {log_response3}")
-                    except pynng.Timeout:
-                        # No anomaly, just continue
-                        pass
-            except Exception as e:
-                print(f"Error on line {i}: {e}")
+    parser = DummyParser()
+    gen = From.log(parser, LOG_PATH, do_process=False)
+    i = 1
+    while True:
+        try:
+            # Step 1: Reader
+            line = next(gen)
+        except StopIteration:
+            break
+        print(f"\n--- Processing line {i}/{total} ---")
+        try:
+            # Step 2: Parser
+            with pynng.Pair0(dial="ipc:///tmp/test_parser_engine.ipc") as parser:
+                parser.send(line.serialize)
+                log_response2 = parser.recv()
+            # Step 3: Detector
+            with pynng.Pair0(dial="ipc:///tmp/test_nvd_engine.ipc", recv_timeout=10) as detector:
+                detector.send(log_response2)
+                try:
+                    log_response3 = detector.recv()
+                    print(f"Anomaly detected: {log_response3}")
+                except pynng.Timeout:
+                    # No anomaly, just continue
+                    pass
+        except Exception as e:
+            print(f"Error on line {i}: {e}")
 
 
 if __name__ == "__main__":

demo/manual_demo_run_tcp.py

@@ -1,7 +1,10 @@
 import pynng
+from detectmatelibrary.helper.from_to import From
+from detectmatelibrary.parsers.dummy_parser import DummyParser
 
 
 LOG_PATH = "/app/demo/data/audit.log"
+DETECTOR_OUT = "/app/demo/data/detector_out.json"
 
 
 def process_logs() -> None:
@@ -10,30 +13,33 @@ def process_logs() -> None:
     with open(LOG_PATH, "r") as f:
         total = sum(1 for _ in f)
     print(f"Processing {total} log lines...")
-    with open(LOG_PATH, "rb") as f:
-        for i, line in enumerate(f, start=1):
-            line = line.rstrip(b"\n")
-            print(f"\n--- Processing line {i}/{total} ---")
-            try:
-                # Step 1: Reader
-                with pynng.Pair0(dial="tcp://reader:8001") as reader:
-                    reader.send(line)
-                    log_response1 = reader.recv()
-                # Step 2: Parser
-                with pynng.Pair0(dial="tcp://parser:8011") as parser:
-                    parser.send(log_response1)
-                    log_response2 = parser.recv()
-                # Step 3: Detector
-                with pynng.Pair0(dial="tcp://detector:8021", recv_timeout=10) as detector:
-                    detector.send(log_response2)
-                    try:
-                        log_response3 = detector.recv()
-                        print(f"Anomaly detected: {log_response3}")
-                    except pynng.Timeout:
-                        # No anomaly, just continue
-                        pass
-            except Exception as e:
-                print(f"Error on line {i}: {e}")
+    parser = DummyParser()
+    gen = From.log(parser, LOG_PATH, do_process=False)
+    i = 1
+    while True:
+        try:
+            # Step 1: Reader
+            line = next(gen)
+        except StopIteration:
+            break
+        print(f"\n--- Processing line {i}/{total} ---")
+        i += 1
+        try:
+            # Step 2: Parser
+            with pynng.Pair0(dial="tcp://parser:8011") as parser:
+                parser.send(line.serialize())
+                log_response2 = parser.recv()
+            # Step 3: Detector
+            with pynng.Pair0(dial="tcp://detector:8021", recv_timeout=10) as detector:
+                detector.send(log_response2)
+                try:
+                    log_response3 = detector.recv()
+                    print(f"Anomaly detected: {log_response3}")
+                except pynng.Timeout:
+                    # No anomaly, just continue
+                    pass
+        except Exception as e:
+            print(f"Error on line {i}: {e}")
 
 
 if __name__ == "__main__":

tests/library_integration/test_pipe_filereader_matcher_nvd.py

@@ -76,7 +76,7 @@ def running_pipeline_services(
             "MatcherParser": {
                 "method_type": "matcher_parser",
                 "auto_config": False,
-                "log_format": "type=<type> msg=audit(<Time>...): <Content>",
+                "log_format": "type=<Type> msg=audit(<Time>:*): <Content>",
                 "time_format": None,
                 "params": {
                     "remove_spaces": True,