Development

README.md

@@ -89,6 +89,7 @@ workspaces/custom_parser/          # workspace root
 │   └── custom_parser.py           # CoreParser-based template
 ├── tests/
 │   └── test_custom_parser.py      # generated from template to test custom_parser
+├── data.json                      # example data to run the code
 ├── LICENSE.md                     # copied from main project
 ├── .gitignore                     # copied from main project
 ├── .pre-commit-config.yaml        # copied from main project

docs/basic_usage.md

@@ -1,9 +1,155 @@
 
 # Getting started:  Basic usage
 
-Basic usage of the DetectMate Library.
+In this section, we will show different examples of the basic usage of the DetectMate Library.
 
+## Parser
 
+In this example, we will use the [`MatcherParser`](parsers/template_matcher.md) to parse audit data from the [AIT Log Data Set V2.0](https://zenodo.org/records/5789064). The code loads the logs, parse them and save the input and output in json files using [`from_to`](helper/from_to.md) module.
+
+```python
+from detectmatelibrary.parsers.template_matcher import MatcherParser
+from detectmatelibrary.helper.from_to import From, To
+
+
+config_dict = {
+    "parsers": {
+        "MatcherParser": {
+            "auto_config": True,
+            "method_type": "matcher_parser",
+            "path_templates": "ait_audit.txt",
+            "log_format": r'type=<Type> msg=audit\(<Time>:<Serial>\): <Content>'
+        }
+    }
+}
+parser = MatcherParser(name="MatcherParser", config=config_dict)
+
+
+for i, log in enumerate(From.log(parser, "audit.log", do_process=False)):
+    To.json(log, "logs.json")
+
+    parsed_log = parser.process(log)
+    To.json(parsed_log, "parsed_log.json")
+
+```
+
+The logs will be saved in `logs.json` in this format:
+
+```json
+{
+"0": {
+        "log": "type=USER_ACCT msg=audit(1642723741.072:375): pid=10125 uid=0 auid=4294967295 ses=4294967295 msg='op=PAM:accounting acct=\"root\" exe=\"/usr/sbin/cron\" hostname=? addr=? terminal=cron res=success'",
+        "__version__": "1.0.0",
+        "hostname": "",
+        "logSource": "",
+        "logID": "0"
+    },
+...
+}
+```
+And the `parsed_log.json`:
+
+```json
+{
+    "0": {
+        "parserID": "MatcherParser",
+        "parsedLogID": "10",
+        "logID": "0",
+        "parsedTimestamp": 1772027171,
+        "logFormatVariables": {
+            "Type": "USER_ACCT",
+            "Serial": "375",
+            "Time": "1642723741.072",
+            "Content": "pid=10125 uid=0 auid=4294967295 ses=4294967295 msg='op=PAM:accounting acct=\"root\" exe=\"/usr/sbin/cron\" hostname=? addr=? terminal=cron res=success'"
+        },
+        "__version__": "1.0.0",
+        "receivedTimestamp": 1772027171,
+        "variables": [
+            "10125",
+            "0",
+            "4294967295",
+            "4294967295",
+            "PAM:accounting",
+            "\"root\"",
+            "\"/usr/sbin/cron\"",
+            "?",
+            "?",
+            "cron",
+            "success"
+        ],
+        "log": "type=USER_ACCT msg=audit(1642723741.072:375): pid=10125 uid=0 auid=4294967295 ses=4294967295 msg='op=PAM:accounting acct=\"root\" exe=\"/usr/sbin/cron\" hostname=? addr=? terminal=cron res=success'",
+        "parserType": "matcher_parser",
+        "EventID": 0,
+        "template": "pid=<*> uid=<*> auid=<*> ses=<*> msg='op=<*> acct=<*> exe=<*> hostname=<*> addr=<*> terminal=<*> res=<*>'"
+    },
+...
+}
+```
+
+## Detector
+
+In this example, we will use the [`RandomDetector`](detectors/random_detector.md) with the parsed logs from the previous example.
+
+```python
+from detectmatelibrary.parsers.template_matcher import MatcherParser
+from detectmatelibrary.helper.from_to import From, To, FromTo
+
+config_dict = {
+    "detectors": {
+        "RandomDetector": {
+            "auto_config": False,
+            "method_type": "random_detector",
+            "params": {},
+            "events": {
+                1: {
+                    "test": {
+                        "params": {},
+                        "variables": [{
+                            "pos": 0,
+                            "name": "process",
+                            "params": {
+                                "threshold": 0.
+                            }
+                        }]
+                    }
+                }
+            }
+        }
+    }
+}
+detector =  RandomDetector(name="RandomDetector", config=config_dict)
+
+for alert in FromTo.json2json(detector, "parsed_log.json", "alerts.json"):
+    if alert is not None:
+        print("Anomaly detected!")
+```
+
+The alerts will be saved in `alerts.json` in this format:
+
+```json
+{
+    "0": {
+        "extractedTimestamps": [
+            1642723752
+        ],
+        "receivedTimestamp": 1772032073,
+        "score": 1.0,
+        "detectionTimestamp": 1772032073,
+        "alertID": "10",
+        "detectorType": "random_detector",
+        "detectorID": "RandomDetector",
+        "description": "",
+        "__version__": "1.0.0",
+        "logIDs": [
+            "6"
+        ],
+        "alertsObtain": {
+            "process": "1.0"
+        }
+    },
+...
+}
+```
 
 
 Go back to [Index](index.md), to previous step: [Installation](installation.md) or to next step: [Create new component](create_components.md).

docs/create_components.md

@@ -5,7 +5,7 @@ DetectMateLibrary includes a small CLI helper to bootstrap standalone workspaces
 for custom parsers and detectors. This is useful if you want to develop and test
 components in isolation while still using the same library and schemas.
 
-### Usage
+## Usage
 
 The CLI entry point is `mate` with a `create` command:
 
@@ -20,7 +20,7 @@ mate create --type <parser|detector> --name <workspace_name> --dir <target_dir>
 | `--dir`  | Directory where the workspace will be created                                                                                                                                 |
 
 
-### What gets generated
+## What gets generated
 
 For example:
 
@@ -37,14 +37,21 @@ workspaces/custom_parser/          # workspace root
 │   └── custom_parser.py           # CoreParser-based template
 ├── tests/
 │   └── test_custom_parser.py      # generated from template to test custom_parser
+├── data.json                      # example data to run the code
 ├── LICENSE.md                     # copied from main project
 ├── .gitignore                     # copied from main project
 ├── .pre-commit-config.yaml        # copied from main project
 ├── pyproject.toml                 # minimal project + dev extras
 └── README.md                      # setup instructions
 ```
 
+## Add component to DetectMateLibrary
 
+To add the new component to the `DetectMateLibrary`, you need to add the component file to the specific folder and update the import paths.
 
+* **Detector**: Add the detector file component in `src/detectmatelibrary/detectors` and the unit tests in `tests/test_detectors`.
+* **Parsers**: Add the parser file component in `src/detectmatelibrary/parsers` and the unit tests in `tests/test_parsers`.
 
-Go back to [Index](index.md), to previous step: [Basic usage](basic_usage.md) or to next step: [Implement new component](implement_components.md).
+Once that is complete, **ensure that all unit tests and the pre-commit process are successful**.
+
+Go back to [Index](index.md), to previous step: [Basic usage](basic_usage.md).

docs/index.md

@@ -11,7 +11,6 @@ List of steps to follow for new users of the library:
 * [Installation](installation.md): steps to install all the components need it.
 * [Basic usage](basic_usage.md): create a basic script with the different components.
 * [Create new component](create_components.md): guide to use the mate commands.
-* [Implement new component](implement_components.md): guide to implement your own component.
 
 ## Components
 

mkdocs.yml

@@ -10,7 +10,6 @@ nav:
     - Installation: installation.md
     - Basic usage: basic_usage.md
     - Create new component: create_components.md
-    - Implement new component: implement_components.md
   - Components:
     - Overall architecture: overall_architecture.md
     - Schemas: schemas.md

src/detectmatelibrary/common/core.py

@@ -119,7 +119,7 @@ def process(self, data: BaseSchema | bytes) -> BaseSchema | bytes | None:
             logger.info(f"<<{self.name}>> returns None")
             return None
 
-        logger.info(f"<<{self.name}>> processed:\n{output_}")
+        logger.debug(f"<<{self.name}>> processed:\n{output_}")
         return SchemaPipeline.postprocess(output_, is_byte=is_byte)
 
     def get_config(self) -> Dict[str, Any]:

src/detectmatelibrary/detectors/new_value_combo_detector.py

@@ -95,14 +95,17 @@ def detect(
         combo_dict = get_combo(configured_variables)
 
         overall_score = 0.0
-        for event_id, event_tracker in self.persistency.get_events_data().items():
+        current_event_id = input_["EventID"]
+        known_events = self.persistency.get_events_data()
+
+        if current_event_id in known_events:
+            event_tracker = known_events[current_event_id]
             for combo_key, multi_tracker in event_tracker.get_data().items():
-                # Get the value tuple for this combo key
                 value_tuple = combo_dict.get(combo_key)
                 if value_tuple is None:
                     continue
                 if value_tuple not in multi_tracker.unique_set:
-                    alerts[f"EventID {event_id}"] = (
+                    alerts[f"EventID {current_event_id} - {combo_key}"] = (
                         f"Unknown value combination: {value_tuple}"
                     )
                     overall_score += 1.0

src/detectmatelibrary/detectors/new_value_detector.py

@@ -59,13 +59,17 @@ def detect(
         configured_variables = get_configured_variables(input_, self.config.events)
         overall_score = 0.0
 
-        for event_id, event_tracker in self.persistency.get_events_data().items():
-            for event_id, multi_tracker in event_tracker.get_data().items():
-                value = configured_variables.get(event_id)
+        current_event_id = input_["EventID"]
+        known_events = self.persistency.get_events_data()
+
+        if current_event_id in known_events:
+            event_tracker = known_events[current_event_id]
+            for var_name, multi_tracker in event_tracker.get_data().items():
+                value = configured_variables.get(var_name)
                 if value is None:
                     continue
                 if value not in multi_tracker.unique_set:
-                    alerts[f"EventID {event_id}"] = (
+                    alerts[f"EventID {current_event_id} - {var_name}"] = (
                         f"Unknown value: '{value}'"
                     )
                     overall_score += 1.0

src/detectmatelibrary/utils/persistency/event_persistency.py

@@ -58,7 +58,23 @@ def get_event_data(self, event_id: int) -> Any | None:
         return data_structure.get_data() if data_structure is not None else None
 
     def get_events_data(self) -> Dict[int, EventDataStructure]:
-        """Retrieve the events' data."""
+        """Retrieve the events data that is currently stored.
+
+        Returns:
+            A dictionary mapping event IDs to their corresponding EventDataStructure instances.
+
+            Example:
+            {
+                1: EventTracker(data={
+                    'var_0': SingleTracker(...),
+                    'var_1': SingleTracker(...),
+                }),
+                2: EventTracker(data={
+                    'var_0': SingleTracker(...)
+                }),
+                ...
+            }
+        """
         return self.events_data
 
     def get_event_template(self, event_id: int) -> str | None:

src/tools/workspace/create_workspace.py

@@ -11,6 +11,10 @@
 TEMPLATE_DIR = BASE_DIR / "workspace" / "templates"
 
 META_FILES = ["LICENSE.md", ".gitignore", ".pre-commit-config.yaml"]
+DATA_FILES = {
+    "parser": "src/tools/workspace/templates/data/logs.json",
+    "detector": "src/tools/workspace/templates/data/parsed_log.json"
+}
 
 
 def copy_file(src: Path, dst: Path) -> None:
@@ -61,6 +65,7 @@ def create_tests(type_: str, name: str, workspace_root: Path, pkg_name: str) ->
     # replace the remaining occurrences of CustomParser/CustomDetector
     # with the new class name (inside the tests)
     content = content.replace(base_class, new_class)
+    content = content.replace(f'"custom_{type_}"', f'"{name}_{type_}"')
     content = content.rstrip() + "\n"
 
     test_file.write_text(content)
@@ -106,6 +111,7 @@ def create_workspace(type_: str, name: str, target_dir: Path) -> None:
         original_class = f"Custom{type_.capitalize()}"
         new_class = camelize(name)
         template_content = template_content.replace(original_class, new_class)
+        template_content = template_content.replace(f"custom_{type_}", f"{name}_{type_}")
 
         target_code_file.write_text(template_content)
 
@@ -116,6 +122,9 @@ def create_workspace(type_: str, name: str, target_dir: Path) -> None:
 
     create_tests(type_=type_, name=name, workspace_root=workspace_root, pkg_name=pkg_name)
 
+    # Copy data
+    copy_file(PROJECT_ROOT / DATA_FILES[type_], workspace_root / "data.json")
+
     # Copy meta/root files
     for file_name in META_FILES:
         src = PROJECT_ROOT / file_name

src/tools/workspace/templates/CustomDetector.py

@@ -2,6 +2,7 @@
 
 from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig
 from detectmatelibrary.utils.data_buffer import BufferMode
+from detectmatelibrary.helper.from_to import From
 from detectmatelibrary import schemas
 
 
@@ -54,3 +55,12 @@ def detect(
             output_["alertsObtain"]["type"] = "Anomaly detected by CustomDetector"  # Additional info
 
         return result
+
+
+if __name__ == "__main__":
+
+    print(detector := CustomDetector())
+
+    print("Running with data...")
+    for alerts in From.json(detector, "data.json"):
+        print(alerts)

src/tools/workspace/templates/CustomParser.py

@@ -1,6 +1,7 @@
 from typing import Any
 
 from detectmatelibrary.common.parser import CoreParser, CoreParserConfig
+from detectmatelibrary.helper.from_to import From
 from detectmatelibrary import schemas
 
 
@@ -43,3 +44,12 @@ def parse(
         output_["EventID"] = 2  # Number of the log template
         output_["variables"].extend(["dummy_variable"])  # Variables found in the log
         output_["template"] = "This is a dummy template"  # Log template
+
+
+if __name__ == "__main__":
+
+    print(parser := CustomParser())
+
+    print("Running with data...")
+    for parsed_log in From.json(parser, "data.json"):
+        print(parsed_log)