Development

.github/workflows/codeql.yml

@@ -0,0 +1,125 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL Advanced"
+
+on:
+  push:
+    branches: [ "development" ]
+  pull_request:
+    branches: [ "development" ]
+  schedule:
+    - cron: '23 23 * * 4'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: actions
+          build-mode: none
+        - language: python
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'rust', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Add any setup steps before running the `github/codeql-action/init` action.
+    # This includes steps like installing compilers or runtimes (`actions/setup-node`
+    # or others). This is typically only required for manual builds.
+    # - name: Setup runtime (example)
+    #   uses: actions/setup-example@v1
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v4
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - name: Run manual build steps
+      if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v4
+      with:
+        category: "/language:${{matrix.language}}"
+
+    - name: Install jq
+      if: runner.os == 'Linux'
+      run: sudo apt-get update && sudo apt-get install -y jq
+
+    - name: Fail if CodeQL found issues
+      shell: bash
+      run: |
+        if [ -f /home/runner/work/DetectMateLibrary/results/python.sarif ]; then
+          SARIF=/home/runner/work/DetectMateLibrary/results/python.sarif
+        elif [ -f /home/runner/work/DetectMateLibrary/results/actions.sarif ]; then
+          SARIF=/home/runner/work/DetectMateLibrary/results/actions.sarif
+        else
+          echo "SARIF file not found. Analysis may have failed."
+          exit 1
+        fi
+        count=$(jq '.runs[].results | length' $SARIF)
+        if [ "$count" -gt 0 ]; then
+          echo "CodeQL found $count issue(s)!"
+          jq -r '.runs[].results[] | "Rule: \(.ruleId)\nSeverity: \(.level)\nMessage: \(.message.text)\nFile: \(.locations[0].physicalLocation.artifactLocation.uri)\nLine: \(.locations[0].physicalLocation.region.startLine)\n---"' $SARIF
+          exit 1
+        else
+          echo "No CodeQL issues found"
+        fi

docs/installation.md

@@ -16,6 +16,12 @@ uv sync
 
 **Result**: the package is installed into the active Python environment and changes to the source tree are reflected immediately.
 
+To install it in a different venv as a library:
+
+```bash
+uv pip install --no-cache-dir <directory_detectmatelibrary>
+```
+
 ## Developer setup
 
 **Purpose**: prepare a development environment with test and lint tooling.

docs/schemas.md

@@ -122,7 +122,7 @@ from detectmatelibrary import schemas
 kwargs = load_somewhere()  # load the dict
 kwargs["log"] = "Test log"
 
-log_schema = LogSchema(kwargs)
+log_schema = schemas.LogSchema(kwargs)
 print(log_schema.log == "Test log")  # True
 ```
 
@@ -132,11 +132,11 @@ print(log_schema.log == "Test log")  # True
 ```python
 from detectmatelibrary import schemas
 
-log_schema = LogSchema()
+log_schema = schemas.LogSchema()
 log_schema.log = "Test log"
 print(log_schema["log"] == log_schema.log)  # True
 
-log_schema2 = LogSchema()
+log_schema2 = schemas.LogSchema()
 print(log_schema == log_schema2)  # False
 
 log_schema2.log = "Test log"
@@ -148,12 +148,12 @@ print(log_schema == log_schema2)  # True
 ```python
 from detectmatelibrary import schemas
 
-log_schema = LogSchema()
+log_schema = schemas.LogSchema()
 log_schema.log = "Test log"
 serialized = log_schema.serialize()
 print(isinstance(serialized, bytes))  # True
 
-new_log_schema = LogSchema()
+new_log_schema = schemas.LogSchema()
 new_log_schema.deserialize(serialized)
 print(new_log_schema.schema_id == log_schema.schema_id)  # True
 ```

pyproject.toml

@@ -3,14 +3,16 @@ name = "detectmatelibrary"
 version = "0.1.0"
 description = "DetectMate Library for log processing components"
 readme = "README.md"
+dynamic = ["authors"]
 requires-python = ">=3.12"
 dependencies = [
-    "drain3>=0.9.11",
     "protobuf>=6.32.1",
     "pydantic>=2.11.7",
     "pyyaml>=6.0.3",
     "regex>=2025.11.3",
     "numpy>=2.3.2",
+    "pandas>=2.3.2",
+    "polars>=1.38.1",
 ]
 
 [dependency-groups]
@@ -21,8 +23,6 @@ dev = [
     "prek>=0.2.8",
     "pytest>=8.4.2",
     "pytest-cov>=6.2.1",
-    "pandas>=2.3.2",
-    "polars>=1.38.1",
 ]
 
 [build-system]

setup.py

@@ -0,0 +1,28 @@
+from setuptools import setup, find_packages
+import tomllib
+
+
+def gather_dependencies(toml_path: str = "pyproject.toml") -> list[str]:
+    with open(toml_path, "rb") as f:
+        data = tomllib.load(f)
+
+    # Try Poetry first
+    poetry_deps = data.get("tool", {}).get("poetry", {}).get("dependencies", {})
+    if poetry_deps:
+        return [f"{dep}{version}" for dep, version in poetry_deps.items()]
+
+    # Fall back to PEP 621
+    project_deps: list[str] = data.get("project", {}).get("dependencies", [])
+    return project_deps
+
+
+setup(
+    name="detectmatelibrary",
+    version="0.1.0",
+    package_dir={"": "src"},
+    packages=find_packages(where="src"),
+    description="DetectMate Library for log processing components",
+    author="voice",
+    author_email="voice@example.com",
+    install_requires=gather_dependencies(),
+)

src/detectmatelibrary/common/_config/__init__.py

@@ -1,5 +1,5 @@
-from detectmatelibrary.common._config._compile import ConfigMethods, generate_detector_config
-from detectmatelibrary.common._config._formats import EventsConfig
+from ._compile import ConfigMethods, generate_detector_config
+from ._formats import EventsConfig
 
 __all__ = ["ConfigMethods", "generate_detector_config", "EventsConfig", "BasicConfig"]
 
@@ -59,6 +59,7 @@ def to_dict(self, method_id: str) -> Dict[str, Any]:
         # Collect all non-meta fields for params
         params = {}
         events_data = None
+        instances_data = None
 
         for field_name, field_value in self:
             # Skip meta fields
@@ -72,6 +73,13 @@ def to_dict(self, method_id: str) -> Dict[str, Any]:
                         events_data = field_value.to_dict()
                     else:
                         events_data = field_value
+            # Handle global instances specially (top-level, not in params)
+            # Serialized as "global" in YAML (Python field is "global_instances")
+            elif field_name == "global_instances" and field_value:
+                instances_data = {
+                    name: inst.to_dict()
+                    for name, inst in field_value.items()
+                }
             else:
                 # All other fields go into params
                 params[field_name] = field_value
@@ -80,6 +88,10 @@ def to_dict(self, method_id: str) -> Dict[str, Any]:
         if params:
             result["params"] = params
 
+        # Add global instances if they exist (serialized as "global" in YAML)
+        if instances_data is not None:
+            result["global"] = instances_data
+
         # Add events if they exist
         if events_data is not None:
             result["events"] = events_data

src/detectmatelibrary/common/_config/_compile.py

@@ -1,4 +1,4 @@
-from detectmatelibrary.common._config._formats import EventsConfig
+from detectmatelibrary.common._config._formats import EventsConfig, _EventInstance
 
 from typing import Any, Dict, List, Sequence, Tuple, Union
 import warnings
@@ -93,8 +93,9 @@ def check_type(config: Dict[str, Any], method_type: str) -> None:
     def process(config: Dict[str, Any]) -> Dict[str, Any]:
         has_params = "params" in config
         has_events = "events" in config
+        has_instances = "global" in config
 
-        if not has_params and not has_events and not config.get("auto_config", False):
+        if not has_params and not has_events and not has_instances and not config.get("auto_config", False):
             warnings.warn(MissingParamsWarning())
 
         if has_params:
@@ -108,11 +109,19 @@ def process(config: Dict[str, Any]) -> Dict[str, Any]:
         if has_events:
             config["events"] = EventsConfig._init(config["events"])
 
+        # Handle "global" key: event-ID-independent global instances
+        # Renamed to "global_instances" to avoid collision with Python keyword
+        if has_instances:
+            config["global_instances"] = {
+                name: _EventInstance._init(**data)
+                for name, data in config.pop("global").items()
+            }
+
         return config
 
 
 def generate_detector_config(
-    variable_selection: Dict[int, List[Union[str, Tuple[str, ...]]]],
+    variable_selection: Dict[int | str, List[Union[str, Tuple[str, ...]]]],
     detector_name: str,
     method_type: str,
     **additional_params: Any
@@ -158,7 +167,7 @@ def generate_detector_config(
     """
     var_pattern = re.compile(r"^var_(\d+)$")
 
-    events_config: Dict[int, Dict[str, Any]] = {}
+    events_config: Dict[int | str, Dict[str, Any]] = {}
 
     for event_id, variable_names in variable_selection.items():
         instances: Dict[str, Any] = {}