Re-organized prometheus exporters and client into a sub directory

.github/workflows/docker.yml

@@ -6,14 +6,14 @@ on:
     paths:
       - 'asap-common/installation/**'
       - 'asap-planner/**'
-      - 'asap-tools/prometheus-client/**'
+      - 'asap-tools/queriers/prometheus-client/**'
       - '.github/workflows/docker.yml'
   pull_request:
     branches: [ main ]
     paths:
       - 'asap-common/installation/**'
       - 'asap-planner/**'
-      - 'asap-tools/prometheus-client/**'
+      - 'asap-tools/queriers/prometheus-client/**'
       - '.github/workflows/docker.yml'
   workflow_dispatch:
 
@@ -49,7 +49,7 @@ jobs:
             .
 
       - name: Build PrometheusClient Docker image
-        working-directory: asap-tools/prometheus-client
+        working-directory: asap-tools/queriers/prometheus-client
         run: |
           docker build \
             -t sketchdb-prometheus-client:latest \

.github/workflows/python.yml

@@ -5,21 +5,21 @@ on:
     branches: [ main ]
     paths:
       - 'asap-sketch-ingest/**'
-      - 'asap-tools/prometheus-client/**'
+      - 'asap-tools/queriers/prometheus-client/**'
       - 'asap-planner/**'
       - 'asap-tools/**'
-      - 'asap-tools/prometheus-exporters/**'
+      - 'asap-tools/data-sources/prometheus-exporters/**'
       - 'asap-tools/execution-utilities/**'
       - 'asap-common/dependencies/py/**'
       - '.github/workflows/python.yml'
   pull_request:
     branches: [ main ]
     paths:
       - 'asap-sketch-ingest/**'
-      - 'asap-tools/prometheus-client/**'
+      - 'asap-tools/queriers/prometheus-client/**'
       - 'asap-planner/**'
       - 'asap-tools/**'
-      - 'asap-tools/prometheus-exporters/**'
+      - 'asap-tools/data-sources/prometheus-exporters/**'
       - 'asap-tools/execution-utilities/**'
       - 'asap-common/dependencies/py/**'
       - '.github/workflows/python.yml'
@@ -49,7 +49,7 @@ jobs:
               - 'asap-sketch-ingest/**'
               - 'asap-common/dependencies/py/**'
             prometheus_client:
-              - 'asap-tools/prometheus-client/**'
+              - 'asap-tools/queriers/prometheus-client/**'
               - 'asap-common/dependencies/py/**'
             controller:
               - 'asap-planner/**'
@@ -58,7 +58,7 @@ jobs:
               - 'asap-tools/**'
               - 'asap-common/dependencies/py/**'
             prometheus_exporters:
-              - 'asap-tools/prometheus-exporters/**'
+              - 'asap-tools/data-sources/prometheus-exporters/**'
               - 'asap-common/dependencies/py/**'
             execution_utilities:
               - 'asap-tools/execution-utilities/**'
@@ -110,19 +110,19 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install black==24.8.0 flake8==6.1.0 mypy types-requests types-PyYAML typing-extensions numpy prometheus-client urllib3
-        if [ -f asap-tools/prometheus-client/requirements.txt ]; then pip install -r asap-tools/prometheus-client/requirements.txt; fi
+        if [ -f asap-tools/queriers/prometheus-client/requirements.txt ]; then pip install -r asap-tools/queriers/prometheus-client/requirements.txt; fi
     - name: Check formatting with Black
-      working-directory: asap-tools/prometheus-client
+      working-directory: asap-tools/queriers/prometheus-client
       run: black --check --diff .
     - name: Lint with flake8
-      working-directory: asap-tools/prometheus-client
+      working-directory: asap-tools/queriers/prometheus-client
       run: |
         # Stop the build if there are Python syntax errors or undefined names
-        flake8 . --config=../../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics
+        flake8 . --config=../../../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics
         # Exit-zero treats all errors as warnings
-        flake8 . --config=../../.flake8 --count --exit-zero --max-complexity=10 --statistics
+        flake8 . --config=../../../.flake8 --count --exit-zero --max-complexity=10 --statistics
     - name: Type check with mypy
-      working-directory: asap-tools/prometheus-client
+      working-directory: asap-tools/queriers/prometheus-client
       run: mypy .
 
   test-controller:
@@ -200,22 +200,22 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install black==24.8.0 flake8==6.1.0 mypy isort
-        if [ -f asap-tools/prometheus-exporters/requirements.txt ]; then pip install -r asap-tools/prometheus-exporters/requirements.txt; fi
+        if [ -f asap-tools/data-sources/prometheus-exporters/requirements.txt ]; then pip install -r asap-tools/data-sources/prometheus-exporters/requirements.txt; fi
     - name: Check formatting with Black
-      working-directory: asap-tools/prometheus-exporters
+      working-directory: asap-tools/data-sources/prometheus-exporters
       run: black --check --diff .
     - name: Check import sorting with isort
-      working-directory: asap-tools/prometheus-exporters
+      working-directory: asap-tools/data-sources/prometheus-exporters
       run: isort --check-only --diff --settings-file .isort.cfg .
     - name: Lint with flake8
-      working-directory: asap-tools/prometheus-exporters
+      working-directory: asap-tools/data-sources/prometheus-exporters
       run: |
         # Stop the build if there are Python syntax errors or undefined names
-        flake8 . --config=../../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics
+        flake8 . --config=../../../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics
         # Exit-zero treats all errors as warnings
-        flake8 . --config=../../.flake8 --count --exit-zero --max-complexity=10 --statistics
+        flake8 . --config=../../../.flake8 --count --exit-zero --max-complexity=10 --statistics
     - name: Type check with mypy
-      working-directory: asap-tools/prometheus-exporters
+      working-directory: asap-tools/data-sources/prometheus-exporters
       run: mypy . --config-file=.mypy.ini
 
   test-execution-utilities:

.pre-commit-config.yaml

@@ -39,8 +39,8 @@ repos:
     hooks:
       - id: isort
         name: isort (python)
-        files: ^asap-tools/prometheus-exporters/.*\.py$
-        args: ["--settings-file", "asap-tools/prometheus-exporters/.isort.cfg"]
+        files: ^asap-tools/data-sources/prometheus-exporters/.*\.py$
+        args: ["--settings-file", "asap-tools/data-sources/prometheus-exporters/.isort.cfg"]
 
   - repo: https://github.com/pycqa/flake8
     rev: 6.1.0
@@ -52,12 +52,12 @@ repos:
     rev: "v1.14.1"
     hooks:
       - id: mypy
-        files: ^asap-tools/prometheus-exporters/.*\.py$
-        args: [--config-file=asap-tools/prometheus-exporters/.mypy.ini]
+        files: ^asap-tools/data-sources/prometheus-exporters/.*\.py$
+        args: [--config-file=asap-tools/data-sources/prometheus-exporters/.mypy.ini]
 
       - id: mypy
-        files: ^asap-tools/prometheus-client/
-        args: [--config-file=asap-tools/prometheus-client/pyproject.toml]
+        files: ^asap-tools/queriers/prometheus-client/
+        args: [--config-file=asap-tools/queriers/prometheus-client/pyproject.toml]
         additional_dependencies:
           - types-requests
           - types-PyYAML

asap-quickstart/set_num_exporters.py

@@ -45,7 +45,7 @@ def update_docker_compose(file_path: str, num_exporters: int) -> None:
     # Generate new exporter services
     exporter_template = """  fake-exporter-{idx}:
     build:
-      context: ../asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter
+      context: ../asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter
     container_name: asap-fake-exporter-{idx}
     hostname: fake-exporter-{idx}
     networks:

asap-tools/README.md

@@ -107,7 +107,7 @@ experiments:
 - mode: prometheus
 ```
 
-With this config, 2 experiments are run independently. In the first experiment, `asap-tools/prometheus-client` only sends queries to ASAP. After this experiment finishes, the infra is torn down. Then the second experiment is set up and `asap-tools/prometheus-client` sends queries only to Prometheus directly. In the second experiment (i.e. when `mode=prometheus`), none of ASAP's components are set up (apart from `asap-tools/prometheus-client`).
+With this config, 2 experiments are run independently. In the first experiment, `asap-tools/queriers/prometheus-client` only sends queries to ASAP. After this experiment finishes, the infra is torn down. Then the second experiment is set up and `asap-tools/queriers/prometheus-client` sends queries only to Prometheus directly. In the second experiment (i.e. when `mode=prometheus`), none of ASAP's components are set up (apart from `asap-tools/queriers/prometheus-client`).
 
 Post-experiment analysis:
 - Use `compare_costs.py` and `compare_latencies.py` from `$REPO_DIR/asap-tools/experiments/post_experiments/`.

asap-tools/components.conf

@@ -11,6 +11,6 @@ asap-planner
 #prometheus-kafka-adapter
 asap-sketch-ingest
 asap-quickstart
-asap-tools/prometheus-exporters
-asap-tools/prometheus-client
+asap-tools/data-sources/prometheus-exporters
+asap-tools/queriers/prometheus-client
 asap-tools/execution-utilities

asap-tools/docs/architecture.md

@@ -107,7 +107,7 @@ Start throughput/health monitors (if enabled)
 Wait for steady state (default 60s)
   ↓
 Run remote monitor (execute queries)
-  ├─ Execute PromQL queries via asap-tools/prometheus-client
+  ├─ Execute PromQL queries via asap-tools/queriers/prometheus-client
   ├─ Monitor process health
   ├─ Profile components (if enabled)
   └─ Record results
@@ -162,7 +162,7 @@ QueryEngine
   ↓
 PromQL Query Results
   ↓
-asap-tools/prometheus-client (logs results)
+asap-tools/queriers/prometheus-client (logs results)
 ``` -->
 
 ### SketchDB Mode with Ingest from Prometheus Remote Write
@@ -196,7 +196,7 @@ Prometheus TSDB
   ↓ (query)
 PromQL Query API
   ↓
-asap-tools/prometheus-client (logs results)
+asap-tools/queriers/prometheus-client (logs results)
 ```
 
 ## Services in the Experiment Framework

asap-tools/docs/deployment.md

@@ -67,7 +67,7 @@ The script executes these phases in order:
   - asap-planner (Docker image)
   - Arroyo (Node.js frontend + Rust binary + Docker image)
   - asap-sketch-ingest (Python scripts)
-  - asap-tools/prometheus-client, asap-tools/prometheus-exporters, asap-tools/execution-utilities, asap-tools/prometheus-benchmark
+  - asap-tools/queriers/prometheus-client, asap-tools/data-sources/prometheus-exporters, asap-tools/execution-utilities, asap-tools/prometheus-benchmark
 
 ### Directory Structure Created
 
@@ -150,8 +150,8 @@ asap-query-engine
 asap-planner
 asap-sketch-ingest
 asap-quickstart
-asap-tools/prometheus-exporters
-asap-tools/prometheus-client
+asap-tools/data-sources/prometheus-exporters
+asap-tools/queriers/prometheus-client
 asap-tools/execution-utilities
 ```
 
@@ -305,11 +305,11 @@ pip install -r requirements.txt  # jinja2 for templating
 ```
 **Deployment:** Python scripts, no Docker image
 
-#### 6. asap-tools/prometheus-client
+#### 6. asap-tools/queriers/prometheus-client
 **What:** Client for executing PromQL queries against Prometheus or SketchDB
 **Build Process:** Language-specific (Python or Rust)
 
-#### 7. asap-tools/prometheus-exporters, asap-tools/execution-utilities, asap-tools/prometheus-benchmark
+#### 7. asap-tools/data-sources/prometheus-exporters, asap-tools/execution-utilities, asap-tools/prometheus-benchmark
 **What:** Various testing and monitoring tools
 **Build Process:** Component-specific
 

asap-tools/docs/troubleshooting.md

@@ -35,7 +35,7 @@ for i in {0..N-1}; do ssh -o StrictHostKeyChecking=no <cloudlab_username>@node"$
 - `experiment_run_e2e.py` script continuously shows "Waiting for X seconds for remote monitor to finish" for multiple minutes (e.g. > 5 min)
 
 **Possible Causes:**
-- Could be a variety of causes, but usually indicates something wrong with `asap-tools/prometheus-client`, `asap-query-engine`, or `remote_monitor.py`
+- Could be a variety of causes, but usually indicates something wrong with `asap-tools/queriers/prometheus-client`, `asap-query-engine`, or `remote_monitor.py`
 
 **Solutions:**
 Look at log files for the above components on the cloudlab nodes. You can also check `docker logs`

asap-tools/experiments/experiment_utils/services/fake_exporters.py

@@ -139,6 +139,7 @@ def _start_bare_metal(
             constants.CLOUDLAB_HOME_DIR,
             "code",
             "asap-tools",
+            "data-sources",
             "prometheus-exporters",
             "fake_exporter",
             "fake_exporter_python",
@@ -395,7 +396,7 @@ def _start_bare_metal(
             )
             cmds.append(cmd)
 
-        cmd_dir = f"{constants.CLOUDLAB_HOME_DIR}/code/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter"
+        cmd_dir = f"{constants.CLOUDLAB_HOME_DIR}/code/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter"
 
         # Dump workload configuration to a file
         os.makedirs(

asap-tools/experiments/experiment_utils/services/prometheus_client_service.py

@@ -82,6 +82,7 @@ def _start_containerized(
             self.provider.get_home_dir(),
             "code",
             "asap-tools",
+            "queriers",
             "prometheus-client",
         )
         template_path = os.path.join(prometheus_client_dir, "docker-compose.yml.j2")
@@ -172,7 +173,11 @@ def _start_bare_metal(
             cmd += " --profile_prometheus_time {}".format(profile_prometheus_time)
 
         cmd_dir = os.path.join(
-            self.provider.get_home_dir(), "code", "asap-tools", "prometheus-client"
+            self.provider.get_home_dir(),
+            "code",
+            "asap-tools",
+            "queriers",
+            "prometheus-client",
         )
         utils.run_cmd(f"cd {cmd_dir}; {cmd}", popen=False)
 

asap-tools/installation/common/setup_dependencies.sh

@@ -23,7 +23,7 @@ else
   source "$HOME/.cargo/env"
 fi
 # (cd "$THIS_DIR/../../experiments/fake_exporter_rust/fake_exporter"; cargo build --release)
-(cd "$THIS_DIR/../../../asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter" && cargo build --release)
+(cd "$THIS_DIR/../../../asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter" && cargo build --release)
 
 DOCKER_DATA_DIR=/scratch/var_lib_docker
 

docs/01-getting-started/architecture.md

@@ -189,7 +189,7 @@ graph LR
 | **asap-planner** | Auto-determines sketch parameters | Python | `asap-planner/` |
 | **Kafka** | Message broker for sketch distribution | Apache Kafka | (external) |
 | **Prometheus** | Time-series database (existing) | Go | (external) |
-| **Exporters** | Generate synthetic metrics for testing | Rust/Python | `asap-tools/prometheus-exporters/` |
+| **Exporters** | Generate synthetic metrics for testing | Rust/Python | `asap-tools/data-sources/prometheus-exporters/` |
 | **asap-tools** | Experimental harness that uses Cloudlab | Python | `asap-tools/` |
 
 **Links to detailed documentation:**
@@ -270,11 +270,14 @@ ASAPQuery/
 │   └── utils/                # Decision logic
 │
 ├── asap-tools/               # Experiment framework & tooling
-│   ├── prometheus-exporters/ # Metric generators
-│   │   ├── fake_exporter/        # Rust/Python fake exporters
-│   │   ├── cluster_data_exporter/  # Real trace data
-│   │   ├── query_cost_exporter/  # Resource metrics
-│   │   └── query_latency_exporter/  # Latency metrics
+│   ├── data-sources/
+│   │   └── prometheus-exporters/ # Metric generators
+│   │       ├── fake_exporter/        # Rust/Python fake exporters
+│   │       ├── cluster_data_exporter/  # Real trace data
+│   │       ├── query_cost_exporter/  # Resource metrics
+│   │       └── query_latency_exporter/  # Latency metrics
+│   ├── queriers/
+│   │   └── prometheus-client/    # PromQL query client
 │   ├── experiments/
 │   │   ├── experiment_run_e2e.py  # Main orchestrator
 │   │   ├── config/           # Hydra configs

docs/02-components/README.md

@@ -10,7 +10,7 @@ This document provides an overview of all ASAP components and links to detailed
 | **Arroyo** | Stream processing for building sketches | Rust (forked) | [Details](arroyo.md) · [Code](https://github.com/ProjectASAP/arroyo) |
 | **asap-sketch-ingest** | Configures Arroyo pipelines from config | Python | [Details](arroyosketch.md) · [Code](../../asap-sketch-ingest/) · [README](../../asap-sketch-ingest/README.md) |
 | **asap-planner** | Auto-determines sketch parameters | Python | [Details](controller.md) · [Code](../../asap-planner/) · [README](../../asap-planner/README.md) |
-| **Exporters** | Generate synthetic metrics for testing | Rust/Python | [Details](exporters.md) · [Code](../../asap-tools/prometheus-exporters/) · [README](../../asap-tools/prometheus-exporters/README.md) |
+| **Exporters** | Generate synthetic metrics for testing | Rust/Python | [Details](exporters.md) · [Code](../../asap-tools/data-sources/prometheus-exporters/) · [README](../../asap-tools/data-sources/prometheus-exporters/README.md) |
 | **asap-tools** | Experiment framework for CloudLab | Python | [Details](utilities.md) · [Code](../../asap-tools/) · [Docs](../../asap-tools/docs/architecture.md) |
 
 ## Component Interaction
@@ -164,5 +164,5 @@ For implementation details, see READMEs co-located with code:
 - [asap-query-engine/docs/](../../asap-query-engine/docs/README.md) - Extensibility guides
 - [asap-planner/README.md](../../asap-planner/README.md) - asap-planner internals
 - [asap-sketch-ingest/README.md](../../asap-sketch-ingest/README.md) - Pipeline config internals
-- [asap-tools/prometheus-exporters/README.md](../../asap-tools/prometheus-exporters/README.md) - Exporter implementations
+- [asap-tools/data-sources/prometheus-exporters/README.md](../../asap-tools/data-sources/prometheus-exporters/README.md) - Exporter implementations
 - [asap-tools/docs/](../../asap-tools/docs/architecture.md) - Experiment framework architecture

docs/03-how-to-guides/development/adapt-pr-after-reorg.md

@@ -15,8 +15,8 @@ This guide walks you through rebasing onto the new structure.
 | `ArroyoSketch/` | `asap-sketch-ingest/` |
 | `Controller/` | `asap-planner/` |
 | `Utilities/` | `asap-tools/` |
-| `PrometheusExporters/` | `asap-tools/prometheus-exporters/` |
-| `PrometheusClient/` | `asap-tools/prometheus-client/` |
+| `PrometheusExporters/` | `asap-tools/data-sources/prometheus-exporters/` |
+| `PrometheusClient/` | `asap-tools/queriers/prometheus-client/` |
 | `ExecutionUtilities/` | `asap-tools/execution-utilities/` |
 | `quickstart/` | `asap-quickstart/` |
 | `sketch-core/` | `asap-common/sketch-core/` |
@@ -122,7 +122,7 @@ git push --force-with-lease origin your-branch-name
 
 Key CI jobs to watch:
 - **Rust** — Cargo build and tests for `asap-query-engine` and `asap-common`
-- **Python** — Linting/type-checking for `asap-tools/prometheus-client` and `asap-planner`
+- **Python** — Linting/type-checking for `asap-tools/queriers/prometheus-client` and `asap-planner`
 - **Docker** — Image builds for `asap-query-engine`, `asap-planner`, `asap-sketch-ingest`
 
 ---
@@ -143,8 +143,8 @@ sed -i 's|CommonDependencies|asap-common|g' your_file.toml
 sed -i 's|"Utilities"|"asap-tools"|g' your_file.py
 sed -i 's|"Controller"|"asap-planner"|g' your_file.py
 sed -i 's|"ArroyoSketch"|"asap-sketch-ingest"|g' your_file.py
-sed -i 's|"PrometheusClient"|"asap-tools/prometheus-client"|g' your_file.py
-sed -i 's|"PrometheusExporters"|"asap-tools/prometheus-exporters"|g' your_file.py
+sed -i 's|"PrometheusClient"|"asap-tools/queriers/prometheus-client"|g' your_file.py
+sed -i 's|"PrometheusExporters"|"asap-tools/data-sources/prometheus-exporters"|g' your_file.py
 
 # Rename in shell scripts
 sed -i 's|/Utilities/|/asap-tools/|g' your_script.sh