diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 94c38ae..ad39f01 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,14 +6,14 @@ on: paths: - 'asap-common/installation/**' - 'asap-planner/**' - - 'asap-tools/prometheus-client/**' + - 'asap-tools/queriers/prometheus-client/**' - '.github/workflows/docker.yml' pull_request: branches: [ main ] paths: - 'asap-common/installation/**' - 'asap-planner/**' - - 'asap-tools/prometheus-client/**' + - 'asap-tools/queriers/prometheus-client/**' - '.github/workflows/docker.yml' workflow_dispatch: @@ -49,7 +49,7 @@ jobs: . - name: Build PrometheusClient Docker image - working-directory: asap-tools/prometheus-client + working-directory: asap-tools/queriers/prometheus-client run: | docker build \ -t sketchdb-prometheus-client:latest \ diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 6ea58e8..bd4ac21 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -5,10 +5,10 @@ on: branches: [ main ] paths: - 'asap-sketch-ingest/**' - - 'asap-tools/prometheus-client/**' + - 'asap-tools/queriers/prometheus-client/**' - 'asap-planner/**' - 'asap-tools/**' - - 'asap-tools/prometheus-exporters/**' + - 'asap-tools/data-sources/prometheus-exporters/**' - 'asap-tools/execution-utilities/**' - 'asap-common/dependencies/py/**' - '.github/workflows/python.yml' @@ -16,10 +16,10 @@ on: branches: [ main ] paths: - 'asap-sketch-ingest/**' - - 'asap-tools/prometheus-client/**' + - 'asap-tools/queriers/prometheus-client/**' - 'asap-planner/**' - 'asap-tools/**' - - 'asap-tools/prometheus-exporters/**' + - 'asap-tools/data-sources/prometheus-exporters/**' - 'asap-tools/execution-utilities/**' - 'asap-common/dependencies/py/**' - '.github/workflows/python.yml' @@ -49,7 +49,7 @@ jobs: - 'asap-sketch-ingest/**' - 'asap-common/dependencies/py/**' prometheus_client: - - 'asap-tools/prometheus-client/**' + - 'asap-tools/queriers/prometheus-client/**' - 'asap-common/dependencies/py/**' controller: - 'asap-planner/**' @@ -58,7 +58,7 @@ jobs: - 'asap-tools/**' - 'asap-common/dependencies/py/**' prometheus_exporters: - - 'asap-tools/prometheus-exporters/**' + - 'asap-tools/data-sources/prometheus-exporters/**' - 'asap-common/dependencies/py/**' execution_utilities: - 'asap-tools/execution-utilities/**' @@ -110,19 +110,19 @@ jobs: run: | python -m pip install --upgrade pip pip install black==24.8.0 flake8==6.1.0 mypy types-requests types-PyYAML typing-extensions numpy prometheus-client urllib3 - if [ -f asap-tools/prometheus-client/requirements.txt ]; then pip install -r asap-tools/prometheus-client/requirements.txt; fi + if [ -f asap-tools/queriers/prometheus-client/requirements.txt ]; then pip install -r asap-tools/queriers/prometheus-client/requirements.txt; fi - name: Check formatting with Black - working-directory: asap-tools/prometheus-client + working-directory: asap-tools/queriers/prometheus-client run: black --check --diff . - name: Lint with flake8 - working-directory: asap-tools/prometheus-client + working-directory: asap-tools/queriers/prometheus-client run: | # Stop the build if there are Python syntax errors or undefined names - flake8 . --config=../../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 . --config=../../../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics # Exit-zero treats all errors as warnings - flake8 . --config=../../.flake8 --count --exit-zero --max-complexity=10 --statistics + flake8 . --config=../../../.flake8 --count --exit-zero --max-complexity=10 --statistics - name: Type check with mypy - working-directory: asap-tools/prometheus-client + working-directory: asap-tools/queriers/prometheus-client run: mypy . test-controller: @@ -200,22 +200,22 @@ jobs: run: | python -m pip install --upgrade pip pip install black==24.8.0 flake8==6.1.0 mypy isort - if [ -f asap-tools/prometheus-exporters/requirements.txt ]; then pip install -r asap-tools/prometheus-exporters/requirements.txt; fi + if [ -f asap-tools/data-sources/prometheus-exporters/requirements.txt ]; then pip install -r asap-tools/data-sources/prometheus-exporters/requirements.txt; fi - name: Check formatting with Black - working-directory: asap-tools/prometheus-exporters + working-directory: asap-tools/data-sources/prometheus-exporters run: black --check --diff . - name: Check import sorting with isort - working-directory: asap-tools/prometheus-exporters + working-directory: asap-tools/data-sources/prometheus-exporters run: isort --check-only --diff --settings-file .isort.cfg . - name: Lint with flake8 - working-directory: asap-tools/prometheus-exporters + working-directory: asap-tools/data-sources/prometheus-exporters run: | # Stop the build if there are Python syntax errors or undefined names - flake8 . --config=../../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 . --config=../../../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics # Exit-zero treats all errors as warnings - flake8 . --config=../../.flake8 --count --exit-zero --max-complexity=10 --statistics + flake8 . --config=../../../.flake8 --count --exit-zero --max-complexity=10 --statistics - name: Type check with mypy - working-directory: asap-tools/prometheus-exporters + working-directory: asap-tools/data-sources/prometheus-exporters run: mypy . --config-file=.mypy.ini test-execution-utilities: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dd4314f..700bbec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,8 +39,8 @@ repos: hooks: - id: isort name: isort (python) - files: ^asap-tools/prometheus-exporters/.*\.py$ - args: ["--settings-file", "asap-tools/prometheus-exporters/.isort.cfg"] + files: ^asap-tools/data-sources/prometheus-exporters/.*\.py$ + args: ["--settings-file", "asap-tools/data-sources/prometheus-exporters/.isort.cfg"] - repo: https://github.com/pycqa/flake8 rev: 6.1.0 @@ -52,12 +52,12 @@ repos: rev: "v1.14.1" hooks: - id: mypy - files: ^asap-tools/prometheus-exporters/.*\.py$ - args: [--config-file=asap-tools/prometheus-exporters/.mypy.ini] + files: ^asap-tools/data-sources/prometheus-exporters/.*\.py$ + args: [--config-file=asap-tools/data-sources/prometheus-exporters/.mypy.ini] - id: mypy - files: ^asap-tools/prometheus-client/ - args: [--config-file=asap-tools/prometheus-client/pyproject.toml] + files: ^asap-tools/queriers/prometheus-client/ + args: [--config-file=asap-tools/queriers/prometheus-client/pyproject.toml] additional_dependencies: - types-requests - types-PyYAML diff --git a/asap-quickstart/set_num_exporters.py b/asap-quickstart/set_num_exporters.py index 194ad9d..3ececec 100755 --- a/asap-quickstart/set_num_exporters.py +++ b/asap-quickstart/set_num_exporters.py @@ -45,7 +45,7 @@ def update_docker_compose(file_path: str, num_exporters: int) -> None: # Generate new exporter services exporter_template = """ fake-exporter-{idx}: build: - context: ../asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter + context: ../asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter container_name: asap-fake-exporter-{idx} hostname: fake-exporter-{idx} networks: diff --git a/asap-tools/README.md b/asap-tools/README.md index 0b411ef..472e679 100644 --- a/asap-tools/README.md +++ b/asap-tools/README.md @@ -107,7 +107,7 @@ experiments: - mode: prometheus ``` -With this config, 2 experiments are run independently. In the first experiment, `asap-tools/prometheus-client` only sends queries to ASAP. After this experiment finishes, the infra is torn down. Then the second experiment is set up and `asap-tools/prometheus-client` sends queries only to Prometheus directly. In the second experiment (i.e. when `mode=prometheus`), none of ASAP's components are set up (apart from `asap-tools/prometheus-client`). +With this config, 2 experiments are run independently. In the first experiment, `asap-tools/queriers/prometheus-client` only sends queries to ASAP. After this experiment finishes, the infra is torn down. Then the second experiment is set up and `asap-tools/queriers/prometheus-client` sends queries only to Prometheus directly. In the second experiment (i.e. when `mode=prometheus`), none of ASAP's components are set up (apart from `asap-tools/queriers/prometheus-client`). Post-experiment analysis: - Use `compare_costs.py` and `compare_latencies.py` from `$REPO_DIR/asap-tools/experiments/post_experiments/`. diff --git a/asap-tools/components.conf b/asap-tools/components.conf index abb3e04..5f325d8 100644 --- a/asap-tools/components.conf +++ b/asap-tools/components.conf @@ -11,6 +11,6 @@ asap-planner #prometheus-kafka-adapter asap-sketch-ingest asap-quickstart -asap-tools/prometheus-exporters -asap-tools/prometheus-client +asap-tools/data-sources/prometheus-exporters +asap-tools/queriers/prometheus-client asap-tools/execution-utilities diff --git a/asap-tools/prometheus-exporters/.gitignore b/asap-tools/data-sources/prometheus-exporters/.gitignore similarity index 100% rename from asap-tools/prometheus-exporters/.gitignore rename to asap-tools/data-sources/prometheus-exporters/.gitignore diff --git a/asap-tools/prometheus-exporters/.isort.cfg b/asap-tools/data-sources/prometheus-exporters/.isort.cfg similarity index 100% rename from asap-tools/prometheus-exporters/.isort.cfg rename to asap-tools/data-sources/prometheus-exporters/.isort.cfg diff --git a/asap-tools/prometheus-exporters/.mypy.ini b/asap-tools/data-sources/prometheus-exporters/.mypy.ini similarity index 100% rename from asap-tools/prometheus-exporters/.mypy.ini rename to asap-tools/data-sources/prometheus-exporters/.mypy.ini diff --git a/asap-tools/prometheus-client/LICENSE b/asap-tools/data-sources/prometheus-exporters/LICENSE similarity index 100% rename from asap-tools/prometheus-client/LICENSE rename to asap-tools/data-sources/prometheus-exporters/LICENSE diff --git a/asap-tools/prometheus-exporters/README.md b/asap-tools/data-sources/prometheus-exporters/README.md similarity index 100% rename from asap-tools/prometheus-exporters/README.md rename to asap-tools/data-sources/prometheus-exporters/README.md diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/.dockerignore b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/.dockerignore similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/.dockerignore rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/.dockerignore diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/.gitignore b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/.gitignore similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/.gitignore rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/.gitignore diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/Cargo.toml b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/Cargo.toml similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/Cargo.toml rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/Cargo.toml diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/Dockerfile b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/Dockerfile similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/Dockerfile rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/Dockerfile diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/README.md b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/README.md similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/README.md rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/README.md diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/docker-compose.yml.j2 b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/docker-compose.yml.j2 similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/docker-compose.yml.j2 rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/docker-compose.yml.j2 diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-msresource-2021-docker-compose.yml b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-msresource-2021-docker-compose.yml similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-msresource-2021-docker-compose.yml rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-msresource-2021-docker-compose.yml diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-msresource-2022-docker-compose.yml b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-msresource-2022-docker-compose.yml similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-msresource-2022-docker-compose.yml rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-msresource-2022-docker-compose.yml diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-node-2021-docker-compose.yml b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-node-2021-docker-compose.yml similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-node-2021-docker-compose.yml rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-node-2021-docker-compose.yml diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-node-2022-docker-compose.yml b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-node-2022-docker-compose.yml similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-node-2022-docker-compose.yml rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-node-2022-docker-compose.yml diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/base-docker-compose.yml b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/docker_compose_frames/base-docker-compose.yml similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/base-docker-compose.yml rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/docker_compose_frames/base-docker-compose.yml diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/google-docker-compose.yml b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/docker_compose_frames/google-docker-compose.yml similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/google-docker-compose.yml rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/docker_compose_frames/google-docker-compose.yml diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/installation/install.sh b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/installation/install.sh similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/installation/install.sh rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/installation/install.sh diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/scripts/generate_docker_compose.py b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/scripts/generate_docker_compose.py similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/scripts/generate_docker_compose.py rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/scripts/generate_docker_compose.py diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/scripts/requirements.txt b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/scripts/requirements.txt similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/scripts/requirements.txt rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/scripts/requirements.txt diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics.rs b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics.rs similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics.rs rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics.rs diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics/ms_resource.rs b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics/ms_resource.rs similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics/ms_resource.rs rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics/ms_resource.rs diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics/node.rs b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics/node.rs similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics/node.rs rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics/node.rs diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/src/google_metrics.rs b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/src/google_metrics.rs similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/src/google_metrics.rs rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/src/google_metrics.rs diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/src/main.rs b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/src/main.rs similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/src/main.rs rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/src/main.rs diff --git a/asap-tools/prometheus-exporters/cluster_data_exporter/src/utilities.rs b/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/src/utilities.rs similarity index 100% rename from asap-tools/prometheus-exporters/cluster_data_exporter/src/utilities.rs rename to asap-tools/data-sources/prometheus-exporters/cluster_data_exporter/src/utilities.rs diff --git a/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/Dockerfile b/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_python/Dockerfile similarity index 100% rename from asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/Dockerfile rename to asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_python/Dockerfile diff --git a/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/docker-compose.yml.j2 b/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_python/docker-compose.yml.j2 similarity index 100% rename from asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/docker-compose.yml.j2 rename to asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_python/docker-compose.yml.j2 diff --git a/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/fake-exporter-python-cli-compose.yml.j2 b/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_python/fake-exporter-python-cli-compose.yml.j2 similarity index 100% rename from asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/fake-exporter-python-cli-compose.yml.j2 rename to asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_python/fake-exporter-python-cli-compose.yml.j2 diff --git a/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/fake_exporter.py b/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_python/fake_exporter.py similarity index 100% rename from asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/fake_exporter.py rename to asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_python/fake_exporter.py diff --git a/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/installation/install.sh b/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_python/installation/install.sh similarity index 100% rename from asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/installation/install.sh rename to asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_python/installation/install.sh diff --git a/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/requirements.txt b/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_python/requirements.txt similarity index 100% rename from asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/requirements.txt rename to asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_python/requirements.txt diff --git a/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Cargo.lock b/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Cargo.lock similarity index 100% rename from asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Cargo.lock rename to asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Cargo.lock diff --git a/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Cargo.toml b/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Cargo.toml similarity index 100% rename from asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Cargo.toml rename to asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Cargo.toml diff --git a/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Dockerfile b/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Dockerfile similarity index 100% rename from asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Dockerfile rename to asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Dockerfile diff --git a/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/docker-compose.yml.j2 b/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/docker-compose.yml.j2 similarity index 100% rename from asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/docker-compose.yml.j2 rename to asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/docker-compose.yml.j2 diff --git a/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/fake-exporter-rust-cli-compose.yml.j2 b/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/fake-exporter-rust-cli-compose.yml.j2 similarity index 100% rename from asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/fake-exporter-rust-cli-compose.yml.j2 rename to asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/fake-exporter-rust-cli-compose.yml.j2 diff --git a/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/installation/install.sh b/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/installation/install.sh similarity index 100% rename from asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/installation/install.sh rename to asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/installation/install.sh diff --git a/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/src/main.rs b/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/src/main.rs similarity index 100% rename from asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/src/main.rs rename to asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/src/main.rs diff --git a/asap-tools/prometheus-exporters/fake_kafka_exporter/Cargo.lock b/asap-tools/data-sources/prometheus-exporters/fake_kafka_exporter/Cargo.lock similarity index 100% rename from asap-tools/prometheus-exporters/fake_kafka_exporter/Cargo.lock rename to asap-tools/data-sources/prometheus-exporters/fake_kafka_exporter/Cargo.lock diff --git a/asap-tools/prometheus-exporters/fake_kafka_exporter/Cargo.toml b/asap-tools/data-sources/prometheus-exporters/fake_kafka_exporter/Cargo.toml similarity index 100% rename from asap-tools/prometheus-exporters/fake_kafka_exporter/Cargo.toml rename to asap-tools/data-sources/prometheus-exporters/fake_kafka_exporter/Cargo.toml diff --git a/asap-tools/prometheus-exporters/fake_kafka_exporter/Dockerfile b/asap-tools/data-sources/prometheus-exporters/fake_kafka_exporter/Dockerfile similarity index 100% rename from asap-tools/prometheus-exporters/fake_kafka_exporter/Dockerfile rename to asap-tools/data-sources/prometheus-exporters/fake_kafka_exporter/Dockerfile diff --git a/asap-tools/prometheus-exporters/fake_kafka_exporter/installation/install.sh b/asap-tools/data-sources/prometheus-exporters/fake_kafka_exporter/installation/install.sh similarity index 100% rename from asap-tools/prometheus-exporters/fake_kafka_exporter/installation/install.sh rename to asap-tools/data-sources/prometheus-exporters/fake_kafka_exporter/installation/install.sh diff --git a/asap-tools/prometheus-exporters/fake_kafka_exporter/src/main.rs b/asap-tools/data-sources/prometheus-exporters/fake_kafka_exporter/src/main.rs similarity index 100% rename from asap-tools/prometheus-exporters/fake_kafka_exporter/src/main.rs rename to asap-tools/data-sources/prometheus-exporters/fake_kafka_exporter/src/main.rs diff --git a/asap-tools/prometheus-exporters/installation/install.sh b/asap-tools/data-sources/prometheus-exporters/installation/install.sh similarity index 100% rename from asap-tools/prometheus-exporters/installation/install.sh rename to asap-tools/data-sources/prometheus-exporters/installation/install.sh diff --git a/asap-tools/prometheus-exporters/query_cost_exporter/QueryCostExporter.py b/asap-tools/data-sources/prometheus-exporters/query_cost_exporter/QueryCostExporter.py similarity index 100% rename from asap-tools/prometheus-exporters/query_cost_exporter/QueryCostExporter.py rename to asap-tools/data-sources/prometheus-exporters/query_cost_exporter/QueryCostExporter.py diff --git a/asap-tools/prometheus-exporters/query_cost_exporter/process/ProcessMonitorHook.py b/asap-tools/data-sources/prometheus-exporters/query_cost_exporter/process/ProcessMonitorHook.py similarity index 100% rename from asap-tools/prometheus-exporters/query_cost_exporter/process/ProcessMonitorHook.py rename to asap-tools/data-sources/prometheus-exporters/query_cost_exporter/process/ProcessMonitorHook.py diff --git a/asap-tools/prometheus-exporters/query_cost_exporter/process/__init__.py b/asap-tools/data-sources/prometheus-exporters/query_cost_exporter/process/__init__.py similarity index 100% rename from asap-tools/prometheus-exporters/query_cost_exporter/process/__init__.py rename to asap-tools/data-sources/prometheus-exporters/query_cost_exporter/process/__init__.py diff --git a/asap-tools/prometheus-exporters/query_cost_exporter/query_cost.py b/asap-tools/data-sources/prometheus-exporters/query_cost_exporter/query_cost.py similarity index 100% rename from asap-tools/prometheus-exporters/query_cost_exporter/query_cost.py rename to asap-tools/data-sources/prometheus-exporters/query_cost_exporter/query_cost.py diff --git a/asap-tools/prometheus-exporters/query_cost_exporter/requirements.txt b/asap-tools/data-sources/prometheus-exporters/query_cost_exporter/requirements.txt similarity index 100% rename from asap-tools/prometheus-exporters/query_cost_exporter/requirements.txt rename to asap-tools/data-sources/prometheus-exporters/query_cost_exporter/requirements.txt diff --git a/asap-tools/prometheus-exporters/query_latency_exporter/QueryLatencyExporter.py b/asap-tools/data-sources/prometheus-exporters/query_latency_exporter/QueryLatencyExporter.py similarity index 100% rename from asap-tools/prometheus-exporters/query_latency_exporter/QueryLatencyExporter.py rename to asap-tools/data-sources/prometheus-exporters/query_latency_exporter/QueryLatencyExporter.py diff --git a/asap-tools/prometheus-exporters/query_latency_exporter/requirements.txt b/asap-tools/data-sources/prometheus-exporters/query_latency_exporter/requirements.txt similarity index 100% rename from asap-tools/prometheus-exporters/query_latency_exporter/requirements.txt rename to asap-tools/data-sources/prometheus-exporters/query_latency_exporter/requirements.txt diff --git a/asap-tools/docs/architecture.md b/asap-tools/docs/architecture.md index 5c49627..0a119e0 100644 --- a/asap-tools/docs/architecture.md +++ b/asap-tools/docs/architecture.md @@ -107,7 +107,7 @@ Start throughput/health monitors (if enabled) Wait for steady state (default 60s) ↓ Run remote monitor (execute queries) - ├─ Execute PromQL queries via asap-tools/prometheus-client + ├─ Execute PromQL queries via asap-tools/queriers/prometheus-client ├─ Monitor process health ├─ Profile components (if enabled) └─ Record results @@ -162,7 +162,7 @@ QueryEngine ↓ PromQL Query Results ↓ -asap-tools/prometheus-client (logs results) +asap-tools/queriers/prometheus-client (logs results) ``` --> ### SketchDB Mode with Ingest from Prometheus Remote Write @@ -196,7 +196,7 @@ Prometheus TSDB ↓ (query) PromQL Query API ↓ -asap-tools/prometheus-client (logs results) +asap-tools/queriers/prometheus-client (logs results) ``` ## Services in the Experiment Framework diff --git a/asap-tools/docs/deployment.md b/asap-tools/docs/deployment.md index bdd62df..8ec76f0 100644 --- a/asap-tools/docs/deployment.md +++ b/asap-tools/docs/deployment.md @@ -67,7 +67,7 @@ The script executes these phases in order: - asap-planner (Docker image) - Arroyo (Node.js frontend + Rust binary + Docker image) - asap-sketch-ingest (Python scripts) - - asap-tools/prometheus-client, asap-tools/prometheus-exporters, asap-tools/execution-utilities, asap-tools/prometheus-benchmark + - asap-tools/queriers/prometheus-client, asap-tools/data-sources/prometheus-exporters, asap-tools/execution-utilities, asap-tools/prometheus-benchmark ### Directory Structure Created @@ -150,8 +150,8 @@ asap-query-engine asap-planner asap-sketch-ingest asap-quickstart -asap-tools/prometheus-exporters -asap-tools/prometheus-client +asap-tools/data-sources/prometheus-exporters +asap-tools/queriers/prometheus-client asap-tools/execution-utilities ``` @@ -305,11 +305,11 @@ pip install -r requirements.txt # jinja2 for templating ``` **Deployment:** Python scripts, no Docker image -#### 6. asap-tools/prometheus-client +#### 6. asap-tools/queriers/prometheus-client **What:** Client for executing PromQL queries against Prometheus or SketchDB **Build Process:** Language-specific (Python or Rust) -#### 7. asap-tools/prometheus-exporters, asap-tools/execution-utilities, asap-tools/prometheus-benchmark +#### 7. asap-tools/data-sources/prometheus-exporters, asap-tools/execution-utilities, asap-tools/prometheus-benchmark **What:** Various testing and monitoring tools **Build Process:** Component-specific diff --git a/asap-tools/docs/troubleshooting.md b/asap-tools/docs/troubleshooting.md index 85da113..99aef08 100644 --- a/asap-tools/docs/troubleshooting.md +++ b/asap-tools/docs/troubleshooting.md @@ -35,7 +35,7 @@ for i in {0..N-1}; do ssh -o StrictHostKeyChecking=no @node"$ - `experiment_run_e2e.py` script continuously shows "Waiting for X seconds for remote monitor to finish" for multiple minutes (e.g. > 5 min) **Possible Causes:** -- Could be a variety of causes, but usually indicates something wrong with `asap-tools/prometheus-client`, `asap-query-engine`, or `remote_monitor.py` +- Could be a variety of causes, but usually indicates something wrong with `asap-tools/queriers/prometheus-client`, `asap-query-engine`, or `remote_monitor.py` **Solutions:** Look at log files for the above components on the cloudlab nodes. You can also check `docker logs` diff --git a/asap-tools/experiments/experiment_utils/services/fake_exporters.py b/asap-tools/experiments/experiment_utils/services/fake_exporters.py index 5a897cd..8d14ac9 100644 --- a/asap-tools/experiments/experiment_utils/services/fake_exporters.py +++ b/asap-tools/experiments/experiment_utils/services/fake_exporters.py @@ -139,6 +139,7 @@ def _start_bare_metal( constants.CLOUDLAB_HOME_DIR, "code", "asap-tools", + "data-sources", "prometheus-exporters", "fake_exporter", "fake_exporter_python", @@ -395,7 +396,7 @@ def _start_bare_metal( ) cmds.append(cmd) - cmd_dir = f"{constants.CLOUDLAB_HOME_DIR}/code/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter" + cmd_dir = f"{constants.CLOUDLAB_HOME_DIR}/code/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter" # Dump workload configuration to a file os.makedirs( diff --git a/asap-tools/experiments/experiment_utils/services/prometheus_client_service.py b/asap-tools/experiments/experiment_utils/services/prometheus_client_service.py index 7a651b2..ac0512b 100644 --- a/asap-tools/experiments/experiment_utils/services/prometheus_client_service.py +++ b/asap-tools/experiments/experiment_utils/services/prometheus_client_service.py @@ -82,6 +82,7 @@ def _start_containerized( self.provider.get_home_dir(), "code", "asap-tools", + "queriers", "prometheus-client", ) template_path = os.path.join(prometheus_client_dir, "docker-compose.yml.j2") @@ -172,7 +173,11 @@ def _start_bare_metal( cmd += " --profile_prometheus_time {}".format(profile_prometheus_time) cmd_dir = os.path.join( - self.provider.get_home_dir(), "code", "asap-tools", "prometheus-client" + self.provider.get_home_dir(), + "code", + "asap-tools", + "queriers", + "prometheus-client", ) utils.run_cmd(f"cd {cmd_dir}; {cmd}", popen=False) diff --git a/asap-tools/installation/common/setup_dependencies.sh b/asap-tools/installation/common/setup_dependencies.sh index 36d340e..5bd7c1b 100755 --- a/asap-tools/installation/common/setup_dependencies.sh +++ b/asap-tools/installation/common/setup_dependencies.sh @@ -23,7 +23,7 @@ else source "$HOME/.cargo/env" fi # (cd "$THIS_DIR/../../experiments/fake_exporter_rust/fake_exporter"; cargo build --release) -(cd "$THIS_DIR/../../../asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter" && cargo build --release) +(cd "$THIS_DIR/../../../asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter" && cargo build --release) DOCKER_DATA_DIR=/scratch/var_lib_docker diff --git a/asap-tools/prometheus-client/.dockerignore b/asap-tools/queriers/prometheus-client/.dockerignore similarity index 100% rename from asap-tools/prometheus-client/.dockerignore rename to asap-tools/queriers/prometheus-client/.dockerignore diff --git a/asap-tools/prometheus-client/.gitignore b/asap-tools/queriers/prometheus-client/.gitignore similarity index 100% rename from asap-tools/prometheus-client/.gitignore rename to asap-tools/queriers/prometheus-client/.gitignore diff --git a/asap-tools/prometheus-client/Dockerfile b/asap-tools/queriers/prometheus-client/Dockerfile similarity index 100% rename from asap-tools/prometheus-client/Dockerfile rename to asap-tools/queriers/prometheus-client/Dockerfile diff --git a/asap-tools/prometheus-exporters/LICENSE b/asap-tools/queriers/prometheus-client/LICENSE similarity index 100% rename from asap-tools/prometheus-exporters/LICENSE rename to asap-tools/queriers/prometheus-client/LICENSE diff --git a/asap-tools/prometheus-client/classes/QueryLatencyExporter.py b/asap-tools/queriers/prometheus-client/classes/QueryLatencyExporter.py similarity index 100% rename from asap-tools/prometheus-client/classes/QueryLatencyExporter.py rename to asap-tools/queriers/prometheus-client/classes/QueryLatencyExporter.py diff --git a/asap-tools/prometheus-client/classes/__init__.py b/asap-tools/queriers/prometheus-client/classes/__init__.py similarity index 100% rename from asap-tools/prometheus-client/classes/__init__.py rename to asap-tools/queriers/prometheus-client/classes/__init__.py diff --git a/asap-tools/prometheus-client/classes/clickhouse_query_client.py b/asap-tools/queriers/prometheus-client/classes/clickhouse_query_client.py similarity index 100% rename from asap-tools/prometheus-client/classes/clickhouse_query_client.py rename to asap-tools/queriers/prometheus-client/classes/clickhouse_query_client.py diff --git a/asap-tools/prometheus-client/classes/config.py b/asap-tools/queriers/prometheus-client/classes/config.py similarity index 100% rename from asap-tools/prometheus-client/classes/config.py rename to asap-tools/queriers/prometheus-client/classes/config.py diff --git a/asap-tools/prometheus-client/classes/prometheus_query_client.py b/asap-tools/queriers/prometheus-client/classes/prometheus_query_client.py similarity index 100% rename from asap-tools/prometheus-client/classes/prometheus_query_client.py rename to asap-tools/queriers/prometheus-client/classes/prometheus_query_client.py diff --git a/asap-tools/prometheus-client/classes/query_client.py b/asap-tools/queriers/prometheus-client/classes/query_client.py similarity index 100% rename from asap-tools/prometheus-client/classes/query_client.py rename to asap-tools/queriers/prometheus-client/classes/query_client.py diff --git a/asap-tools/prometheus-client/classes/query_client_factory.py b/asap-tools/queriers/prometheus-client/classes/query_client_factory.py similarity index 100% rename from asap-tools/prometheus-client/classes/query_client_factory.py rename to asap-tools/queriers/prometheus-client/classes/query_client_factory.py diff --git a/asap-tools/prometheus-client/classes/query_template.py b/asap-tools/queriers/prometheus-client/classes/query_template.py similarity index 100% rename from asap-tools/prometheus-client/classes/query_template.py rename to asap-tools/queriers/prometheus-client/classes/query_template.py diff --git a/asap-tools/prometheus-client/docker-compose.yml.j2 b/asap-tools/queriers/prometheus-client/docker-compose.yml.j2 similarity index 100% rename from asap-tools/prometheus-client/docker-compose.yml.j2 rename to asap-tools/queriers/prometheus-client/docker-compose.yml.j2 diff --git a/asap-tools/prometheus-client/installation/install.sh b/asap-tools/queriers/prometheus-client/installation/install.sh similarity index 100% rename from asap-tools/prometheus-client/installation/install.sh rename to asap-tools/queriers/prometheus-client/installation/install.sh diff --git a/asap-tools/prometheus-client/installation/setup_dependencies.sh b/asap-tools/queriers/prometheus-client/installation/setup_dependencies.sh similarity index 100% rename from asap-tools/prometheus-client/installation/setup_dependencies.sh rename to asap-tools/queriers/prometheus-client/installation/setup_dependencies.sh diff --git a/asap-tools/prometheus-client/main_prometheus_client.py b/asap-tools/queriers/prometheus-client/main_prometheus_client.py similarity index 100% rename from asap-tools/prometheus-client/main_prometheus_client.py rename to asap-tools/queriers/prometheus-client/main_prometheus_client.py diff --git a/asap-tools/prometheus-client/pyproject.toml b/asap-tools/queriers/prometheus-client/pyproject.toml similarity index 100% rename from asap-tools/prometheus-client/pyproject.toml rename to asap-tools/queriers/prometheus-client/pyproject.toml diff --git a/asap-tools/prometheus-client/requirements.txt b/asap-tools/queriers/prometheus-client/requirements.txt similarity index 100% rename from asap-tools/prometheus-client/requirements.txt rename to asap-tools/queriers/prometheus-client/requirements.txt diff --git a/asap-tools/prometheus-client/similarity_scores.py b/asap-tools/queriers/prometheus-client/similarity_scores.py similarity index 100% rename from asap-tools/prometheus-client/similarity_scores.py rename to asap-tools/queriers/prometheus-client/similarity_scores.py diff --git a/asap-tools/prometheus-client/type_aliases.py b/asap-tools/queriers/prometheus-client/type_aliases.py similarity index 100% rename from asap-tools/prometheus-client/type_aliases.py rename to asap-tools/queriers/prometheus-client/type_aliases.py diff --git a/docs/01-getting-started/architecture.md b/docs/01-getting-started/architecture.md index 5011ede..5910f41 100644 --- a/docs/01-getting-started/architecture.md +++ b/docs/01-getting-started/architecture.md @@ -189,7 +189,7 @@ graph LR | **asap-planner** | Auto-determines sketch parameters | Python | `asap-planner/` | | **Kafka** | Message broker for sketch distribution | Apache Kafka | (external) | | **Prometheus** | Time-series database (existing) | Go | (external) | -| **Exporters** | Generate synthetic metrics for testing | Rust/Python | `asap-tools/prometheus-exporters/` | +| **Exporters** | Generate synthetic metrics for testing | Rust/Python | `asap-tools/data-sources/prometheus-exporters/` | | **asap-tools** | Experimental harness that uses Cloudlab | Python | `asap-tools/` | **Links to detailed documentation:** @@ -270,11 +270,14 @@ ASAPQuery/ │ └── utils/ # Decision logic │ ├── asap-tools/ # Experiment framework & tooling -│ ├── prometheus-exporters/ # Metric generators -│ │ ├── fake_exporter/ # Rust/Python fake exporters -│ │ ├── cluster_data_exporter/ # Real trace data -│ │ ├── query_cost_exporter/ # Resource metrics -│ │ └── query_latency_exporter/ # Latency metrics +│ ├── data-sources/ +│ │ └── prometheus-exporters/ # Metric generators +│ │ ├── fake_exporter/ # Rust/Python fake exporters +│ │ ├── cluster_data_exporter/ # Real trace data +│ │ ├── query_cost_exporter/ # Resource metrics +│ │ └── query_latency_exporter/ # Latency metrics +│ ├── queriers/ +│ │ └── prometheus-client/ # PromQL query client │ ├── experiments/ │ │ ├── experiment_run_e2e.py # Main orchestrator │ │ ├── config/ # Hydra configs diff --git a/docs/02-components/README.md b/docs/02-components/README.md index 0f96f78..9c8929b 100644 --- a/docs/02-components/README.md +++ b/docs/02-components/README.md @@ -10,7 +10,7 @@ This document provides an overview of all ASAP components and links to detailed | **Arroyo** | Stream processing for building sketches | Rust (forked) | [Details](arroyo.md) · [Code](https://github.com/ProjectASAP/arroyo) | | **asap-sketch-ingest** | Configures Arroyo pipelines from config | Python | [Details](arroyosketch.md) · [Code](../../asap-sketch-ingest/) · [README](../../asap-sketch-ingest/README.md) | | **asap-planner** | Auto-determines sketch parameters | Python | [Details](controller.md) · [Code](../../asap-planner/) · [README](../../asap-planner/README.md) | -| **Exporters** | Generate synthetic metrics for testing | Rust/Python | [Details](exporters.md) · [Code](../../asap-tools/prometheus-exporters/) · [README](../../asap-tools/prometheus-exporters/README.md) | +| **Exporters** | Generate synthetic metrics for testing | Rust/Python | [Details](exporters.md) · [Code](../../asap-tools/data-sources/prometheus-exporters/) · [README](../../asap-tools/data-sources/prometheus-exporters/README.md) | | **asap-tools** | Experiment framework for CloudLab | Python | [Details](utilities.md) · [Code](../../asap-tools/) · [Docs](../../asap-tools/docs/architecture.md) | ## Component Interaction @@ -164,5 +164,5 @@ For implementation details, see READMEs co-located with code: - [asap-query-engine/docs/](../../asap-query-engine/docs/README.md) - Extensibility guides - [asap-planner/README.md](../../asap-planner/README.md) - asap-planner internals - [asap-sketch-ingest/README.md](../../asap-sketch-ingest/README.md) - Pipeline config internals -- [asap-tools/prometheus-exporters/README.md](../../asap-tools/prometheus-exporters/README.md) - Exporter implementations +- [asap-tools/data-sources/prometheus-exporters/README.md](../../asap-tools/data-sources/prometheus-exporters/README.md) - Exporter implementations - [asap-tools/docs/](../../asap-tools/docs/architecture.md) - Experiment framework architecture diff --git a/docs/03-how-to-guides/development/adapt-pr-after-reorg.md b/docs/03-how-to-guides/development/adapt-pr-after-reorg.md index 8cc134c..e25abf7 100644 --- a/docs/03-how-to-guides/development/adapt-pr-after-reorg.md +++ b/docs/03-how-to-guides/development/adapt-pr-after-reorg.md @@ -15,8 +15,8 @@ This guide walks you through rebasing onto the new structure. | `ArroyoSketch/` | `asap-sketch-ingest/` | | `Controller/` | `asap-planner/` | | `Utilities/` | `asap-tools/` | -| `PrometheusExporters/` | `asap-tools/prometheus-exporters/` | -| `PrometheusClient/` | `asap-tools/prometheus-client/` | +| `PrometheusExporters/` | `asap-tools/data-sources/prometheus-exporters/` | +| `PrometheusClient/` | `asap-tools/queriers/prometheus-client/` | | `ExecutionUtilities/` | `asap-tools/execution-utilities/` | | `quickstart/` | `asap-quickstart/` | | `sketch-core/` | `asap-common/sketch-core/` | @@ -122,7 +122,7 @@ git push --force-with-lease origin your-branch-name Key CI jobs to watch: - **Rust** — Cargo build and tests for `asap-query-engine` and `asap-common` -- **Python** — Linting/type-checking for `asap-tools/prometheus-client` and `asap-planner` +- **Python** — Linting/type-checking for `asap-tools/queriers/prometheus-client` and `asap-planner` - **Docker** — Image builds for `asap-query-engine`, `asap-planner`, `asap-sketch-ingest` --- @@ -143,8 +143,8 @@ sed -i 's|CommonDependencies|asap-common|g' your_file.toml sed -i 's|"Utilities"|"asap-tools"|g' your_file.py sed -i 's|"Controller"|"asap-planner"|g' your_file.py sed -i 's|"ArroyoSketch"|"asap-sketch-ingest"|g' your_file.py -sed -i 's|"PrometheusClient"|"asap-tools/prometheus-client"|g' your_file.py -sed -i 's|"PrometheusExporters"|"asap-tools/prometheus-exporters"|g' your_file.py +sed -i 's|"PrometheusClient"|"asap-tools/queriers/prometheus-client"|g' your_file.py +sed -i 's|"PrometheusExporters"|"asap-tools/data-sources/prometheus-exporters"|g' your_file.py # Rename in shell scripts sed -i 's|/Utilities/|/asap-tools/|g' your_script.sh diff --git a/docs/03-how-to-guides/operations/manual-stack-run-clickhouse.md b/docs/03-how-to-guides/operations/manual-stack-run-clickhouse.md index bc022d4..439b3ca 100644 --- a/docs/03-how-to-guides/operations/manual-stack-run-clickhouse.md +++ b/docs/03-how-to-guides/operations/manual-stack-run-clickhouse.md @@ -7,7 +7,7 @@ This guide covers running the ASAP stack manually with Clickhouse for developmen - Kafka installed - Arroyo built at `~/code/arroyo/target/release/arroyo` - asap-query-engine built at `~/code/asap-query-engine/target/release/query_engine_rust` -- asap-tools/prometheus-exporters built (fake_kafka_exporter) +- asap-tools/data-sources/prometheus-exporters built (fake_kafka_exporter) - Clickhouse installed and accessible ## Directory Structure @@ -18,7 +18,7 @@ This guide covers running the ASAP stack manually with Clickhouse for developmen │ ├── config.yaml # Arroyo cluster config │ └── run_arroyosketch.py # Creates sources, sinks, and pipelines ├── asap-query-engine/ # Query interception layer -├── asap-tools/prometheus-exporters/ # Data generators +├── asap-tools/data-sources/prometheus-exporters/ # Data generators │ └── fake_kafka_exporter/ └── asap-tools/experiments/ # Automated experiment framework ``` @@ -110,7 +110,7 @@ Wait for Kafka to be ready, then create topics: The fake_kafka_exporter generates synthetic data and writes directly to Kafka: ```bash -cd ~/code/asap-tools/prometheus-exporters/fake_kafka_exporter +cd ~/code/asap-tools/data-sources/prometheus-exporters/fake_kafka_exporter ./target/release/fake_kafka_exporter \ --kafka-topic raw_data_topic \ --metadata-columns hostname,datacenter \ @@ -208,7 +208,7 @@ Wait for Kafka to be ready, then create topics: Same as baseline - exporter writes to Kafka: ```bash -cd ~/code/asap-tools/prometheus-exporters/fake_kafka_exporter +cd ~/code/asap-tools/data-sources/prometheus-exporters/fake_kafka_exporter ./target/release/fake_kafka_exporter \ --kafka-topic raw_data_topic \ --metadata-columns hostname,datacenter \ diff --git a/docs/03-how-to-guides/operations/manual-stack-run-prometheus.md b/docs/03-how-to-guides/operations/manual-stack-run-prometheus.md index 9066cca..8d08403 100644 --- a/docs/03-how-to-guides/operations/manual-stack-run-prometheus.md +++ b/docs/03-how-to-guides/operations/manual-stack-run-prometheus.md @@ -7,7 +7,7 @@ This guide covers running the ASAP stack manually with Prometheus for developmen - Kafka installed - Arroyo built at `~/code/arroyo/target/release/arroyo` - asap-query-engine built at `~/code/asap-query-engine/target/release/query_engine_rust` -- asap-tools/prometheus-exporters built (fake_exporter_rust) +- asap-tools/data-sources/prometheus-exporters built (fake_exporter_rust) - Prometheus installed and accessible ## Directory Structure @@ -18,7 +18,7 @@ This guide covers running the ASAP stack manually with Prometheus for developmen │ ├── config.yaml # Arroyo cluster config │ └── run_arroyosketch.py # Creates sources, sinks, and pipelines ├── asap-query-engine/ # Query interception layer -├── asap-tools/prometheus-exporters/ # Data generators +├── asap-tools/data-sources/prometheus-exporters/ # Data generators │ └── fake_exporter/ └── asap-tools/experiments/ # Automated experiment framework ``` @@ -80,7 +80,7 @@ Baseline mode runs queries directly against Prometheus or Clickhouse without ASA The fake exporter generates synthetic metrics exposed at `/metrics`: ```bash -cd ~/code/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter +cd ~/code/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter ./target/release/fake_exporter \ --num-labels 3 \ --cardinality 100 \ @@ -142,7 +142,7 @@ Wait for Kafka to be ready, then create topics: Same as baseline - exporter just exposes `/metrics`: ```bash -cd ~/code/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter +cd ~/code/asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter ./target/release/fake_exporter \ --num-labels 3 \ --cardinality 100 \ diff --git a/docs/README.md b/docs/README.md index 2699e63..bdc6389 100644 --- a/docs/README.md +++ b/docs/README.md @@ -56,4 +56,4 @@ Technical details co-located with code: - [asap-tools/Experiments](../asap-tools/docs/architecture.md) - Experiment framework architecture - [asap-planner](../asap-planner/README.md) - Controller internals - [asap-sketch-ingest](../asap-sketch-ingest/README.md) - Pipeline configuration -- [asap-tools/prometheus-exporters](../asap-tools/prometheus-exporters/README.md) - Exporter implementations +- [asap-tools/data-sources/prometheus-exporters](../asap-tools/data-sources/prometheus-exporters/README.md) - Exporter implementations