diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1191162..94c38ae 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,16 +4,16 @@ on: push: branches: [ main ] paths: - - 'CommonDependencies/installation/**' - - 'Controller/**' - - 'PrometheusClient/**' + - 'asap-common/installation/**' + - 'asap-planner/**' + - 'asap-tools/prometheus-client/**' - '.github/workflows/docker.yml' pull_request: branches: [ main ] paths: - - 'CommonDependencies/installation/**' - - 'Controller/**' - - 'PrometheusClient/**' + - 'asap-common/installation/**' + - 'asap-planner/**' + - 'asap-tools/prometheus-client/**' - '.github/workflows/docker.yml' workflow_dispatch: @@ -28,7 +28,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Build base image - working-directory: CommonDependencies/installation + working-directory: asap-common/installation run: | docker build \ -t sketchdb-base:latest \ @@ -41,7 +41,7 @@ jobs: docker run --rm sketchdb-base:latest pip list - name: Build Controller Docker image - working-directory: Controller + working-directory: asap-planner run: | docker build \ -t sketchdb-controller:latest \ @@ -49,7 +49,7 @@ jobs: . - name: Build PrometheusClient Docker image - working-directory: PrometheusClient + working-directory: asap-tools/prometheus-client run: | docker build \ -t sketchdb-prometheus-client:latest \ diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index fc0e377..6ea58e8 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -4,24 +4,24 @@ on: push: branches: [ main ] paths: - - 'ArroyoSketch/**' - - 'PrometheusClient/**' - - 'Controller/**' - - 'Utilities/**' - - 'PrometheusExporters/**' - - 'ExecutionUtilities/**' - - 'CommonDependencies/dependencies/py/**' + - 'asap-sketch-ingest/**' + - 'asap-tools/prometheus-client/**' + - 'asap-planner/**' + - 'asap-tools/**' + - 'asap-tools/prometheus-exporters/**' + - 'asap-tools/execution-utilities/**' + - 'asap-common/dependencies/py/**' - '.github/workflows/python.yml' pull_request: branches: [ main ] paths: - - 'ArroyoSketch/**' - - 'PrometheusClient/**' - - 'Controller/**' - - 'Utilities/**' - - 'PrometheusExporters/**' - - 'ExecutionUtilities/**' - - 'CommonDependencies/dependencies/py/**' + - 'asap-sketch-ingest/**' + - 'asap-tools/prometheus-client/**' + - 'asap-planner/**' + - 'asap-tools/**' + - 'asap-tools/prometheus-exporters/**' + - 'asap-tools/execution-utilities/**' + - 'asap-common/dependencies/py/**' - '.github/workflows/python.yml' workflow_dispatch: @@ -46,23 +46,23 @@ jobs: with: filters: | arroyo_sketch: - - 'ArroyoSketch/**' - - 'CommonDependencies/dependencies/py/**' + - 'asap-sketch-ingest/**' + - 'asap-common/dependencies/py/**' prometheus_client: - - 'PrometheusClient/**' - - 'CommonDependencies/dependencies/py/**' + - 'asap-tools/prometheus-client/**' + - 'asap-common/dependencies/py/**' controller: - - 'Controller/**' - - 'CommonDependencies/dependencies/py/**' + - 'asap-planner/**' + - 'asap-common/dependencies/py/**' utilities: - - 'Utilities/**' - - 'CommonDependencies/dependencies/py/**' + - 'asap-tools/**' + - 'asap-common/dependencies/py/**' prometheus_exporters: - - 'PrometheusExporters/**' - - 'CommonDependencies/dependencies/py/**' + - 'asap-tools/prometheus-exporters/**' + - 'asap-common/dependencies/py/**' execution_utilities: - - 'ExecutionUtilities/**' - - 'CommonDependencies/dependencies/py/**' + - 'asap-tools/execution-utilities/**' + - 'asap-common/dependencies/py/**' test-arroyo-sketch: needs: detect-changes @@ -81,12 +81,12 @@ jobs: run: | python -m pip install --upgrade pip pip install black==24.8.0 flake8==6.1.0 - if [ -f ArroyoSketch/requirements.txt ]; then pip install -r ArroyoSketch/requirements.txt; fi + if [ -f asap-sketch-ingest/requirements.txt ]; then pip install -r asap-sketch-ingest/requirements.txt; fi - name: Check formatting with Black - working-directory: ArroyoSketch + working-directory: asap-sketch-ingest run: black --check --diff . - name: Lint with flake8 - working-directory: ArroyoSketch + working-directory: asap-sketch-ingest run: | # Stop the build if there are Python syntax errors or undefined names flake8 . --config=../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics @@ -110,19 +110,19 @@ jobs: run: | python -m pip install --upgrade pip pip install black==24.8.0 flake8==6.1.0 mypy types-requests types-PyYAML typing-extensions numpy prometheus-client urllib3 - if [ -f PrometheusClient/requirements.txt ]; then pip install -r PrometheusClient/requirements.txt; fi + if [ -f asap-tools/prometheus-client/requirements.txt ]; then pip install -r asap-tools/prometheus-client/requirements.txt; fi - name: Check formatting with Black - working-directory: PrometheusClient + working-directory: asap-tools/prometheus-client run: black --check --diff . - name: Lint with flake8 - working-directory: PrometheusClient + working-directory: asap-tools/prometheus-client run: | # Stop the build if there are Python syntax errors or undefined names - flake8 . --config=../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 . --config=../../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics # Exit-zero treats all errors as warnings - flake8 . --config=../.flake8 --count --exit-zero --max-complexity=10 --statistics + flake8 . --config=../../.flake8 --count --exit-zero --max-complexity=10 --statistics - name: Type check with mypy - working-directory: PrometheusClient + working-directory: asap-tools/prometheus-client run: mypy . test-controller: @@ -142,12 +142,12 @@ jobs: run: | python -m pip install --upgrade pip pip install black==24.8.0 flake8==6.1.0 - if [ -f Controller/requirements.txt ]; then pip install -r Controller/requirements.txt; fi + if [ -f asap-planner/requirements.txt ]; then pip install -r asap-planner/requirements.txt; fi - name: Check formatting with Black - working-directory: Controller + working-directory: asap-planner run: black --check --diff . - name: Lint with flake8 - working-directory: Controller + working-directory: asap-planner run: | # Stop the build if there are Python syntax errors or undefined names flake8 . --config=../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics @@ -171,12 +171,12 @@ jobs: run: | python -m pip install --upgrade pip pip install black==24.8.0 flake8==6.1.0 - if [ -f Utilities/requirements.txt ]; then pip install -r Utilities/requirements.txt; fi + if [ -f asap-tools/requirements.txt ]; then pip install -r asap-tools/requirements.txt; fi - name: Check formatting with Black - working-directory: Utilities + working-directory: asap-tools run: black --check --diff . - name: Lint with flake8 - working-directory: Utilities + working-directory: asap-tools run: | # Stop the build if there are Python syntax errors or undefined names flake8 . --config=../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics @@ -200,22 +200,22 @@ jobs: run: | python -m pip install --upgrade pip pip install black==24.8.0 flake8==6.1.0 mypy isort - if [ -f PrometheusExporters/requirements.txt ]; then pip install -r PrometheusExporters/requirements.txt; fi + if [ -f asap-tools/prometheus-exporters/requirements.txt ]; then pip install -r asap-tools/prometheus-exporters/requirements.txt; fi - name: Check formatting with Black - working-directory: PrometheusExporters + working-directory: asap-tools/prometheus-exporters run: black --check --diff . - name: Check import sorting with isort - working-directory: PrometheusExporters + working-directory: asap-tools/prometheus-exporters run: isort --check-only --diff --settings-file .isort.cfg . - name: Lint with flake8 - working-directory: PrometheusExporters + working-directory: asap-tools/prometheus-exporters run: | # Stop the build if there are Python syntax errors or undefined names - flake8 . --config=../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 . --config=../../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics # Exit-zero treats all errors as warnings - flake8 . --config=../.flake8 --count --exit-zero --max-complexity=10 --statistics + flake8 . --config=../../.flake8 --count --exit-zero --max-complexity=10 --statistics - name: Type check with mypy - working-directory: PrometheusExporters + working-directory: asap-tools/prometheus-exporters run: mypy . --config-file=.mypy.ini test-execution-utilities: @@ -235,14 +235,14 @@ jobs: run: | python -m pip install --upgrade pip pip install black==24.8.0 flake8==6.1.0 - if [ -f ExecutionUtilities/requirements.txt ]; then pip install -r ExecutionUtilities/requirements.txt; fi + if [ -f asap-tools/execution-utilities/requirements.txt ]; then pip install -r asap-tools/execution-utilities/requirements.txt; fi - name: Check formatting with Black - working-directory: ExecutionUtilities + working-directory: asap-tools/execution-utilities run: black --check --diff . - name: Lint with flake8 - working-directory: ExecutionUtilities + working-directory: asap-tools/execution-utilities run: | # Stop the build if there are Python syntax errors or undefined names - flake8 . --config=../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 . --config=../../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics # Exit-zero treats all errors as warnings - flake8 . --config=../.flake8 --count --exit-zero --max-complexity=10 --statistics + flake8 . --config=../../.flake8 --count --exit-zero --max-complexity=10 --statistics diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 143865b..bc2155d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -4,18 +4,18 @@ on: push: branches: [ main ] paths: - - 'QueryEngineRust/**' - - 'CommonDependencies/dependencies/rs/**' - - 'CommonDependencies/tests/**' - - 'sketch-core/**' + - 'asap-query-engine/**' + - 'asap-common/dependencies/rs/**' + - 'asap-common/tests/**' + - 'asap-common/sketch-core/**' - '.github/workflows/rust.yml' pull_request: branches: [ main ] paths: - - 'QueryEngineRust/**' - - 'CommonDependencies/dependencies/rs/**' - - 'CommonDependencies/tests/**' - - 'sketch-core/**' + - 'asap-query-engine/**' + - 'asap-common/dependencies/rs/**' + - 'asap-common/tests/**' + - 'asap-common/sketch-core/**' - '.github/workflows/rust.yml' workflow_dispatch: @@ -49,30 +49,30 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - QueryEngineRust/target + asap-query-engine/target key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Check formatting run: cargo fmt -- --check - working-directory: QueryEngineRust + working-directory: asap-query-engine - name: Check formatting (sketch_db_common) run: cargo fmt -- --check - working-directory: CommonDependencies/dependencies/rs/sketch_db_common + working-directory: asap-common/dependencies/rs/sketch_db_common - name: Run clippy run: cargo clippy --all-targets --all-features -- -D warnings - working-directory: QueryEngineRust + working-directory: asap-query-engine env: RUSTC_WRAPPER: sccache - name: Check formatting (sketch-core) run: cargo fmt -- --check - working-directory: sketch-core + working-directory: asap-common/sketch-core - name: Run clippy (sketch-core) run: cargo clippy --all-targets --all-features -- -D warnings - working-directory: sketch-core + working-directory: asap-common/sketch-core env: RUSTC_WRAPPER: sccache @@ -101,33 +101,33 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - QueryEngineRust/target + asap-query-engine/target key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Run QueryEngineRust tests run: cargo test - working-directory: QueryEngineRust + working-directory: asap-query-engine env: RUSTC_WRAPPER: sccache - name: Run sql_utilities tests run: cargo test - working-directory: CommonDependencies/dependencies/rs/sql_utilities + working-directory: asap-common/dependencies/rs/sql_utilities env: RUSTC_WRAPPER: sccache - name: Run promql_utilities tests run: cargo test - working-directory: CommonDependencies/dependencies/rs/promql_utilities + working-directory: asap-common/dependencies/rs/promql_utilities env: RUSTC_WRAPPER: sccache - name: Run sketch-core tests run: cargo test - working-directory: sketch-core + working-directory: asap-common/sketch-core - name: Run sketch_db_common tests run: cargo test - working-directory: CommonDependencies/dependencies/rs/sketch_db_common + working-directory: asap-common/dependencies/rs/sketch_db_common env: RUSTC_WRAPPER: sccache @@ -144,7 +144,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Build Docker image - run: docker build -f QueryEngineRust/Dockerfile -t sketchdb-queryengine-rust:latest . + run: docker build -f asap-query-engine/Dockerfile -t sketchdb-queryengine-rust:latest . # run: | # echo "${{ secrets.PRIVATE_REPO_TOKEN }}" > /tmp/git_token # docker build --secret id=git_token,src=/tmp/git_token -f QueryEngineRust/Dockerfile -t sketchdb-queryengine-rust:latest . diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fed70d5..dd4314f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,8 +39,8 @@ repos: hooks: - id: isort name: isort (python) - files: ^PrometheusExporters/.*\.py$ - args: ["--settings-file", "PrometheusExporters/.isort.cfg"] + files: ^asap-tools/prometheus-exporters/.*\.py$ + args: ["--settings-file", "asap-tools/prometheus-exporters/.isort.cfg"] - repo: https://github.com/pycqa/flake8 rev: 6.1.0 @@ -52,12 +52,12 @@ repos: rev: "v1.14.1" hooks: - id: mypy - files: ^PrometheusExporters/.*\.py$ - args: [--config-file=PrometheusExporters/.mypy.ini] + files: ^asap-tools/prometheus-exporters/.*\.py$ + args: [--config-file=asap-tools/prometheus-exporters/.mypy.ini] - id: mypy - files: ^PrometheusClient/ - args: [--config-file=PrometheusClient/pyproject.toml] + files: ^asap-tools/prometheus-client/ + args: [--config-file=asap-tools/prometheus-client/pyproject.toml] additional_dependencies: - types-requests - types-PyYAML @@ -72,108 +72,108 @@ repos: - id: cargo-fmt name: cargo fmt description: Format Rust files with rustfmt. - entry: bash -c 'cargo fmt --manifest-path QueryEngineRust/Cargo.toml -- --check' + entry: bash -c 'cargo fmt --manifest-path asap-query-engine/Cargo.toml -- --check' language: system - files: ^QueryEngineRust/.*\.rs$ + files: ^asap-query-engine/.*\.rs$ pass_filenames: false - id: cargo-check name: cargo check description: Check the package for errors. - entry: bash -c 'cargo check --manifest-path QueryEngineRust/Cargo.toml --all' + entry: bash -c 'cargo check --manifest-path asap-query-engine/Cargo.toml --all' language: system - files: ^QueryEngineRust/.*\.rs$ + files: ^asap-query-engine/.*\.rs$ pass_filenames: false - id: cargo-clippy name: cargo clippy description: Lint Rust sources - entry: bash -c 'cargo clippy --manifest-path QueryEngineRust/Cargo.toml --all-targets --all-features -- -D warnings' + entry: bash -c 'cargo clippy --manifest-path asap-query-engine/Cargo.toml --all-targets --all-features -- -D warnings' language: system - files: ^QueryEngineRust/.*\.rs$ + files: ^asap-query-engine/.*\.rs$ pass_filenames: false - id: cargo-fmt-datafusion-summary-library name: cargo fmt (datafusion_summary_library) description: Format datafusion_summary_libraryRust files with rustfmt. - entry: bash -c 'cargo fmt --manifest-path CommonDependencies/dependencies/rs/datafusion_summary_library/Cargo.toml -- --check' + entry: bash -c 'cargo fmt --manifest-path asap-common/dependencies/rs/datafusion_summary_library/Cargo.toml -- --check' language: system - files: ^CommonDependencies/dependencies/rs/datafusion_summary_library/.*\.rs$ + files: ^asap-common/dependencies/rs/datafusion_summary_library/.*\.rs$ pass_filenames: false - id: cargo-test-datafusion-summary-library name: cargo test (datafusion_summary_library) description: Run datafusion_summary_library tests. - entry: bash -c 'cargo test --manifest-path CommonDependencies/dependencies/rs/datafusion_summary_library/Cargo.toml' + entry: bash -c 'cargo test --manifest-path asap-common/dependencies/rs/datafusion_summary_library/Cargo.toml' language: system - files: ^CommonDependencies/dependencies/rs/datafusion_summary_library/.*\.rs$ + files: ^asap-common/dependencies/rs/datafusion_summary_library/.*\.rs$ pass_filenames: false - id: cargo-fmt-sql-utilities name: cargo fmt (sql_utilities) description: Format sql_utilities Rust files with rustfmt. - entry: bash -c 'cargo fmt --manifest-path CommonDependencies/dependencies/rs/sql_utilities/Cargo.toml -- --check' + entry: bash -c 'cargo fmt --manifest-path asap-common/dependencies/rs/sql_utilities/Cargo.toml -- --check' language: system - files: ^CommonDependencies/dependencies/rs/sql_utilities/.*\.rs$ + files: ^asap-common/dependencies/rs/sql_utilities/.*\.rs$ pass_filenames: false - id: cargo-test-sql-utilities name: cargo test (sql_utilities) description: Run sql_utilities tests. - entry: bash -c 'cargo test --manifest-path CommonDependencies/dependencies/rs/sql_utilities/Cargo.toml' + entry: bash -c 'cargo test --manifest-path asap-common/dependencies/rs/sql_utilities/Cargo.toml' language: system - files: ^CommonDependencies/dependencies/rs/sql_utilities/.*\.rs$ + files: ^asap-common/dependencies/rs/sql_utilities/.*\.rs$ pass_filenames: false - id: cargo-fmt-promql-utilities name: cargo fmt (promql_utilities) description: Format promql_utilities Rust files with rustfmt. - entry: bash -c 'cargo fmt --manifest-path CommonDependencies/dependencies/rs/promql_utilities/Cargo.toml -- --check' + entry: bash -c 'cargo fmt --manifest-path asap-common/dependencies/rs/promql_utilities/Cargo.toml -- --check' language: system - files: ^CommonDependencies/dependencies/rs/promql_utilities/.*\.rs$ + files: ^asap-common/dependencies/rs/promql_utilities/.*\.rs$ pass_filenames: false - id: cargo-test-promql-utilities name: cargo test (promql_utilities) description: Run promql_utilities tests. - entry: bash -c 'cargo test --manifest-path CommonDependencies/dependencies/rs/promql_utilities/Cargo.toml' + entry: bash -c 'cargo test --manifest-path asap-common/dependencies/rs/promql_utilities/Cargo.toml' language: system - files: ^CommonDependencies/dependencies/rs/promql_utilities/.*\.rs$ + files: ^asap-common/dependencies/rs/promql_utilities/.*\.rs$ pass_filenames: false - id: cargo-fmt-sketch-core name: cargo fmt (sketch-core) description: Format sketch-core Rust files with rustfmt. - entry: bash -c 'cargo fmt --manifest-path sketch-core/Cargo.toml -- --check' + entry: bash -c 'cargo fmt --manifest-path asap-common/sketch-core/Cargo.toml -- --check' language: system - files: ^sketch-core/.*\.rs$ + files: ^asap-common/sketch-core/.*\.rs$ pass_filenames: false - id: cargo-clippy-sketch-core name: cargo clippy (sketch-core) description: Lint sketch-core Rust sources. - entry: bash -c 'cargo clippy --manifest-path sketch-core/Cargo.toml --all-targets --all-features -- -D warnings' + entry: bash -c 'cargo clippy --manifest-path asap-common/sketch-core/Cargo.toml --all-targets --all-features -- -D warnings' language: system - files: ^sketch-core/.*\.rs$ + files: ^asap-common/sketch-core/.*\.rs$ pass_filenames: false - id: cargo-test-sketch-core name: cargo test (sketch-core) description: Run sketch-core tests. - entry: bash -c 'cargo test --manifest-path sketch-core/Cargo.toml' + entry: bash -c 'cargo test --manifest-path asap-common/sketch-core/Cargo.toml' language: system - files: ^sketch-core/.*\.rs$ + files: ^asap-common/sketch-core/.*\.rs$ - id: cargo-fmt-sketch-db-common name: cargo fmt (sketch_db_common) description: Format sketch_db_common Rust files with rustfmt. - entry: bash -c 'cargo fmt --manifest-path CommonDependencies/dependencies/rs/sketch_db_common/Cargo.toml -- --check' + entry: bash -c 'cargo fmt --manifest-path asap-common/dependencies/rs/sketch_db_common/Cargo.toml -- --check' language: system - files: ^CommonDependencies/dependencies/rs/sketch_db_common/.*\.rs$ + files: ^asap-common/dependencies/rs/sketch_db_common/.*\.rs$ pass_filenames: false - id: cargo-test-sketch-db-common name: cargo test (sketch_db_common) description: Run sketch_db_common tests. - entry: bash -c 'cargo test --manifest-path CommonDependencies/dependencies/rs/sketch_db_common/Cargo.toml' + entry: bash -c 'cargo test --manifest-path asap-common/dependencies/rs/sketch_db_common/Cargo.toml' language: system - files: ^CommonDependencies/dependencies/rs/sketch_db_common/.*\.rs$ + files: ^asap-common/dependencies/rs/sketch_db_common/.*\.rs$ pass_filenames: false diff --git a/Cargo.toml b/Cargo.toml index f6da817..be74035 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,4 @@ [workspace] -members = ["sketch-core", "QueryEngineRust"] +members = ["asap-common/sketch-core", "asap-query-engine"] exclude = ["sketchlib-rust"] resolver = "2" diff --git a/README.md b/README.md index c1a889b..cb8d95f 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,13 @@ Future versions of ASAPQuery will accelerate queries against other observability **Try ASAPQuery in 5 minutes** with our self-contained demo: ```bash -cd quickstart +cd asap-quickstart docker compose up -d ``` Open http://localhost:3000 and see ASAPQuery vs Prometheus side-by-side! -Full quickstart instructions at [**Quickstart Guide**](quickstart/README.md) +Full quickstart instructions at [**Quickstart Guide**](asap-quickstart/README.md) ## Why ASAPQuery? @@ -44,23 +44,23 @@ ASAPQuery uses **streaming sketches** to: ## Architecture -ASAPQuery has four main components: the **Controller** generates sketch configurations from your query workload, **ArroyoSketch** deploys streaming pipelines in **Arroyo** that continuously build sketches from live Prometheus metrics, and **QueryEngineRust** intercepts PromQL queries and serves them from those pre-computed sketches. +ASAPQuery has four main components: the **asap-planner** generates sketch configurations from your query workload, **asap-sketch-ingest** deploys streaming pipelines in **Arroyo** that continuously build sketches from live Prometheus metrics, and **asap-query-engine** intercepts PromQL queries and serves them from those pre-computed sketches. ### Components -- **[Controller](Controller/)** - Analyzes a PromQL query workload and auto-generates sketch configurations for ArroyoSketch and QueryEngineRust -- **[ArroyoSketch](ArroyoSketch/)** - Deploys Arroyo streaming pipelines that continuously compute and publish sketches from live metrics +- **[asap-planner](asap-planner/)** - Analyzes a PromQL query workload and auto-generates sketch configurations for asap-sketch-ingest and asap-query-engine +- **[asap-sketch-ingest](asap-sketch-ingest/)** - Deploys Arroyo streaming pipelines that continuously compute and publish sketches from live metrics - **[arroyo](arroyo/)** - Fork of the [Arroyo](https://github.com/ArroyoSystems/Arroyo) stream processing engine that runs the sketch-building SQL pipelines -- **[QueryEngineRust](QueryEngineRust/)** - Intercepts incoming PromQL queries and serves them from pre-computed sketches, falling back to Prometheus for unsupported queries +- **[asap-query-engine](asap-query-engine/)** - Intercepts incoming PromQL queries and serves them from pre-computed sketches, falling back to Prometheus for unsupported queries ### Repository Structure ``` -├── quickstart/ # Self-contained demo (start here!) -├── Controller/ # Auto-configuration service -├── ArroyoSketch/ # Arroyo pipeline deployer +├── asap-quickstart/ # Self-contained demo (start here!) +├── asap-planner/ # Auto-configuration service +├── asap-sketch-ingest/ # Arroyo pipeline deployer ├── arroyo/ # Fork of Arroyo stream processing engine -└── QueryEngineRust/ # Query serving engine +└── asap-query-engine/ # Query serving engine ``` ## Coming soon diff --git a/Utilities/LICENSE b/Utilities/LICENSE deleted file mode 100644 index 404d657..0000000 --- a/Utilities/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2025 SketchDB - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/CommonDependencies/.gitignore b/asap-common/.gitignore similarity index 100% rename from CommonDependencies/.gitignore rename to asap-common/.gitignore diff --git a/ArroyoSketch/tests/__init__.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/__init__.py similarity index 100% rename from ArroyoSketch/tests/__init__.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/__init__.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/ast_matching/PromQLPattern.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/ast_matching/PromQLPattern.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/ast_matching/PromQLPattern.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/ast_matching/PromQLPattern.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/ast_matching/PromQLPatternBuilder.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/ast_matching/PromQLPatternBuilder.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/ast_matching/PromQLPatternBuilder.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/ast_matching/PromQLPatternBuilder.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/__init__.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/ast_matching/__init__.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/__init__.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/ast_matching/__init__.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/data_model/KeyByLabelNames.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/data_model/KeyByLabelNames.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/data_model/KeyByLabelNames.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/data_model/KeyByLabelNames.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/data_model/__init__.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/data_model/__init__.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/data_model/__init__.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/data_model/__init__.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_logics/__init__.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/query_logics/__init__.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_logics/__init__.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/query_logics/__init__.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_logics/enums.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/query_logics/enums.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_logics/enums.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/query_logics/enums.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_logics/logics.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/query_logics/logics.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_logics/logics.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/query_logics/logics.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_logics/parsing.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/query_logics/parsing.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_logics/parsing.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/query_logics/parsing.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_results/__init__.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/query_results/__init__.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_results/__init__.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/query_results/__init__.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_results/classes.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/query_results/classes.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_results/classes.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/query_results/classes.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/__init__.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/__init__.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/__init__.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/__init__.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/base.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/base.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/base.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/base.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/factory.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/factory.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/factory.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/factory.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/jsonl_serializer.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/jsonl_serializer.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/jsonl_serializer.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/jsonl_serializer.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/parquet_serializer.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/parquet_serializer.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/parquet_serializer.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/query_results/serializers/parquet_serializer.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/streaming_config/MetricConfig.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/streaming_config/MetricConfig.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/streaming_config/MetricConfig.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/streaming_config/MetricConfig.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/streaming_config/SQLTableConfig.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/streaming_config/SQLTableConfig.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/streaming_config/SQLTableConfig.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/streaming_config/SQLTableConfig.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/streaming_config/StreamingAggregationConfig.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/streaming_config/StreamingAggregationConfig.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/streaming_config/StreamingAggregationConfig.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/streaming_config/StreamingAggregationConfig.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/streaming_config/__init__.py b/asap-common/dependencies/py/promql_utilities/promql_utilities/streaming_config/__init__.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/streaming_config/__init__.py rename to asap-common/dependencies/py/promql_utilities/promql_utilities/streaming_config/__init__.py diff --git a/CommonDependencies/dependencies/py/promql_utilities/setup.py b/asap-common/dependencies/py/promql_utilities/setup.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/setup.py rename to asap-common/dependencies/py/promql_utilities/setup.py diff --git a/CommonDependencies/dependencies/rs/datafusion_summary_library/Cargo.lock b/asap-common/dependencies/rs/datafusion_summary_library/Cargo.lock similarity index 100% rename from CommonDependencies/dependencies/rs/datafusion_summary_library/Cargo.lock rename to asap-common/dependencies/rs/datafusion_summary_library/Cargo.lock diff --git a/CommonDependencies/dependencies/rs/datafusion_summary_library/Cargo.toml b/asap-common/dependencies/rs/datafusion_summary_library/Cargo.toml similarity index 100% rename from CommonDependencies/dependencies/rs/datafusion_summary_library/Cargo.toml rename to asap-common/dependencies/rs/datafusion_summary_library/Cargo.toml diff --git a/CommonDependencies/dependencies/rs/datafusion_summary_library/src/lib.rs b/asap-common/dependencies/rs/datafusion_summary_library/src/lib.rs similarity index 100% rename from CommonDependencies/dependencies/rs/datafusion_summary_library/src/lib.rs rename to asap-common/dependencies/rs/datafusion_summary_library/src/lib.rs diff --git a/CommonDependencies/dependencies/rs/datafusion_summary_library/src/physical/hll.rs b/asap-common/dependencies/rs/datafusion_summary_library/src/physical/hll.rs similarity index 100% rename from CommonDependencies/dependencies/rs/datafusion_summary_library/src/physical/hll.rs rename to asap-common/dependencies/rs/datafusion_summary_library/src/physical/hll.rs diff --git a/CommonDependencies/dependencies/rs/datafusion_summary_library/src/physical/mod.rs b/asap-common/dependencies/rs/datafusion_summary_library/src/physical/mod.rs similarity index 100% rename from CommonDependencies/dependencies/rs/datafusion_summary_library/src/physical/mod.rs rename to asap-common/dependencies/rs/datafusion_summary_library/src/physical/mod.rs diff --git a/CommonDependencies/dependencies/rs/datafusion_summary_library/src/physical/planner.rs b/asap-common/dependencies/rs/datafusion_summary_library/src/physical/planner.rs similarity index 100% rename from CommonDependencies/dependencies/rs/datafusion_summary_library/src/physical/planner.rs rename to asap-common/dependencies/rs/datafusion_summary_library/src/physical/planner.rs diff --git a/CommonDependencies/dependencies/rs/datafusion_summary_library/src/physical/summary_infer_exec.rs b/asap-common/dependencies/rs/datafusion_summary_library/src/physical/summary_infer_exec.rs similarity index 100% rename from CommonDependencies/dependencies/rs/datafusion_summary_library/src/physical/summary_infer_exec.rs rename to asap-common/dependencies/rs/datafusion_summary_library/src/physical/summary_infer_exec.rs diff --git a/CommonDependencies/dependencies/rs/datafusion_summary_library/src/physical/summary_insert_exec.rs b/asap-common/dependencies/rs/datafusion_summary_library/src/physical/summary_insert_exec.rs similarity index 100% rename from CommonDependencies/dependencies/rs/datafusion_summary_library/src/physical/summary_insert_exec.rs rename to asap-common/dependencies/rs/datafusion_summary_library/src/physical/summary_insert_exec.rs diff --git a/CommonDependencies/dependencies/rs/datafusion_summary_library/src/sketch_operators.rs b/asap-common/dependencies/rs/datafusion_summary_library/src/sketch_operators.rs similarity index 100% rename from CommonDependencies/dependencies/rs/datafusion_summary_library/src/sketch_operators.rs rename to asap-common/dependencies/rs/datafusion_summary_library/src/sketch_operators.rs diff --git a/CommonDependencies/dependencies/rs/promql_utilities/.gitignore b/asap-common/dependencies/rs/promql_utilities/.gitignore similarity index 100% rename from CommonDependencies/dependencies/rs/promql_utilities/.gitignore rename to asap-common/dependencies/rs/promql_utilities/.gitignore diff --git a/CommonDependencies/dependencies/rs/promql_utilities/Cargo.lock b/asap-common/dependencies/rs/promql_utilities/Cargo.lock similarity index 100% rename from CommonDependencies/dependencies/rs/promql_utilities/Cargo.lock rename to asap-common/dependencies/rs/promql_utilities/Cargo.lock diff --git a/CommonDependencies/dependencies/rs/promql_utilities/Cargo.toml b/asap-common/dependencies/rs/promql_utilities/Cargo.toml similarity index 100% rename from CommonDependencies/dependencies/rs/promql_utilities/Cargo.toml rename to asap-common/dependencies/rs/promql_utilities/Cargo.toml diff --git a/CommonDependencies/dependencies/rs/promql_utilities/src/ast_matching/mod.rs b/asap-common/dependencies/rs/promql_utilities/src/ast_matching/mod.rs similarity index 100% rename from CommonDependencies/dependencies/rs/promql_utilities/src/ast_matching/mod.rs rename to asap-common/dependencies/rs/promql_utilities/src/ast_matching/mod.rs diff --git a/CommonDependencies/dependencies/rs/promql_utilities/src/ast_matching/promql_pattern.rs b/asap-common/dependencies/rs/promql_utilities/src/ast_matching/promql_pattern.rs similarity index 100% rename from CommonDependencies/dependencies/rs/promql_utilities/src/ast_matching/promql_pattern.rs rename to asap-common/dependencies/rs/promql_utilities/src/ast_matching/promql_pattern.rs diff --git a/CommonDependencies/dependencies/rs/promql_utilities/src/ast_matching/promql_pattern_builder.rs b/asap-common/dependencies/rs/promql_utilities/src/ast_matching/promql_pattern_builder.rs similarity index 100% rename from CommonDependencies/dependencies/rs/promql_utilities/src/ast_matching/promql_pattern_builder.rs rename to asap-common/dependencies/rs/promql_utilities/src/ast_matching/promql_pattern_builder.rs diff --git a/CommonDependencies/dependencies/rs/promql_utilities/src/ast_matching/promql_pattern_factory.rs b/asap-common/dependencies/rs/promql_utilities/src/ast_matching/promql_pattern_factory.rs similarity index 100% rename from CommonDependencies/dependencies/rs/promql_utilities/src/ast_matching/promql_pattern_factory.rs rename to asap-common/dependencies/rs/promql_utilities/src/ast_matching/promql_pattern_factory.rs diff --git a/CommonDependencies/dependencies/rs/promql_utilities/src/data_model/key_by_label_names.rs b/asap-common/dependencies/rs/promql_utilities/src/data_model/key_by_label_names.rs similarity index 100% rename from CommonDependencies/dependencies/rs/promql_utilities/src/data_model/key_by_label_names.rs rename to asap-common/dependencies/rs/promql_utilities/src/data_model/key_by_label_names.rs diff --git a/CommonDependencies/dependencies/rs/promql_utilities/src/data_model/mod.rs b/asap-common/dependencies/rs/promql_utilities/src/data_model/mod.rs similarity index 100% rename from CommonDependencies/dependencies/rs/promql_utilities/src/data_model/mod.rs rename to asap-common/dependencies/rs/promql_utilities/src/data_model/mod.rs diff --git a/CommonDependencies/dependencies/rs/promql_utilities/src/lib.rs b/asap-common/dependencies/rs/promql_utilities/src/lib.rs similarity index 100% rename from CommonDependencies/dependencies/rs/promql_utilities/src/lib.rs rename to asap-common/dependencies/rs/promql_utilities/src/lib.rs diff --git a/CommonDependencies/dependencies/rs/promql_utilities/src/query_logics/enums.rs b/asap-common/dependencies/rs/promql_utilities/src/query_logics/enums.rs similarity index 100% rename from CommonDependencies/dependencies/rs/promql_utilities/src/query_logics/enums.rs rename to asap-common/dependencies/rs/promql_utilities/src/query_logics/enums.rs diff --git a/CommonDependencies/dependencies/rs/promql_utilities/src/query_logics/logics.rs b/asap-common/dependencies/rs/promql_utilities/src/query_logics/logics.rs similarity index 100% rename from CommonDependencies/dependencies/rs/promql_utilities/src/query_logics/logics.rs rename to asap-common/dependencies/rs/promql_utilities/src/query_logics/logics.rs diff --git a/CommonDependencies/dependencies/rs/promql_utilities/src/query_logics/mod.rs b/asap-common/dependencies/rs/promql_utilities/src/query_logics/mod.rs similarity index 100% rename from CommonDependencies/dependencies/rs/promql_utilities/src/query_logics/mod.rs rename to asap-common/dependencies/rs/promql_utilities/src/query_logics/mod.rs diff --git a/CommonDependencies/dependencies/rs/promql_utilities/src/query_logics/parsing.rs b/asap-common/dependencies/rs/promql_utilities/src/query_logics/parsing.rs similarity index 100% rename from CommonDependencies/dependencies/rs/promql_utilities/src/query_logics/parsing.rs rename to asap-common/dependencies/rs/promql_utilities/src/query_logics/parsing.rs diff --git a/CommonDependencies/dependencies/rs/sketch_db_common/Cargo.lock b/asap-common/dependencies/rs/sketch_db_common/Cargo.lock similarity index 100% rename from CommonDependencies/dependencies/rs/sketch_db_common/Cargo.lock rename to asap-common/dependencies/rs/sketch_db_common/Cargo.lock diff --git a/CommonDependencies/dependencies/rs/sketch_db_common/Cargo.toml b/asap-common/dependencies/rs/sketch_db_common/Cargo.toml similarity index 100% rename from CommonDependencies/dependencies/rs/sketch_db_common/Cargo.toml rename to asap-common/dependencies/rs/sketch_db_common/Cargo.toml diff --git a/CommonDependencies/dependencies/rs/sketch_db_common/src/aggregation_config.rs b/asap-common/dependencies/rs/sketch_db_common/src/aggregation_config.rs similarity index 100% rename from CommonDependencies/dependencies/rs/sketch_db_common/src/aggregation_config.rs rename to asap-common/dependencies/rs/sketch_db_common/src/aggregation_config.rs diff --git a/CommonDependencies/dependencies/rs/sketch_db_common/src/enums.rs b/asap-common/dependencies/rs/sketch_db_common/src/enums.rs similarity index 100% rename from CommonDependencies/dependencies/rs/sketch_db_common/src/enums.rs rename to asap-common/dependencies/rs/sketch_db_common/src/enums.rs diff --git a/CommonDependencies/dependencies/rs/sketch_db_common/src/lib.rs b/asap-common/dependencies/rs/sketch_db_common/src/lib.rs similarity index 100% rename from CommonDependencies/dependencies/rs/sketch_db_common/src/lib.rs rename to asap-common/dependencies/rs/sketch_db_common/src/lib.rs diff --git a/CommonDependencies/dependencies/rs/sketch_db_common/src/traits.rs b/asap-common/dependencies/rs/sketch_db_common/src/traits.rs similarity index 100% rename from CommonDependencies/dependencies/rs/sketch_db_common/src/traits.rs rename to asap-common/dependencies/rs/sketch_db_common/src/traits.rs diff --git a/CommonDependencies/dependencies/rs/sketch_db_common/src/utils.rs b/asap-common/dependencies/rs/sketch_db_common/src/utils.rs similarity index 100% rename from CommonDependencies/dependencies/rs/sketch_db_common/src/utils.rs rename to asap-common/dependencies/rs/sketch_db_common/src/utils.rs diff --git a/CommonDependencies/dependencies/rs/sql_utilities/.gitignore b/asap-common/dependencies/rs/sql_utilities/.gitignore similarity index 100% rename from CommonDependencies/dependencies/rs/sql_utilities/.gitignore rename to asap-common/dependencies/rs/sql_utilities/.gitignore diff --git a/CommonDependencies/dependencies/rs/sql_utilities/Cargo.lock b/asap-common/dependencies/rs/sql_utilities/Cargo.lock similarity index 100% rename from CommonDependencies/dependencies/rs/sql_utilities/Cargo.lock rename to asap-common/dependencies/rs/sql_utilities/Cargo.lock diff --git a/CommonDependencies/dependencies/rs/sql_utilities/Cargo.toml b/asap-common/dependencies/rs/sql_utilities/Cargo.toml similarity index 100% rename from CommonDependencies/dependencies/rs/sql_utilities/Cargo.toml rename to asap-common/dependencies/rs/sql_utilities/Cargo.toml diff --git a/CommonDependencies/dependencies/rs/sql_utilities/src/ast_matching/mod.rs b/asap-common/dependencies/rs/sql_utilities/src/ast_matching/mod.rs similarity index 100% rename from CommonDependencies/dependencies/rs/sql_utilities/src/ast_matching/mod.rs rename to asap-common/dependencies/rs/sql_utilities/src/ast_matching/mod.rs diff --git a/CommonDependencies/dependencies/rs/sql_utilities/src/ast_matching/sqlhelper.rs b/asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlhelper.rs similarity index 100% rename from CommonDependencies/dependencies/rs/sql_utilities/src/ast_matching/sqlhelper.rs rename to asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlhelper.rs diff --git a/CommonDependencies/dependencies/rs/sql_utilities/src/ast_matching/sqlparser_test.rs b/asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlparser_test.rs similarity index 100% rename from CommonDependencies/dependencies/rs/sql_utilities/src/ast_matching/sqlparser_test.rs rename to asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlparser_test.rs diff --git a/CommonDependencies/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_matcher.rs b/asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_matcher.rs similarity index 100% rename from CommonDependencies/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_matcher.rs rename to asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_matcher.rs diff --git a/CommonDependencies/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_parser.rs b/asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_parser.rs similarity index 100% rename from CommonDependencies/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_parser.rs rename to asap-common/dependencies/rs/sql_utilities/src/ast_matching/sqlpattern_parser.rs diff --git a/CommonDependencies/dependencies/rs/sql_utilities/src/lib.rs b/asap-common/dependencies/rs/sql_utilities/src/lib.rs similarity index 100% rename from CommonDependencies/dependencies/rs/sql_utilities/src/lib.rs rename to asap-common/dependencies/rs/sql_utilities/src/lib.rs diff --git a/CommonDependencies/installation/Dockerfile b/asap-common/installation/Dockerfile similarity index 100% rename from CommonDependencies/installation/Dockerfile rename to asap-common/installation/Dockerfile diff --git a/CommonDependencies/installation/install.sh b/asap-common/installation/install.sh similarity index 100% rename from CommonDependencies/installation/install.sh rename to asap-common/installation/install.sh diff --git a/CommonDependencies/installation/requirements.txt b/asap-common/installation/requirements.txt similarity index 100% rename from CommonDependencies/installation/requirements.txt rename to asap-common/installation/requirements.txt diff --git a/CommonDependencies/installation/setup_dependencies.sh b/asap-common/installation/setup_dependencies.sh similarity index 100% rename from CommonDependencies/installation/setup_dependencies.sh rename to asap-common/installation/setup_dependencies.sh diff --git a/sketch-core/Cargo.toml b/asap-common/sketch-core/Cargo.toml similarity index 100% rename from sketch-core/Cargo.toml rename to asap-common/sketch-core/Cargo.toml diff --git a/sketch-core/src/count_min.rs b/asap-common/sketch-core/src/count_min.rs similarity index 100% rename from sketch-core/src/count_min.rs rename to asap-common/sketch-core/src/count_min.rs diff --git a/sketch-core/src/count_min_with_heap.rs b/asap-common/sketch-core/src/count_min_with_heap.rs similarity index 100% rename from sketch-core/src/count_min_with_heap.rs rename to asap-common/sketch-core/src/count_min_with_heap.rs diff --git a/sketch-core/src/delta_set_aggregator.rs b/asap-common/sketch-core/src/delta_set_aggregator.rs similarity index 100% rename from sketch-core/src/delta_set_aggregator.rs rename to asap-common/sketch-core/src/delta_set_aggregator.rs diff --git a/sketch-core/src/hydra_kll.rs b/asap-common/sketch-core/src/hydra_kll.rs similarity index 100% rename from sketch-core/src/hydra_kll.rs rename to asap-common/sketch-core/src/hydra_kll.rs diff --git a/sketch-core/src/kll.rs b/asap-common/sketch-core/src/kll.rs similarity index 100% rename from sketch-core/src/kll.rs rename to asap-common/sketch-core/src/kll.rs diff --git a/sketch-core/src/lib.rs b/asap-common/sketch-core/src/lib.rs similarity index 100% rename from sketch-core/src/lib.rs rename to asap-common/sketch-core/src/lib.rs diff --git a/sketch-core/src/set_aggregator.rs b/asap-common/sketch-core/src/set_aggregator.rs similarity index 100% rename from sketch-core/src/set_aggregator.rs rename to asap-common/sketch-core/src/set_aggregator.rs diff --git a/CommonDependencies/tests/compare_matched_tokens/README.md b/asap-common/tests/compare_matched_tokens/README.md similarity index 100% rename from CommonDependencies/tests/compare_matched_tokens/README.md rename to asap-common/tests/compare_matched_tokens/README.md diff --git a/CommonDependencies/tests/compare_matched_tokens/ast_matching_comparison.md b/asap-common/tests/compare_matched_tokens/ast_matching_comparison.md similarity index 100% rename from CommonDependencies/tests/compare_matched_tokens/ast_matching_comparison.md rename to asap-common/tests/compare_matched_tokens/ast_matching_comparison.md diff --git a/CommonDependencies/tests/compare_matched_tokens/comparison_tests/result_comparator.py b/asap-common/tests/compare_matched_tokens/comparison_tests/result_comparator.py similarity index 100% rename from CommonDependencies/tests/compare_matched_tokens/comparison_tests/result_comparator.py rename to asap-common/tests/compare_matched_tokens/comparison_tests/result_comparator.py diff --git a/CommonDependencies/tests/compare_matched_tokens/python_tests/pattern_tests.py b/asap-common/tests/compare_matched_tokens/python_tests/pattern_tests.py similarity index 99% rename from CommonDependencies/tests/compare_matched_tokens/python_tests/pattern_tests.py rename to asap-common/tests/compare_matched_tokens/python_tests/pattern_tests.py index 3900a9f..26fd1cd 100644 --- a/CommonDependencies/tests/compare_matched_tokens/python_tests/pattern_tests.py +++ b/asap-common/tests/compare_matched_tokens/python_tests/pattern_tests.py @@ -8,7 +8,7 @@ sys.path.append( os.path.join( os.path.dirname(__file__), - "../../../CommonDependencies/dependencies/py/promql_utilities", + "../../../dependencies/py/promql_utilities", ) ) diff --git a/CommonDependencies/tests/compare_matched_tokens/python_tests/test_data.py b/asap-common/tests/compare_matched_tokens/python_tests/test_data.py similarity index 100% rename from CommonDependencies/tests/compare_matched_tokens/python_tests/test_data.py rename to asap-common/tests/compare_matched_tokens/python_tests/test_data.py diff --git a/CommonDependencies/tests/compare_matched_tokens/python_tests/test_runner.py b/asap-common/tests/compare_matched_tokens/python_tests/test_runner.py similarity index 100% rename from CommonDependencies/tests/compare_matched_tokens/python_tests/test_runner.py rename to asap-common/tests/compare_matched_tokens/python_tests/test_runner.py diff --git a/CommonDependencies/tests/compare_matched_tokens/rust_tests/Cargo.lock b/asap-common/tests/compare_matched_tokens/rust_tests/Cargo.lock similarity index 100% rename from CommonDependencies/tests/compare_matched_tokens/rust_tests/Cargo.lock rename to asap-common/tests/compare_matched_tokens/rust_tests/Cargo.lock diff --git a/CommonDependencies/tests/rust_pattern_matching/Cargo.toml b/asap-common/tests/compare_matched_tokens/rust_tests/Cargo.toml similarity index 85% rename from CommonDependencies/tests/rust_pattern_matching/Cargo.toml rename to asap-common/tests/compare_matched_tokens/rust_tests/Cargo.toml index 6712b6b..e38e483 100644 --- a/CommonDependencies/tests/rust_pattern_matching/Cargo.toml +++ b/asap-common/tests/compare_matched_tokens/rust_tests/Cargo.toml @@ -17,4 +17,4 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } [dependencies.promql_utilities] -path = "../../../CommonDependencies/dependencies/rs/promql_utilities" +path = "../../../dependencies/rs/promql_utilities" diff --git a/CommonDependencies/tests/compare_matched_tokens/rust_tests/src/main.rs b/asap-common/tests/compare_matched_tokens/rust_tests/src/main.rs similarity index 100% rename from CommonDependencies/tests/compare_matched_tokens/rust_tests/src/main.rs rename to asap-common/tests/compare_matched_tokens/rust_tests/src/main.rs diff --git a/CommonDependencies/tests/compare_matched_tokens/rust_tests/src/pattern_tests.rs b/asap-common/tests/compare_matched_tokens/rust_tests/src/pattern_tests.rs similarity index 100% rename from CommonDependencies/tests/compare_matched_tokens/rust_tests/src/pattern_tests.rs rename to asap-common/tests/compare_matched_tokens/rust_tests/src/pattern_tests.rs diff --git a/CommonDependencies/tests/compare_matched_tokens/rust_tests/src/test_data.rs b/asap-common/tests/compare_matched_tokens/rust_tests/src/test_data.rs similarity index 100% rename from CommonDependencies/tests/compare_matched_tokens/rust_tests/src/test_data.rs rename to asap-common/tests/compare_matched_tokens/rust_tests/src/test_data.rs diff --git a/CommonDependencies/tests/compare_matched_tokens/utilities/master_test_runner.py b/asap-common/tests/compare_matched_tokens/utilities/master_test_runner.py similarity index 100% rename from CommonDependencies/tests/compare_matched_tokens/utilities/master_test_runner.py rename to asap-common/tests/compare_matched_tokens/utilities/master_test_runner.py diff --git a/CommonDependencies/tests/compare_patterns/Cargo.lock b/asap-common/tests/compare_patterns/Cargo.lock similarity index 100% rename from CommonDependencies/tests/compare_patterns/Cargo.lock rename to asap-common/tests/compare_patterns/Cargo.lock diff --git a/CommonDependencies/tests/compare_patterns/Cargo.toml b/asap-common/tests/compare_patterns/Cargo.toml similarity index 100% rename from CommonDependencies/tests/compare_patterns/Cargo.toml rename to asap-common/tests/compare_patterns/Cargo.toml diff --git a/CommonDependencies/tests/compare_patterns/README.md b/asap-common/tests/compare_patterns/README.md similarity index 100% rename from CommonDependencies/tests/compare_patterns/README.md rename to asap-common/tests/compare_patterns/README.md diff --git a/CommonDependencies/tests/compare_patterns/compare_serialized_patterns.py b/asap-common/tests/compare_patterns/compare_serialized_patterns.py similarity index 100% rename from CommonDependencies/tests/compare_patterns/compare_serialized_patterns.py rename to asap-common/tests/compare_patterns/compare_serialized_patterns.py diff --git a/CommonDependencies/tests/compare_patterns/python_generate_patterns.py b/asap-common/tests/compare_patterns/python_generate_patterns.py similarity index 97% rename from CommonDependencies/tests/compare_patterns/python_generate_patterns.py rename to asap-common/tests/compare_patterns/python_generate_patterns.py index 3ca7bea..52a8235 100644 --- a/CommonDependencies/tests/compare_patterns/python_generate_patterns.py +++ b/asap-common/tests/compare_patterns/python_generate_patterns.py @@ -10,7 +10,7 @@ root = os.path.dirname(__file__) sys.path.append( os.path.abspath( - os.path.join(root, "../../CommonDependencies/dependencies/py/promql_utilities") + os.path.join(root, "../../dependencies/py/promql_utilities") ) ) diff --git a/CommonDependencies/tests/compare_patterns/src/main.rs b/asap-common/tests/compare_patterns/src/main.rs similarity index 100% rename from CommonDependencies/tests/compare_patterns/src/main.rs rename to asap-common/tests/compare_patterns/src/main.rs diff --git a/CommonDependencies/tests/rust_pattern_matching/Cargo.lock b/asap-common/tests/rust_pattern_matching/Cargo.lock similarity index 100% rename from CommonDependencies/tests/rust_pattern_matching/Cargo.lock rename to asap-common/tests/rust_pattern_matching/Cargo.lock diff --git a/CommonDependencies/tests/compare_matched_tokens/rust_tests/Cargo.toml b/asap-common/tests/rust_pattern_matching/Cargo.toml similarity index 85% rename from CommonDependencies/tests/compare_matched_tokens/rust_tests/Cargo.toml rename to asap-common/tests/rust_pattern_matching/Cargo.toml index b472386..14b2980 100644 --- a/CommonDependencies/tests/compare_matched_tokens/rust_tests/Cargo.toml +++ b/asap-common/tests/rust_pattern_matching/Cargo.toml @@ -17,4 +17,4 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } [dependencies.promql_utilities] -path = "../../../../CommonDependencies/dependencies/rs/promql_utilities" +path = "../../dependencies/rs/promql_utilities" diff --git a/CommonDependencies/tests/rust_pattern_matching/src/main.rs b/asap-common/tests/rust_pattern_matching/src/main.rs similarity index 100% rename from CommonDependencies/tests/rust_pattern_matching/src/main.rs rename to asap-common/tests/rust_pattern_matching/src/main.rs diff --git a/Controller/.gitignore b/asap-planner/.gitignore similarity index 100% rename from Controller/.gitignore rename to asap-planner/.gitignore diff --git a/Controller/Dockerfile b/asap-planner/Dockerfile similarity index 100% rename from Controller/Dockerfile rename to asap-planner/Dockerfile diff --git a/ArroyoSketch/LICENSE b/asap-planner/LICENSE similarity index 100% rename from ArroyoSketch/LICENSE rename to asap-planner/LICENSE diff --git a/Controller/README.md b/asap-planner/README.md similarity index 97% rename from Controller/README.md rename to asap-planner/README.md index 8be2f8b..e3fc3d0 100644 --- a/Controller/README.md +++ b/asap-planner/README.md @@ -7,7 +7,7 @@ The Controller is ASAP's auto-configuration service that determines optimal sket Given a workload of PromQL queries, the Controller: 1. Analyzes each query to determine which sketch algorithm to use 2. Computes sketch parameters (size, accuracy) based on SLAs -3. Generates `streaming_config.yaml` for ArroyoSketch +3. Generates `streaming_config.yaml` for asap-sketch-ingest 4. Generates `inference_config.yaml` for QueryEngine This automation eliminates manual configuration and ensures sketches meet performance targets. @@ -70,7 +70,7 @@ metrics: ### Output Files -**streaming_config.yaml** (for ArroyoSketch): +**streaming_config.yaml** (for asap-sketch-ingest): ```yaml sketches: - metric: "http_request_duration_seconds" diff --git a/Controller/classes/SingleQueryConfig.py b/asap-planner/classes/SingleQueryConfig.py similarity index 100% rename from Controller/classes/SingleQueryConfig.py rename to asap-planner/classes/SingleQueryConfig.py diff --git a/Controller/classes/WorkloadConfig.py b/asap-planner/classes/WorkloadConfig.py similarity index 100% rename from Controller/classes/WorkloadConfig.py rename to asap-planner/classes/WorkloadConfig.py diff --git a/Controller/controller-cli-compose.yml.j2 b/asap-planner/controller-cli-compose.yml.j2 similarity index 100% rename from Controller/controller-cli-compose.yml.j2 rename to asap-planner/controller-cli-compose.yml.j2 diff --git a/Controller/docker-compose.yml.j2 b/asap-planner/docker-compose.yml.j2 similarity index 100% rename from Controller/docker-compose.yml.j2 rename to asap-planner/docker-compose.yml.j2 diff --git a/Controller/installation/install.sh b/asap-planner/installation/install.sh similarity index 100% rename from Controller/installation/install.sh rename to asap-planner/installation/install.sh diff --git a/Controller/main_controller.py b/asap-planner/main_controller.py similarity index 100% rename from Controller/main_controller.py rename to asap-planner/main_controller.py diff --git a/Controller/requirements.txt b/asap-planner/requirements.txt similarity index 100% rename from Controller/requirements.txt rename to asap-planner/requirements.txt diff --git a/Controller/utils/logics.py b/asap-planner/utils/logics.py similarity index 100% rename from Controller/utils/logics.py rename to asap-planner/utils/logics.py diff --git a/Controller/utils/test_logics.py b/asap-planner/utils/test_logics.py similarity index 100% rename from Controller/utils/test_logics.py rename to asap-planner/utils/test_logics.py diff --git a/QueryEngineRust/.cargo/config.toml b/asap-query-engine/.cargo/config.toml similarity index 100% rename from QueryEngineRust/.cargo/config.toml rename to asap-query-engine/.cargo/config.toml diff --git a/QueryEngineRust/.gitignore b/asap-query-engine/.gitignore similarity index 100% rename from QueryEngineRust/.gitignore rename to asap-query-engine/.gitignore diff --git a/QueryEngineRust/Cargo.toml b/asap-query-engine/Cargo.toml similarity index 77% rename from QueryEngineRust/Cargo.toml rename to asap-query-engine/Cargo.toml index 567bcc0..73780ac 100644 --- a/QueryEngineRust/Cargo.toml +++ b/asap-query-engine/Cargo.toml @@ -4,11 +4,11 @@ version = "0.1.0" edition = "2021" [dependencies] -sketch-core = { path = "../sketch-core" } +sketch-core = { path = "../asap-common/sketch-core" } form_urlencoded = "1.2" -promql_utilities = { path = "../CommonDependencies/dependencies/rs/promql_utilities" } -sql_utilities = { path = "../CommonDependencies/dependencies/rs/sql_utilities" } -sketch_db_common = { path = "../CommonDependencies/dependencies/rs/sketch_db_common" } +promql_utilities = { path = "../asap-common/dependencies/rs/promql_utilities" } +sql_utilities = { path = "../asap-common/dependencies/rs/sql_utilities" } +sketch_db_common = { path = "../asap-common/dependencies/rs/sketch_db_common" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" @@ -40,7 +40,7 @@ hex = "0.4" sqlparser = "0.59.0" datafusion = "43" arrow = "53.4.1" -datafusion_summary_library = { path = "../CommonDependencies/dependencies/rs/datafusion_summary_library" } +datafusion_summary_library = { path = "../asap-common/dependencies/rs/datafusion_summary_library" } futures = "0.3" prost = "0.13" snap = "1" diff --git a/QueryEngineRust/Dockerfile b/asap-query-engine/Dockerfile similarity index 80% rename from QueryEngineRust/Dockerfile rename to asap-query-engine/Dockerfile index c43d757..14f8bdf 100644 --- a/QueryEngineRust/Dockerfile +++ b/asap-query-engine/Dockerfile @@ -8,23 +8,23 @@ LABEL description="QueryEngine Rust service for SketchDB" WORKDIR /code -# Copy the CommonDependencies directory -COPY CommonDependencies ./CommonDependencies +# Copy the asap-common directory +COPY asap-common ./asap-common -# Copy path dependencies of QueryEngineRust -COPY sketch-core ./sketch-core +# Copy path dependencies of asap-query-engine +COPY asap-common/sketch-core ./asap-common/sketch-core COPY Cargo.toml ./ COPY Cargo.lock ./ -COPY QueryEngineRust/Cargo.toml ./QueryEngineRust/ -# COPY QueryEngineRust/.cargo ./QueryEngineRust/.cargo +COPY asap-query-engine/Cargo.toml ./asap-query-engine/ +# COPY asap-query-engine/.cargo ./asap-query-engine/.cargo # Create a dummy main.rs to build dependencies -RUN mkdir -p QueryEngineRust/src && echo "fn main() {}" > QueryEngineRust/src/main.rs +RUN mkdir -p asap-query-engine/src && echo "fn main() {}" > asap-query-engine/src/main.rs # Build dependencies (this layer will be cached) # Uses BuildKit secret mount to pass git credentials without baking into a layer -WORKDIR /code/QueryEngineRust +WORKDIR /code/asap-query-engine RUN cargo build --release && rm -rf src/ # RUN --mount=type=secret,id=git_token \ # if [ -f /run/secrets/git_token ]; then \ @@ -33,7 +33,7 @@ RUN cargo build --release && rm -rf src/ # cargo build --release && rm -rf src/ # Copy source code -COPY QueryEngineRust/src ./src +COPY asap-query-engine/src ./src # Build the actual application RUN touch src/main.rs && cargo build --release diff --git a/CommonDependencies/LICENSE b/asap-query-engine/LICENSE similarity index 100% rename from CommonDependencies/LICENSE rename to asap-query-engine/LICENSE diff --git a/QueryEngineRust/docker-compose.yml.j2 b/asap-query-engine/docker-compose.yml.j2 similarity index 100% rename from QueryEngineRust/docker-compose.yml.j2 rename to asap-query-engine/docker-compose.yml.j2 diff --git a/QueryEngineRust/docs/README.md b/asap-query-engine/docs/README.md similarity index 100% rename from QueryEngineRust/docs/README.md rename to asap-query-engine/docs/README.md diff --git a/QueryEngineRust/docs/adding-fallback-backend.md b/asap-query-engine/docs/adding-fallback-backend.md similarity index 100% rename from QueryEngineRust/docs/adding-fallback-backend.md rename to asap-query-engine/docs/adding-fallback-backend.md diff --git a/QueryEngineRust/docs/adding-protocol-adapter.md b/asap-query-engine/docs/adding-protocol-adapter.md similarity index 100% rename from QueryEngineRust/docs/adding-protocol-adapter.md rename to asap-query-engine/docs/adding-protocol-adapter.md diff --git a/QueryEngineRust/docs/adding-protocol-server.md b/asap-query-engine/docs/adding-protocol-server.md similarity index 100% rename from QueryEngineRust/docs/adding-protocol-server.md rename to asap-query-engine/docs/adding-protocol-server.md diff --git a/QueryEngineRust/docs/promsketch-integration.md b/asap-query-engine/docs/promsketch-integration.md similarity index 100% rename from QueryEngineRust/docs/promsketch-integration.md rename to asap-query-engine/docs/promsketch-integration.md diff --git a/QueryEngineRust/examples/promql/inference_config.yaml b/asap-query-engine/examples/promql/inference_config.yaml similarity index 100% rename from QueryEngineRust/examples/promql/inference_config.yaml rename to asap-query-engine/examples/promql/inference_config.yaml diff --git a/QueryEngineRust/examples/promql/sketch_config.yaml b/asap-query-engine/examples/promql/sketch_config.yaml similarity index 100% rename from QueryEngineRust/examples/promql/sketch_config.yaml rename to asap-query-engine/examples/promql/sketch_config.yaml diff --git a/QueryEngineRust/examples/promql/streaming_config.yaml b/asap-query-engine/examples/promql/streaming_config.yaml similarity index 100% rename from QueryEngineRust/examples/promql/streaming_config.yaml rename to asap-query-engine/examples/promql/streaming_config.yaml diff --git a/QueryEngineRust/examples/sql/inference_config.yaml b/asap-query-engine/examples/sql/inference_config.yaml similarity index 100% rename from QueryEngineRust/examples/sql/inference_config.yaml rename to asap-query-engine/examples/sql/inference_config.yaml diff --git a/QueryEngineRust/examples/sql/streaming_config.yaml b/asap-query-engine/examples/sql/streaming_config.yaml similarity index 100% rename from QueryEngineRust/examples/sql/streaming_config.yaml rename to asap-query-engine/examples/sql/streaming_config.yaml diff --git a/QueryEngineRust/installation/install.sh b/asap-query-engine/installation/install.sh similarity index 82% rename from QueryEngineRust/installation/install.sh rename to asap-query-engine/installation/install.sh index dfce4dc..724da26 100755 --- a/QueryEngineRust/installation/install.sh +++ b/asap-query-engine/installation/install.sh @@ -13,6 +13,6 @@ cargo build --release echo "Building QueryEngine Rust Docker image..." cd "$(dirname "$PARENT_DIR")" -docker build . -f QueryEngineRust/Dockerfile -t sketchdb-queryengine-rust:latest +docker build . -f asap-query-engine/Dockerfile -t sketchdb-queryengine-rust:latest echo "QueryEngine Rust Docker image built successfully: sketchdb-queryengine-rust:latest" diff --git a/QueryEngineRust/query-engine-rust-cli-compose.yml.j2 b/asap-query-engine/query-engine-rust-cli-compose.yml.j2 similarity index 100% rename from QueryEngineRust/query-engine-rust-cli-compose.yml.j2 rename to asap-query-engine/query-engine-rust-cli-compose.yml.j2 diff --git a/QueryEngineRust/rustfmt.toml b/asap-query-engine/rustfmt.toml similarity index 100% rename from QueryEngineRust/rustfmt.toml rename to asap-query-engine/rustfmt.toml diff --git a/QueryEngineRust/src/bin/run.sh b/asap-query-engine/src/bin/run.sh similarity index 100% rename from QueryEngineRust/src/bin/run.sh rename to asap-query-engine/src/bin/run.sh diff --git a/QueryEngineRust/src/bin/show_logical_plans.rs b/asap-query-engine/src/bin/show_logical_plans.rs similarity index 100% rename from QueryEngineRust/src/bin/show_logical_plans.rs rename to asap-query-engine/src/bin/show_logical_plans.rs diff --git a/QueryEngineRust/src/bin/test_offline_precomputes.rs b/asap-query-engine/src/bin/test_offline_precomputes.rs similarity index 100% rename from QueryEngineRust/src/bin/test_offline_precomputes.rs rename to asap-query-engine/src/bin/test_offline_precomputes.rs diff --git a/QueryEngineRust/src/commenting_out_flink_diff b/asap-query-engine/src/commenting_out_flink_diff similarity index 100% rename from QueryEngineRust/src/commenting_out_flink_diff rename to asap-query-engine/src/commenting_out_flink_diff diff --git a/QueryEngineRust/src/data_model/aggregation_config.rs b/asap-query-engine/src/data_model/aggregation_config.rs similarity index 100% rename from QueryEngineRust/src/data_model/aggregation_config.rs rename to asap-query-engine/src/data_model/aggregation_config.rs diff --git a/QueryEngineRust/src/data_model/aggregation_reference.rs b/asap-query-engine/src/data_model/aggregation_reference.rs similarity index 100% rename from QueryEngineRust/src/data_model/aggregation_reference.rs rename to asap-query-engine/src/data_model/aggregation_reference.rs diff --git a/QueryEngineRust/src/data_model/enums.rs b/asap-query-engine/src/data_model/enums.rs similarity index 100% rename from QueryEngineRust/src/data_model/enums.rs rename to asap-query-engine/src/data_model/enums.rs diff --git a/QueryEngineRust/src/data_model/inference_config.rs b/asap-query-engine/src/data_model/inference_config.rs similarity index 100% rename from QueryEngineRust/src/data_model/inference_config.rs rename to asap-query-engine/src/data_model/inference_config.rs diff --git a/QueryEngineRust/src/data_model/key_by_label_values.rs b/asap-query-engine/src/data_model/key_by_label_values.rs similarity index 100% rename from QueryEngineRust/src/data_model/key_by_label_values.rs rename to asap-query-engine/src/data_model/key_by_label_values.rs diff --git a/QueryEngineRust/src/data_model/measurement.rs b/asap-query-engine/src/data_model/measurement.rs similarity index 100% rename from QueryEngineRust/src/data_model/measurement.rs rename to asap-query-engine/src/data_model/measurement.rs diff --git a/QueryEngineRust/src/data_model/mod.rs b/asap-query-engine/src/data_model/mod.rs similarity index 100% rename from QueryEngineRust/src/data_model/mod.rs rename to asap-query-engine/src/data_model/mod.rs diff --git a/QueryEngineRust/src/data_model/precomputed_output.rs b/asap-query-engine/src/data_model/precomputed_output.rs similarity index 100% rename from QueryEngineRust/src/data_model/precomputed_output.rs rename to asap-query-engine/src/data_model/precomputed_output.rs diff --git a/QueryEngineRust/src/data_model/promql_schema.rs b/asap-query-engine/src/data_model/promql_schema.rs similarity index 100% rename from QueryEngineRust/src/data_model/promql_schema.rs rename to asap-query-engine/src/data_model/promql_schema.rs diff --git a/QueryEngineRust/src/data_model/query_config.rs b/asap-query-engine/src/data_model/query_config.rs similarity index 100% rename from QueryEngineRust/src/data_model/query_config.rs rename to asap-query-engine/src/data_model/query_config.rs diff --git a/QueryEngineRust/src/data_model/streaming_config.rs b/asap-query-engine/src/data_model/streaming_config.rs similarity index 100% rename from QueryEngineRust/src/data_model/streaming_config.rs rename to asap-query-engine/src/data_model/streaming_config.rs diff --git a/QueryEngineRust/src/data_model/traits.rs b/asap-query-engine/src/data_model/traits.rs similarity index 100% rename from QueryEngineRust/src/data_model/traits.rs rename to asap-query-engine/src/data_model/traits.rs diff --git a/QueryEngineRust/src/drivers/ingest/kafka.rs b/asap-query-engine/src/drivers/ingest/kafka.rs similarity index 100% rename from QueryEngineRust/src/drivers/ingest/kafka.rs rename to asap-query-engine/src/drivers/ingest/kafka.rs diff --git a/QueryEngineRust/src/drivers/ingest/mod.rs b/asap-query-engine/src/drivers/ingest/mod.rs similarity index 100% rename from QueryEngineRust/src/drivers/ingest/mod.rs rename to asap-query-engine/src/drivers/ingest/mod.rs diff --git a/QueryEngineRust/src/drivers/ingest/prometheus_remote_write.rs b/asap-query-engine/src/drivers/ingest/prometheus_remote_write.rs similarity index 100% rename from QueryEngineRust/src/drivers/ingest/prometheus_remote_write.rs rename to asap-query-engine/src/drivers/ingest/prometheus_remote_write.rs diff --git a/QueryEngineRust/src/drivers/ingest/victoriametrics_remote_write.rs b/asap-query-engine/src/drivers/ingest/victoriametrics_remote_write.rs similarity index 100% rename from QueryEngineRust/src/drivers/ingest/victoriametrics_remote_write.rs rename to asap-query-engine/src/drivers/ingest/victoriametrics_remote_write.rs diff --git a/QueryEngineRust/src/drivers/mod.rs b/asap-query-engine/src/drivers/mod.rs similarity index 100% rename from QueryEngineRust/src/drivers/mod.rs rename to asap-query-engine/src/drivers/mod.rs diff --git a/QueryEngineRust/src/drivers/query/adapters/clickhouse_http.rs b/asap-query-engine/src/drivers/query/adapters/clickhouse_http.rs similarity index 100% rename from QueryEngineRust/src/drivers/query/adapters/clickhouse_http.rs rename to asap-query-engine/src/drivers/query/adapters/clickhouse_http.rs diff --git a/QueryEngineRust/src/drivers/query/adapters/config.rs b/asap-query-engine/src/drivers/query/adapters/config.rs similarity index 100% rename from QueryEngineRust/src/drivers/query/adapters/config.rs rename to asap-query-engine/src/drivers/query/adapters/config.rs diff --git a/QueryEngineRust/src/drivers/query/adapters/elastic_http.rs b/asap-query-engine/src/drivers/query/adapters/elastic_http.rs similarity index 100% rename from QueryEngineRust/src/drivers/query/adapters/elastic_http.rs rename to asap-query-engine/src/drivers/query/adapters/elastic_http.rs diff --git a/QueryEngineRust/src/drivers/query/adapters/factory.rs b/asap-query-engine/src/drivers/query/adapters/factory.rs similarity index 100% rename from QueryEngineRust/src/drivers/query/adapters/factory.rs rename to asap-query-engine/src/drivers/query/adapters/factory.rs diff --git a/QueryEngineRust/src/drivers/query/adapters/mod.rs b/asap-query-engine/src/drivers/query/adapters/mod.rs similarity index 100% rename from QueryEngineRust/src/drivers/query/adapters/mod.rs rename to asap-query-engine/src/drivers/query/adapters/mod.rs diff --git a/QueryEngineRust/src/drivers/query/adapters/prometheus_http.rs b/asap-query-engine/src/drivers/query/adapters/prometheus_http.rs similarity index 100% rename from QueryEngineRust/src/drivers/query/adapters/prometheus_http.rs rename to asap-query-engine/src/drivers/query/adapters/prometheus_http.rs diff --git a/QueryEngineRust/src/drivers/query/adapters/traits.rs b/asap-query-engine/src/drivers/query/adapters/traits.rs similarity index 100% rename from QueryEngineRust/src/drivers/query/adapters/traits.rs rename to asap-query-engine/src/drivers/query/adapters/traits.rs diff --git a/QueryEngineRust/src/drivers/query/fallback/clickhouse.rs b/asap-query-engine/src/drivers/query/fallback/clickhouse.rs similarity index 100% rename from QueryEngineRust/src/drivers/query/fallback/clickhouse.rs rename to asap-query-engine/src/drivers/query/fallback/clickhouse.rs diff --git a/QueryEngineRust/src/drivers/query/fallback/elastic.rs b/asap-query-engine/src/drivers/query/fallback/elastic.rs similarity index 100% rename from QueryEngineRust/src/drivers/query/fallback/elastic.rs rename to asap-query-engine/src/drivers/query/fallback/elastic.rs diff --git a/QueryEngineRust/src/drivers/query/fallback/mod.rs b/asap-query-engine/src/drivers/query/fallback/mod.rs similarity index 100% rename from QueryEngineRust/src/drivers/query/fallback/mod.rs rename to asap-query-engine/src/drivers/query/fallback/mod.rs diff --git a/QueryEngineRust/src/drivers/query/fallback/prometheus.rs b/asap-query-engine/src/drivers/query/fallback/prometheus.rs similarity index 100% rename from QueryEngineRust/src/drivers/query/fallback/prometheus.rs rename to asap-query-engine/src/drivers/query/fallback/prometheus.rs diff --git a/QueryEngineRust/src/drivers/query/mod.rs b/asap-query-engine/src/drivers/query/mod.rs similarity index 100% rename from QueryEngineRust/src/drivers/query/mod.rs rename to asap-query-engine/src/drivers/query/mod.rs diff --git a/QueryEngineRust/src/drivers/query/servers/http.rs b/asap-query-engine/src/drivers/query/servers/http.rs similarity index 100% rename from QueryEngineRust/src/drivers/query/servers/http.rs rename to asap-query-engine/src/drivers/query/servers/http.rs diff --git a/QueryEngineRust/src/drivers/query/servers/mod.rs b/asap-query-engine/src/drivers/query/servers/mod.rs similarity index 100% rename from QueryEngineRust/src/drivers/query/servers/mod.rs rename to asap-query-engine/src/drivers/query/servers/mod.rs diff --git a/QueryEngineRust/src/engines/logical/mod.rs b/asap-query-engine/src/engines/logical/mod.rs similarity index 100% rename from QueryEngineRust/src/engines/logical/mod.rs rename to asap-query-engine/src/engines/logical/mod.rs diff --git a/QueryEngineRust/src/engines/logical/plan_builder.rs b/asap-query-engine/src/engines/logical/plan_builder.rs similarity index 100% rename from QueryEngineRust/src/engines/logical/plan_builder.rs rename to asap-query-engine/src/engines/logical/plan_builder.rs diff --git a/QueryEngineRust/src/engines/mod.rs b/asap-query-engine/src/engines/mod.rs similarity index 100% rename from QueryEngineRust/src/engines/mod.rs rename to asap-query-engine/src/engines/mod.rs diff --git a/QueryEngineRust/src/engines/physical/accumulator_serde.rs b/asap-query-engine/src/engines/physical/accumulator_serde.rs similarity index 100% rename from QueryEngineRust/src/engines/physical/accumulator_serde.rs rename to asap-query-engine/src/engines/physical/accumulator_serde.rs diff --git a/QueryEngineRust/src/engines/physical/conversion.rs b/asap-query-engine/src/engines/physical/conversion.rs similarity index 100% rename from QueryEngineRust/src/engines/physical/conversion.rs rename to asap-query-engine/src/engines/physical/conversion.rs diff --git a/QueryEngineRust/src/engines/physical/mod.rs b/asap-query-engine/src/engines/physical/mod.rs similarity index 100% rename from QueryEngineRust/src/engines/physical/mod.rs rename to asap-query-engine/src/engines/physical/mod.rs diff --git a/QueryEngineRust/src/engines/physical/planner.rs b/asap-query-engine/src/engines/physical/planner.rs similarity index 100% rename from QueryEngineRust/src/engines/physical/planner.rs rename to asap-query-engine/src/engines/physical/planner.rs diff --git a/QueryEngineRust/src/engines/physical/precomputed_summary_read_exec.rs b/asap-query-engine/src/engines/physical/precomputed_summary_read_exec.rs similarity index 100% rename from QueryEngineRust/src/engines/physical/precomputed_summary_read_exec.rs rename to asap-query-engine/src/engines/physical/precomputed_summary_read_exec.rs diff --git a/QueryEngineRust/src/engines/physical/summary_infer_exec.rs b/asap-query-engine/src/engines/physical/summary_infer_exec.rs similarity index 100% rename from QueryEngineRust/src/engines/physical/summary_infer_exec.rs rename to asap-query-engine/src/engines/physical/summary_infer_exec.rs diff --git a/QueryEngineRust/src/engines/physical/summary_merge_multiple_exec.rs b/asap-query-engine/src/engines/physical/summary_merge_multiple_exec.rs similarity index 100% rename from QueryEngineRust/src/engines/physical/summary_merge_multiple_exec.rs rename to asap-query-engine/src/engines/physical/summary_merge_multiple_exec.rs diff --git a/QueryEngineRust/src/engines/query_result.rs b/asap-query-engine/src/engines/query_result.rs similarity index 100% rename from QueryEngineRust/src/engines/query_result.rs rename to asap-query-engine/src/engines/query_result.rs diff --git a/QueryEngineRust/src/engines/simple_engine.rs b/asap-query-engine/src/engines/simple_engine.rs similarity index 100% rename from QueryEngineRust/src/engines/simple_engine.rs rename to asap-query-engine/src/engines/simple_engine.rs diff --git a/QueryEngineRust/src/engines/window_merger.rs b/asap-query-engine/src/engines/window_merger.rs similarity index 100% rename from QueryEngineRust/src/engines/window_merger.rs rename to asap-query-engine/src/engines/window_merger.rs diff --git a/QueryEngineRust/src/lib.rs b/asap-query-engine/src/lib.rs similarity index 100% rename from QueryEngineRust/src/lib.rs rename to asap-query-engine/src/lib.rs diff --git a/QueryEngineRust/src/main.rs b/asap-query-engine/src/main.rs similarity index 100% rename from QueryEngineRust/src/main.rs rename to asap-query-engine/src/main.rs diff --git a/QueryEngineRust/src/precompute_operators/count_min_sketch_accumulator.rs b/asap-query-engine/src/precompute_operators/count_min_sketch_accumulator.rs similarity index 100% rename from QueryEngineRust/src/precompute_operators/count_min_sketch_accumulator.rs rename to asap-query-engine/src/precompute_operators/count_min_sketch_accumulator.rs diff --git a/QueryEngineRust/src/precompute_operators/count_min_sketch_with_heap_accumulator.rs b/asap-query-engine/src/precompute_operators/count_min_sketch_with_heap_accumulator.rs similarity index 100% rename from QueryEngineRust/src/precompute_operators/count_min_sketch_with_heap_accumulator.rs rename to asap-query-engine/src/precompute_operators/count_min_sketch_with_heap_accumulator.rs diff --git a/QueryEngineRust/src/precompute_operators/datasketches_kll_accumulator.rs b/asap-query-engine/src/precompute_operators/datasketches_kll_accumulator.rs similarity index 100% rename from QueryEngineRust/src/precompute_operators/datasketches_kll_accumulator.rs rename to asap-query-engine/src/precompute_operators/datasketches_kll_accumulator.rs diff --git a/QueryEngineRust/src/precompute_operators/delta_set_aggregator_accumulator.rs b/asap-query-engine/src/precompute_operators/delta_set_aggregator_accumulator.rs similarity index 100% rename from QueryEngineRust/src/precompute_operators/delta_set_aggregator_accumulator.rs rename to asap-query-engine/src/precompute_operators/delta_set_aggregator_accumulator.rs diff --git a/QueryEngineRust/src/precompute_operators/hydra_kll_accumulator.rs b/asap-query-engine/src/precompute_operators/hydra_kll_accumulator.rs similarity index 100% rename from QueryEngineRust/src/precompute_operators/hydra_kll_accumulator.rs rename to asap-query-engine/src/precompute_operators/hydra_kll_accumulator.rs diff --git a/QueryEngineRust/src/precompute_operators/increase_accumulator.rs b/asap-query-engine/src/precompute_operators/increase_accumulator.rs similarity index 100% rename from QueryEngineRust/src/precompute_operators/increase_accumulator.rs rename to asap-query-engine/src/precompute_operators/increase_accumulator.rs diff --git a/QueryEngineRust/src/precompute_operators/min_max_accumulator.rs b/asap-query-engine/src/precompute_operators/min_max_accumulator.rs similarity index 100% rename from QueryEngineRust/src/precompute_operators/min_max_accumulator.rs rename to asap-query-engine/src/precompute_operators/min_max_accumulator.rs diff --git a/QueryEngineRust/src/precompute_operators/mod.rs b/asap-query-engine/src/precompute_operators/mod.rs similarity index 100% rename from QueryEngineRust/src/precompute_operators/mod.rs rename to asap-query-engine/src/precompute_operators/mod.rs diff --git a/QueryEngineRust/src/precompute_operators/multiple_increase_accumulator.rs b/asap-query-engine/src/precompute_operators/multiple_increase_accumulator.rs similarity index 100% rename from QueryEngineRust/src/precompute_operators/multiple_increase_accumulator.rs rename to asap-query-engine/src/precompute_operators/multiple_increase_accumulator.rs diff --git a/QueryEngineRust/src/precompute_operators/multiple_min_max_accumulator.rs b/asap-query-engine/src/precompute_operators/multiple_min_max_accumulator.rs similarity index 100% rename from QueryEngineRust/src/precompute_operators/multiple_min_max_accumulator.rs rename to asap-query-engine/src/precompute_operators/multiple_min_max_accumulator.rs diff --git a/QueryEngineRust/src/precompute_operators/multiple_sum_accumulator.rs b/asap-query-engine/src/precompute_operators/multiple_sum_accumulator.rs similarity index 100% rename from QueryEngineRust/src/precompute_operators/multiple_sum_accumulator.rs rename to asap-query-engine/src/precompute_operators/multiple_sum_accumulator.rs diff --git a/QueryEngineRust/src/precompute_operators/set_aggregator_accumulator.rs b/asap-query-engine/src/precompute_operators/set_aggregator_accumulator.rs similarity index 100% rename from QueryEngineRust/src/precompute_operators/set_aggregator_accumulator.rs rename to asap-query-engine/src/precompute_operators/set_aggregator_accumulator.rs diff --git a/QueryEngineRust/src/precompute_operators/sum_accumulator.rs b/asap-query-engine/src/precompute_operators/sum_accumulator.rs similarity index 100% rename from QueryEngineRust/src/precompute_operators/sum_accumulator.rs rename to asap-query-engine/src/precompute_operators/sum_accumulator.rs diff --git a/QueryEngineRust/src/stores/mod.rs b/asap-query-engine/src/stores/mod.rs similarity index 100% rename from QueryEngineRust/src/stores/mod.rs rename to asap-query-engine/src/stores/mod.rs diff --git a/QueryEngineRust/src/stores/promsketch_store/config.rs b/asap-query-engine/src/stores/promsketch_store/config.rs similarity index 100% rename from QueryEngineRust/src/stores/promsketch_store/config.rs rename to asap-query-engine/src/stores/promsketch_store/config.rs diff --git a/QueryEngineRust/src/stores/promsketch_store/metrics.rs b/asap-query-engine/src/stores/promsketch_store/metrics.rs similarity index 100% rename from QueryEngineRust/src/stores/promsketch_store/metrics.rs rename to asap-query-engine/src/stores/promsketch_store/metrics.rs diff --git a/QueryEngineRust/src/stores/promsketch_store/mod.rs b/asap-query-engine/src/stores/promsketch_store/mod.rs similarity index 100% rename from QueryEngineRust/src/stores/promsketch_store/mod.rs rename to asap-query-engine/src/stores/promsketch_store/mod.rs diff --git a/QueryEngineRust/src/stores/promsketch_store/query.rs b/asap-query-engine/src/stores/promsketch_store/query.rs similarity index 100% rename from QueryEngineRust/src/stores/promsketch_store/query.rs rename to asap-query-engine/src/stores/promsketch_store/query.rs diff --git a/QueryEngineRust/src/stores/promsketch_store/series.rs b/asap-query-engine/src/stores/promsketch_store/series.rs similarity index 100% rename from QueryEngineRust/src/stores/promsketch_store/series.rs rename to asap-query-engine/src/stores/promsketch_store/series.rs diff --git a/QueryEngineRust/src/stores/promsketch_store/store.rs b/asap-query-engine/src/stores/promsketch_store/store.rs similarity index 100% rename from QueryEngineRust/src/stores/promsketch_store/store.rs rename to asap-query-engine/src/stores/promsketch_store/store.rs diff --git a/QueryEngineRust/src/stores/promsketch_store/types.rs b/asap-query-engine/src/stores/promsketch_store/types.rs similarity index 100% rename from QueryEngineRust/src/stores/promsketch_store/types.rs rename to asap-query-engine/src/stores/promsketch_store/types.rs diff --git a/QueryEngineRust/src/stores/simple_map_store/global.rs b/asap-query-engine/src/stores/simple_map_store/global.rs similarity index 100% rename from QueryEngineRust/src/stores/simple_map_store/global.rs rename to asap-query-engine/src/stores/simple_map_store/global.rs diff --git a/QueryEngineRust/src/stores/simple_map_store/mod.rs b/asap-query-engine/src/stores/simple_map_store/mod.rs similarity index 100% rename from QueryEngineRust/src/stores/simple_map_store/mod.rs rename to asap-query-engine/src/stores/simple_map_store/mod.rs diff --git a/QueryEngineRust/src/stores/simple_map_store/per_key.rs b/asap-query-engine/src/stores/simple_map_store/per_key.rs similarity index 100% rename from QueryEngineRust/src/stores/simple_map_store/per_key.rs rename to asap-query-engine/src/stores/simple_map_store/per_key.rs diff --git a/QueryEngineRust/src/stores/traits.rs b/asap-query-engine/src/stores/traits.rs similarity index 100% rename from QueryEngineRust/src/stores/traits.rs rename to asap-query-engine/src/stores/traits.rs diff --git a/QueryEngineRust/src/tests/clickhouse_forwarding_tests.rs b/asap-query-engine/src/tests/clickhouse_forwarding_tests.rs similarity index 100% rename from QueryEngineRust/src/tests/clickhouse_forwarding_tests.rs rename to asap-query-engine/src/tests/clickhouse_forwarding_tests.rs diff --git a/QueryEngineRust/src/tests/datafusion/accumulator_serde_tests.rs b/asap-query-engine/src/tests/datafusion/accumulator_serde_tests.rs similarity index 100% rename from QueryEngineRust/src/tests/datafusion/accumulator_serde_tests.rs rename to asap-query-engine/src/tests/datafusion/accumulator_serde_tests.rs diff --git a/QueryEngineRust/src/tests/datafusion/mod.rs b/asap-query-engine/src/tests/datafusion/mod.rs similarity index 100% rename from QueryEngineRust/src/tests/datafusion/mod.rs rename to asap-query-engine/src/tests/datafusion/mod.rs diff --git a/QueryEngineRust/src/tests/datafusion/plan_builder_regression_tests.rs b/asap-query-engine/src/tests/datafusion/plan_builder_regression_tests.rs similarity index 100% rename from QueryEngineRust/src/tests/datafusion/plan_builder_regression_tests.rs rename to asap-query-engine/src/tests/datafusion/plan_builder_regression_tests.rs diff --git a/QueryEngineRust/src/tests/datafusion/plan_execution_dual_input_tests.rs b/asap-query-engine/src/tests/datafusion/plan_execution_dual_input_tests.rs similarity index 100% rename from QueryEngineRust/src/tests/datafusion/plan_execution_dual_input_tests.rs rename to asap-query-engine/src/tests/datafusion/plan_execution_dual_input_tests.rs diff --git a/QueryEngineRust/src/tests/datafusion/plan_execution_temporal_tests.rs b/asap-query-engine/src/tests/datafusion/plan_execution_temporal_tests.rs similarity index 100% rename from QueryEngineRust/src/tests/datafusion/plan_execution_temporal_tests.rs rename to asap-query-engine/src/tests/datafusion/plan_execution_temporal_tests.rs diff --git a/QueryEngineRust/src/tests/datafusion/plan_execution_tests.rs b/asap-query-engine/src/tests/datafusion/plan_execution_tests.rs similarity index 100% rename from QueryEngineRust/src/tests/datafusion/plan_execution_tests.rs rename to asap-query-engine/src/tests/datafusion/plan_execution_tests.rs diff --git a/QueryEngineRust/src/tests/elastic_forwarding_tests.rs b/asap-query-engine/src/tests/elastic_forwarding_tests.rs similarity index 100% rename from QueryEngineRust/src/tests/elastic_forwarding_tests.rs rename to asap-query-engine/src/tests/elastic_forwarding_tests.rs diff --git a/QueryEngineRust/src/tests/mod.rs b/asap-query-engine/src/tests/mod.rs similarity index 100% rename from QueryEngineRust/src/tests/mod.rs rename to asap-query-engine/src/tests/mod.rs diff --git a/QueryEngineRust/src/tests/prometheus_forwarding_tests.rs b/asap-query-engine/src/tests/prometheus_forwarding_tests.rs similarity index 100% rename from QueryEngineRust/src/tests/prometheus_forwarding_tests.rs rename to asap-query-engine/src/tests/prometheus_forwarding_tests.rs diff --git a/QueryEngineRust/src/tests/query_equivalence_tests.rs b/asap-query-engine/src/tests/query_equivalence_tests.rs similarity index 100% rename from QueryEngineRust/src/tests/query_equivalence_tests.rs rename to asap-query-engine/src/tests/query_equivalence_tests.rs diff --git a/QueryEngineRust/src/tests/test_utilities/comparison.rs b/asap-query-engine/src/tests/test_utilities/comparison.rs similarity index 100% rename from QueryEngineRust/src/tests/test_utilities/comparison.rs rename to asap-query-engine/src/tests/test_utilities/comparison.rs diff --git a/QueryEngineRust/src/tests/test_utilities/config_builders.rs b/asap-query-engine/src/tests/test_utilities/config_builders.rs similarity index 100% rename from QueryEngineRust/src/tests/test_utilities/config_builders.rs rename to asap-query-engine/src/tests/test_utilities/config_builders.rs diff --git a/QueryEngineRust/src/tests/test_utilities/engine_factories.rs b/asap-query-engine/src/tests/test_utilities/engine_factories.rs similarity index 100% rename from QueryEngineRust/src/tests/test_utilities/engine_factories.rs rename to asap-query-engine/src/tests/test_utilities/engine_factories.rs diff --git a/QueryEngineRust/src/tests/test_utilities/mod.rs b/asap-query-engine/src/tests/test_utilities/mod.rs similarity index 100% rename from QueryEngineRust/src/tests/test_utilities/mod.rs rename to asap-query-engine/src/tests/test_utilities/mod.rs diff --git a/QueryEngineRust/src/tests/trait_design_tests.rs b/asap-query-engine/src/tests/trait_design_tests.rs similarity index 100% rename from QueryEngineRust/src/tests/trait_design_tests.rs rename to asap-query-engine/src/tests/trait_design_tests.rs diff --git a/QueryEngineRust/src/utils/file_io.rs b/asap-query-engine/src/utils/file_io.rs similarity index 100% rename from QueryEngineRust/src/utils/file_io.rs rename to asap-query-engine/src/utils/file_io.rs diff --git a/QueryEngineRust/src/utils/http.rs b/asap-query-engine/src/utils/http.rs similarity index 100% rename from QueryEngineRust/src/utils/http.rs rename to asap-query-engine/src/utils/http.rs diff --git a/QueryEngineRust/src/utils/mod.rs b/asap-query-engine/src/utils/mod.rs similarity index 100% rename from QueryEngineRust/src/utils/mod.rs rename to asap-query-engine/src/utils/mod.rs diff --git a/QueryEngineRust/src/utils/precompute_dumper.rs b/asap-query-engine/src/utils/precompute_dumper.rs similarity index 100% rename from QueryEngineRust/src/utils/precompute_dumper.rs rename to asap-query-engine/src/utils/precompute_dumper.rs diff --git a/QueryEngineRust/src/utils/promql.rs b/asap-query-engine/src/utils/promql.rs similarity index 100% rename from QueryEngineRust/src/utils/promql.rs rename to asap-query-engine/src/utils/promql.rs diff --git a/QueryEngineRust/tests/e2e_promsketch.rs b/asap-query-engine/tests/e2e_promsketch.rs similarity index 100% rename from QueryEngineRust/tests/e2e_promsketch.rs rename to asap-query-engine/tests/e2e_promsketch.rs diff --git a/quickstart/.gitignore b/asap-quickstart/.gitignore similarity index 100% rename from quickstart/.gitignore rename to asap-quickstart/.gitignore diff --git a/quickstart/README.md b/asap-quickstart/README.md similarity index 93% rename from quickstart/README.md rename to asap-quickstart/README.md index d89d557..11c5901 100644 --- a/quickstart/README.md +++ b/asap-quickstart/README.md @@ -14,9 +14,9 @@ This quickstart simulates a typical monitoring deployment with components you mi Then it adds ASAPQuery's components on top: - **Query Engine** - Prometheus-compatible API with sketch-based acceleration -- **Arroyo + ArroyoSketch** - Streaming engine with pipelines configured for building sketches +- **Arroyo + asap-sketch-ingest** - Streaming engine with pipelines configured for building sketches - **Kafka** - Message broker for streaming data from Arroyo to the Query Engine -- **Controller** - Automatically configures sketches from PromQL queries +- **asap-planner** - Automatically configures sketches from PromQL queries Once you run the quickstart, you will see a pre-configured Grafana dashboard that compares Prometheues and ASAPQuery side-by-side. You will see **visually indistinguishable** results from Prometheus and ASAPQuery, with ASAPQuery being 100x faster @@ -30,7 +30,7 @@ Once you run the quickstart, you will see a pre-configured Grafana dashboard tha ### Step 1: Start the Demo ```bash -cd quickstart +cd asap-quickstart docker compose up -d ``` @@ -77,7 +77,7 @@ docker compose down ### Changing the cardinality of ingested data -To change the cardinality of data exported by each exporter, use [set_data_cardinality.py](quickstart/set_data_cardinality.py). +To change the cardinality of data exported by each exporter, use [set_data_cardinality.py](set_data_cardinality.py). Run `python3 set_data_cardinality.py ` where `M` is the number of labels (currently configured to 3) and `N` is the number of unique values per label (currently configured to 30). The script changes all the `fake-exporter-*` services in `docker-compose.yml` to have: ```yaml - "--num-labels=M" # Number of label dimensions @@ -88,7 +88,7 @@ Run `python3 set_data_cardinality.py ` where `M` is the number of labels To modify the queries in the Grafana dashboard and run ASAPQuery against those: -#### 1. Edit the Controller Config +#### 1. Edit the asap-planner Config Edit `config/controller-config.yaml`: diff --git a/quickstart/config/arroyo-config.yaml b/asap-quickstart/config/arroyo-config.yaml similarity index 100% rename from quickstart/config/arroyo-config.yaml rename to asap-quickstart/config/arroyo-config.yaml diff --git a/quickstart/config/controller-config.yaml b/asap-quickstart/config/controller-config.yaml similarity index 100% rename from quickstart/config/controller-config.yaml rename to asap-quickstart/config/controller-config.yaml diff --git a/quickstart/config/grafana/provisioning/dashboards/asap-comparison.json b/asap-quickstart/config/grafana/provisioning/dashboards/asap-comparison.json similarity index 100% rename from quickstart/config/grafana/provisioning/dashboards/asap-comparison.json rename to asap-quickstart/config/grafana/provisioning/dashboards/asap-comparison.json diff --git a/quickstart/config/grafana/provisioning/dashboards/default.yml b/asap-quickstart/config/grafana/provisioning/dashboards/default.yml similarity index 100% rename from quickstart/config/grafana/provisioning/dashboards/default.yml rename to asap-quickstart/config/grafana/provisioning/dashboards/default.yml diff --git a/quickstart/config/grafana/provisioning/datasources/datasources.yml b/asap-quickstart/config/grafana/provisioning/datasources/datasources.yml similarity index 100% rename from quickstart/config/grafana/provisioning/datasources/datasources.yml rename to asap-quickstart/config/grafana/provisioning/datasources/datasources.yml diff --git a/quickstart/config/prometheus.yml b/asap-quickstart/config/prometheus.yml similarity index 100% rename from quickstart/config/prometheus.yml rename to asap-quickstart/config/prometheus.yml diff --git a/quickstart/docker-compose.yml b/asap-quickstart/docker-compose.yml similarity index 93% rename from quickstart/docker-compose.yml rename to asap-quickstart/docker-compose.yml index 589b12b..4d0dcba 100644 --- a/quickstart/docker-compose.yml +++ b/asap-quickstart/docker-compose.yml @@ -1,8 +1,8 @@ -name: asap-turboquery-quickstart +name: asapquery-quickstart # Docker Compose file for pattern-based fake exporters demo # Uses deterministic time-based patterns instead of random data -# This makes visual comparison between Prometheus and TurboQuery easier +# This makes visual comparison between Prometheus and ASAPQuery easier networks: asap-network: @@ -16,7 +16,7 @@ volumes: kafka-data: prometheus-data: grafana-data: - controller-output: + asap-planner-output: services: ############################################################################# @@ -137,7 +137,7 @@ services: timeout: 5s retries: 5 depends_on: - arroyosketch: + asap-sketch-ingest: condition: service_completed_successfully restart: no @@ -169,32 +169,32 @@ services: # INIT CONTAINERS ############################################################################# - controller: + asap-planner: image: ghcr.io/projectasap/asap-controller:v0.1.0 - container_name: asap-controller - hostname: controller + container_name: asap-planner + hostname: asap-planner networks: - asap-network command: - "--input_config=/config/controller-config.yaml" - - "--output_dir=/controller-output" + - "--output_dir=/asap-planner-output" - "--prometheus_scrape_interval=1" - "--streaming_engine=arroyo" - "--range-duration=300" - "--step=10" volumes: - ./config/controller-config.yaml:/config/controller-config.yaml:ro - - controller-output:/controller-output + - asap-planner-output:/asap-planner-output restart: "no" - arroyosketch: + asap-sketch-ingest: image: ghcr.io/projectasap/asap-arroyosketch:v0.1.0 - container_name: asap-arroyosketch - hostname: arroyosketch + container_name: asap-sketch-ingest + hostname: asap-sketch-ingest networks: - asap-network command: - - "--config_file_path=/controller-output/streaming_config.yaml" + - "--config_file_path=/asap-planner-output/streaming_config.yaml" - "--source_type=prometheus_remote_write" - "--prometheus_base_port=9091" - "--prometheus_path=/receive" @@ -203,14 +203,14 @@ services: - "--output_kafka_topic=flink_output" - "--output_format=json" - "--pipeline_name=asap-demo" - - "--output_dir=/arroyosketch-output" + - "--output_dir=/asap-sketch-ingest-output" - "--arroyo_url=http://arroyo:5115/api/v1" - "--bootstrap_servers=kafka:9092" volumes: - - controller-output:/controller-output:ro - - ./output/arroyosketch:/arroyosketch-output + - asap-planner-output:/asap-planner-output:ro + - ./output/asap-sketch-ingest:/asap-sketch-ingest-output depends_on: - controller: + asap-planner: condition: service_completed_successfully arroyo: condition: service_healthy @@ -232,14 +232,14 @@ services: - RUST_LOG=INFO - RUST_BACKTRACE=1 volumes: - - controller-output:/controller-output:ro + - asap-planner-output:/asap-planner-output:ro - ./output/queryengine:/app/outputs command: - "--kafka-topic=flink_output" - "--kafka-broker=kafka:9092" - "--input-format=json" - - "--config=/controller-output/inference_config.yaml" - - "--streaming-config=/controller-output/streaming_config.yaml" + - "--config=/asap-planner-output/inference_config.yaml" + - "--streaming-config=/asap-planner-output/streaming_config.yaml" - "--prometheus-server=http://prometheus:9090" - "--prometheus-scrape-interval=1" - "--streaming-engine=arroyo" @@ -251,7 +251,7 @@ services: - "--decompress-json" - "--forward-unsupported-queries" depends_on: - arroyosketch: + asap-sketch-ingest: condition: service_completed_successfully kafka: condition: service_healthy diff --git a/quickstart/generate_dashboards.py b/asap-quickstart/generate_dashboards.py similarity index 100% rename from quickstart/generate_dashboards.py rename to asap-quickstart/generate_dashboards.py diff --git a/quickstart/set_data_cardinality.py b/asap-quickstart/set_data_cardinality.py similarity index 100% rename from quickstart/set_data_cardinality.py rename to asap-quickstart/set_data_cardinality.py diff --git a/quickstart/set_num_exporters.py b/asap-quickstart/set_num_exporters.py similarity index 98% rename from quickstart/set_num_exporters.py rename to asap-quickstart/set_num_exporters.py index 9065f3e..194ad9d 100755 --- a/quickstart/set_num_exporters.py +++ b/asap-quickstart/set_num_exporters.py @@ -45,7 +45,7 @@ def update_docker_compose(file_path: str, num_exporters: int) -> None: # Generate new exporter services exporter_template = """ fake-exporter-{idx}: build: - context: ../PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter + context: ../asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter container_name: asap-fake-exporter-{idx} hostname: fake-exporter-{idx} networks: diff --git a/ArroyoSketch/.gitignore b/asap-sketch-ingest/.gitignore similarity index 100% rename from ArroyoSketch/.gitignore rename to asap-sketch-ingest/.gitignore diff --git a/ArroyoSketch/Dockerfile b/asap-sketch-ingest/Dockerfile similarity index 100% rename from ArroyoSketch/Dockerfile rename to asap-sketch-ingest/Dockerfile diff --git a/Controller/LICENSE b/asap-sketch-ingest/LICENSE similarity index 100% rename from Controller/LICENSE rename to asap-sketch-ingest/LICENSE diff --git a/ArroyoSketch/README.md b/asap-sketch-ingest/README.md similarity index 83% rename from ArroyoSketch/README.md rename to asap-sketch-ingest/README.md index 576b181..cfda240 100644 --- a/ArroyoSketch/README.md +++ b/asap-sketch-ingest/README.md @@ -1,22 +1,22 @@ -# ArroyoSketch +# asap-sketch-ingest -ArroyoSketch is the pipeline configurator that creates Arroyo streaming pipelines from configuration files. +asap-sketch-ingest is the pipeline configurator that creates Arroyo streaming pipelines from configuration files. ## Purpose -Given `streaming_config.yaml` (generated by Controller), ArroyoSketch: +Given `streaming_config.yaml` (generated by asap-planner), asap-sketch-ingest: 1. Renders SQL query templates using Jinja2 2. Creates Arroyo pipelines via REST API 3. Configures sketch-building UDFs with parameters 4. Sets up connections to Kafka for sketch output -This automation eliminates manual pipeline creation and ensures consistency with Controller decisions. +This automation eliminates manual pipeline creation and ensures consistency with asap-planner decisions. ## How It Works ### Input: streaming_config.yaml -The Controller generates this file describing which sketches to build: +The asap-planner generates this file describing which sketches to build: **TODO** @@ -89,7 +89,7 @@ cd arroyo docker compose up ``` -2. Run ArroyoSketch: +2. Run asap-sketch-ingest: ```bash python run_arroyosketch.py --config test_config.yaml ``` diff --git a/ArroyoSketch/arroyo-compose.yml b/asap-sketch-ingest/arroyo-compose.yml similarity index 100% rename from ArroyoSketch/arroyo-compose.yml rename to asap-sketch-ingest/arroyo-compose.yml diff --git a/ArroyoSketch/arroyosketch-cli-compose.yml.j2 b/asap-sketch-ingest/arroyosketch-cli-compose.yml.j2 similarity index 100% rename from ArroyoSketch/arroyosketch-cli-compose.yml.j2 rename to asap-sketch-ingest/arroyosketch-cli-compose.yml.j2 diff --git a/ArroyoSketch/config.yaml b/asap-sketch-ingest/config.yaml similarity index 100% rename from ArroyoSketch/config.yaml rename to asap-sketch-ingest/config.yaml diff --git a/ArroyoSketch/delete_pipeline.py b/asap-sketch-ingest/delete_pipeline.py similarity index 100% rename from ArroyoSketch/delete_pipeline.py rename to asap-sketch-ingest/delete_pipeline.py diff --git a/ArroyoSketch/examples/configs/streaming_config.yaml b/asap-sketch-ingest/examples/configs/streaming_config.yaml similarity index 100% rename from ArroyoSketch/examples/configs/streaming_config.yaml rename to asap-sketch-ingest/examples/configs/streaming_config.yaml diff --git a/ArroyoSketch/examples/configs/test_promql_streaming_config.yaml b/asap-sketch-ingest/examples/configs/test_promql_streaming_config.yaml similarity index 100% rename from ArroyoSketch/examples/configs/test_promql_streaming_config.yaml rename to asap-sketch-ingest/examples/configs/test_promql_streaming_config.yaml diff --git a/ArroyoSketch/examples/configs/test_sql_streaming_config.yaml b/asap-sketch-ingest/examples/configs/test_sql_streaming_config.yaml similarity index 100% rename from ArroyoSketch/examples/configs/test_sql_streaming_config.yaml rename to asap-sketch-ingest/examples/configs/test_sql_streaming_config.yaml diff --git a/ArroyoSketch/examples/inputs/fake_metric_total_10.json b/asap-sketch-ingest/examples/inputs/fake_metric_total_10.json similarity index 100% rename from ArroyoSketch/examples/inputs/fake_metric_total_10.json rename to asap-sketch-ingest/examples/inputs/fake_metric_total_10.json diff --git a/ArroyoSketch/examples/inputs/fake_metric_total_10_2.json b/asap-sketch-ingest/examples/inputs/fake_metric_total_10_2.json similarity index 100% rename from ArroyoSketch/examples/inputs/fake_metric_total_10_2.json rename to asap-sketch-ingest/examples/inputs/fake_metric_total_10_2.json diff --git a/ArroyoSketch/examples/inputs/value.json b/asap-sketch-ingest/examples/inputs/value.json similarity index 100% rename from ArroyoSketch/examples/inputs/value.json rename to asap-sketch-ingest/examples/inputs/value.json diff --git a/ArroyoSketch/examples/json/connection_profile.json b/asap-sketch-ingest/examples/json/connection_profile.json similarity index 100% rename from ArroyoSketch/examples/json/connection_profile.json rename to asap-sketch-ingest/examples/json/connection_profile.json diff --git a/ArroyoSketch/examples/json/connection_table.json b/asap-sketch-ingest/examples/json/connection_table.json similarity index 100% rename from ArroyoSketch/examples/json/connection_table.json rename to asap-sketch-ingest/examples/json/connection_table.json diff --git a/ArroyoSketch/examples/json/connection_table_sink.json b/asap-sketch-ingest/examples/json/connection_table_sink.json similarity index 100% rename from ArroyoSketch/examples/json/connection_table_sink.json rename to asap-sketch-ingest/examples/json/connection_table_sink.json diff --git a/ArroyoSketch/examples/json/pipeline.json b/asap-sketch-ingest/examples/json/pipeline.json similarity index 100% rename from ArroyoSketch/examples/json/pipeline.json rename to asap-sketch-ingest/examples/json/pipeline.json diff --git a/ArroyoSketch/examples/sql/create_fake_metric.sql b/asap-sketch-ingest/examples/sql/create_fake_metric.sql similarity index 100% rename from ArroyoSketch/examples/sql/create_fake_metric.sql rename to asap-sketch-ingest/examples/sql/create_fake_metric.sql diff --git a/ArroyoSketch/examples/sql/create_value.sql b/asap-sketch-ingest/examples/sql/create_value.sql similarity index 100% rename from ArroyoSketch/examples/sql/create_value.sql rename to asap-sketch-ingest/examples/sql/create_value.sql diff --git a/ArroyoSketch/examples/sql/tumbling_window.sql b/asap-sketch-ingest/examples/sql/tumbling_window.sql similarity index 100% rename from ArroyoSketch/examples/sql/tumbling_window.sql rename to asap-sketch-ingest/examples/sql/tumbling_window.sql diff --git a/ArroyoSketch/installation/setup_dependencies.sh b/asap-sketch-ingest/installation/setup_dependencies.sh similarity index 100% rename from ArroyoSketch/installation/setup_dependencies.sh rename to asap-sketch-ingest/installation/setup_dependencies.sh diff --git a/ArroyoSketch/run_arroyosketch.py b/asap-sketch-ingest/run_arroyosketch.py similarity index 100% rename from ArroyoSketch/run_arroyosketch.py rename to asap-sketch-ingest/run_arroyosketch.py diff --git a/ArroyoSketch/templates/hashed_key_udfs/countminsketch_sum.rs b/asap-sketch-ingest/templates/hashed_key_udfs/countminsketch_sum.rs similarity index 100% rename from ArroyoSketch/templates/hashed_key_udfs/countminsketch_sum.rs rename to asap-sketch-ingest/templates/hashed_key_udfs/countminsketch_sum.rs diff --git a/ArroyoSketch/templates/hashed_key_udfs/multipleincrease_.rs b/asap-sketch-ingest/templates/hashed_key_udfs/multipleincrease_.rs similarity index 100% rename from ArroyoSketch/templates/hashed_key_udfs/multipleincrease_.rs rename to asap-sketch-ingest/templates/hashed_key_udfs/multipleincrease_.rs diff --git a/ArroyoSketch/templates/hashed_key_udfs/multipleminmax_max.rs b/asap-sketch-ingest/templates/hashed_key_udfs/multipleminmax_max.rs similarity index 100% rename from ArroyoSketch/templates/hashed_key_udfs/multipleminmax_max.rs rename to asap-sketch-ingest/templates/hashed_key_udfs/multipleminmax_max.rs diff --git a/ArroyoSketch/templates/hashed_key_udfs/multipleminmax_min.rs b/asap-sketch-ingest/templates/hashed_key_udfs/multipleminmax_min.rs similarity index 100% rename from ArroyoSketch/templates/hashed_key_udfs/multipleminmax_min.rs rename to asap-sketch-ingest/templates/hashed_key_udfs/multipleminmax_min.rs diff --git a/ArroyoSketch/templates/hashed_key_udfs/multiplesum_count.rs b/asap-sketch-ingest/templates/hashed_key_udfs/multiplesum_count.rs similarity index 100% rename from ArroyoSketch/templates/hashed_key_udfs/multiplesum_count.rs rename to asap-sketch-ingest/templates/hashed_key_udfs/multiplesum_count.rs diff --git a/ArroyoSketch/templates/hashed_key_udfs/multiplesum_sum.rs b/asap-sketch-ingest/templates/hashed_key_udfs/multiplesum_sum.rs similarity index 100% rename from ArroyoSketch/templates/hashed_key_udfs/multiplesum_sum.rs rename to asap-sketch-ingest/templates/hashed_key_udfs/multiplesum_sum.rs diff --git a/ArroyoSketch/templates/hashed_key_udfs/old/countminsketch_sum.rs b/asap-sketch-ingest/templates/hashed_key_udfs/old/countminsketch_sum.rs similarity index 100% rename from ArroyoSketch/templates/hashed_key_udfs/old/countminsketch_sum.rs rename to asap-sketch-ingest/templates/hashed_key_udfs/old/countminsketch_sum.rs diff --git a/ArroyoSketch/templates/hashed_key_udfs/old/string_to_hash.rs b/asap-sketch-ingest/templates/hashed_key_udfs/old/string_to_hash.rs similarity index 100% rename from ArroyoSketch/templates/hashed_key_udfs/old/string_to_hash.rs rename to asap-sketch-ingest/templates/hashed_key_udfs/old/string_to_hash.rs diff --git a/ArroyoSketch/templates/hashed_key_udfs/string_to_hash.rs b/asap-sketch-ingest/templates/hashed_key_udfs/string_to_hash.rs similarity index 100% rename from ArroyoSketch/templates/hashed_key_udfs/string_to_hash.rs rename to asap-sketch-ingest/templates/hashed_key_udfs/string_to_hash.rs diff --git a/ArroyoSketch/templates/json/connection_profile.j2 b/asap-sketch-ingest/templates/json/connection_profile.j2 similarity index 100% rename from ArroyoSketch/templates/json/connection_profile.j2 rename to asap-sketch-ingest/templates/json/connection_profile.j2 diff --git a/ArroyoSketch/templates/json/connection_table_file.j2 b/asap-sketch-ingest/templates/json/connection_table_file.j2 similarity index 100% rename from ArroyoSketch/templates/json/connection_table_file.j2 rename to asap-sketch-ingest/templates/json/connection_table_file.j2 diff --git a/ArroyoSketch/templates/json/connection_table_kafka.j2 b/asap-sketch-ingest/templates/json/connection_table_kafka.j2 similarity index 100% rename from ArroyoSketch/templates/json/connection_table_kafka.j2 rename to asap-sketch-ingest/templates/json/connection_table_kafka.j2 diff --git a/ArroyoSketch/templates/json/connection_table_kafka_sql.j2 b/asap-sketch-ingest/templates/json/connection_table_kafka_sql.j2 similarity index 100% rename from ArroyoSketch/templates/json/connection_table_kafka_sql.j2 rename to asap-sketch-ingest/templates/json/connection_table_kafka_sql.j2 diff --git a/ArroyoSketch/templates/json/connection_table_prometheus_remote_write.j2 b/asap-sketch-ingest/templates/json/connection_table_prometheus_remote_write.j2 similarity index 100% rename from ArroyoSketch/templates/json/connection_table_prometheus_remote_write.j2 rename to asap-sketch-ingest/templates/json/connection_table_prometheus_remote_write.j2 diff --git a/ArroyoSketch/templates/json/connection_table_prometheus_remote_write_optimized.j2 b/asap-sketch-ingest/templates/json/connection_table_prometheus_remote_write_optimized.j2 similarity index 100% rename from ArroyoSketch/templates/json/connection_table_prometheus_remote_write_optimized.j2 rename to asap-sketch-ingest/templates/json/connection_table_prometheus_remote_write_optimized.j2 diff --git a/ArroyoSketch/templates/json/connection_table_sink.j2 b/asap-sketch-ingest/templates/json/connection_table_sink.j2 similarity index 100% rename from ArroyoSketch/templates/json/connection_table_sink.j2 rename to asap-sketch-ingest/templates/json/connection_table_sink.j2 diff --git a/ArroyoSketch/templates/json/pipeline.j2 b/asap-sketch-ingest/templates/json/pipeline.j2 similarity index 100% rename from ArroyoSketch/templates/json/pipeline.j2 rename to asap-sketch-ingest/templates/json/pipeline.j2 diff --git a/ArroyoSketch/templates/sql/distinct_windowed_labels.j2 b/asap-sketch-ingest/templates/sql/distinct_windowed_labels.j2 similarity index 100% rename from ArroyoSketch/templates/sql/distinct_windowed_labels.j2 rename to asap-sketch-ingest/templates/sql/distinct_windowed_labels.j2 diff --git a/ArroyoSketch/templates/sql/distinct_windowed_labels_deltasetaggregator.j2 b/asap-sketch-ingest/templates/sql/distinct_windowed_labels_deltasetaggregator.j2 similarity index 100% rename from ArroyoSketch/templates/sql/distinct_windowed_labels_deltasetaggregator.j2 rename to asap-sketch-ingest/templates/sql/distinct_windowed_labels_deltasetaggregator.j2 diff --git a/ArroyoSketch/templates/sql/single_arg_value_aggregation.j2 b/asap-sketch-ingest/templates/sql/single_arg_value_aggregation.j2 similarity index 100% rename from ArroyoSketch/templates/sql/single_arg_value_aggregation.j2 rename to asap-sketch-ingest/templates/sql/single_arg_value_aggregation.j2 diff --git a/ArroyoSketch/templates/sql/single_windowed_aggregation.j2 b/asap-sketch-ingest/templates/sql/single_windowed_aggregation.j2 similarity index 100% rename from ArroyoSketch/templates/sql/single_windowed_aggregation.j2 rename to asap-sketch-ingest/templates/sql/single_windowed_aggregation.j2 diff --git a/ArroyoSketch/templates/udfs/countminsketch_count.rs.j2 b/asap-sketch-ingest/templates/udfs/countminsketch_count.rs.j2 similarity index 100% rename from ArroyoSketch/templates/udfs/countminsketch_count.rs.j2 rename to asap-sketch-ingest/templates/udfs/countminsketch_count.rs.j2 diff --git a/ArroyoSketch/templates/udfs/countminsketch_sum.rs.j2 b/asap-sketch-ingest/templates/udfs/countminsketch_sum.rs.j2 similarity index 100% rename from ArroyoSketch/templates/udfs/countminsketch_sum.rs.j2 rename to asap-sketch-ingest/templates/udfs/countminsketch_sum.rs.j2 diff --git a/ArroyoSketch/templates/udfs/countminsketchwithheap_topk.rs.j2 b/asap-sketch-ingest/templates/udfs/countminsketchwithheap_topk.rs.j2 similarity index 100% rename from ArroyoSketch/templates/udfs/countminsketchwithheap_topk.rs.j2 rename to asap-sketch-ingest/templates/udfs/countminsketchwithheap_topk.rs.j2 diff --git a/ArroyoSketch/templates/udfs/datasketcheskll_.rs.j2 b/asap-sketch-ingest/templates/udfs/datasketcheskll_.rs.j2 similarity index 100% rename from ArroyoSketch/templates/udfs/datasketcheskll_.rs.j2 rename to asap-sketch-ingest/templates/udfs/datasketcheskll_.rs.j2 diff --git a/ArroyoSketch/templates/udfs/deltasetaggregator_.rs.j2 b/asap-sketch-ingest/templates/udfs/deltasetaggregator_.rs.j2 similarity index 100% rename from ArroyoSketch/templates/udfs/deltasetaggregator_.rs.j2 rename to asap-sketch-ingest/templates/udfs/deltasetaggregator_.rs.j2 diff --git a/ArroyoSketch/templates/udfs/gzip_compress.rs b/asap-sketch-ingest/templates/udfs/gzip_compress.rs similarity index 100% rename from ArroyoSketch/templates/udfs/gzip_compress.rs rename to asap-sketch-ingest/templates/udfs/gzip_compress.rs diff --git a/ArroyoSketch/templates/udfs/hydrakll_.rs.j2 b/asap-sketch-ingest/templates/udfs/hydrakll_.rs.j2 similarity index 100% rename from ArroyoSketch/templates/udfs/hydrakll_.rs.j2 rename to asap-sketch-ingest/templates/udfs/hydrakll_.rs.j2 diff --git a/ArroyoSketch/templates/udfs/multipleincrease_.rs b/asap-sketch-ingest/templates/udfs/multipleincrease_.rs similarity index 100% rename from ArroyoSketch/templates/udfs/multipleincrease_.rs rename to asap-sketch-ingest/templates/udfs/multipleincrease_.rs diff --git a/ArroyoSketch/templates/udfs/multipleminmax_max.rs b/asap-sketch-ingest/templates/udfs/multipleminmax_max.rs similarity index 100% rename from ArroyoSketch/templates/udfs/multipleminmax_max.rs rename to asap-sketch-ingest/templates/udfs/multipleminmax_max.rs diff --git a/ArroyoSketch/templates/udfs/multipleminmax_min.rs b/asap-sketch-ingest/templates/udfs/multipleminmax_min.rs similarity index 100% rename from ArroyoSketch/templates/udfs/multipleminmax_min.rs rename to asap-sketch-ingest/templates/udfs/multipleminmax_min.rs diff --git a/ArroyoSketch/templates/udfs/multiplesum_count.rs b/asap-sketch-ingest/templates/udfs/multiplesum_count.rs similarity index 100% rename from ArroyoSketch/templates/udfs/multiplesum_count.rs rename to asap-sketch-ingest/templates/udfs/multiplesum_count.rs diff --git a/ArroyoSketch/templates/udfs/multiplesum_sum.rs b/asap-sketch-ingest/templates/udfs/multiplesum_sum.rs similarity index 100% rename from ArroyoSketch/templates/udfs/multiplesum_sum.rs rename to asap-sketch-ingest/templates/udfs/multiplesum_sum.rs diff --git a/ArroyoSketch/templates/udfs/setaggregator_.rs b/asap-sketch-ingest/templates/udfs/setaggregator_.rs similarity index 100% rename from ArroyoSketch/templates/udfs/setaggregator_.rs rename to asap-sketch-ingest/templates/udfs/setaggregator_.rs diff --git a/CommonDependencies/dependencies/py/promql_utilities/promql_utilities/ast_matching/__init__.py b/asap-sketch-ingest/tests/__init__.py similarity index 100% rename from CommonDependencies/dependencies/py/promql_utilities/promql_utilities/ast_matching/__init__.py rename to asap-sketch-ingest/tests/__init__.py diff --git a/ArroyoSketch/tests/test_integration.py b/asap-sketch-ingest/tests/test_integration.py similarity index 100% rename from ArroyoSketch/tests/test_integration.py rename to asap-sketch-ingest/tests/test_integration.py diff --git a/ArroyoSketch/tests/test_sql_schema.py b/asap-sketch-ingest/tests/test_sql_schema.py similarity index 100% rename from ArroyoSketch/tests/test_sql_schema.py rename to asap-sketch-ingest/tests/test_sql_schema.py diff --git a/ArroyoSketch/utils/arroyo_utils.py b/asap-sketch-ingest/utils/arroyo_utils.py similarity index 100% rename from ArroyoSketch/utils/arroyo_utils.py rename to asap-sketch-ingest/utils/arroyo_utils.py diff --git a/ArroyoSketch/utils/http_utils.py b/asap-sketch-ingest/utils/http_utils.py similarity index 100% rename from ArroyoSketch/utils/http_utils.py rename to asap-sketch-ingest/utils/http_utils.py diff --git a/ArroyoSketch/utils/jinja_utils.py b/asap-sketch-ingest/utils/jinja_utils.py similarity index 100% rename from ArroyoSketch/utils/jinja_utils.py rename to asap-sketch-ingest/utils/jinja_utils.py diff --git a/ArroyoSketch/validate_udfs.py b/asap-sketch-ingest/validate_udfs.py similarity index 100% rename from ArroyoSketch/validate_udfs.py rename to asap-sketch-ingest/validate_udfs.py diff --git a/Utilities/.gitignore b/asap-tools/.gitignore similarity index 100% rename from Utilities/.gitignore rename to asap-tools/.gitignore diff --git a/ExecutionUtilities/LICENSE b/asap-tools/LICENSE similarity index 100% rename from ExecutionUtilities/LICENSE rename to asap-tools/LICENSE diff --git a/Utilities/README.md b/asap-tools/README.md similarity index 88% rename from Utilities/README.md rename to asap-tools/README.md index 47d6db1..0b411ef 100644 --- a/Utilities/README.md +++ b/asap-tools/README.md @@ -22,7 +22,7 @@ A framework for running end-to-end distributed systems experiments on CloudLab, This takes approximately 1 hour and sets up all infrastructure on CloudLab nodes. ```bash -cd $$REPO_ROOT/Utilities +cd $$REPO_ROOT/asap-tools ./deploy_from_scratch.sh # Example: @@ -39,10 +39,10 @@ cd $$REPO_ROOT/Utilities ### Running an Experiment -**Important:** Must be run from the `Utilities/experiments/` directory. +**Important:** Must be run from the `asap-tools/experiments/` directory. ```bash -cd $$REPO_ROOT/Utilities/experiments +cd $$REPO_ROOT/asap-tools/experiments python experiment_run_e2e.py \ experiment_type= \ @@ -85,10 +85,10 @@ After making code changes locally, use `deploy_changes.sh` to sync and rebuild o ```bash # Make your changes -vim QueryEngineRust/src/main.rs +vim asap-query-engine/src/main.rs # Deploy changes -cd $REPO_ROOT/Utilities +cd $REPO_ROOT/asap-tools ./deploy_changes.sh # Example: @@ -99,7 +99,7 @@ cd $REPO_ROOT/Utilities ### Comparing cost/latency for ASAP vs Prometheus -In your experiment config yaml file (e.g. `$REPO_DIR/Utilities/experiments/config/experiment_type/simple_config.yaml`), set: +In your experiment config yaml file (e.g. `$REPO_DIR/asap-tools/experiments/config/experiment_type/simple_config.yaml`), set: ```yaml experiments: @@ -107,10 +107,10 @@ experiments: - mode: prometheus ``` -With this config, 2 experiments are run independently. In the first experiment, `PrometheusClient` only sends queries to ASAP. After this experiment finishes, the infra is torn down. Then the second experiment is set up and `PrometheusClient` sends queries only to Prometheus directly. In the second experiment (i.e. when `mode=prometheus`), none of ASAP's components are set up (apart from `PrometheusClient`). +With this config, 2 experiments are run independently. In the first experiment, `asap-tools/prometheus-client` only sends queries to ASAP. After this experiment finishes, the infra is torn down. Then the second experiment is set up and `asap-tools/prometheus-client` sends queries only to Prometheus directly. In the second experiment (i.e. when `mode=prometheus`), none of ASAP's components are set up (apart from `asap-tools/prometheus-client`). Post-experiment analysis: -- Use `compare_costs.py` and `compare_latencies.py` from `$REPO_DIR/Utilities/experiments/post_experiments/`. +- Use `compare_costs.py` and `compare_latencies.py` from `$REPO_DIR/asap-tools/experiments/post_experiments/`. - `run_compare_latencies.sh` is an easy wrapper around `compare_latencies.py` ### Comparing query accuracy for ASAP vs Prometheus @@ -125,7 +125,7 @@ experiments: With this config, only one experiment is run. In the same experiment, `PrometheusClient` sends a query to ASAP and then immediately after that, sends a query to Prometheus too. Post-experiment analysis: -- Use `calculate_fidelity.py` from `$REPO_DIR/Utilities/experiments/post_experiments/`. +- Use `calculate_fidelity.py` from `$REPO_DIR/asap-tools/experiments/post_experiments/`. - `run_calculate_fidelity.sh` is an easy wrapper around `calculate_fidelity.py` ### Debugging with Verbose Logging @@ -192,7 +192,7 @@ See [docs/troubleshooting.md](docs/troubleshooting.md) for more detailed debuggi ## Project Structure ``` -Utilities/ +asap-tools/ ├── deploy_from_scratch.sh # Initial CloudLab setup ├── deploy_changes.sh # Incremental code deployment ├── components.conf # Components to sync diff --git a/Utilities/asap-cli/Cargo.lock b/asap-tools/asap-cli/Cargo.lock similarity index 100% rename from Utilities/asap-cli/Cargo.lock rename to asap-tools/asap-cli/Cargo.lock diff --git a/Utilities/asap-cli/Cargo.toml b/asap-tools/asap-cli/Cargo.toml similarity index 100% rename from Utilities/asap-cli/Cargo.toml rename to asap-tools/asap-cli/Cargo.toml diff --git a/Utilities/asap-cli/src/docker_util.rs b/asap-tools/asap-cli/src/docker_util.rs similarity index 89% rename from Utilities/asap-cli/src/docker_util.rs rename to asap-tools/asap-cli/src/docker_util.rs index 5fd2d35..f37ad07 100644 --- a/Utilities/asap-cli/src/docker_util.rs +++ b/asap-tools/asap-cli/src/docker_util.rs @@ -26,8 +26,8 @@ const CONTROLLER_CONTAINER_NAME: &str = "sketchdb-controller"; const ARROYOSKETCH_CONTAINER_NAME: &str = "asap-arroyosketch"; /// Get the ProjectASAP root directory based on the binary's location -/// Binary is at: ProjectASAP/Utilities/asap-cli/target/debug/asap-cli -/// So we go up 4 levels: debug -> target -> asap-cli -> Utilities -> ProjectASAP +/// Binary is at: ProjectASAP/asap-tools/asap-cli/target/debug/asap-cli +/// So we go up 4 levels: debug -> target -> asap-cli -> asap-tools -> ProjectASAP fn get_project_root() -> Result> { let binary_path = std::env::current_exe()?; @@ -36,7 +36,7 @@ fn get_project_root() -> Result> .and_then(|p| p.parent()) // Remove 'debug' .and_then(|p| p.parent()) // Remove 'target' .and_then(|p| p.parent()) // Remove 'asap-cli' - .and_then(|p| p.parent()) // Remove 'Utilities' + .and_then(|p| p.parent()) // Remove 'asap-tools' .ok_or("Failed to determine project root from binary location")?; Ok(project_root.to_path_buf()) @@ -123,15 +123,15 @@ pub async fn generate_query_engine_compose( let code_dir = project_root.to_string_lossy(); // Paths for template and script - let queryengine_dir = format!("{}/QueryEngineRust", code_dir); + let queryengine_dir = format!("{}/asap-query-engine", code_dir); let template_path = format!("{}/query-engine-rust-cli-compose.yml.j2", queryengine_dir); - let helper_script = format!("{}/Utilities/experiments/generate_queryengine_compose.py", code_dir); + let helper_script = format!("{}/asap-tools/experiments/generate_queryengine_compose.py", code_dir); // Compose file output path (inside ProjectASAP) - let compose_output_path = format!("{}/Utilities/docker/generated_compose_files/query-engine-compose.yml", code_dir); + let compose_output_path = format!("{}/asap-tools/docker/generated_compose_files/query-engine-compose.yml", code_dir); // Ensure compose output directory exists - tokio::fs::create_dir_all(format!("{}/Utilities/docker/generated_compose_files", code_dir)).await?; + tokio::fs::create_dir_all(format!("{}/asap-tools/docker/generated_compose_files", code_dir)).await?; // Experiment output paths (outside ProjectASAP, absolute paths for Docker volume mounts) let controller_remote_output_dir = format!("{}/controller_output", experiment_outputs_abs_path); @@ -227,15 +227,15 @@ pub async fn generate_controller_compose( let code_dir = project_root.to_string_lossy(); // Paths for template and script - let controller_dir = format!("{}/Controller", code_dir); + let controller_dir = format!("{}/asap-planner", code_dir); let template_path = format!("{}/controller-cli-compose.yml.j2", controller_dir); - let helper_script = format!("{}/Utilities/experiments/generate_controller_compose.py", code_dir); + let helper_script = format!("{}/asap-tools/experiments/generate_controller_compose.py", code_dir); // Compose file output path (inside ProjectASAP) - let compose_output_path = format!("{}/Utilities/docker/generated_compose_files/controller-compose.yml", code_dir); + let compose_output_path = format!("{}/asap-tools/docker/generated_compose_files/controller-compose.yml", code_dir); // Ensure compose output directory exists - tokio::fs::create_dir_all(format!("{}/Utilities/docker/generated_compose_files", code_dir)).await?; + tokio::fs::create_dir_all(format!("{}/asap-tools/docker/generated_compose_files", code_dir)).await?; // Controller output directory (outside ProjectASAP, absolute path for Docker volume mounts) let controller_remote_output_dir = format!("{}/controller_output", experiment_outputs_abs_path); @@ -276,7 +276,7 @@ async fn call_generate_queryengine_compose_script( ) -> Result<(), Box> { // Use uv run with the generate_compose project directory (absolute path) let project_root = get_project_root()?; - let uv_project = format!("{}/Utilities/asap-cli/uv_configs/generate_compose", project_root.to_string_lossy()); + let uv_project = format!("{}/asap-tools/asap-cli/uv_configs/generate_compose", project_root.to_string_lossy()); let mut cmd = Command::new("uv"); cmd.arg("run") @@ -342,7 +342,7 @@ async fn call_generate_controller_compose_script( ) -> Result<(), Box> { // Use uv run with the generate_compose project directory (absolute path) let project_root = get_project_root()?; - let uv_project = format!("{}/Utilities/asap-cli/uv_configs/generate_compose", project_root.to_string_lossy()); + let uv_project = format!("{}/asap-tools/asap-cli/uv_configs/generate_compose", project_root.to_string_lossy()); let mut cmd = Command::new("uv"); cmd.arg("run") @@ -378,7 +378,7 @@ async fn call_generate_fake_exporter_compose_script( ) -> Result<(), Box> { // Use uv run with the generate_compose project directory (absolute path) let project_root = get_project_root()?; - let uv_project = format!("{}/Utilities/asap-cli/uv_configs/generate_compose", project_root.to_string_lossy()); + let uv_project = format!("{}/asap-tools/asap-cli/uv_configs/generate_compose", project_root.to_string_lossy()); let mut cmd = Command::new("uv"); cmd.arg("run") @@ -462,8 +462,8 @@ pub async fn generate_fake_exporters_compose( // Paths for template and script let fake_exporter_dir = match language { - "python" => format!("{}/PrometheusExporters/fake_exporter/fake_exporter_python", code_dir), - "rust" => format!("{}/PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter", code_dir), + "python" => format!("{}/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python", code_dir), + "rust" => format!("{}/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter", code_dir), _ => return Err(format!("Unsupported fake exporter language: {}", language).into()), }; @@ -474,11 +474,11 @@ pub async fn generate_fake_exporters_compose( _ => return Err(format!("Unsupported fake exporter language for CLI template: {}", language).into()), }; let template_path = format!("{}/{}", fake_exporter_dir, template_filename); - let helper_script = format!("{}/Utilities/experiments/generate_fake_exporter_compose.py", code_dir); + let helper_script = format!("{}/asap-tools/experiments/generate_fake_exporter_compose.py", code_dir); // Ensure compose output directories exist - tokio::fs::create_dir_all(format!("{}/Utilities/docker/generated_compose_files", code_dir)).await?; - tokio::fs::create_dir_all(format!("{}/Utilities/docker/generated_compose_files/fake_exporter_composes", code_dir)).await?; + tokio::fs::create_dir_all(format!("{}/asap-tools/docker/generated_compose_files", code_dir)).await?; + tokio::fs::create_dir_all(format!("{}/asap-tools/docker/generated_compose_files/fake_exporter_composes", code_dir)).await?; // Experiment output paths let experiment_output_dir = format!("{}/fake_exporter_output", experiment_outputs_abs_path); @@ -490,7 +490,7 @@ pub async fn generate_fake_exporters_compose( let port = start_port + i; let container_name = format!("sketchdb-fake-exporter-{}-{}", port, language); let compose_name = format!("fake-exporter-compose-{}-{}.yml", port, language); - let compose_output_path = format!("{}/Utilities/docker/generated_compose_files/fake_exporter_composes/{}", code_dir, compose_name); + let compose_output_path = format!("{}/asap-tools/docker/generated_compose_files/fake_exporter_composes/{}", code_dir, compose_name); let args = FakeExporterComposeArgs { template_path: template_path.clone(), @@ -522,7 +522,7 @@ async fn generate_master_fake_exporters_compose( compose_files: &[String], ) -> Result<(), Box> { let project_root = get_project_root()?; - let master_compose_path = format!("{}/Utilities/docker/generated_compose_files/fake-exporter-compose.yml", project_root.to_string_lossy()); + let master_compose_path = format!("{}/asap-tools/docker/generated_compose_files/fake-exporter-compose.yml", project_root.to_string_lossy()); // Create YAML content with merged services let mut content = String::from("# Master compose file for all fake exporters\n"); @@ -531,7 +531,7 @@ async fn generate_master_fake_exporters_compose( // Read each individual compose file and extract the service definition for (idx, compose_file) in compose_files.iter().enumerate() { - let compose_path = format!("{}/Utilities/docker/generated_compose_files/fake_exporter_composes/{}", project_root.to_string_lossy(), compose_file); + let compose_path = format!("{}/asap-tools/docker/generated_compose_files/fake_exporter_composes/{}", project_root.to_string_lossy(), compose_file); let compose_content = tokio::fs::read_to_string(&compose_path).await?; // Parse the YAML to extract the fake-exporter service @@ -583,13 +583,13 @@ pub async fn generate_arroyosketch_compose( let code_dir = project_root.to_string_lossy(); // Paths - let arroyosketch_dir = format!("{}/ArroyoSketch", code_dir); + let arroyosketch_dir = format!("{}/asap-sketch-ingest", code_dir); let template_path = format!("{}/arroyosketch-cli-compose.yml.j2", arroyosketch_dir); - let helper_script = format!("{}/Utilities/experiments/generate_arroyosketch_compose.py", code_dir); - let compose_output_path = format!("{}/Utilities/docker/generated_compose_files/arroyosketch-compose.yml", code_dir); + let helper_script = format!("{}/asap-tools/experiments/generate_arroyosketch_compose.py", code_dir); + let compose_output_path = format!("{}/asap-tools/docker/generated_compose_files/arroyosketch-compose.yml", code_dir); // Ensure compose output directory exists - tokio::fs::create_dir_all(format!("{}/Utilities/docker/generated_compose_files", code_dir)).await?; + tokio::fs::create_dir_all(format!("{}/asap-tools/docker/generated_compose_files", code_dir)).await?; // Experiment output paths (absolute) let controller_output_dir = format!("{}/controller_output", experiment_outputs_abs_path); @@ -629,7 +629,7 @@ async fn call_generate_arroyosketch_compose_script( helper_script: String, ) -> Result<(), Box> { let project_root = get_project_root()?; - let uv_project = format!("{}/Utilities/asap-cli/uv_configs/generate_compose", project_root.to_string_lossy()); + let uv_project = format!("{}/asap-tools/asap-cli/uv_configs/generate_compose", project_root.to_string_lossy()); let mut cmd = Command::new("uv"); cmd.arg("run") @@ -667,7 +667,7 @@ async fn call_generate_arroyosketch_compose_script( /// Starts ProjectASAP by bringing up containers using asap-docker-compose.yml pub async fn asap_up() -> Result<(), Box> { let project_root = get_project_root()?; - let compose_path = format!("{}/Utilities/docker/asap-docker-compose.yml", project_root.to_string_lossy()); + let compose_path = format!("{}/asap-tools/docker/asap-docker-compose.yml", project_root.to_string_lossy()); let mut cmd = Command::new("docker"); println!("Starting ProjectASAP..."); @@ -689,7 +689,7 @@ pub async fn asap_up() -> Result<(), Box> { pub async fn asap_down() -> Result<(), Box> { let project_root = get_project_root()?; - let compose_path = format!("{}/Utilities/docker/asap-docker-compose.yml", project_root.to_string_lossy()); + let compose_path = format!("{}/asap-tools/docker/asap-docker-compose.yml", project_root.to_string_lossy()); let mut cmd = Command::new("docker"); cmd.arg("compose") diff --git a/Utilities/asap-cli/src/main.rs b/asap-tools/asap-cli/src/main.rs similarity index 95% rename from Utilities/asap-cli/src/main.rs rename to asap-tools/asap-cli/src/main.rs index 9010beb..cae2179 100644 --- a/Utilities/asap-cli/src/main.rs +++ b/asap-tools/asap-cli/src/main.rs @@ -11,7 +11,7 @@ const EXPERIMENT_CONFIG_SUFFIX: &str = ".yaml"; lazy_static! { pub static ref PROJECT_ROOT_DIR: PathBuf = get_project_root().unwrap(); - pub static ref HYDRA_CONFIGS_RELATIVE_PATH: PathBuf = PathBuf::from("Utilities/experiments/config"); + pub static ref HYDRA_CONFIGS_RELATIVE_PATH: PathBuf = PathBuf::from("asap-tools/experiments/config"); } #[derive(Parser)] @@ -60,25 +60,25 @@ struct StartArgs { } /// Get the ProjectASAP root directory based on the binary's location -/// Binary is at: ProjectASAP/Utilities/asap-cli/target/debug/asap-cli -/// So we go up 4 levels: debug -> target -> asap-cli -> Utilities -> ProjectASAP +/// Binary is at: ProjectASAP/asap-tools/asap-cli/target/debug/asap-cli +/// So we go up 4 levels: debug -> target -> asap-cli -> asap-tools -> ProjectASAP fn get_project_root() -> Result> { let binary_path = std::env::current_exe()?; - // Go up 4 levels: asap-cli -> debug -> target -> asap-cli -> Utilities -> ProjectASAP + // Go up 4 levels: asap-cli -> debug -> target -> asap-cli -> asap-tools -> ProjectASAP let project_root = binary_path .parent() // Remove 'asap-cli' binary .and_then(|p| p.parent()) // Remove 'debug' .and_then(|p| p.parent()) // Remove 'target' .and_then(|p| p.parent()) // Remove 'asap-cli' - .and_then(|p| p.parent()) // Remove 'Utilities' + .and_then(|p| p.parent()) // Remove 'asap-tools' .ok_or("Failed to determine project root from binary location")?; Ok(project_root.to_path_buf()) } /// Gets the path to the experiment config yaml file using the given experiment -/// type and the expected location in Utilities/experiments/config/experiment_type/ +/// type and the expected location in asap-tools/experiments/config/experiment_type/ pub fn get_experiment_config_path(experiment_type: &str) -> Result> { let experiment_types_directory = PathBuf::from("experiment_type"); @@ -86,7 +86,7 @@ pub fn get_experiment_config_path(experiment_type: &str) -> Result.yaml + // ROOT/ + /asap-tools/experiments/config/ + /experiment_type/ + /.yaml let experiment_config_path = PROJECT_ROOT_DIR.join(HYDRA_CONFIGS_RELATIVE_PATH.as_path()) .join(experiment_types_directory.as_path()) .join(experiment_config_filename.as_path()); @@ -103,7 +103,7 @@ async fn generate_configs_and_compose_files( // Get project root based on binary location let project_root = get_project_root()?; - let experiments_dir = project_root.join("Utilities/experiments"); + let experiments_dir = project_root.join("asap-tools/experiments"); println!("Generating configuration and compose files..."); @@ -196,7 +196,7 @@ async fn generate_configs_and_compose_files( // 3. Generate prometheus config println!(" - Generating prometheus config..."); - let prometheus_output_dir = project_root.join("Utilities/docker/prometheus"); + let prometheus_output_dir = project_root.join("asap-tools/docker/prometheus"); tokio::fs::create_dir_all(&prometheus_output_dir).await?; let num_nodes = config.cloudlab.num_nodes.unwrap_or(1); @@ -228,7 +228,7 @@ async fn generate_configs_and_compose_files( // are already being used in the fake-exporters-compose.yml file. // The prometheus config will need manual adjustment or post-processing. - let uv_project = project_root.join("Utilities/asap-cli/uv_configs/generate_prometheus_config"); + let uv_project = project_root.join("asap-tools/asap-cli/uv_configs/generate_prometheus_config"); // For local deployment, node_offset is 0 // For CloudLab, this would be read from config @@ -375,8 +375,8 @@ async fn fix_prometheus_targets_for_docker( /// Configure Grafana after containers are running async fn configure_grafana(experiment_type: &str, experiment_name: Option<&str>) -> Result<(), Box> { let project_root = get_project_root()?; - let experiments_dir = project_root.join("Utilities/experiments"); - let uv_project = project_root.join("Utilities/asap-cli/uv_configs/grafana_config"); + let experiments_dir = project_root.join("asap-tools/experiments"); + let uv_project = project_root.join("asap-tools/asap-cli/uv_configs/grafana_config"); let experiment_type_arg: String = format!("experiment_type={}", experiment_type); let experiment_name_arg: String; diff --git a/Utilities/asap-cli/src/util.rs b/asap-tools/asap-cli/src/util.rs similarity index 100% rename from Utilities/asap-cli/src/util.rs rename to asap-tools/asap-cli/src/util.rs diff --git a/Utilities/asap-cli/uv_configs/generate_compose/pyproject.toml b/asap-tools/asap-cli/uv_configs/generate_compose/pyproject.toml similarity index 100% rename from Utilities/asap-cli/uv_configs/generate_compose/pyproject.toml rename to asap-tools/asap-cli/uv_configs/generate_compose/pyproject.toml diff --git a/Utilities/asap-cli/uv_configs/generate_compose/uv.lock b/asap-tools/asap-cli/uv_configs/generate_compose/uv.lock similarity index 100% rename from Utilities/asap-cli/uv_configs/generate_compose/uv.lock rename to asap-tools/asap-cli/uv_configs/generate_compose/uv.lock diff --git a/Utilities/asap-cli/uv_configs/generate_prometheus_config/pyproject.toml b/asap-tools/asap-cli/uv_configs/generate_prometheus_config/pyproject.toml similarity index 100% rename from Utilities/asap-cli/uv_configs/generate_prometheus_config/pyproject.toml rename to asap-tools/asap-cli/uv_configs/generate_prometheus_config/pyproject.toml diff --git a/Utilities/asap-cli/uv_configs/generate_prometheus_config/uv.lock b/asap-tools/asap-cli/uv_configs/generate_prometheus_config/uv.lock similarity index 100% rename from Utilities/asap-cli/uv_configs/generate_prometheus_config/uv.lock rename to asap-tools/asap-cli/uv_configs/generate_prometheus_config/uv.lock diff --git a/Utilities/asap-cli/uv_configs/grafana_config/pyproject.toml b/asap-tools/asap-cli/uv_configs/grafana_config/pyproject.toml similarity index 100% rename from Utilities/asap-cli/uv_configs/grafana_config/pyproject.toml rename to asap-tools/asap-cli/uv_configs/grafana_config/pyproject.toml diff --git a/Utilities/cloudlab_setup/multi_node/constants.sh b/asap-tools/cloudlab_setup/multi_node/constants.sh similarity index 100% rename from Utilities/cloudlab_setup/multi_node/constants.sh rename to asap-tools/cloudlab_setup/multi_node/constants.sh diff --git a/Utilities/cloudlab_setup/multi_node/oneshot_only_rsync.sh b/asap-tools/cloudlab_setup/multi_node/oneshot_only_rsync.sh similarity index 100% rename from Utilities/cloudlab_setup/multi_node/oneshot_only_rsync.sh rename to asap-tools/cloudlab_setup/multi_node/oneshot_only_rsync.sh diff --git a/Utilities/cloudlab_setup/multi_node/oneshot_rsync_and_selective_install_internal.sh b/asap-tools/cloudlab_setup/multi_node/oneshot_rsync_and_selective_install_internal.sh similarity index 100% rename from Utilities/cloudlab_setup/multi_node/oneshot_rsync_and_selective_install_internal.sh rename to asap-tools/cloudlab_setup/multi_node/oneshot_rsync_and_selective_install_internal.sh diff --git a/Utilities/cloudlab_setup/multi_node/oneshot_setup.sh b/asap-tools/cloudlab_setup/multi_node/oneshot_setup.sh similarity index 100% rename from Utilities/cloudlab_setup/multi_node/oneshot_setup.sh rename to asap-tools/cloudlab_setup/multi_node/oneshot_setup.sh diff --git a/Utilities/cloudlab_setup/multi_node/utils.sh b/asap-tools/cloudlab_setup/multi_node/utils.sh similarity index 100% rename from Utilities/cloudlab_setup/multi_node/utils.sh rename to asap-tools/cloudlab_setup/multi_node/utils.sh diff --git a/Utilities/cloudlab_setup/single_node/constants.sh b/asap-tools/cloudlab_setup/single_node/constants.sh similarity index 100% rename from Utilities/cloudlab_setup/single_node/constants.sh rename to asap-tools/cloudlab_setup/single_node/constants.sh diff --git a/Utilities/cloudlab_setup/single_node/local_storage_helper.sh b/asap-tools/cloudlab_setup/single_node/local_storage_helper.sh similarity index 100% rename from Utilities/cloudlab_setup/single_node/local_storage_helper.sh rename to asap-tools/cloudlab_setup/single_node/local_storage_helper.sh diff --git a/Utilities/cloudlab_setup/single_node/rsync.sh b/asap-tools/cloudlab_setup/single_node/rsync.sh similarity index 100% rename from Utilities/cloudlab_setup/single_node/rsync.sh rename to asap-tools/cloudlab_setup/single_node/rsync.sh diff --git a/Utilities/cloudlab_setup/single_node/rsync_and_selective_install_internal.sh b/asap-tools/cloudlab_setup/single_node/rsync_and_selective_install_internal.sh similarity index 89% rename from Utilities/cloudlab_setup/single_node/rsync_and_selective_install_internal.sh rename to asap-tools/cloudlab_setup/single_node/rsync_and_selective_install_internal.sh index 76c0dc2..9a2d6d6 100755 --- a/Utilities/cloudlab_setup/single_node/rsync_and_selective_install_internal.sh +++ b/asap-tools/cloudlab_setup/single_node/rsync_and_selective_install_internal.sh @@ -55,17 +55,17 @@ else echo "" echo "Installing synced internal components on $HOSTNAME..." - # Filter out Utilities component as it doesn't have an installation directory + # Filter out asap-tools component as it doesn't have an installation directory INSTALLABLE_COMPONENTS=() for COMPONENT in "${SYNCED_COMPONENTS[@]}"; do - if [ "$COMPONENT" != "Utilities" ]; then + if [ "$COMPONENT" != "asap-tools" ]; then INSTALLABLE_COMPONENTS+=("$COMPONENT") fi done if [ ${#INSTALLABLE_COMPONENTS[@]} -gt 0 ]; then # Use the existing only_install_internal_components.sh script on the remote - ssh $OPTIONS $USERNAME@$HOSTNAME "cd $DESTINATION_DIR/Utilities/installation && ./only_install_internal_components.sh ${INSTALLABLE_COMPONENTS[*]}" + ssh $OPTIONS $USERNAME@$HOSTNAME "cd $DESTINATION_DIR/asap-tools/installation && ./only_install_internal_components.sh ${INSTALLABLE_COMPONENTS[*]}" echo "Installation complete for $HOSTNAME" else echo "No installable components to process for $HOSTNAME" diff --git a/Utilities/cloudlab_setup/single_node/setup_storage.sh b/asap-tools/cloudlab_setup/single_node/setup_storage.sh similarity index 100% rename from Utilities/cloudlab_setup/single_node/setup_storage.sh rename to asap-tools/cloudlab_setup/single_node/setup_storage.sh diff --git a/Utilities/components.conf b/asap-tools/components.conf similarity index 58% rename from Utilities/components.conf rename to asap-tools/components.conf index 86c01dd..cd242f4 100644 --- a/Utilities/components.conf +++ b/asap-tools/components.conf @@ -2,18 +2,16 @@ # This file defines the list of all components used across the project # Used by installation and deployment scripts to maintain consistency -Utilities -CommonDependencies +asap-tools +asap-common #FlinkSketch #QueryEngine -sketch-core sketchlib-rust -QueryEngineRust -PrometheusClient -Controller +asap-query-engine +asap-planner #prometheus-kafka-adapter -ArroyoSketch -ExecutionUtilities -PrometheusExporters -prometheus-benchmark -quickstart +asap-sketch-ingest +asap-quickstart +asap-tools/prometheus-exporters +asap-tools/prometheus-client +asap-tools/execution-utilities diff --git a/Utilities/deploy_changes.sh b/asap-tools/deploy_changes.sh similarity index 100% rename from Utilities/deploy_changes.sh rename to asap-tools/deploy_changes.sh diff --git a/Utilities/deploy_from_scratch.sh b/asap-tools/deploy_from_scratch.sh similarity index 100% rename from Utilities/deploy_from_scratch.sh rename to asap-tools/deploy_from_scratch.sh diff --git a/Utilities/docker/arroyo-cli-compose.yml b/asap-tools/docker/arroyo-cli-compose.yml similarity index 92% rename from Utilities/docker/arroyo-cli-compose.yml rename to asap-tools/docker/arroyo-cli-compose.yml index c8eddc4..a33f3b6 100644 --- a/Utilities/docker/arroyo-cli-compose.yml +++ b/asap-tools/docker/arroyo-cli-compose.yml @@ -8,7 +8,7 @@ services: ports: - "5115:5115" volumes: - - ../../ArroyoSketch/config.yaml:/config.yaml + - ../../asap-sketch-ingest/config.yaml:/config.yaml command: ["--config", "/config.yaml", "cluster"] environment: - ARROYO__API__RUN_HTTP_PORT=5115 diff --git a/Utilities/docker/asap-docker-compose.yml b/asap-tools/docker/asap-docker-compose.yml similarity index 100% rename from Utilities/docker/asap-docker-compose.yml rename to asap-tools/docker/asap-docker-compose.yml diff --git a/Utilities/docker/grafana/grafana-compose.yml b/asap-tools/docker/grafana/grafana-compose.yml similarity index 100% rename from Utilities/docker/grafana/grafana-compose.yml rename to asap-tools/docker/grafana/grafana-compose.yml diff --git a/Utilities/docker/kafka/kafka-compose.yml b/asap-tools/docker/kafka/kafka-compose.yml similarity index 100% rename from Utilities/docker/kafka/kafka-compose.yml rename to asap-tools/docker/kafka/kafka-compose.yml diff --git a/Utilities/docker/prometheus/prometheus-compose.yml b/asap-tools/docker/prometheus/prometheus-compose.yml similarity index 100% rename from Utilities/docker/prometheus/prometheus-compose.yml rename to asap-tools/docker/prometheus/prometheus-compose.yml diff --git a/Utilities/docs/architecture.md b/asap-tools/docs/architecture.md similarity index 99% rename from Utilities/docs/architecture.md rename to asap-tools/docs/architecture.md index 81d9bdd..5c49627 100644 --- a/Utilities/docs/architecture.md +++ b/asap-tools/docs/architecture.md @@ -107,7 +107,7 @@ Start throughput/health monitors (if enabled) Wait for steady state (default 60s) ↓ Run remote monitor (execute queries) - ├─ Execute PromQL queries via PrometheusClient + ├─ Execute PromQL queries via asap-tools/prometheus-client ├─ Monitor process health ├─ Profile components (if enabled) └─ Record results @@ -162,7 +162,7 @@ QueryEngine ↓ PromQL Query Results ↓ -PrometheusClient (logs results) +asap-tools/prometheus-client (logs results) ``` --> ### SketchDB Mode with Ingest from Prometheus Remote Write @@ -196,7 +196,7 @@ Prometheus TSDB ↓ (query) PromQL Query API ↓ -PrometheusClient (logs results) +asap-tools/prometheus-client (logs results) ``` ## Services in the Experiment Framework diff --git a/Utilities/docs/configuration.md b/asap-tools/docs/configuration.md similarity index 100% rename from Utilities/docs/configuration.md rename to asap-tools/docs/configuration.md diff --git a/Utilities/docs/deployment.md b/asap-tools/docs/deployment.md similarity index 85% rename from Utilities/docs/deployment.md rename to asap-tools/docs/deployment.md index 5ee7ebb..bdd62df 100644 --- a/Utilities/docs/deployment.md +++ b/asap-tools/docs/deployment.md @@ -28,7 +28,7 @@ All deployment scripts are **Linux-only** and have been tested on Ubuntu. They a **Usage:** ```bash -cd $REPO_ROOT/Utilities +cd $REPO_ROOT/asap-tools ./deploy_from_scratch.sh # Example: @@ -62,28 +62,25 @@ The script executes these phases in order: #### Phase 4: Internal Components Installation - Runs on all CloudLab nodes in parallel - Builds and installs project-specific code: - - CommonDependencies (base Docker image) - - QueryEngineRust (Rust binary + Docker image) - - Controller (Docker image) + - asap-common (base Docker image) + - asap-query-engine (Rust binary + Docker image) + - asap-planner (Docker image) - Arroyo (Node.js frontend + Rust binary + Docker image) - - ArroyoSketch (Python scripts) - - PrometheusClient, PrometheusExporters, ExecutionUtilities, prometheus-benchmark + - asap-sketch-ingest (Python scripts) + - asap-tools/prometheus-client, asap-tools/prometheus-exporters, asap-tools/execution-utilities, asap-tools/prometheus-benchmark ### Directory Structure Created ``` /scratch/sketch_db_for_prometheus/ ├── code/ # All component source code -│ ├── Utilities/ -│ ├── QueryEngineRust/ -│ ├── Controller/ -│ ├── ArroyoSketch/ +│ ├── asap-tools/ +│ ├── asap-query-engine/ +│ ├── asap-planner/ +│ ├── asap-sketch-ingest/ │ ├── arroyo/ -│ ├── CommonDependencies/ -│ ├── PrometheusClient/ -│ ├── PrometheusExporters/ -│ ├── ExecutionUtilities/ -│ └── prometheus-benchmark/ +│ ├── asap-common/ +│ └── asap-tools/prometheus-benchmark/ ├── prometheus/ # Prometheus binaries ├── kafka/ # Kafka installation ├── flink/ # Flink installation @@ -100,10 +97,10 @@ The script executes these phases in order: **Usage:** ```bash -cd $REPO_ROOT/Utilities +cd $REPO_ROOT/asap-tools # Make your code changes -vim QueryEngineRust/src/main.rs +vim asap-query-engine/src/main.rs # Deploy changes ./deploy_changes.sh @@ -126,13 +123,13 @@ vim QueryEngineRust/src/main.rs 3. **Example Scenario:** ```bash - # Edit QueryEngineRust only - vim QueryEngineRust/src/query_executor.rs + # Edit asap-query-engine only + vim asap-query-engine/src/query_executor.rs # Deploy changes ./deploy_changes.sh 10 myuser sketchdb.utah.cloudlab.us - # Result: Only QueryEngineRust is rebuilt and reinstalled + # Result: Only asap-query-engine is rebuilt and reinstalled # Takes ~5 minutes instead of 60 minutes ``` @@ -140,22 +137,22 @@ vim QueryEngineRust/src/main.rs ### components.conf -Located at: `Utilities/components.conf` +Located at: `asap-tools/components.conf` This file defines which components get synced to CloudLab. Each component must be a directory in `$REPO_ROOT/`. **Currently Enabled Components:** ```bash -Utilities -CommonDependencies -QueryEngineRust -PrometheusClient -Controller -ArroyoSketch -ExecutionUtilities -arroyo -PrometheusExporters -prometheus-benchmark +asap-tools +asap-common +sketchlib-rust +asap-query-engine +asap-planner +asap-sketch-ingest +asap-quickstart +asap-tools/prometheus-exporters +asap-tools/prometheus-client +asap-tools/execution-utilities ``` **Deprecated Components (commented out):** @@ -245,30 +242,30 @@ These are third-party software installed from source or packages: These are project-specific components built as Docker images or binaries: -#### 1. CommonDependencies +#### 1. asap-common **What:** Base Docker image with shared dependencies **Build Process:** ```bash -cd CommonDependencies +cd asap-common docker build -t sketchdb-base:latest . ``` **Purpose:** Shared base layer for other Docker images to reduce build time -#### 2. QueryEngineRust +#### 2. asap-query-engine **What:** Rust-based query engine for executing PromQL queries over sketches **Build Process:** ```bash -cd QueryEngineRust +cd asap-query-engine cargo build --release # Compile Rust binary docker build -t sketchdb-queryengine-rust:latest . # Build Docker image ``` **Deployment:** Can run as Docker container or bare-metal binary -#### 3. Controller +#### 3. asap-planner **What:** Service that generates sketch configurations based on query patterns **Build Process:** ```bash -cd Controller +cd asap-planner docker build -t sketchdb-controller:latest . ``` **Deployment:** Docker container only @@ -299,20 +296,20 @@ cargo install refinery_cli - Web console (Node.js/React frontend) - Controller and workers (Rust binaries) -#### 5. ArroyoSketch -**What:** Python scripts for deploying ArroyoSketch pipelines +#### 5. asap-sketch-ingest +**What:** Python scripts for deploying asap-sketch-ingest pipelines **Build Process:** ```bash -cd ArroyoSketch +cd asap-sketch-ingest pip install -r requirements.txt # jinja2 for templating ``` **Deployment:** Python scripts, no Docker image -#### 6. PrometheusClient +#### 6. asap-tools/prometheus-client **What:** Client for executing PromQL queries against Prometheus or SketchDB **Build Process:** Language-specific (Python or Rust) -#### 7. PrometheusExporters, ExecutionUtilities, prometheus-benchmark +#### 7. asap-tools/prometheus-exporters, asap-tools/execution-utilities, asap-tools/prometheus-benchmark **What:** Various testing and monitoring tools **Build Process:** Component-specific @@ -322,8 +319,8 @@ pip install -r requirements.txt # jinja2 for templating | Script | Location | Purpose | |--------|----------|---------| -| `deploy_from_scratch.sh` | `Utilities/` | Full deployment | -| `deploy_changes.sh` | `Utilities/` | Incremental updates | +| `deploy_from_scratch.sh` | `asap-tools/` | Full deployment | +| `deploy_changes.sh` | `asap-tools/` | Incremental updates | | `oneshot_setup.sh` | `cloudlab_setup/multi_node/` | Storage + rsync | | `oneshot_rsync_and_selective_install_internal.sh` | `cloudlab_setup/multi_node/` | Incremental sync + install | @@ -343,7 +340,7 @@ pip install -r requirements.txt # jinja2 for templating | Script | Location | Purpose | |--------|----------|---------| -| `shared_utils.sh` | `Utilities/` | Shared functions (rsync, component loading) | +| `shared_utils.sh` | `asap-tools/` | Shared functions (rsync, component loading) | | `utils.sh` | `cloudlab_setup/multi_node/` | Parallel execution helper | | `constants.sh` | `cloudlab_setup/multi_node/` | Hostname configuration | | `constants.sh` | `cloudlab_setup/single_node/` | Remote path configuration | @@ -366,11 +363,11 @@ pip install -r requirements.txt # jinja2 for templating **Solutions:** ```bash # Force rebuild by touching files -cd $REPO_ROOT/QueryEngineRust +cd $REPO_ROOT/asap-query-engine touch src/main.rs # Run deploy_changes again -cd $REPO_ROOT/Utilities +cd $REPO_ROOT/asap-tools ./deploy_changes.sh 10 myuser sketchdb.utah.cloudlab.us ``` @@ -411,7 +408,7 @@ Run `deploy_from_scratch.sh` ## Known Problems -1. Modifying `CommonDependencies` and then using `deploy_changes.sh` will only rebuild the `sketchdb-base` Docker image, not all the images that depend on this. +1. Modifying `asap-common` and then using `deploy_changes.sh` will only rebuild the `sketchdb-base` Docker image, not all the images that depend on this. Workaround: Run `deploy_from_scratch.sh` ## Advanced Usage diff --git a/Utilities/docs/troubleshooting.md b/asap-tools/docs/troubleshooting.md similarity index 92% rename from Utilities/docs/troubleshooting.md rename to asap-tools/docs/troubleshooting.md index 8b5c940..85da113 100644 --- a/Utilities/docs/troubleshooting.md +++ b/asap-tools/docs/troubleshooting.md @@ -35,7 +35,7 @@ for i in {0..N-1}; do ssh -o StrictHostKeyChecking=no @node"$ - `experiment_run_e2e.py` script continuously shows "Waiting for X seconds for remote monitor to finish" for multiple minutes (e.g. > 5 min) **Possible Causes:** -- Could be a variety of causes, but usually indicates something wrong with `PrometheusClient`, `QueryEngineRust`, or `remote_monitor.py` +- Could be a variety of causes, but usually indicates something wrong with `asap-tools/prometheus-client`, `asap-query-engine`, or `remote_monitor.py` **Solutions:** Look at log files for the above components on the cloudlab nodes. You can also check `docker logs` @@ -87,14 +87,14 @@ python experiment_run_e2e.py \ Kafka has three places where message size limits need to be configured (see [PR #154](https://github.com/ProjectASAP/asap-internal/pull/154)): -1. **Kafka broker configuration** (`Utilities/installation/kafka/install.sh`): +1. **Kafka broker configuration** (`asap-tools/installation/kafka/install.sh`): - Set `message.max.bytes` to desired limit (e.g., `20971520` for 20MB) - Set `replica.fetch.max.bytes` to the same limit -2. **Kafka topic configuration** (`Utilities/experiments/experiment_utils/services/kafka.py`): +2. **Kafka topic configuration** (`asap-tools/experiments/experiment_utils/services/kafka.py`): - Update `max.message.bytes` in the topic creation command -3. **Arroyo connection profile** (`ArroyoSketch/templates/json/connection_profile.j2`): +3. **Arroyo connection profile** (`asap-sketch-ingest/templates/json/connection_profile.j2`): - Add connection properties with `message.max.bytes` and `batch.size` **Example values:** diff --git a/ExecutionUtilities/.gitignore b/asap-tools/execution-utilities/.gitignore similarity index 100% rename from ExecutionUtilities/.gitignore rename to asap-tools/execution-utilities/.gitignore diff --git a/PrometheusClient/LICENSE b/asap-tools/execution-utilities/LICENSE similarity index 100% rename from PrometheusClient/LICENSE rename to asap-tools/execution-utilities/LICENSE diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/README.md b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/README.md similarity index 79% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/README.md rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/README.md index 1680e91..bba0a72 100644 --- a/ExecutionUtilities/clickhouse-benchmark-pipeline/README.md +++ b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/README.md @@ -8,27 +8,27 @@ Install and run Kafka and ClickHouse natively (no Docker): ```bash # Install Kafka (one-time) -cd /path/to/Utilities/installation/kafka -./install.sh /path/to/Utilities/installation/kafka +cd /path/to/asap-tools/installation/kafka +./install.sh /path/to/asap-tools/installation/kafka # Install ClickHouse (one-time) -cd /path/to/Utilities/installation/clickhouse -./install.sh /path/to/Utilities/installation/clickhouse +cd /path/to/asap-tools/installation/clickhouse +./install.sh /path/to/asap-tools/installation/clickhouse ``` ## Usage ```bash # 1. Start Kafka (in a terminal) -cd /path/to/Utilities/installation/kafka +cd /path/to/asap-tools/installation/kafka ./run.sh kafka/ # 2. Create Kafka topic kafka/bin/kafka-topics.sh --create --topic hits --bootstrap-server localhost:9092 # 3. Start ClickHouse (in another terminal) -cd /path/to/Utilities/installation/clickhouse -./run.sh /path/to/Utilities/installation/clickhouse +cd /path/to/asap-tools/installation/clickhouse +./run.sh /path/to/asap-tools/installation/clickhouse # 4. Initialize ClickHouse tables ./scripts/init_clickhouse.sh @@ -61,7 +61,7 @@ Edit `config.env` to change defaults. Environment variables override config valu Follow instructions to install Elasticsearch: ```bash -cd /path/to/Utilities/installation/elastic +cd /path/to/asap-tools/installation/elastic ``` ## Configuration diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/benchmark_importer/download_clickbench_data.py b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/benchmark_importer/download_clickbench_data.py similarity index 100% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/benchmark_importer/download_clickbench_data.py rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/benchmark_importer/download_clickbench_data.py diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/benchmark_importer/download_h2o_data.py b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/benchmark_importer/download_h2o_data.py similarity index 100% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/benchmark_importer/download_h2o_data.py rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/benchmark_importer/download_h2o_data.py diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/benchmark_importer/requirements.txt b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/benchmark_importer/requirements.txt similarity index 100% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/benchmark_importer/requirements.txt rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/benchmark_importer/requirements.txt diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/benchmark_queries.sql b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/benchmark_queries.sql similarity index 100% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/benchmark_queries.sql rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/benchmark_queries.sql diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/clickhouse/clickbench_init.sql b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/clickhouse/clickbench_init.sql similarity index 100% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/clickhouse/clickbench_init.sql rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/clickhouse/clickbench_init.sql diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/clickhouse/h2o_init.sql b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/clickhouse/h2o_init.sql similarity index 100% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/clickhouse/h2o_init.sql rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/clickhouse/h2o_init.sql diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/config.env b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/config.env similarity index 93% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/config.env rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/config.env index 0ca2c41..83f5ff9 100644 --- a/ExecutionUtilities/clickhouse-benchmark-pipeline/config.env +++ b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/config.env @@ -6,7 +6,7 @@ PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Kafka KAFKA_BROKER=localhost:9092 # Native Kafka KAFKA_TOPIC=hits -KAFKA_HOME="${PROJECT_DIR}/../../Utilities/installation/kafka/kafka" +KAFKA_HOME="${PROJECT_DIR}/../../installation/kafka/kafka" # ClickHouse CLICKHOUSE_HOST=localhost diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/data_exporter/Cargo.lock b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/data_exporter/Cargo.lock similarity index 100% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/data_exporter/Cargo.lock rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/data_exporter/Cargo.lock diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/data_exporter/Cargo.toml b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/data_exporter/Cargo.toml similarity index 100% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/data_exporter/Cargo.toml rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/data_exporter/Cargo.toml diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/data_exporter/src/main.rs b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/data_exporter/src/main.rs similarity index 100% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/data_exporter/src/main.rs rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/data_exporter/src/main.rs diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/h2o_elastic_queries.sql b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/h2o_elastic_queries.sql similarity index 100% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/h2o_elastic_queries.sql rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/h2o_elastic_queries.sql diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/h2o_queries.sql b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/h2o_queries.sql similarity index 100% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/h2o_queries.sql rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/h2o_queries.sql diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/scripts/check_data.sh b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/scripts/check_data.sh similarity index 100% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/scripts/check_data.sh rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/scripts/check_data.sh diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/scripts/check_elastic_data.sh b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/scripts/check_elastic_data.sh similarity index 100% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/scripts/check_elastic_data.sh rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/scripts/check_elastic_data.sh diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/scripts/generate_data.sh b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/scripts/generate_data.sh similarity index 98% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/scripts/generate_data.sh rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/scripts/generate_data.sh index df145e2..6ded961 100755 --- a/ExecutionUtilities/clickhouse-benchmark-pipeline/scripts/generate_data.sh +++ b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/scripts/generate_data.sh @@ -21,7 +21,7 @@ IMPORTER_DIR="${PROJECT_DIR}/benchmark_importer" DATA_DIR="${IMPORTER_DIR}/data" # Hardcoded Kafka Path -KAFKA_HOME="${PROJECT_DIR}/../../Utilities/installation/kafka/kafka" +KAFKA_HOME="${PROJECT_DIR}/../../installation/kafka/kafka" KAFKA_BIN="${KAFKA_HOME}/bin" # Check Mode diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/scripts/init_clickhouse.sh b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/scripts/init_clickhouse.sh similarity index 93% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/scripts/init_clickhouse.sh rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/scripts/init_clickhouse.sh index e6fe485..1d34355 100755 --- a/ExecutionUtilities/clickhouse-benchmark-pipeline/scripts/init_clickhouse.sh +++ b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/scripts/init_clickhouse.sh @@ -10,7 +10,7 @@ source "${PROJECT_DIR}/config.env" set +a # --- CONFIGURATION --- -CH_INSTALL_DIR="${PROJECT_DIR}/../../Utilities/installation/clickhouse/clickhouse" +CH_INSTALL_DIR="${PROJECT_DIR}/../../installation/clickhouse/clickhouse" CLICKHOUSE_BIN="${CH_INSTALL_DIR}/clickhouse" # Set Defaults if config variables are missing diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/scripts/init_elastic.sh b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/scripts/init_elastic.sh similarity index 100% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/scripts/init_elastic.sh rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/scripts/init_elastic.sh diff --git a/ExecutionUtilities/clickhouse-benchmark-pipeline/scripts/run_benchmark.sh b/asap-tools/execution-utilities/clickhouse-benchmark-pipeline/scripts/run_benchmark.sh similarity index 100% rename from ExecutionUtilities/clickhouse-benchmark-pipeline/scripts/run_benchmark.sh rename to asap-tools/execution-utilities/clickhouse-benchmark-pipeline/scripts/run_benchmark.sh diff --git a/ExecutionUtilities/csv_to_prometheus/__init__.py b/asap-tools/execution-utilities/csv_to_prometheus/__init__.py similarity index 100% rename from ExecutionUtilities/csv_to_prometheus/__init__.py rename to asap-tools/execution-utilities/csv_to_prometheus/__init__.py diff --git a/ExecutionUtilities/csv_to_prometheus/csv_to_prometheus.py b/asap-tools/execution-utilities/csv_to_prometheus/csv_to_prometheus.py similarity index 100% rename from ExecutionUtilities/csv_to_prometheus/csv_to_prometheus.py rename to asap-tools/execution-utilities/csv_to_prometheus/csv_to_prometheus.py diff --git a/ExecutionUtilities/csv_to_prometheus/setup.py b/asap-tools/execution-utilities/csv_to_prometheus/setup.py similarity index 100% rename from ExecutionUtilities/csv_to_prometheus/setup.py rename to asap-tools/execution-utilities/csv_to_prometheus/setup.py diff --git a/ExecutionUtilities/high-throughput-kafka-producer-single-threaded/Cargo.lock b/asap-tools/execution-utilities/high-throughput-kafka-producer-single-threaded/Cargo.lock similarity index 100% rename from ExecutionUtilities/high-throughput-kafka-producer-single-threaded/Cargo.lock rename to asap-tools/execution-utilities/high-throughput-kafka-producer-single-threaded/Cargo.lock diff --git a/ExecutionUtilities/high-throughput-kafka-producer-single-threaded/Cargo.toml b/asap-tools/execution-utilities/high-throughput-kafka-producer-single-threaded/Cargo.toml similarity index 100% rename from ExecutionUtilities/high-throughput-kafka-producer-single-threaded/Cargo.toml rename to asap-tools/execution-utilities/high-throughput-kafka-producer-single-threaded/Cargo.toml diff --git a/ExecutionUtilities/high-throughput-kafka-producer-single-threaded/src/main.rs b/asap-tools/execution-utilities/high-throughput-kafka-producer-single-threaded/src/main.rs similarity index 100% rename from ExecutionUtilities/high-throughput-kafka-producer-single-threaded/src/main.rs rename to asap-tools/execution-utilities/high-throughput-kafka-producer-single-threaded/src/main.rs diff --git a/ExecutionUtilities/high-throughput-kafka-producer/Cargo.lock b/asap-tools/execution-utilities/high-throughput-kafka-producer/Cargo.lock similarity index 100% rename from ExecutionUtilities/high-throughput-kafka-producer/Cargo.lock rename to asap-tools/execution-utilities/high-throughput-kafka-producer/Cargo.lock diff --git a/ExecutionUtilities/high-throughput-kafka-producer/Cargo.toml b/asap-tools/execution-utilities/high-throughput-kafka-producer/Cargo.toml similarity index 100% rename from ExecutionUtilities/high-throughput-kafka-producer/Cargo.toml rename to asap-tools/execution-utilities/high-throughput-kafka-producer/Cargo.toml diff --git a/ExecutionUtilities/high-throughput-kafka-producer/code_for_reference/ProducerPerformance.java b/asap-tools/execution-utilities/high-throughput-kafka-producer/code_for_reference/ProducerPerformance.java similarity index 100% rename from ExecutionUtilities/high-throughput-kafka-producer/code_for_reference/ProducerPerformance.java rename to asap-tools/execution-utilities/high-throughput-kafka-producer/code_for_reference/ProducerPerformance.java diff --git a/ExecutionUtilities/high-throughput-kafka-producer/src/main.rs b/asap-tools/execution-utilities/high-throughput-kafka-producer/src/main.rs similarity index 100% rename from ExecutionUtilities/high-throughput-kafka-producer/src/main.rs rename to asap-tools/execution-utilities/high-throughput-kafka-producer/src/main.rs diff --git a/ExecutionUtilities/kafka-consumer/Cargo.lock b/asap-tools/execution-utilities/kafka-consumer/Cargo.lock similarity index 100% rename from ExecutionUtilities/kafka-consumer/Cargo.lock rename to asap-tools/execution-utilities/kafka-consumer/Cargo.lock diff --git a/ExecutionUtilities/kafka-consumer/Cargo.toml b/asap-tools/execution-utilities/kafka-consumer/Cargo.toml similarity index 100% rename from ExecutionUtilities/kafka-consumer/Cargo.toml rename to asap-tools/execution-utilities/kafka-consumer/Cargo.toml diff --git a/ExecutionUtilities/kafka-consumer/src/main.rs b/asap-tools/execution-utilities/kafka-consumer/src/main.rs similarity index 100% rename from ExecutionUtilities/kafka-consumer/src/main.rs rename to asap-tools/execution-utilities/kafka-consumer/src/main.rs diff --git a/ExecutionUtilities/kafka_toy_data_generator/main.py b/asap-tools/execution-utilities/kafka_toy_data_generator/main.py similarity index 100% rename from ExecutionUtilities/kafka_toy_data_generator/main.py rename to asap-tools/execution-utilities/kafka_toy_data_generator/main.py diff --git a/ExecutionUtilities/kafka_toy_data_generator/requirements.txt b/asap-tools/execution-utilities/kafka_toy_data_generator/requirements.txt similarity index 100% rename from ExecutionUtilities/kafka_toy_data_generator/requirements.txt rename to asap-tools/execution-utilities/kafka_toy_data_generator/requirements.txt diff --git a/Utilities/experiments/CONFIG_PARAMETERS_REFERENCE.md b/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md similarity index 100% rename from Utilities/experiments/CONFIG_PARAMETERS_REFERENCE.md rename to asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md diff --git a/Utilities/experiments/HYDRA_CONFIG_USAGE.md b/asap-tools/experiments/HYDRA_CONFIG_USAGE.md similarity index 100% rename from Utilities/experiments/HYDRA_CONFIG_USAGE.md rename to asap-tools/experiments/HYDRA_CONFIG_USAGE.md diff --git a/Utilities/experiments/README_WORKLOAD_GENERATOR.md b/asap-tools/experiments/README_WORKLOAD_GENERATOR.md similarity index 100% rename from Utilities/experiments/README_WORKLOAD_GENERATOR.md rename to asap-tools/experiments/README_WORKLOAD_GENERATOR.md diff --git a/Utilities/experiments/arroyo_throughput_monitor.py b/asap-tools/experiments/arroyo_throughput_monitor.py similarity index 100% rename from Utilities/experiments/arroyo_throughput_monitor.py rename to asap-tools/experiments/arroyo_throughput_monitor.py diff --git a/PrometheusExporters/query_cost_exporter/process/ProcessMonitorHook.py b/asap-tools/experiments/classes/ProcessMonitorHook.py similarity index 100% rename from PrometheusExporters/query_cost_exporter/process/ProcessMonitorHook.py rename to asap-tools/experiments/classes/ProcessMonitorHook.py diff --git a/Utilities/experiments/classes/QueryCostExporter.py b/asap-tools/experiments/classes/QueryCostExporter.py similarity index 100% rename from Utilities/experiments/classes/QueryCostExporter.py rename to asap-tools/experiments/classes/QueryCostExporter.py diff --git a/PrometheusExporters/query_cost_exporter/process/__init__.py b/asap-tools/experiments/classes/__init__.py similarity index 100% rename from PrometheusExporters/query_cost_exporter/process/__init__.py rename to asap-tools/experiments/classes/__init__.py diff --git a/Utilities/experiments/classes/process_monitor.py b/asap-tools/experiments/classes/process_monitor.py similarity index 100% rename from Utilities/experiments/classes/process_monitor.py rename to asap-tools/experiments/classes/process_monitor.py diff --git a/PrometheusExporters/query_cost_exporter/query_cost.py b/asap-tools/experiments/classes/query_cost.py similarity index 100% rename from PrometheusExporters/query_cost_exporter/query_cost.py rename to asap-tools/experiments/classes/query_cost.py diff --git a/Utilities/experiments/cluster_data_exporter/Cargo.toml b/asap-tools/experiments/cluster_data_exporter/Cargo.toml similarity index 100% rename from Utilities/experiments/cluster_data_exporter/Cargo.toml rename to asap-tools/experiments/cluster_data_exporter/Cargo.toml diff --git a/Utilities/experiments/cluster_data_exporter/README.txt b/asap-tools/experiments/cluster_data_exporter/README.txt similarity index 100% rename from Utilities/experiments/cluster_data_exporter/README.txt rename to asap-tools/experiments/cluster_data_exporter/README.txt diff --git a/Utilities/experiments/cluster_data_exporter/bin/alibaba/sort_and_format.sh b/asap-tools/experiments/cluster_data_exporter/bin/alibaba/sort_and_format.sh similarity index 100% rename from Utilities/experiments/cluster_data_exporter/bin/alibaba/sort_and_format.sh rename to asap-tools/experiments/cluster_data_exporter/bin/alibaba/sort_and_format.sh diff --git a/Utilities/experiments/cluster_data_exporter/src/alibaba_metrics.rs b/asap-tools/experiments/cluster_data_exporter/src/alibaba_metrics.rs similarity index 100% rename from Utilities/experiments/cluster_data_exporter/src/alibaba_metrics.rs rename to asap-tools/experiments/cluster_data_exporter/src/alibaba_metrics.rs diff --git a/Utilities/experiments/cluster_data_exporter/src/alibaba_metrics/ms_resource.rs b/asap-tools/experiments/cluster_data_exporter/src/alibaba_metrics/ms_resource.rs similarity index 100% rename from Utilities/experiments/cluster_data_exporter/src/alibaba_metrics/ms_resource.rs rename to asap-tools/experiments/cluster_data_exporter/src/alibaba_metrics/ms_resource.rs diff --git a/Utilities/experiments/cluster_data_exporter/src/alibaba_metrics/node.rs b/asap-tools/experiments/cluster_data_exporter/src/alibaba_metrics/node.rs similarity index 100% rename from Utilities/experiments/cluster_data_exporter/src/alibaba_metrics/node.rs rename to asap-tools/experiments/cluster_data_exporter/src/alibaba_metrics/node.rs diff --git a/Utilities/experiments/cluster_data_exporter/src/google_metrics.rs b/asap-tools/experiments/cluster_data_exporter/src/google_metrics.rs similarity index 100% rename from Utilities/experiments/cluster_data_exporter/src/google_metrics.rs rename to asap-tools/experiments/cluster_data_exporter/src/google_metrics.rs diff --git a/Utilities/experiments/cluster_data_exporter/src/main.rs b/asap-tools/experiments/cluster_data_exporter/src/main.rs similarity index 100% rename from Utilities/experiments/cluster_data_exporter/src/main.rs rename to asap-tools/experiments/cluster_data_exporter/src/main.rs diff --git a/Utilities/experiments/cluster_data_exporter/src/utilities.rs b/asap-tools/experiments/cluster_data_exporter/src/utilities.rs similarity index 100% rename from Utilities/experiments/cluster_data_exporter/src/utilities.rs rename to asap-tools/experiments/cluster_data_exporter/src/utilities.rs diff --git a/Utilities/experiments/config/config.yaml b/asap-tools/experiments/config/config.yaml similarity index 100% rename from Utilities/experiments/config/config.yaml rename to asap-tools/experiments/config/config.yaml diff --git a/Utilities/experiments/constants.py b/asap-tools/experiments/constants.py similarity index 100% rename from Utilities/experiments/constants.py rename to asap-tools/experiments/constants.py diff --git a/Utilities/experiments/datasets/README.md b/asap-tools/experiments/datasets/README.md similarity index 100% rename from Utilities/experiments/datasets/README.md rename to asap-tools/experiments/datasets/README.md diff --git a/Utilities/experiments/datasets/download_google_cluster_2019.sh b/asap-tools/experiments/datasets/download_google_cluster_2019.sh similarity index 89% rename from Utilities/experiments/datasets/download_google_cluster_2019.sh rename to asap-tools/experiments/datasets/download_google_cluster_2019.sh index f84d6e6..79f9e7f 100755 --- a/Utilities/experiments/datasets/download_google_cluster_2019.sh +++ b/asap-tools/experiments/datasets/download_google_cluster_2019.sh @@ -9,7 +9,7 @@ num_nodes=$1 USERNAME=$2 HOSTNAME_SUFFIX=$3 -CMD="cd /scratch/sketch_db_for_prometheus/code/Utilities/experiments/datasets; " +CMD="cd /scratch/sketch_db_for_prometheus/code/asap-tools/experiments/datasets; " CMD=$CMD"curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-linux-x86_64.tar.gz; " CMD=$CMD"tar -xf google-cloud-cli-linux-x86_64.tar.gz; " diff --git a/Utilities/experiments/datasets/example.json b/asap-tools/experiments/datasets/example.json similarity index 100% rename from Utilities/experiments/datasets/example.json rename to asap-tools/experiments/datasets/example.json diff --git a/Utilities/experiments/dumb_kafka_consumer_arroyo.py b/asap-tools/experiments/dumb_kafka_consumer_arroyo.py similarity index 100% rename from Utilities/experiments/dumb_kafka_consumer_arroyo.py rename to asap-tools/experiments/dumb_kafka_consumer_arroyo.py diff --git a/Utilities/experiments/experiment_only_ingest_path.py b/asap-tools/experiments/experiment_only_ingest_path.py similarity index 99% rename from Utilities/experiments/experiment_only_ingest_path.py rename to asap-tools/experiments/experiment_only_ingest_path.py index 79277fb..05e6a4e 100644 --- a/Utilities/experiments/experiment_only_ingest_path.py +++ b/asap-tools/experiments/experiment_only_ingest_path.py @@ -506,7 +506,7 @@ def start_resource_monitoring( f"--node_offset {node_offset} " ) - cmd_dir = os.path.join(provider.get_home_dir(), "code", "Utilities", "experiments") + cmd_dir = os.path.join(provider.get_home_dir(), "code", "asap-tools", "experiments") cmd += f" > {experiment_output_dir}/remote_monitor.out 2>&1 &" diff --git a/Utilities/experiments/experiment_run_e2e.py b/asap-tools/experiments/experiment_run_e2e.py similarity index 100% rename from Utilities/experiments/experiment_run_e2e.py rename to asap-tools/experiments/experiment_run_e2e.py diff --git a/Utilities/experiments/experiment_run_exporters_and_prometheus.py b/asap-tools/experiments/experiment_run_exporters_and_prometheus.py similarity index 100% rename from Utilities/experiments/experiment_run_exporters_and_prometheus.py rename to asap-tools/experiments/experiment_run_exporters_and_prometheus.py diff --git a/Utilities/experiments/experiment_run_grafana_demo.py b/asap-tools/experiments/experiment_run_grafana_demo.py similarity index 100% rename from Utilities/experiments/experiment_run_grafana_demo.py rename to asap-tools/experiments/experiment_run_grafana_demo.py diff --git a/Utilities/experiments/experiment_run_sketchdboffline.py b/asap-tools/experiments/experiment_run_sketchdboffline.py similarity index 100% rename from Utilities/experiments/experiment_run_sketchdboffline.py rename to asap-tools/experiments/experiment_run_sketchdboffline.py diff --git a/Utilities/experiments/experiment_teardown_everything.py b/asap-tools/experiments/experiment_teardown_everything.py similarity index 100% rename from Utilities/experiments/experiment_teardown_everything.py rename to asap-tools/experiments/experiment_teardown_everything.py diff --git a/Utilities/experiments/experiment_utils/__init__.py b/asap-tools/experiments/experiment_utils/__init__.py similarity index 100% rename from Utilities/experiments/experiment_utils/__init__.py rename to asap-tools/experiments/experiment_utils/__init__.py diff --git a/Utilities/experiments/experiment_utils/config.py b/asap-tools/experiments/experiment_utils/config.py similarity index 100% rename from Utilities/experiments/experiment_utils/config.py rename to asap-tools/experiments/experiment_utils/config.py diff --git a/Utilities/experiments/experiment_utils/core.py b/asap-tools/experiments/experiment_utils/core.py similarity index 100% rename from Utilities/experiments/experiment_utils/core.py rename to asap-tools/experiments/experiment_utils/core.py diff --git a/Utilities/experiments/experiment_utils/providers/__init__.py b/asap-tools/experiments/experiment_utils/providers/__init__.py similarity index 100% rename from Utilities/experiments/experiment_utils/providers/__init__.py rename to asap-tools/experiments/experiment_utils/providers/__init__.py diff --git a/Utilities/experiments/experiment_utils/providers/base.py b/asap-tools/experiments/experiment_utils/providers/base.py similarity index 100% rename from Utilities/experiments/experiment_utils/providers/base.py rename to asap-tools/experiments/experiment_utils/providers/base.py diff --git a/Utilities/experiments/experiment_utils/providers/cloudlab.py b/asap-tools/experiments/experiment_utils/providers/cloudlab.py similarity index 100% rename from Utilities/experiments/experiment_utils/providers/cloudlab.py rename to asap-tools/experiments/experiment_utils/providers/cloudlab.py diff --git a/Utilities/experiments/experiment_utils/providers/cloudlab_local.py b/asap-tools/experiments/experiment_utils/providers/cloudlab_local.py similarity index 100% rename from Utilities/experiments/experiment_utils/providers/cloudlab_local.py rename to asap-tools/experiments/experiment_utils/providers/cloudlab_local.py diff --git a/Utilities/experiments/experiment_utils/providers/factory.py b/asap-tools/experiments/experiment_utils/providers/factory.py similarity index 100% rename from Utilities/experiments/experiment_utils/providers/factory.py rename to asap-tools/experiments/experiment_utils/providers/factory.py diff --git a/Utilities/experiments/experiment_utils/services/__init__.py b/asap-tools/experiments/experiment_utils/services/__init__.py similarity index 100% rename from Utilities/experiments/experiment_utils/services/__init__.py rename to asap-tools/experiments/experiment_utils/services/__init__.py diff --git a/Utilities/experiments/experiment_utils/services/arroyo.py b/asap-tools/experiments/experiment_utils/services/arroyo.py similarity index 97% rename from Utilities/experiments/experiment_utils/services/arroyo.py rename to asap-tools/experiments/experiment_utils/services/arroyo.py index 5dfba18..bc8148d 100644 --- a/Utilities/experiments/experiment_utils/services/arroyo.py +++ b/asap-tools/experiments/experiment_utils/services/arroyo.py @@ -65,7 +65,9 @@ def stop(self, **kwargs) -> None: def stop_all_jobs(self) -> None: """Stop all running Arroyo jobs.""" cmd = "python3 delete_pipeline.py --all_pipelines" - cmd_dir = os.path.join(self.provider.get_home_dir(), "code", "ArroyoSketch") + cmd_dir = os.path.join( + self.provider.get_home_dir(), "code", "asap-sketch-ingest" + ) self.provider.execute_command( node_idx=self.node_offset, cmd=cmd, @@ -157,7 +159,9 @@ def run_arroyosketch( # Add optimized source flag if enabled if enable_optimized_remote_write: cmd += " --prometheus_remote_write_source optimized" - cmd_dir = os.path.join(constants.CLOUDLAB_HOME_DIR, "code", "ArroyoSketch") + cmd_dir = os.path.join( + constants.CLOUDLAB_HOME_DIR, "code", "asap-sketch-ingest" + ) if avoid_long_ssh: # Run in background to avoid long SSH connection @@ -211,7 +215,9 @@ def stop_arroyosketch(self, pipeline_id: str) -> None: pipeline_id: ID of the pipeline to stop """ cmd = "python3 delete_pipeline.py --pipeline_id {}".format(pipeline_id) - cmd_dir = os.path.join(constants.CLOUDLAB_HOME_DIR, "code", "ArroyoSketch") + cmd_dir = os.path.join( + constants.CLOUDLAB_HOME_DIR, "code", "asap-sketch-ingest" + ) self.provider.execute_command( node_idx=self.node_offset, cmd=cmd, @@ -303,7 +309,7 @@ def is_healthy(self) -> bool: def _start_bare_metal(self, experiment_output_dir: str, **kwargs) -> None: """Start Arroyo cluster using bare metal deployment (original implementation).""" arroyo_config_file_path = os.path.join( - constants.CLOUDLAB_HOME_DIR, "code", "ArroyoSketch", "config.yaml" + constants.CLOUDLAB_HOME_DIR, "code", "asap-sketch-ingest", "config.yaml" ) arroyo_bin_path = os.path.join( constants.CLOUDLAB_HOME_DIR, "code", "arroyo", "target", "release", "arroyo" @@ -326,7 +332,7 @@ def _start_bare_metal(self, experiment_output_dir: str, **kwargs) -> None: def _start_containerized(self, experiment_output_dir: str, **kwargs) -> None: """Start Arroyo cluster using Docker container deployment.""" arroyo_config_file_path = os.path.join( - constants.CLOUDLAB_HOME_DIR, "code", "ArroyoSketch", "config.yaml" + constants.CLOUDLAB_HOME_DIR, "code", "asap-sketch-ingest", "config.yaml" ) arroyo_output_file = os.path.join(experiment_output_dir, "arroyo_cluster.out") diff --git a/Utilities/experiments/experiment_utils/services/arroyo_throughput_monitor.py b/asap-tools/experiments/experiment_utils/services/arroyo_throughput_monitor.py similarity index 99% rename from Utilities/experiments/experiment_utils/services/arroyo_throughput_monitor.py rename to asap-tools/experiments/experiment_utils/services/arroyo_throughput_monitor.py index 2809cd5..7fb1839 100644 --- a/Utilities/experiments/experiment_utils/services/arroyo_throughput_monitor.py +++ b/asap-tools/experiments/experiment_utils/services/arroyo_throughput_monitor.py @@ -51,7 +51,7 @@ def start(self, pipeline_id: str, experiment_output_dir: str, **kwargs) -> None: self.monitor_script_path = os.path.join( self.provider.get_home_dir(), "code", - "Utilities", + "asap-tools", "experiments", "arroyo_throughput_monitor.py", ) diff --git a/Utilities/experiments/experiment_utils/services/base.py b/asap-tools/experiments/experiment_utils/services/base.py similarity index 100% rename from Utilities/experiments/experiment_utils/services/base.py rename to asap-tools/experiments/experiment_utils/services/base.py diff --git a/Utilities/experiments/experiment_utils/services/cluster_data_exporter.py b/asap-tools/experiments/experiment_utils/services/cluster_data_exporter.py similarity index 100% rename from Utilities/experiments/experiment_utils/services/cluster_data_exporter.py rename to asap-tools/experiments/experiment_utils/services/cluster_data_exporter.py diff --git a/Utilities/experiments/experiment_utils/services/docker-compose.victoriametrics.yml.j2 b/asap-tools/experiments/experiment_utils/services/docker-compose.victoriametrics.yml.j2 similarity index 100% rename from Utilities/experiments/experiment_utils/services/docker-compose.victoriametrics.yml.j2 rename to asap-tools/experiments/experiment_utils/services/docker-compose.victoriametrics.yml.j2 diff --git a/Utilities/experiments/experiment_utils/services/docker_prometheus.py b/asap-tools/experiments/experiment_utils/services/docker_prometheus.py similarity index 100% rename from Utilities/experiments/experiment_utils/services/docker_prometheus.py rename to asap-tools/experiments/experiment_utils/services/docker_prometheus.py diff --git a/Utilities/experiments/experiment_utils/services/docker_victoriametrics.py b/asap-tools/experiments/experiment_utils/services/docker_victoriametrics.py similarity index 100% rename from Utilities/experiments/experiment_utils/services/docker_victoriametrics.py rename to asap-tools/experiments/experiment_utils/services/docker_victoriametrics.py diff --git a/Utilities/experiments/experiment_utils/services/fake_exporters.py b/asap-tools/experiments/experiment_utils/services/fake_exporters.py similarity index 99% rename from Utilities/experiments/experiment_utils/services/fake_exporters.py rename to asap-tools/experiments/experiment_utils/services/fake_exporters.py index 58f1904..5a897cd 100644 --- a/Utilities/experiments/experiment_utils/services/fake_exporters.py +++ b/asap-tools/experiments/experiment_utils/services/fake_exporters.py @@ -138,7 +138,8 @@ def _start_bare_metal( cmd_dir = os.path.join( constants.CLOUDLAB_HOME_DIR, "code", - "PrometheusExporters", + "asap-tools", + "prometheus-exporters", "fake_exporter", "fake_exporter_python", ) @@ -394,7 +395,7 @@ def _start_bare_metal( ) cmds.append(cmd) - cmd_dir = f"{constants.CLOUDLAB_HOME_DIR}/code/PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter" + cmd_dir = f"{constants.CLOUDLAB_HOME_DIR}/code/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter" # Dump workload configuration to a file os.makedirs( diff --git a/Utilities/experiments/experiment_utils/services/flink.py b/asap-tools/experiments/experiment_utils/services/flink.py similarity index 100% rename from Utilities/experiments/experiment_utils/services/flink.py rename to asap-tools/experiments/experiment_utils/services/flink.py diff --git a/Utilities/experiments/experiment_utils/services/grafana.py b/asap-tools/experiments/experiment_utils/services/grafana.py similarity index 98% rename from Utilities/experiments/experiment_utils/services/grafana.py rename to asap-tools/experiments/experiment_utils/services/grafana.py index e67ab56..84ede9e 100644 --- a/Utilities/experiments/experiment_utils/services/grafana.py +++ b/asap-tools/experiments/experiment_utils/services/grafana.py @@ -127,7 +127,7 @@ def configure_dashboard(self, experiment_type: str, experiment_name: str) -> boo # Use the CloudLab home directory pattern like other services cmd_dir = os.path.join( - self.provider.get_home_dir(), "code", "Utilities", "experiments" + self.provider.get_home_dir(), "code", "asap-tools", "experiments" ) print(f"Calling grafana_config.py: {cmd}") diff --git a/Utilities/experiments/experiment_utils/services/kafka.py b/asap-tools/experiments/experiment_utils/services/kafka.py similarity index 100% rename from Utilities/experiments/experiment_utils/services/kafka.py rename to asap-tools/experiments/experiment_utils/services/kafka.py diff --git a/Utilities/experiments/experiment_utils/services/misc.py b/asap-tools/experiments/experiment_utils/services/misc.py similarity index 98% rename from Utilities/experiments/experiment_utils/services/misc.py rename to asap-tools/experiments/experiment_utils/services/misc.py index f805659..1e4e340 100644 --- a/Utilities/experiments/experiment_utils/services/misc.py +++ b/asap-tools/experiments/experiment_utils/services/misc.py @@ -244,7 +244,7 @@ def _start_bare_metal( ) if punting: cmd += " --enable-punting" - cmd_dir = os.path.join(self.provider.get_home_dir(), "code", "Controller") + cmd_dir = os.path.join(self.provider.get_home_dir(), "code", "asap-planner") self.provider.execute_command( node_idx=self.node_offset, cmd=cmd, @@ -263,7 +263,7 @@ def _start_containerized( punting: bool, ): controller_dir = os.path.join( - self.provider.get_home_dir(), "code", "Controller" + self.provider.get_home_dir(), "code", "asap-planner" ) template_path = os.path.join(controller_dir, "docker-compose.yml.j2") @@ -273,7 +273,7 @@ def _start_containerized( helper_script = os.path.join( self.provider.get_home_dir(), "code", - "Utilities", + "asap-tools", "experiments", "generate_controller_compose.py", ) @@ -380,7 +380,7 @@ def start(self, experiment_output_dir: str, **kwargs) -> None: os.path.join(experiment_output_dir, "dumb_kafka_consumer_output.json"), ) cmd_dir = os.path.join( - self.provider.get_home_dir(), "code", "Utilities", "experiments" + self.provider.get_home_dir(), "code", "asap-tools", "experiments" ) self.provider.execute_command( node_idx=self.node_offset, diff --git a/Utilities/experiments/experiment_utils/services/monitoring.py b/asap-tools/experiments/experiment_utils/services/monitoring.py similarity index 100% rename from Utilities/experiments/experiment_utils/services/monitoring.py rename to asap-tools/experiments/experiment_utils/services/monitoring.py diff --git a/Utilities/experiments/experiment_utils/services/prometheus.py b/asap-tools/experiments/experiment_utils/services/prometheus.py similarity index 100% rename from Utilities/experiments/experiment_utils/services/prometheus.py rename to asap-tools/experiments/experiment_utils/services/prometheus.py diff --git a/Utilities/experiments/experiment_utils/services/prometheus_client_service.py b/asap-tools/experiments/experiment_utils/services/prometheus_client_service.py similarity index 97% rename from Utilities/experiments/experiment_utils/services/prometheus_client_service.py rename to asap-tools/experiments/experiment_utils/services/prometheus_client_service.py index 43e7604..7a651b2 100644 --- a/Utilities/experiments/experiment_utils/services/prometheus_client_service.py +++ b/asap-tools/experiments/experiment_utils/services/prometheus_client_service.py @@ -81,7 +81,8 @@ def _start_containerized( prometheus_client_dir = os.path.join( self.provider.get_home_dir(), "code", - "PrometheusClient", + "asap-tools", + "prometheus-client", ) template_path = os.path.join(prometheus_client_dir, "docker-compose.yml.j2") remote_compose_file = os.path.join( @@ -91,7 +92,7 @@ def _start_containerized( helper_script = os.path.join( self.provider.get_home_dir(), "code", - "Utilities", + "asap-tools", "experiments", "generate_prometheus_client_compose.py", ) @@ -170,7 +171,9 @@ def _start_bare_metal( if profile_prometheus_time is not None: cmd += " --profile_prometheus_time {}".format(profile_prometheus_time) - cmd_dir = os.path.join(self.provider.get_home_dir(), "code", "PrometheusClient") + cmd_dir = os.path.join( + self.provider.get_home_dir(), "code", "asap-tools", "prometheus-client" + ) utils.run_cmd(f"cd {cmd_dir}; {cmd}", popen=False) return diff --git a/Utilities/experiments/experiment_utils/services/prometheus_health_monitor.py b/asap-tools/experiments/experiment_utils/services/prometheus_health_monitor.py similarity index 99% rename from Utilities/experiments/experiment_utils/services/prometheus_health_monitor.py rename to asap-tools/experiments/experiment_utils/services/prometheus_health_monitor.py index 2503ddf..662b470 100644 --- a/Utilities/experiments/experiment_utils/services/prometheus_health_monitor.py +++ b/asap-tools/experiments/experiment_utils/services/prometheus_health_monitor.py @@ -47,7 +47,7 @@ def start( self.monitor_script_path = os.path.join( self.provider.get_home_dir(), "code", - "Utilities", + "asap-tools", "experiments", "prometheus_health_monitor.py", ) diff --git a/Utilities/experiments/experiment_utils/services/prometheus_kafka_adapter.py b/asap-tools/experiments/experiment_utils/services/prometheus_kafka_adapter.py similarity index 100% rename from Utilities/experiments/experiment_utils/services/prometheus_kafka_adapter.py rename to asap-tools/experiments/experiment_utils/services/prometheus_kafka_adapter.py diff --git a/Utilities/experiments/experiment_utils/services/prometheus_throughput_monitor.py b/asap-tools/experiments/experiment_utils/services/prometheus_throughput_monitor.py similarity index 99% rename from Utilities/experiments/experiment_utils/services/prometheus_throughput_monitor.py rename to asap-tools/experiments/experiment_utils/services/prometheus_throughput_monitor.py index a9ee63c..e91e5a9 100644 --- a/Utilities/experiments/experiment_utils/services/prometheus_throughput_monitor.py +++ b/asap-tools/experiments/experiment_utils/services/prometheus_throughput_monitor.py @@ -49,7 +49,7 @@ def start( self.monitor_script_path = os.path.join( self.provider.get_home_dir(), "code", - "Utilities", + "asap-tools", "experiments", "prometheus_throughput_monitor.py", ) diff --git a/Utilities/experiments/experiment_utils/services/query_engine.py b/asap-tools/experiments/experiment_utils/services/query_engine.py similarity index 99% rename from Utilities/experiments/experiment_utils/services/query_engine.py rename to asap-tools/experiments/experiment_utils/services/query_engine.py index 65959ef..a88e34a 100644 --- a/Utilities/experiments/experiment_utils/services/query_engine.py +++ b/asap-tools/experiments/experiment_utils/services/query_engine.py @@ -216,7 +216,7 @@ def _start_containerized( helper_script = os.path.join( constants.CLOUDLAB_HOME_DIR, "code", - "Utilities", + "asap-tools", "experiments", "generate_queryengine_compose.py", ) @@ -551,7 +551,9 @@ def _start_bare_metal( cmd += "--dump-precomputes " cmd += "> {}/query_engine_rust.out 2>&1 &".format(output_dir) - cmd_dir = os.path.join(self.provider.get_home_dir(), "code", "QueryEngineRust") + cmd_dir = os.path.join( + self.provider.get_home_dir(), "code", "asap-query-engine" + ) self.provider.execute_command( node_idx=self.node_offset, cmd=cmd, @@ -586,14 +588,14 @@ def _start_containerized( # Paths on remote CloudLab node queryengine_dir = os.path.join( - constants.CLOUDLAB_HOME_DIR, "code", "QueryEngineRust" + constants.CLOUDLAB_HOME_DIR, "code", "asap-query-engine" ) template_path = os.path.join(queryengine_dir, "docker-compose.yml.j2") remote_compose_file = os.path.join(output_dir, "docker-compose.yml") helper_script = os.path.join( constants.CLOUDLAB_HOME_DIR, "code", - "Utilities", + "asap-tools", "experiments", "generate_queryengine_compose.py", ) diff --git a/Utilities/experiments/experiment_utils/services/remote_monitor_service.py b/asap-tools/experiments/experiment_utils/services/remote_monitor_service.py similarity index 98% rename from Utilities/experiments/experiment_utils/services/remote_monitor_service.py rename to asap-tools/experiments/experiment_utils/services/remote_monitor_service.py index 21e3adc..527c83d 100644 --- a/Utilities/experiments/experiment_utils/services/remote_monitor_service.py +++ b/asap-tools/experiments/experiment_utils/services/remote_monitor_service.py @@ -120,7 +120,7 @@ def start( ) cmd_dir = os.path.join( - self.provider.get_home_dir(), "code", "Utilities", "experiments" + self.provider.get_home_dir(), "code", "asap-tools", "experiments" ) cmd += " > {}/remote_monitor.out 2>&1".format(experiment_output_dir) @@ -218,7 +218,7 @@ def start( cmd += " --profile_prometheus_time {}".format(profile_prometheus_time) cmd_dir = os.path.join( - self.provider.get_home_dir(), "code", "Utilities", "experiments" + self.provider.get_home_dir(), "code", "asap-tools", "experiments" ) cmd += " > {}/remote_monitor.out 2>&1".format(experiment_output_dir) diff --git a/Utilities/experiments/experiment_utils/services/system_exporters.py b/asap-tools/experiments/experiment_utils/services/system_exporters.py similarity index 100% rename from Utilities/experiments/experiment_utils/services/system_exporters.py rename to asap-tools/experiments/experiment_utils/services/system_exporters.py diff --git a/Utilities/experiments/experiment_utils/sync.py b/asap-tools/experiments/experiment_utils/sync.py similarity index 100% rename from Utilities/experiments/experiment_utils/sync.py rename to asap-tools/experiments/experiment_utils/sync.py diff --git a/Utilities/experiments/export_prometheus_data.py b/asap-tools/experiments/export_prometheus_data.py similarity index 100% rename from Utilities/experiments/export_prometheus_data.py rename to asap-tools/experiments/export_prometheus_data.py diff --git a/Utilities/experiments/generate_arroyosketch_compose.py b/asap-tools/experiments/generate_arroyosketch_compose.py similarity index 100% rename from Utilities/experiments/generate_arroyosketch_compose.py rename to asap-tools/experiments/generate_arroyosketch_compose.py diff --git a/Utilities/experiments/generate_controller_compose.py b/asap-tools/experiments/generate_controller_compose.py similarity index 100% rename from Utilities/experiments/generate_controller_compose.py rename to asap-tools/experiments/generate_controller_compose.py diff --git a/Utilities/experiments/generate_fake_exporter_compose.py b/asap-tools/experiments/generate_fake_exporter_compose.py similarity index 100% rename from Utilities/experiments/generate_fake_exporter_compose.py rename to asap-tools/experiments/generate_fake_exporter_compose.py diff --git a/Utilities/experiments/generate_prometheus_client_compose.py b/asap-tools/experiments/generate_prometheus_client_compose.py similarity index 100% rename from Utilities/experiments/generate_prometheus_client_compose.py rename to asap-tools/experiments/generate_prometheus_client_compose.py diff --git a/Utilities/experiments/generate_prometheus_config.py b/asap-tools/experiments/generate_prometheus_config.py similarity index 100% rename from Utilities/experiments/generate_prometheus_config.py rename to asap-tools/experiments/generate_prometheus_config.py diff --git a/Utilities/experiments/generate_queryengine_compose.py b/asap-tools/experiments/generate_queryengine_compose.py similarity index 100% rename from Utilities/experiments/generate_queryengine_compose.py rename to asap-tools/experiments/generate_queryengine_compose.py diff --git a/Utilities/experiments/generate_victoriametrics_config.py b/asap-tools/experiments/generate_victoriametrics_config.py similarity index 100% rename from Utilities/experiments/generate_victoriametrics_config.py rename to asap-tools/experiments/generate_victoriametrics_config.py diff --git a/Utilities/experiments/generate_workload.py b/asap-tools/experiments/generate_workload.py similarity index 100% rename from Utilities/experiments/generate_workload.py rename to asap-tools/experiments/generate_workload.py diff --git a/Utilities/experiments/grafana_config.py b/asap-tools/experiments/grafana_config.py similarity index 100% rename from Utilities/experiments/grafana_config.py rename to asap-tools/experiments/grafana_config.py diff --git a/Utilities/experiments/old_experiment_scripts/calculate_cost.py b/asap-tools/experiments/old_experiment_scripts/calculate_cost.py similarity index 100% rename from Utilities/experiments/old_experiment_scripts/calculate_cost.py rename to asap-tools/experiments/old_experiment_scripts/calculate_cost.py diff --git a/Utilities/experiments/old_experiment_scripts/copy_prometheus_data.py b/asap-tools/experiments/old_experiment_scripts/copy_prometheus_data.py similarity index 100% rename from Utilities/experiments/old_experiment_scripts/copy_prometheus_data.py rename to asap-tools/experiments/old_experiment_scripts/copy_prometheus_data.py diff --git a/Utilities/experiments/old_experiment_scripts/experiment_all_cost_sla_vs_nodes.py b/asap-tools/experiments/old_experiment_scripts/experiment_all_cost_sla_vs_nodes.py similarity index 100% rename from Utilities/experiments/old_experiment_scripts/experiment_all_cost_sla_vs_nodes.py rename to asap-tools/experiments/old_experiment_scripts/experiment_all_cost_sla_vs_nodes.py diff --git a/Utilities/experiments/old_experiment_scripts/experiment_collect_prom_data.py b/asap-tools/experiments/old_experiment_scripts/experiment_collect_prom_data.py similarity index 100% rename from Utilities/experiments/old_experiment_scripts/experiment_collect_prom_data.py rename to asap-tools/experiments/old_experiment_scripts/experiment_collect_prom_data.py diff --git a/Utilities/experiments/old_experiment_scripts/experiment_prom_cost_sla_vs_nodes_time.py b/asap-tools/experiments/old_experiment_scripts/experiment_prom_cost_sla_vs_nodes_time.py similarity index 100% rename from Utilities/experiments/old_experiment_scripts/experiment_prom_cost_sla_vs_nodes_time.py rename to asap-tools/experiments/old_experiment_scripts/experiment_prom_cost_sla_vs_nodes_time.py diff --git a/Utilities/experiments/old_experiment_scripts/fake_exporter.py b/asap-tools/experiments/old_experiment_scripts/fake_exporter.py similarity index 100% rename from Utilities/experiments/old_experiment_scripts/fake_exporter.py rename to asap-tools/experiments/old_experiment_scripts/fake_exporter.py diff --git a/Utilities/experiments/old_experiment_scripts/fake_exporter_rust/fake_exporter/Cargo.toml b/asap-tools/experiments/old_experiment_scripts/fake_exporter_rust/fake_exporter/Cargo.toml similarity index 100% rename from Utilities/experiments/old_experiment_scripts/fake_exporter_rust/fake_exporter/Cargo.toml rename to asap-tools/experiments/old_experiment_scripts/fake_exporter_rust/fake_exporter/Cargo.toml diff --git a/Utilities/experiments/old_experiment_scripts/fake_exporter_rust/fake_exporter/src/main.rs b/asap-tools/experiments/old_experiment_scripts/fake_exporter_rust/fake_exporter/src/main.rs similarity index 100% rename from Utilities/experiments/old_experiment_scripts/fake_exporter_rust/fake_exporter/src/main.rs rename to asap-tools/experiments/old_experiment_scripts/fake_exporter_rust/fake_exporter/src/main.rs diff --git a/Utilities/experiments/old_experiment_scripts/old_prometheus_throughput_monitor.py b/asap-tools/experiments/old_experiment_scripts/old_prometheus_throughput_monitor.py similarity index 100% rename from Utilities/experiments/old_experiment_scripts/old_prometheus_throughput_monitor.py rename to asap-tools/experiments/old_experiment_scripts/old_prometheus_throughput_monitor.py diff --git a/Utilities/experiments/old_experiment_scripts/prometheus_config/recording_rules/blackbox-exporter.yml b/asap-tools/experiments/old_experiment_scripts/prometheus_config/recording_rules/blackbox-exporter.yml similarity index 100% rename from Utilities/experiments/old_experiment_scripts/prometheus_config/recording_rules/blackbox-exporter.yml rename to asap-tools/experiments/old_experiment_scripts/prometheus_config/recording_rules/blackbox-exporter.yml diff --git a/Utilities/experiments/old_experiment_scripts/prometheus_config/recording_rules/custom_recording_rules.yml b/asap-tools/experiments/old_experiment_scripts/prometheus_config/recording_rules/custom_recording_rules.yml similarity index 100% rename from Utilities/experiments/old_experiment_scripts/prometheus_config/recording_rules/custom_recording_rules.yml rename to asap-tools/experiments/old_experiment_scripts/prometheus_config/recording_rules/custom_recording_rules.yml diff --git a/Utilities/experiments/old_experiment_scripts/prometheus_config/recording_rules/google-cadvisor.yml b/asap-tools/experiments/old_experiment_scripts/prometheus_config/recording_rules/google-cadvisor.yml similarity index 100% rename from Utilities/experiments/old_experiment_scripts/prometheus_config/recording_rules/google-cadvisor.yml rename to asap-tools/experiments/old_experiment_scripts/prometheus_config/recording_rules/google-cadvisor.yml diff --git a/Utilities/experiments/old_experiment_scripts/prometheus_config/recording_rules/node-exporter.yml b/asap-tools/experiments/old_experiment_scripts/prometheus_config/recording_rules/node-exporter.yml similarity index 100% rename from Utilities/experiments/old_experiment_scripts/prometheus_config/recording_rules/node-exporter.yml rename to asap-tools/experiments/old_experiment_scripts/prometheus_config/recording_rules/node-exporter.yml diff --git a/Utilities/experiments/old_experiment_scripts/prometheus_config/template.prometheus.yml b/asap-tools/experiments/old_experiment_scripts/prometheus_config/template.prometheus.yml similarity index 100% rename from Utilities/experiments/old_experiment_scripts/prometheus_config/template.prometheus.yml rename to asap-tools/experiments/old_experiment_scripts/prometheus_config/template.prometheus.yml diff --git a/Utilities/experiments/old_experiment_scripts/setup.py b/asap-tools/experiments/old_experiment_scripts/setup.py similarity index 100% rename from Utilities/experiments/old_experiment_scripts/setup.py rename to asap-tools/experiments/old_experiment_scripts/setup.py diff --git a/Utilities/experiments/old_experiment_scripts/vertical_scalability_grid_search.py b/asap-tools/experiments/old_experiment_scripts/vertical_scalability_grid_search.py similarity index 100% rename from Utilities/experiments/old_experiment_scripts/vertical_scalability_grid_search.py rename to asap-tools/experiments/old_experiment_scripts/vertical_scalability_grid_search.py diff --git a/Utilities/experiments/post_experiment/README_plot_cardinality_vs_benefit.md b/asap-tools/experiments/post_experiment/README_plot_cardinality_vs_benefit.md similarity index 100% rename from Utilities/experiments/post_experiment/README_plot_cardinality_vs_benefit.md rename to asap-tools/experiments/post_experiment/README_plot_cardinality_vs_benefit.md diff --git a/Utilities/experiments/post_experiment/analyze_latencies.py b/asap-tools/experiments/post_experiment/analyze_latencies.py similarity index 100% rename from Utilities/experiments/post_experiment/analyze_latencies.py rename to asap-tools/experiments/post_experiment/analyze_latencies.py diff --git a/Utilities/experiments/post_experiment/analyze_monitor_output.py b/asap-tools/experiments/post_experiment/analyze_monitor_output.py similarity index 100% rename from Utilities/experiments/post_experiment/analyze_monitor_output.py rename to asap-tools/experiments/post_experiment/analyze_monitor_output.py diff --git a/Utilities/experiments/post_experiment/analyze_throughput.py b/asap-tools/experiments/post_experiment/analyze_throughput.py similarity index 100% rename from Utilities/experiments/post_experiment/analyze_throughput.py rename to asap-tools/experiments/post_experiment/analyze_throughput.py diff --git a/Utilities/experiments/post_experiment/calculate_fidelity.py b/asap-tools/experiments/post_experiment/calculate_fidelity.py similarity index 100% rename from Utilities/experiments/post_experiment/calculate_fidelity.py rename to asap-tools/experiments/post_experiment/calculate_fidelity.py diff --git a/Utilities/experiments/post_experiment/compare_costs.py b/asap-tools/experiments/post_experiment/compare_costs.py similarity index 100% rename from Utilities/experiments/post_experiment/compare_costs.py rename to asap-tools/experiments/post_experiment/compare_costs.py diff --git a/Utilities/experiments/post_experiment/compare_latencies.py b/asap-tools/experiments/post_experiment/compare_latencies.py similarity index 100% rename from Utilities/experiments/post_experiment/compare_latencies.py rename to asap-tools/experiments/post_experiment/compare_latencies.py diff --git a/Utilities/experiments/post_experiment/plot_cardinality_vs_benefit.py b/asap-tools/experiments/post_experiment/plot_cardinality_vs_benefit.py similarity index 100% rename from Utilities/experiments/post_experiment/plot_cardinality_vs_benefit.py rename to asap-tools/experiments/post_experiment/plot_cardinality_vs_benefit.py diff --git a/Utilities/experiments/post_experiment/plot_comparison_bars.py b/asap-tools/experiments/post_experiment/plot_comparison_bars.py similarity index 100% rename from Utilities/experiments/post_experiment/plot_comparison_bars.py rename to asap-tools/experiments/post_experiment/plot_comparison_bars.py diff --git a/Utilities/experiments/post_experiment/plot_latency_distribution.py b/asap-tools/experiments/post_experiment/plot_latency_distribution.py similarity index 100% rename from Utilities/experiments/post_experiment/plot_latency_distribution.py rename to asap-tools/experiments/post_experiment/plot_latency_distribution.py diff --git a/Utilities/experiments/post_experiment/plot_latency_metrics.py b/asap-tools/experiments/post_experiment/plot_latency_metrics.py similarity index 100% rename from Utilities/experiments/post_experiment/plot_latency_metrics.py rename to asap-tools/experiments/post_experiment/plot_latency_metrics.py diff --git a/Utilities/experiments/post_experiment/plot_scale_vs_metrics.py b/asap-tools/experiments/post_experiment/plot_scale_vs_metrics.py similarity index 100% rename from Utilities/experiments/post_experiment/plot_scale_vs_metrics.py rename to asap-tools/experiments/post_experiment/plot_scale_vs_metrics.py diff --git a/Utilities/experiments/post_experiment/read_dumped_precomputes.py b/asap-tools/experiments/post_experiment/read_dumped_precomputes.py similarity index 100% rename from Utilities/experiments/post_experiment/read_dumped_precomputes.py rename to asap-tools/experiments/post_experiment/read_dumped_precomputes.py diff --git a/Utilities/experiments/post_experiment/results_loader.py b/asap-tools/experiments/post_experiment/results_loader.py similarity index 100% rename from Utilities/experiments/post_experiment/results_loader.py rename to asap-tools/experiments/post_experiment/results_loader.py diff --git a/Utilities/experiments/post_experiment/run_analyze_latencies.sh b/asap-tools/experiments/post_experiment/run_analyze_latencies.sh similarity index 100% rename from Utilities/experiments/post_experiment/run_analyze_latencies.sh rename to asap-tools/experiments/post_experiment/run_analyze_latencies.sh diff --git a/Utilities/experiments/post_experiment/run_calculate_fidelity.sh b/asap-tools/experiments/post_experiment/run_calculate_fidelity.sh similarity index 100% rename from Utilities/experiments/post_experiment/run_calculate_fidelity.sh rename to asap-tools/experiments/post_experiment/run_calculate_fidelity.sh diff --git a/Utilities/experiments/post_experiment/run_compare_latencies.sh b/asap-tools/experiments/post_experiment/run_compare_latencies.sh similarity index 100% rename from Utilities/experiments/post_experiment/run_compare_latencies.sh rename to asap-tools/experiments/post_experiment/run_compare_latencies.sh diff --git a/Utilities/experiments/post_experiment/run_plot_latency_distribution.sh b/asap-tools/experiments/post_experiment/run_plot_latency_distribution.sh similarity index 100% rename from Utilities/experiments/post_experiment/run_plot_latency_distribution.sh rename to asap-tools/experiments/post_experiment/run_plot_latency_distribution.sh diff --git a/Utilities/experiments/prometheus_health_monitor.py b/asap-tools/experiments/prometheus_health_monitor.py similarity index 100% rename from Utilities/experiments/prometheus_health_monitor.py rename to asap-tools/experiments/prometheus_health_monitor.py diff --git a/Utilities/experiments/prometheus_throughput_monitor.py b/asap-tools/experiments/prometheus_throughput_monitor.py similarity index 100% rename from Utilities/experiments/prometheus_throughput_monitor.py rename to asap-tools/experiments/prometheus_throughput_monitor.py diff --git a/Utilities/experiments/remote_monitor.py b/asap-tools/experiments/remote_monitor.py similarity index 100% rename from Utilities/experiments/remote_monitor.py rename to asap-tools/experiments/remote_monitor.py diff --git a/Utilities/experiments/reset_prometheus.py b/asap-tools/experiments/reset_prometheus.py similarity index 100% rename from Utilities/experiments/reset_prometheus.py rename to asap-tools/experiments/reset_prometheus.py diff --git a/Utilities/experiments/run_export_prometheus_data_for_experiment.sh b/asap-tools/experiments/run_export_prometheus_data_for_experiment.sh similarity index 100% rename from Utilities/experiments/run_export_prometheus_data_for_experiment.sh rename to asap-tools/experiments/run_export_prometheus_data_for_experiment.sh diff --git a/Utilities/experiments/utils.py b/asap-tools/experiments/utils.py similarity index 100% rename from Utilities/experiments/utils.py rename to asap-tools/experiments/utils.py diff --git a/Utilities/install_components.sh b/asap-tools/install_components.sh similarity index 100% rename from Utilities/install_components.sh rename to asap-tools/install_components.sh diff --git a/Utilities/installation/arroyo/install.sh b/asap-tools/installation/arroyo/install.sh similarity index 100% rename from Utilities/installation/arroyo/install.sh rename to asap-tools/installation/arroyo/install.sh diff --git a/Utilities/installation/asprof/install.sh b/asap-tools/installation/asprof/install.sh similarity index 100% rename from Utilities/installation/asprof/install.sh rename to asap-tools/installation/asprof/install.sh diff --git a/Utilities/installation/asprof/setup_dependencies.sh b/asap-tools/installation/asprof/setup_dependencies.sh similarity index 100% rename from Utilities/installation/asprof/setup_dependencies.sh rename to asap-tools/installation/asprof/setup_dependencies.sh diff --git a/Utilities/installation/benchmarks/install.sh b/asap-tools/installation/benchmarks/install.sh similarity index 100% rename from Utilities/installation/benchmarks/install.sh rename to asap-tools/installation/benchmarks/install.sh diff --git a/Utilities/installation/benchmarks/setup_dependencies.sh b/asap-tools/installation/benchmarks/setup_dependencies.sh similarity index 100% rename from Utilities/installation/benchmarks/setup_dependencies.sh rename to asap-tools/installation/benchmarks/setup_dependencies.sh diff --git a/Utilities/installation/clickhouse/.gitignore b/asap-tools/installation/clickhouse/.gitignore similarity index 100% rename from Utilities/installation/clickhouse/.gitignore rename to asap-tools/installation/clickhouse/.gitignore diff --git a/Utilities/installation/clickhouse/install.sh b/asap-tools/installation/clickhouse/install.sh similarity index 100% rename from Utilities/installation/clickhouse/install.sh rename to asap-tools/installation/clickhouse/install.sh diff --git a/Utilities/installation/clickhouse/run.sh b/asap-tools/installation/clickhouse/run.sh similarity index 79% rename from Utilities/installation/clickhouse/run.sh rename to asap-tools/installation/clickhouse/run.sh index 7b8ed4f..17d2fe2 100755 --- a/Utilities/installation/clickhouse/run.sh +++ b/asap-tools/installation/clickhouse/run.sh @@ -9,7 +9,7 @@ CLICKHOUSE_DIR=$1 # Load config from benchmark pipeline SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -CONFIG_FILE="${SCRIPT_DIR}/../../../ExecutionUtilities/clickhouse-benchmark-pipeline/config.env" +CONFIG_FILE="${SCRIPT_DIR}/../../../asap-tools/execution-utilities/clickhouse-benchmark-pipeline/config.env" if [ -f "$CONFIG_FILE" ]; then set -a source "$CONFIG_FILE" diff --git a/Utilities/installation/common/setup_dependencies.sh b/asap-tools/installation/common/setup_dependencies.sh similarity index 94% rename from Utilities/installation/common/setup_dependencies.sh rename to asap-tools/installation/common/setup_dependencies.sh index b30b997..36d340e 100755 --- a/Utilities/installation/common/setup_dependencies.sh +++ b/asap-tools/installation/common/setup_dependencies.sh @@ -23,7 +23,7 @@ else source "$HOME/.cargo/env" fi # (cd "$THIS_DIR/../../experiments/fake_exporter_rust/fake_exporter"; cargo build --release) -(cd "$THIS_DIR/../../../PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter" && cargo build --release) +(cd "$THIS_DIR/../../../asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter" && cargo build --release) DOCKER_DATA_DIR=/scratch/var_lib_docker diff --git a/Utilities/installation/elastic/README.md b/asap-tools/installation/elastic/README.md similarity index 100% rename from Utilities/installation/elastic/README.md rename to asap-tools/installation/elastic/README.md diff --git a/Utilities/installation/empty_experiment_outputs.sh b/asap-tools/installation/empty_experiment_outputs.sh similarity index 100% rename from Utilities/installation/empty_experiment_outputs.sh rename to asap-tools/installation/empty_experiment_outputs.sh diff --git a/Utilities/installation/empty_flink_logs.sh b/asap-tools/installation/empty_flink_logs.sh similarity index 100% rename from Utilities/installation/empty_flink_logs.sh rename to asap-tools/installation/empty_flink_logs.sh diff --git a/Utilities/installation/exporters/install.sh b/asap-tools/installation/exporters/install.sh similarity index 100% rename from Utilities/installation/exporters/install.sh rename to asap-tools/installation/exporters/install.sh diff --git a/Utilities/installation/exporters/setup_dependencies.sh b/asap-tools/installation/exporters/setup_dependencies.sh similarity index 100% rename from Utilities/installation/exporters/setup_dependencies.sh rename to asap-tools/installation/exporters/setup_dependencies.sh diff --git a/Utilities/installation/flink/install.sh b/asap-tools/installation/flink/install.sh similarity index 100% rename from Utilities/installation/flink/install.sh rename to asap-tools/installation/flink/install.sh diff --git a/Utilities/installation/grafana/install.sh b/asap-tools/installation/grafana/install.sh similarity index 100% rename from Utilities/installation/grafana/install.sh rename to asap-tools/installation/grafana/install.sh diff --git a/Utilities/installation/grafana/setup_dependencies.sh b/asap-tools/installation/grafana/setup_dependencies.sh similarity index 100% rename from Utilities/installation/grafana/setup_dependencies.sh rename to asap-tools/installation/grafana/setup_dependencies.sh diff --git a/Utilities/installation/install_external_components.sh b/asap-tools/installation/install_external_components.sh similarity index 100% rename from Utilities/installation/install_external_components.sh rename to asap-tools/installation/install_external_components.sh diff --git a/Utilities/installation/kafka/.gitignore b/asap-tools/installation/kafka/.gitignore similarity index 100% rename from Utilities/installation/kafka/.gitignore rename to asap-tools/installation/kafka/.gitignore diff --git a/Utilities/installation/kafka/install.sh b/asap-tools/installation/kafka/install.sh similarity index 100% rename from Utilities/installation/kafka/install.sh rename to asap-tools/installation/kafka/install.sh diff --git a/Utilities/installation/kafka/run.sh b/asap-tools/installation/kafka/run.sh similarity index 100% rename from Utilities/installation/kafka/run.sh rename to asap-tools/installation/kafka/run.sh diff --git a/Utilities/installation/kafka/setup_dependencies.sh b/asap-tools/installation/kafka/setup_dependencies.sh similarity index 100% rename from Utilities/installation/kafka/setup_dependencies.sh rename to asap-tools/installation/kafka/setup_dependencies.sh diff --git a/Utilities/installation/oneshot_only_install_internal_components.sh b/asap-tools/installation/oneshot_only_install_internal_components.sh similarity index 91% rename from Utilities/installation/oneshot_only_install_internal_components.sh rename to asap-tools/installation/oneshot_only_install_internal_components.sh index b9743b4..ce631ec 100755 --- a/Utilities/installation/oneshot_only_install_internal_components.sh +++ b/asap-tools/installation/oneshot_only_install_internal_components.sh @@ -20,7 +20,7 @@ HOSTNAME_PREFIX="node" install_internal_cmd() { local username="$1" local hostname="$2" - local cd_cmd="cd /scratch/sketch_db_for_prometheus/code/Utilities/installation" + local cd_cmd="cd /scratch/sketch_db_for_prometheus/code/asap-tools/installation" local internal_cmd="./only_install_internal_components.sh all" ssh $OPTIONS $username@$hostname "$cd_cmd && $internal_cmd" diff --git a/Utilities/installation/oneshot_setup.sh b/asap-tools/installation/oneshot_setup.sh similarity index 92% rename from Utilities/installation/oneshot_setup.sh rename to asap-tools/installation/oneshot_setup.sh index ed6aec0..94e53dc 100755 --- a/Utilities/installation/oneshot_setup.sh +++ b/asap-tools/installation/oneshot_setup.sh @@ -13,7 +13,7 @@ NODE_OFFSET=${4:-0} THIS_DIR=$(dirname "$(readlink -f "$0")") OPTIONS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -CD_CMD="cd /scratch/sketch_db_for_prometheus/code/Utilities/installation; " +CD_CMD="cd /scratch/sketch_db_for_prometheus/code/asap-tools/installation; " EXTERNAL_CMD="./install_external_components.sh /scratch/sketch_db_for_prometheus/ all; " INTERNAL_CMD="./setup_internal_components.sh all;" diff --git a/Utilities/installation/only_install_internal_components.sh b/asap-tools/installation/only_install_internal_components.sh similarity index 100% rename from Utilities/installation/only_install_internal_components.sh rename to asap-tools/installation/only_install_internal_components.sh diff --git a/Utilities/installation/prometheus/install.sh b/asap-tools/installation/prometheus/install.sh similarity index 100% rename from Utilities/installation/prometheus/install.sh rename to asap-tools/installation/prometheus/install.sh diff --git a/Utilities/installation/prometheus_kafka_adapter/install.sh b/asap-tools/installation/prometheus_kafka_adapter/install.sh similarity index 100% rename from Utilities/installation/prometheus_kafka_adapter/install.sh rename to asap-tools/installation/prometheus_kafka_adapter/install.sh diff --git a/Utilities/installation/prometheus_kafka_adapter/run.sh b/asap-tools/installation/prometheus_kafka_adapter/run.sh similarity index 100% rename from Utilities/installation/prometheus_kafka_adapter/run.sh rename to asap-tools/installation/prometheus_kafka_adapter/run.sh diff --git a/Utilities/installation/prometheus_kafka_adapter/setup_dependencies.sh b/asap-tools/installation/prometheus_kafka_adapter/setup_dependencies.sh similarity index 100% rename from Utilities/installation/prometheus_kafka_adapter/setup_dependencies.sh rename to asap-tools/installation/prometheus_kafka_adapter/setup_dependencies.sh diff --git a/Utilities/installation/recompile_flink.sh b/asap-tools/installation/recompile_flink.sh similarity index 83% rename from Utilities/installation/recompile_flink.sh rename to asap-tools/installation/recompile_flink.sh index 73d6b24..91b9e0c 100755 --- a/Utilities/installation/recompile_flink.sh +++ b/asap-tools/installation/recompile_flink.sh @@ -10,7 +10,7 @@ HOSTNAME_SUFFIX=$2 THIS_DIR=$(dirname "$(readlink -f "$0")") -CMD="cd /scratch/sketch_db_for_prometheus/code/Utilities/installation; " +CMD="cd /scratch/sketch_db_for_prometheus/code/asap-tools/installation; " CMD=$CMD"./setup_internal_components.sh FlinkSketch;" echo "Running command: $CMD" diff --git a/Utilities/installation/setup_internal_components.sh b/asap-tools/installation/setup_internal_components.sh similarity index 100% rename from Utilities/installation/setup_internal_components.sh rename to asap-tools/installation/setup_internal_components.sh diff --git a/Utilities/installation/utils.sh b/asap-tools/installation/utils.sh similarity index 100% rename from Utilities/installation/utils.sh rename to asap-tools/installation/utils.sh diff --git a/PrometheusClient/.dockerignore b/asap-tools/prometheus-client/.dockerignore similarity index 100% rename from PrometheusClient/.dockerignore rename to asap-tools/prometheus-client/.dockerignore diff --git a/PrometheusClient/.gitignore b/asap-tools/prometheus-client/.gitignore similarity index 100% rename from PrometheusClient/.gitignore rename to asap-tools/prometheus-client/.gitignore diff --git a/PrometheusClient/Dockerfile b/asap-tools/prometheus-client/Dockerfile similarity index 100% rename from PrometheusClient/Dockerfile rename to asap-tools/prometheus-client/Dockerfile diff --git a/PrometheusExporters/LICENSE b/asap-tools/prometheus-client/LICENSE similarity index 100% rename from PrometheusExporters/LICENSE rename to asap-tools/prometheus-client/LICENSE diff --git a/PrometheusClient/classes/QueryLatencyExporter.py b/asap-tools/prometheus-client/classes/QueryLatencyExporter.py similarity index 100% rename from PrometheusClient/classes/QueryLatencyExporter.py rename to asap-tools/prometheus-client/classes/QueryLatencyExporter.py diff --git a/PrometheusClient/classes/__init__.py b/asap-tools/prometheus-client/classes/__init__.py similarity index 100% rename from PrometheusClient/classes/__init__.py rename to asap-tools/prometheus-client/classes/__init__.py diff --git a/PrometheusClient/classes/clickhouse_query_client.py b/asap-tools/prometheus-client/classes/clickhouse_query_client.py similarity index 100% rename from PrometheusClient/classes/clickhouse_query_client.py rename to asap-tools/prometheus-client/classes/clickhouse_query_client.py diff --git a/PrometheusClient/classes/config.py b/asap-tools/prometheus-client/classes/config.py similarity index 100% rename from PrometheusClient/classes/config.py rename to asap-tools/prometheus-client/classes/config.py diff --git a/PrometheusClient/classes/prometheus_query_client.py b/asap-tools/prometheus-client/classes/prometheus_query_client.py similarity index 100% rename from PrometheusClient/classes/prometheus_query_client.py rename to asap-tools/prometheus-client/classes/prometheus_query_client.py diff --git a/PrometheusClient/classes/query_client.py b/asap-tools/prometheus-client/classes/query_client.py similarity index 100% rename from PrometheusClient/classes/query_client.py rename to asap-tools/prometheus-client/classes/query_client.py diff --git a/PrometheusClient/classes/query_client_factory.py b/asap-tools/prometheus-client/classes/query_client_factory.py similarity index 100% rename from PrometheusClient/classes/query_client_factory.py rename to asap-tools/prometheus-client/classes/query_client_factory.py diff --git a/PrometheusClient/classes/query_template.py b/asap-tools/prometheus-client/classes/query_template.py similarity index 100% rename from PrometheusClient/classes/query_template.py rename to asap-tools/prometheus-client/classes/query_template.py diff --git a/PrometheusClient/docker-compose.yml.j2 b/asap-tools/prometheus-client/docker-compose.yml.j2 similarity index 100% rename from PrometheusClient/docker-compose.yml.j2 rename to asap-tools/prometheus-client/docker-compose.yml.j2 diff --git a/PrometheusClient/installation/install.sh b/asap-tools/prometheus-client/installation/install.sh similarity index 100% rename from PrometheusClient/installation/install.sh rename to asap-tools/prometheus-client/installation/install.sh diff --git a/PrometheusClient/installation/setup_dependencies.sh b/asap-tools/prometheus-client/installation/setup_dependencies.sh similarity index 100% rename from PrometheusClient/installation/setup_dependencies.sh rename to asap-tools/prometheus-client/installation/setup_dependencies.sh diff --git a/PrometheusClient/main_prometheus_client.py b/asap-tools/prometheus-client/main_prometheus_client.py similarity index 100% rename from PrometheusClient/main_prometheus_client.py rename to asap-tools/prometheus-client/main_prometheus_client.py diff --git a/PrometheusClient/pyproject.toml b/asap-tools/prometheus-client/pyproject.toml similarity index 92% rename from PrometheusClient/pyproject.toml rename to asap-tools/prometheus-client/pyproject.toml index 000d249..154e704 100644 --- a/PrometheusClient/pyproject.toml +++ b/asap-tools/prometheus-client/pyproject.toml @@ -19,7 +19,7 @@ exclude = ''' [tool.mypy] python_version = "3.8" -mypy_path = "../../../CommonDependencies/py" +mypy_path = "../../asap-common/py" # STRICT MODE ENABLED # This enables all 14 strict type checking flags diff --git a/PrometheusClient/requirements.txt b/asap-tools/prometheus-client/requirements.txt similarity index 100% rename from PrometheusClient/requirements.txt rename to asap-tools/prometheus-client/requirements.txt diff --git a/PrometheusClient/similarity_scores.py b/asap-tools/prometheus-client/similarity_scores.py similarity index 100% rename from PrometheusClient/similarity_scores.py rename to asap-tools/prometheus-client/similarity_scores.py diff --git a/PrometheusClient/type_aliases.py b/asap-tools/prometheus-client/type_aliases.py similarity index 100% rename from PrometheusClient/type_aliases.py rename to asap-tools/prometheus-client/type_aliases.py diff --git a/PrometheusExporters/.gitignore b/asap-tools/prometheus-exporters/.gitignore similarity index 100% rename from PrometheusExporters/.gitignore rename to asap-tools/prometheus-exporters/.gitignore diff --git a/PrometheusExporters/.isort.cfg b/asap-tools/prometheus-exporters/.isort.cfg similarity index 100% rename from PrometheusExporters/.isort.cfg rename to asap-tools/prometheus-exporters/.isort.cfg diff --git a/PrometheusExporters/.mypy.ini b/asap-tools/prometheus-exporters/.mypy.ini similarity index 100% rename from PrometheusExporters/.mypy.ini rename to asap-tools/prometheus-exporters/.mypy.ini diff --git a/QueryEngineRust/LICENSE b/asap-tools/prometheus-exporters/LICENSE similarity index 100% rename from QueryEngineRust/LICENSE rename to asap-tools/prometheus-exporters/LICENSE diff --git a/PrometheusExporters/README.md b/asap-tools/prometheus-exporters/README.md similarity index 100% rename from PrometheusExporters/README.md rename to asap-tools/prometheus-exporters/README.md diff --git a/PrometheusExporters/cluster_data_exporter/.dockerignore b/asap-tools/prometheus-exporters/cluster_data_exporter/.dockerignore similarity index 100% rename from PrometheusExporters/cluster_data_exporter/.dockerignore rename to asap-tools/prometheus-exporters/cluster_data_exporter/.dockerignore diff --git a/PrometheusExporters/cluster_data_exporter/.gitignore b/asap-tools/prometheus-exporters/cluster_data_exporter/.gitignore similarity index 100% rename from PrometheusExporters/cluster_data_exporter/.gitignore rename to asap-tools/prometheus-exporters/cluster_data_exporter/.gitignore diff --git a/PrometheusExporters/cluster_data_exporter/Cargo.toml b/asap-tools/prometheus-exporters/cluster_data_exporter/Cargo.toml similarity index 100% rename from PrometheusExporters/cluster_data_exporter/Cargo.toml rename to asap-tools/prometheus-exporters/cluster_data_exporter/Cargo.toml diff --git a/PrometheusExporters/cluster_data_exporter/Dockerfile b/asap-tools/prometheus-exporters/cluster_data_exporter/Dockerfile similarity index 100% rename from PrometheusExporters/cluster_data_exporter/Dockerfile rename to asap-tools/prometheus-exporters/cluster_data_exporter/Dockerfile diff --git a/PrometheusExporters/cluster_data_exporter/README.md b/asap-tools/prometheus-exporters/cluster_data_exporter/README.md similarity index 100% rename from PrometheusExporters/cluster_data_exporter/README.md rename to asap-tools/prometheus-exporters/cluster_data_exporter/README.md diff --git a/PrometheusExporters/cluster_data_exporter/docker-compose.yml.j2 b/asap-tools/prometheus-exporters/cluster_data_exporter/docker-compose.yml.j2 similarity index 100% rename from PrometheusExporters/cluster_data_exporter/docker-compose.yml.j2 rename to asap-tools/prometheus-exporters/cluster_data_exporter/docker-compose.yml.j2 diff --git a/PrometheusExporters/cluster_data_exporter/docker_compose_frames/alibaba-msresource-2021-docker-compose.yml b/asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-msresource-2021-docker-compose.yml similarity index 100% rename from PrometheusExporters/cluster_data_exporter/docker_compose_frames/alibaba-msresource-2021-docker-compose.yml rename to asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-msresource-2021-docker-compose.yml diff --git a/PrometheusExporters/cluster_data_exporter/docker_compose_frames/alibaba-msresource-2022-docker-compose.yml b/asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-msresource-2022-docker-compose.yml similarity index 100% rename from PrometheusExporters/cluster_data_exporter/docker_compose_frames/alibaba-msresource-2022-docker-compose.yml rename to asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-msresource-2022-docker-compose.yml diff --git a/PrometheusExporters/cluster_data_exporter/docker_compose_frames/alibaba-node-2021-docker-compose.yml b/asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-node-2021-docker-compose.yml similarity index 100% rename from PrometheusExporters/cluster_data_exporter/docker_compose_frames/alibaba-node-2021-docker-compose.yml rename to asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-node-2021-docker-compose.yml diff --git a/PrometheusExporters/cluster_data_exporter/docker_compose_frames/alibaba-node-2022-docker-compose.yml b/asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-node-2022-docker-compose.yml similarity index 100% rename from PrometheusExporters/cluster_data_exporter/docker_compose_frames/alibaba-node-2022-docker-compose.yml rename to asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/alibaba-node-2022-docker-compose.yml diff --git a/PrometheusExporters/cluster_data_exporter/docker_compose_frames/base-docker-compose.yml b/asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/base-docker-compose.yml similarity index 100% rename from PrometheusExporters/cluster_data_exporter/docker_compose_frames/base-docker-compose.yml rename to asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/base-docker-compose.yml diff --git a/PrometheusExporters/cluster_data_exporter/docker_compose_frames/google-docker-compose.yml b/asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/google-docker-compose.yml similarity index 100% rename from PrometheusExporters/cluster_data_exporter/docker_compose_frames/google-docker-compose.yml rename to asap-tools/prometheus-exporters/cluster_data_exporter/docker_compose_frames/google-docker-compose.yml diff --git a/PrometheusExporters/cluster_data_exporter/installation/install.sh b/asap-tools/prometheus-exporters/cluster_data_exporter/installation/install.sh similarity index 100% rename from PrometheusExporters/cluster_data_exporter/installation/install.sh rename to asap-tools/prometheus-exporters/cluster_data_exporter/installation/install.sh diff --git a/PrometheusExporters/cluster_data_exporter/scripts/generate_docker_compose.py b/asap-tools/prometheus-exporters/cluster_data_exporter/scripts/generate_docker_compose.py similarity index 100% rename from PrometheusExporters/cluster_data_exporter/scripts/generate_docker_compose.py rename to asap-tools/prometheus-exporters/cluster_data_exporter/scripts/generate_docker_compose.py diff --git a/PrometheusExporters/cluster_data_exporter/scripts/requirements.txt b/asap-tools/prometheus-exporters/cluster_data_exporter/scripts/requirements.txt similarity index 100% rename from PrometheusExporters/cluster_data_exporter/scripts/requirements.txt rename to asap-tools/prometheus-exporters/cluster_data_exporter/scripts/requirements.txt diff --git a/PrometheusExporters/cluster_data_exporter/src/alibaba_metrics.rs b/asap-tools/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics.rs similarity index 100% rename from PrometheusExporters/cluster_data_exporter/src/alibaba_metrics.rs rename to asap-tools/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics.rs diff --git a/PrometheusExporters/cluster_data_exporter/src/alibaba_metrics/ms_resource.rs b/asap-tools/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics/ms_resource.rs similarity index 100% rename from PrometheusExporters/cluster_data_exporter/src/alibaba_metrics/ms_resource.rs rename to asap-tools/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics/ms_resource.rs diff --git a/PrometheusExporters/cluster_data_exporter/src/alibaba_metrics/node.rs b/asap-tools/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics/node.rs similarity index 100% rename from PrometheusExporters/cluster_data_exporter/src/alibaba_metrics/node.rs rename to asap-tools/prometheus-exporters/cluster_data_exporter/src/alibaba_metrics/node.rs diff --git a/PrometheusExporters/cluster_data_exporter/src/google_metrics.rs b/asap-tools/prometheus-exporters/cluster_data_exporter/src/google_metrics.rs similarity index 100% rename from PrometheusExporters/cluster_data_exporter/src/google_metrics.rs rename to asap-tools/prometheus-exporters/cluster_data_exporter/src/google_metrics.rs diff --git a/PrometheusExporters/cluster_data_exporter/src/main.rs b/asap-tools/prometheus-exporters/cluster_data_exporter/src/main.rs similarity index 100% rename from PrometheusExporters/cluster_data_exporter/src/main.rs rename to asap-tools/prometheus-exporters/cluster_data_exporter/src/main.rs diff --git a/PrometheusExporters/cluster_data_exporter/src/utilities.rs b/asap-tools/prometheus-exporters/cluster_data_exporter/src/utilities.rs similarity index 100% rename from PrometheusExporters/cluster_data_exporter/src/utilities.rs rename to asap-tools/prometheus-exporters/cluster_data_exporter/src/utilities.rs diff --git a/PrometheusExporters/fake_exporter/fake_exporter_python/Dockerfile b/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/Dockerfile similarity index 100% rename from PrometheusExporters/fake_exporter/fake_exporter_python/Dockerfile rename to asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/Dockerfile diff --git a/PrometheusExporters/fake_exporter/fake_exporter_python/docker-compose.yml.j2 b/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/docker-compose.yml.j2 similarity index 100% rename from PrometheusExporters/fake_exporter/fake_exporter_python/docker-compose.yml.j2 rename to asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/docker-compose.yml.j2 diff --git a/PrometheusExporters/fake_exporter/fake_exporter_python/fake-exporter-python-cli-compose.yml.j2 b/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/fake-exporter-python-cli-compose.yml.j2 similarity index 100% rename from PrometheusExporters/fake_exporter/fake_exporter_python/fake-exporter-python-cli-compose.yml.j2 rename to asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/fake-exporter-python-cli-compose.yml.j2 diff --git a/PrometheusExporters/fake_exporter/fake_exporter_python/fake_exporter.py b/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/fake_exporter.py similarity index 100% rename from PrometheusExporters/fake_exporter/fake_exporter_python/fake_exporter.py rename to asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/fake_exporter.py diff --git a/PrometheusExporters/fake_exporter/fake_exporter_python/installation/install.sh b/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/installation/install.sh similarity index 100% rename from PrometheusExporters/fake_exporter/fake_exporter_python/installation/install.sh rename to asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/installation/install.sh diff --git a/PrometheusExporters/fake_exporter/fake_exporter_python/requirements.txt b/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/requirements.txt similarity index 100% rename from PrometheusExporters/fake_exporter/fake_exporter_python/requirements.txt rename to asap-tools/prometheus-exporters/fake_exporter/fake_exporter_python/requirements.txt diff --git a/PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter/Cargo.lock b/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Cargo.lock similarity index 100% rename from PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter/Cargo.lock rename to asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Cargo.lock diff --git a/PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter/Cargo.toml b/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Cargo.toml similarity index 100% rename from PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter/Cargo.toml rename to asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Cargo.toml diff --git a/PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter/Dockerfile b/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Dockerfile similarity index 100% rename from PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter/Dockerfile rename to asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Dockerfile diff --git a/PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter/docker-compose.yml.j2 b/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/docker-compose.yml.j2 similarity index 100% rename from PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter/docker-compose.yml.j2 rename to asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/docker-compose.yml.j2 diff --git a/PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter/fake-exporter-rust-cli-compose.yml.j2 b/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/fake-exporter-rust-cli-compose.yml.j2 similarity index 100% rename from PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter/fake-exporter-rust-cli-compose.yml.j2 rename to asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/fake-exporter-rust-cli-compose.yml.j2 diff --git a/PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter/installation/install.sh b/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/installation/install.sh similarity index 100% rename from PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter/installation/install.sh rename to asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/installation/install.sh diff --git a/PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter/src/main.rs b/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/src/main.rs similarity index 100% rename from PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter/src/main.rs rename to asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/src/main.rs diff --git a/PrometheusExporters/fake_kafka_exporter/Cargo.lock b/asap-tools/prometheus-exporters/fake_kafka_exporter/Cargo.lock similarity index 100% rename from PrometheusExporters/fake_kafka_exporter/Cargo.lock rename to asap-tools/prometheus-exporters/fake_kafka_exporter/Cargo.lock diff --git a/PrometheusExporters/fake_kafka_exporter/Cargo.toml b/asap-tools/prometheus-exporters/fake_kafka_exporter/Cargo.toml similarity index 100% rename from PrometheusExporters/fake_kafka_exporter/Cargo.toml rename to asap-tools/prometheus-exporters/fake_kafka_exporter/Cargo.toml diff --git a/PrometheusExporters/fake_kafka_exporter/Dockerfile b/asap-tools/prometheus-exporters/fake_kafka_exporter/Dockerfile similarity index 100% rename from PrometheusExporters/fake_kafka_exporter/Dockerfile rename to asap-tools/prometheus-exporters/fake_kafka_exporter/Dockerfile diff --git a/PrometheusExporters/fake_kafka_exporter/installation/install.sh b/asap-tools/prometheus-exporters/fake_kafka_exporter/installation/install.sh similarity index 100% rename from PrometheusExporters/fake_kafka_exporter/installation/install.sh rename to asap-tools/prometheus-exporters/fake_kafka_exporter/installation/install.sh diff --git a/PrometheusExporters/fake_kafka_exporter/src/main.rs b/asap-tools/prometheus-exporters/fake_kafka_exporter/src/main.rs similarity index 100% rename from PrometheusExporters/fake_kafka_exporter/src/main.rs rename to asap-tools/prometheus-exporters/fake_kafka_exporter/src/main.rs diff --git a/PrometheusExporters/installation/install.sh b/asap-tools/prometheus-exporters/installation/install.sh similarity index 100% rename from PrometheusExporters/installation/install.sh rename to asap-tools/prometheus-exporters/installation/install.sh diff --git a/PrometheusExporters/query_cost_exporter/QueryCostExporter.py b/asap-tools/prometheus-exporters/query_cost_exporter/QueryCostExporter.py similarity index 100% rename from PrometheusExporters/query_cost_exporter/QueryCostExporter.py rename to asap-tools/prometheus-exporters/query_cost_exporter/QueryCostExporter.py diff --git a/Utilities/experiments/classes/ProcessMonitorHook.py b/asap-tools/prometheus-exporters/query_cost_exporter/process/ProcessMonitorHook.py similarity index 100% rename from Utilities/experiments/classes/ProcessMonitorHook.py rename to asap-tools/prometheus-exporters/query_cost_exporter/process/ProcessMonitorHook.py diff --git a/Utilities/experiments/classes/__init__.py b/asap-tools/prometheus-exporters/query_cost_exporter/process/__init__.py similarity index 100% rename from Utilities/experiments/classes/__init__.py rename to asap-tools/prometheus-exporters/query_cost_exporter/process/__init__.py diff --git a/Utilities/experiments/classes/query_cost.py b/asap-tools/prometheus-exporters/query_cost_exporter/query_cost.py similarity index 100% rename from Utilities/experiments/classes/query_cost.py rename to asap-tools/prometheus-exporters/query_cost_exporter/query_cost.py diff --git a/PrometheusExporters/query_cost_exporter/requirements.txt b/asap-tools/prometheus-exporters/query_cost_exporter/requirements.txt similarity index 100% rename from PrometheusExporters/query_cost_exporter/requirements.txt rename to asap-tools/prometheus-exporters/query_cost_exporter/requirements.txt diff --git a/PrometheusExporters/query_latency_exporter/QueryLatencyExporter.py b/asap-tools/prometheus-exporters/query_latency_exporter/QueryLatencyExporter.py similarity index 100% rename from PrometheusExporters/query_latency_exporter/QueryLatencyExporter.py rename to asap-tools/prometheus-exporters/query_latency_exporter/QueryLatencyExporter.py diff --git a/PrometheusExporters/query_latency_exporter/requirements.txt b/asap-tools/prometheus-exporters/query_latency_exporter/requirements.txt similarity index 100% rename from PrometheusExporters/query_latency_exporter/requirements.txt rename to asap-tools/prometheus-exporters/query_latency_exporter/requirements.txt diff --git a/Utilities/root_files.conf b/asap-tools/root_files.conf similarity index 100% rename from Utilities/root_files.conf rename to asap-tools/root_files.conf diff --git a/Utilities/shared_utils.sh b/asap-tools/shared_utils.sh similarity index 100% rename from Utilities/shared_utils.sh rename to asap-tools/shared_utils.sh diff --git a/docs/01-getting-started/architecture.md b/docs/01-getting-started/architecture.md index c0496b5..c41a859 100644 --- a/docs/01-getting-started/architecture.md +++ b/docs/01-getting-started/architecture.md @@ -183,14 +183,14 @@ graph LR | Component | Purpose | Technology | Location | |-----------|---------|------------|----------| -| **QueryEngineRust** | Answers PromQL queries using sketches | Rust | `QueryEngineRust/` | +| **asap-query-engine** | Answers PromQL queries using sketches | Rust | `asap-query-engine/` | | **Arroyo** | Stream processing for building sketches | Rust (forked) | `arroyo/` | -| **ArroyoSketch** | Configures Arroyo pipelines from config | Python | `ArroyoSketch/` | -| **Controller** | Auto-determines sketch parameters | Python | `Controller/` | +| **asap-sketch-ingest** | Configures Arroyo pipelines from config | Python | `asap-sketch-ingest/` | +| **asap-planner** | Auto-determines sketch parameters | Python | `asap-planner/` | | **Kafka** | Message broker for sketch distribution | Apache Kafka | (external) | | **Prometheus** | Time-series database (existing) | Go | (external) | -| **Exporters** | Generate synthetic metrics for testing | Rust/Python | `PrometheusExporters/` | -| **Utilities** | Experimental harness that uses Cloudlab | Python | `Utilities/` | +| **Exporters** | Generate synthetic metrics for testing | Rust/Python | `asap-tools/prometheus-exporters/` | +| **asap-tools** | Experimental harness that uses Cloudlab | Python | `asap-tools/` | **Links to detailed documentation:** - [QueryEngineRust](../02-components/query-engine.md) @@ -222,13 +222,13 @@ graph LR ## Technology Stack ### Core Languages -- **Rust** - QueryEngine, Arroyo, some exporters +- **Rust** - asap-query-engine, Arroyo, some exporters - Tokio for async runtime - Axum for HTTP server - Serde for serialization - DataSketches (dsrs) for sketch algorithms -- **Python** - Controller, ArroyoSketch, experiment framework +- **Python** - asap-planner, asap-sketch-ingest, experiment framework - PyYAML for config parsing - Jinja2 for SQL templates - Requests for HTTP clients @@ -249,7 +249,7 @@ graph LR ``` ASAPQuery/ -├── QueryEngineRust/ # Rust query processor +├── asap-query-engine/ # Rust query processor │ ├── src/ │ │ ├── drivers/ # Ingest, query adapters, servers │ │ ├── engines/ # Query execution (SimpleEngine) @@ -265,35 +265,36 @@ ASAPQuery/ │ ├── prometheus_remote_write_with_schema/ │ └── prometheus_remote_write_optimized/ │ -├── ArroyoSketch/ # Pipeline configurator +├── asap-sketch-ingest/ # Pipeline configurator │ ├── run_arroyosketch.py # Main script │ ├── templates/ # Jinja2 SQL templates │ └── utils/ # Arroyo API client │ -├── Controller/ # Auto-configuration service +├── asap-planner/ # Auto-configuration service │ ├── main_controller.py # Entry point │ ├── classes/ # Config data structures │ └── utils/ # Decision logic │ -├── PrometheusExporters/ # Metric generators -│ ├── fake_exporter/ # Rust/Python fake exporters -│ ├── cluster_data_exporter/ # Real trace data -│ ├── query_cost_exporter/ # Resource metrics -│ └── query_latency_exporter/ # Latency metrics -│ -├── Utilities/ # Experiment framework +├── asap-tools/ # Experiment framework & tooling +│ ├── prometheus-exporters/ # Metric generators +│ │ ├── fake_exporter/ # Rust/Python fake exporters +│ │ ├── cluster_data_exporter/ # Real trace data +│ │ ├── query_cost_exporter/ # Resource metrics +│ │ └── query_latency_exporter/ # Latency metrics │ ├── experiments/ │ │ ├── experiment_run_e2e.py # Main orchestrator │ │ ├── config/ # Hydra configs │ │ ├── experiment_utils/ # Services, providers │ │ └── post_experiment/ # Analysis scripts -│ └── docs/ # Utilities dev docs +│ └── docs/ # asap-tools dev docs │ -├── CommonDependencies/ # Shared libraries -│ ├── promql_utilities/ # PromQL parsing (Rust/Python) -│ └── sql_utilities/ # SQL utilities +├── asap-common/ # Shared libraries +│ ├── dependencies/ +│ │ ├── rs/ # Rust shared crates +│ │ └── py/ # Python shared packages +│ └── sketch-core/ # Core sketch library (Rust) │ -├── quickstart/ # Self-contained demo +├── asap-quickstart/ # Self-contained demo │ ├── docker-compose.yml # Demo stack │ └── config/ # Demo configs │ diff --git a/docs/02-components/README.md b/docs/02-components/README.md index 9c9b205..7d64c8c 100644 --- a/docs/02-components/README.md +++ b/docs/02-components/README.md @@ -6,12 +6,12 @@ This document provides an overview of all ASAP components and links to detailed | Component | Purpose | Technology | Links | |-----------|---------|------------|-------| -| **QueryEngineRust** | Answers PromQL queries using sketches | Rust | [Details](query-engine.md) · [Code](../../QueryEngineRust/) · [Dev Docs](../../QueryEngineRust/docs/README.md) | +| **asap-query-engine** | Answers PromQL queries using sketches | Rust | [Details](query-engine.md) · [Code](../../asap-query-engine/) · [Dev Docs](../../asap-query-engine/docs/README.md) | | **Arroyo** | Stream processing for building sketches | Rust (forked) | [Details](arroyo.md) · [Code](../../arroyo/) | -| **ArroyoSketch** | Configures Arroyo pipelines from config | Python | [Details](arroyosketch.md) · [Code](../../ArroyoSketch/) · [README](../../ArroyoSketch/README.md) | -| **Controller** | Auto-determines sketch parameters | Python | [Details](controller.md) · [Code](../../Controller/) · [README](../../Controller/README.md) | -| **Exporters** | Generate synthetic metrics for testing | Rust/Python | [Details](exporters.md) · [Code](../../PrometheusExporters/) · [README](../../PrometheusExporters/README.md) | -| **Utilities** | Experiment framework for CloudLab | Python | [Details](utilities.md) · [Code](../../Utilities/) · [Docs](../../Utilities/docs/architecture.md) | +| **asap-sketch-ingest** | Configures Arroyo pipelines from config | Python | [Details](arroyosketch.md) · [Code](../../asap-sketch-ingest/) · [README](../../asap-sketch-ingest/README.md) | +| **asap-planner** | Auto-determines sketch parameters | Python | [Details](controller.md) · [Code](../../asap-planner/) · [README](../../asap-planner/README.md) | +| **Exporters** | Generate synthetic metrics for testing | Rust/Python | [Details](exporters.md) · [Code](../../asap-tools/prometheus-exporters/) · [README](../../asap-tools/prometheus-exporters/README.md) | +| **asap-tools** | Experiment framework for CloudLab | Python | [Details](utilities.md) · [Code](../../asap-tools/) · [Docs](../../asap-tools/docs/architecture.md) | ## Component Interaction @@ -19,8 +19,8 @@ This document provides an overview of all ASAP components and links to detailed graph TB subgraph "Configuration (Offline)" U[User] -->|edits| CC[controller-config.yaml] - CC --> C[Controller] - C -->|streaming_config.yaml| AS[ArroyoSketch] + CC --> C[asap-planner] + C -->|streaming_config.yaml| AS[asap-sketch-ingest] C -->|inference_config.yaml| Q AS -->|create pipelines| A end @@ -40,7 +40,7 @@ graph TB end subgraph "Experiments (Research)" - EXP[Utilities] -->|deploy & run| E + EXP[asap-tools] -->|deploy & run| E EXP -->|deploy & run| P EXP -->|deploy & run| A EXP -->|collect results| EXP @@ -59,7 +59,7 @@ graph TB These run continuously to serve queries: -- **[QueryEngineRust](query-engine.md)** - Answers PromQL queries using sketches +- **[asap-query-engine](query-engine.md)** - Answers PromQL queries using sketches - Consumes sketches from Kafka - Implements Prometheus HTTP API - Forwards unsupported queries to Prometheus @@ -73,12 +73,12 @@ These run continuously to serve queries: These run once to set up the system: -- **[Controller](controller.md)** - Determines optimal sketch parameters +- **[asap-planner](controller.md)** - Determines optimal sketch parameters - Analyzes query workload - Selects sketch algorithms - Generates configs for Arroyo and QueryEngine -- **[ArroyoSketch](arroyosketch.md)** - Creates Arroyo pipelines +- **[asap-sketch-ingest](arroyosketch.md)** - Creates Arroyo pipelines - Reads streaming_config.yaml - Renders SQL templates - Creates pipelines via Arroyo API @@ -92,7 +92,7 @@ These are used for development and experiments: - Real trace data exporters - Performance monitoring exporters -- **[Utilities](utilities.md)** - Experiment orchestration +- **[asap-tools](utilities.md)** - Experiment orchestration - Deploy ASAP to CloudLab - Run controlled experiments - Collect and analyze results @@ -103,7 +103,7 @@ These are used for development and experiments: Performance-critical components written in Rust: -- **QueryEngineRust** - Sub-millisecond query execution +- **asap-query-engine** - Sub-millisecond query execution - **Arroyo** - High-throughput stream processing - **Fake Exporters** - Fast metric generation @@ -111,37 +111,37 @@ Performance-critical components written in Rust: Configuration and orchestration in Python: -- **Controller** - Query analysis and config generation -- **ArroyoSketch** - Pipeline configuration -- **Utilities** - Experiment framework +- **asap-planner** - Query analysis and config generation +- **asap-sketch-ingest** - Pipeline configuration +- **asap-tools** - Experiment framework - **Python Exporters** - Simpler metric generators ## Component Dependencies ``` -QueryEngineRust +asap-query-engine ├── Kafka (runtime) - Consumes sketches ├── Prometheus (runtime, optional) - Fallback queries -└── inference_config.yaml (config) - From Controller +└── inference_config.yaml (config) - From asap-planner Arroyo ├── Prometheus (runtime) - Remote write source ├── Kafka (runtime) - Sketch output -└── SQL pipelines (config) - From ArroyoSketch +└── SQL pipelines (config) - From asap-sketch-ingest -ArroyoSketch +asap-sketch-ingest ├── Arroyo (runtime) - Creates pipelines via API -└── streaming_config.yaml (config) - From Controller +└── streaming_config.yaml (config) - From asap-planner -Controller +asap-planner ├── controller-config.yaml (input) - User-provided -├── streaming_config.yaml (output) - For ArroyoSketch -└── inference_config.yaml (output) - For QueryEngine +├── streaming_config.yaml (output) - For asap-sketch-ingest +└── inference_config.yaml (output) - For asap-query-engine Exporters └── (standalone, no dependencies) -Utilities +asap-tools ├── All components (deploys and orchestrates) └── Hydra configs (experiment specifications) ``` @@ -150,19 +150,19 @@ Utilities ### Detailed Component Docs -- [QueryEngineRust](query-engine.md) - Query processor deep dive +- [asap-query-engine](query-engine.md) - Query processor deep dive - [Arroyo](arroyo.md) - Streaming engine + ASAP customizations -- [ArroyoSketch](arroyosketch.md) - Pipeline configurator -- [Controller](controller.md) - Auto-configuration service +- [asap-sketch-ingest](arroyosketch.md) - Pipeline configurator +- [asap-planner](controller.md) - Auto-configuration service - [Exporters](exporters.md) - Metric generators -- [Utilities](utilities.md) - Experiment framework +- [asap-tools](utilities.md) - Experiment framework ### Component-Specific READMEs For implementation details, see READMEs co-located with code: -- [QueryEngineRust/docs/](../../QueryEngineRust/docs/README.md) - Extensibility guides -- [Controller/README.md](../../Controller/README.md) - Controller internals -- [ArroyoSketch/README.md](../../ArroyoSketch/README.md) - Pipeline config internals -- [PrometheusExporters/README.md](../../PrometheusExporters/README.md) - Exporter implementations -- [Utilities/docs/](../../Utilities/docs/architecture.md) - Experiment framework architecture +- [asap-query-engine/docs/](../../asap-query-engine/docs/README.md) - Extensibility guides +- [asap-planner/README.md](../../asap-planner/README.md) - asap-planner internals +- [asap-sketch-ingest/README.md](../../asap-sketch-ingest/README.md) - Pipeline config internals +- [asap-tools/prometheus-exporters/README.md](../../asap-tools/prometheus-exporters/README.md) - Exporter implementations +- [asap-tools/docs/](../../asap-tools/docs/architecture.md) - Experiment framework architecture diff --git a/docs/03-how-to-guides/development/adapt-pr-after-reorg.md b/docs/03-how-to-guides/development/adapt-pr-after-reorg.md new file mode 100644 index 0000000..8cc134c --- /dev/null +++ b/docs/03-how-to-guides/development/adapt-pr-after-reorg.md @@ -0,0 +1,154 @@ +# How to Adapt an Open PR After the Repo Reorganization + +In [issue #129](https://github.com/ProjectASAP/ASAPQuery/issues/129), all top-level directories were renamed to use lowercase, hyphenated `asap-` prefixes. If you opened a PR before this reorganization was merged, your branch will have conflicts and stale path references that need to be updated. + +This guide walks you through rebasing onto the new structure. + +--- + +## Directory Renames + +| Old name | New name | +|---|---| +| `QueryEngineRust/` | `asap-query-engine/` | +| `CommonDependencies/` | `asap-common/` | +| `ArroyoSketch/` | `asap-sketch-ingest/` | +| `Controller/` | `asap-planner/` | +| `Utilities/` | `asap-tools/` | +| `PrometheusExporters/` | `asap-tools/prometheus-exporters/` | +| `PrometheusClient/` | `asap-tools/prometheus-client/` | +| `ExecutionUtilities/` | `asap-tools/execution-utilities/` | +| `quickstart/` | `asap-quickstart/` | +| `sketch-core/` | `asap-common/sketch-core/` | + +--- + +## Step 1: Rebase onto main + +```bash +git fetch origin +git rebase origin/main +``` + +Git will likely report conflicts because files it knew as (e.g.) `QueryEngineRust/src/main.rs` no longer exist at that path — they are now at `asap-query-engine/src/main.rs`. + +--- + +## Step 2: Resolve file-move conflicts + +When Git cannot automatically apply your commits because the files moved, you need to tell it where the files went. + +For each conflicted file, check where it now lives: + +```bash +# See which files Git is confused about +git status + +# Find where a file ended up after the rename +git log --diff-filter=R --summary origin/main | grep "rename" +``` + +Then manually apply your changes to the new path: + +```bash +# Example: your change was to QueryEngineRust/src/foo.rs +# Apply it to the new location +git checkout HEAD -- asap-query-engine/src/foo.rs # get current version +# Then re-apply your edits manually, or use: +git show ORIG_HEAD:QueryEngineRust/src/foo.rs # see your old version +``` + +--- + +## Step 3: Update path references in your changed files + +After resolving file moves, search your branch's changes for any remaining hardcoded old paths. + +```bash +# Check your diff for old names +git diff origin/main | grep -E "QueryEngineRust|CommonDependencies|ArroyoSketch|Controller/|Utilities/|PrometheusExporters|PrometheusClient|ExecutionUtilities|sketch-core" +``` + +Apply the substitutions from the table above to any hits. Common places to check: + +- **Cargo.toml** `path =` dependencies +- **Python** `os.path.join(...)` calls +- **Shell scripts** `cd` commands and path variables +- **Docker** `build.context`, volume mounts, `COPY` directives +- **GitHub Actions** `working-directory` and `paths` triggers +- **`.pre-commit-config.yaml`** manifest paths and file regex patterns + +--- + +## Step 4: Check sub-directory renames within asap-common + +`sketch-core` moved from being a top-level directory to living inside `asap-common/`: + +```toml +# Old +path = "../../sketch-core" + +# New +path = "../../asap-common/sketch-core" +``` + +`CommonDependencies/dependencies/` is now `asap-common/dependencies/`: + +```toml +# Old (Cargo.toml) +path = "../CommonDependencies/dependencies/rs/promql_utilities" + +# New +path = "../asap-common/dependencies/rs/promql_utilities" +``` + +```python +# Old (pyproject.toml / Python imports) +mypy_path = "../../CommonDependencies/py" + +# New +mypy_path = "../../asap-common/py" +``` + +--- + +## Step 5: Verify CI passes + +Push your rebased branch and check that all CI jobs pass: + +```bash +git push --force-with-lease origin your-branch-name +``` + +Key CI jobs to watch: +- **Rust** — Cargo build and tests for `asap-query-engine` and `asap-common` +- **Python** — Linting/type-checking for `asap-tools/prometheus-client` and `asap-planner` +- **Docker** — Image builds for `asap-query-engine`, `asap-planner`, `asap-sketch-ingest` + +--- + +## Quick reference: sed commands for bulk renaming + +If your PR touches many files, these one-liners can handle the mechanical substitutions: + +```bash +# Run from repo root on files in your PR +# Adjust the file glob to match what your PR touches + +# Rename directory references in Rust/TOML files +sed -i 's|QueryEngineRust|asap-query-engine|g' your_file.toml +sed -i 's|CommonDependencies|asap-common|g' your_file.toml + +# Rename in Python files +sed -i 's|"Utilities"|"asap-tools"|g' your_file.py +sed -i 's|"Controller"|"asap-planner"|g' your_file.py +sed -i 's|"ArroyoSketch"|"asap-sketch-ingest"|g' your_file.py +sed -i 's|"PrometheusClient"|"asap-tools/prometheus-client"|g' your_file.py +sed -i 's|"PrometheusExporters"|"asap-tools/prometheus-exporters"|g' your_file.py + +# Rename in shell scripts +sed -i 's|/Utilities/|/asap-tools/|g' your_script.sh +sed -i 's|/Controller/|/asap-planner/|g' your_script.sh +``` + +Always review the diff after running `sed` to make sure it didn't touch comment prose you intended to keep. diff --git a/docs/03-how-to-guides/development/add-new-sketch.md b/docs/03-how-to-guides/development/add-new-sketch.md index 11c1258..b88ec20 100644 --- a/docs/03-how-to-guides/development/add-new-sketch.md +++ b/docs/03-how-to-guides/development/add-new-sketch.md @@ -1,10 +1,10 @@ # How to Add a New Sketch Algorithm -Adding a new sketch requires changes to 3 components: CommonDependencies (sketch selection logic), ArroyoSketch (UDF for building sketches), and QueryEngineRust (deserialization and query logic). +Adding a new sketch requires changes to 3 components: asap-common (sketch selection logic), asap-sketch-ingest (UDF for building sketches), and asap-query-engine (deserialization and query logic). -## Step 1: CommonDependencies - Define Sketch Mapping +## Step 1: asap-common - Define Sketch Mapping -**File**: `CommonDependencies/dependencies/py/promql_utilities/promql_utilities/query_logics/logics.py` +**File**: `asap-common/dependencies/py/promql_utilities/promql_utilities/query_logics/logics.py` **What to modify**: - `map_statistic_to_precompute_operator()` - Add mapping from statistic to your sketch name @@ -14,9 +14,9 @@ Adding a new sketch requires changes to 3 components: CommonDependencies (sketch --- -## Step 2: ArroyoSketch - Create Sketch UDF +## Step 2: asap-sketch-ingest - Create Sketch UDF -**File to create**: `ArroyoSketch/templates/udfs/yoursketchname_[subop].rs.j2` (or `.rs` if no template vars) +**File to create**: `asap-sketch-ingest/templates/udfs/yoursketchname_[subop].rs.j2` (or `.rs` if no template vars) **What to implement**: - Rust UDF function using `#[udf]` macro @@ -29,16 +29,16 @@ Adding a new sketch requires changes to 3 components: CommonDependencies (sketch **Validate**: Run `python validate_udfs.py` to check UDF compiles. **Reference examples**: -- `ArroyoSketch/templates/udfs/datasketcheskll_.rs.j2` -- `ArroyoSketch/templates/udfs/countminsketch_sum.rs.j2` +- `asap-sketch-ingest/templates/udfs/datasketcheskll_.rs.j2` +- `asap-sketch-ingest/templates/udfs/countminsketch_sum.rs.j2` --- -## Step 3: QueryEngineRust - Implement Accumulator +## Step 3: asap-query-engine - Implement Accumulator ### 3.1 Create Accumulator File -**File to create**: `QueryEngineRust/src/precompute_operators/your_sketch_accumulator.rs` +**File to create**: `asap-query-engine/src/precompute_operators/your_sketch_accumulator.rs` **What to implement**: - `YourSketchAccumulator` struct with sketch state @@ -55,7 +55,7 @@ Adding a new sketch requires changes to 3 components: CommonDependencies (sketch ### 3.2 Register Accumulator -**File to modify**: `QueryEngineRust/src/precompute_operators/mod.rs` +**File to modify**: `asap-query-engine/src/precompute_operators/mod.rs` **What to add**: ```rust @@ -65,34 +65,34 @@ pub use your_sketch_accumulator::*; ### 3.3 Add Deserialization Dispatcher -**Files to search**: Look for "DatasketchesKLL" pattern in `QueryEngineRust/src/stores/` or `QueryEngineRust/src/drivers/ingest/` +**Files to search**: Look for "DatasketchesKLL" pattern in `asap-query-engine/src/stores/` or `asap-query-engine/src/drivers/ingest/` **What to add**: Match case for your sketch name calling `YourSketchAccumulator::deserialize_from_bytes_arroyo(buffer)`. **Reference examples**: -- `QueryEngineRust/src/precompute_operators/datasketches_kll_accumulator.rs` -- `QueryEngineRust/src/precompute_operators/count_min_sketch_accumulator.rs` +- `asap-query-engine/src/precompute_operators/datasketches_kll_accumulator.rs` +- `asap-query-engine/src/precompute_operators/count_min_sketch_accumulator.rs` --- -## Step 4: Controller - Sketch Parameters (Optional) +## Step 4: asap-planner - Sketch Parameters (Optional) -**File to modify**: `Controller/classes/StreamingAggregationConfig.py` or `Controller/utils/logics.py` +**File to modify**: `asap-planner/classes/StreamingAggregationConfig.py` or `asap-planner/utils/logics.py` **What to add**: - Custom sketch parameters (size, epsilon, etc.) in `get_sketch_parameters()` or similar - SLA-based parameter computation in `compute_sketch_parameters()` if needed -**Usually**: Controller picks up sketch automatically from CommonDependencies mapping. +**Usually**: asap-planner picks up sketch automatically from asap-common mapping. --- ## Testing Checklist - [ ] `validate_udfs.py` passes (ArroyoSketch) -- [ ] `cargo build --release` succeeds (QueryEngineRust) -- [ ] `cargo test` passes (QueryEngineRust) -- [ ] End-to-end: Controller → ArroyoSketch → Arroyo → Kafka → QueryEngine → Query result +- [ ] `cargo build --release` succeeds (asap-query-engine) +- [ ] `cargo test` passes (asap-query-engine) +- [ ] End-to-end: asap-planner → asap-sketch-ingest → Arroyo → Kafka → QueryEngine → Query result --- @@ -100,8 +100,8 @@ pub use your_sketch_accumulator::*; | Component | Format | Example | |-----------|--------|---------| -| CommonDependencies mapping | PascalCase | `DatasketchesKLL` | -| ArroyoSketch UDF filename | lowercase_subop | `datasketcheskll_.rs.j2` | +| asap-common mapping | PascalCase | `DatasketchesKLL` | +| asap-sketch-ingest UDF filename | lowercase_subop | `datasketcheskll_.rs.j2` | | QueryEngine accumulator | PascalCase + Accumulator | `DatasketchesKLLAccumulator` | | `get_accumulator_type()` return | Must match mapping | `"DatasketchesKLL"` | diff --git a/docs/03-how-to-guides/operations/manual-stack-run-clickhouse.md b/docs/03-how-to-guides/operations/manual-stack-run-clickhouse.md index 2568969..bc022d4 100644 --- a/docs/03-how-to-guides/operations/manual-stack-run-clickhouse.md +++ b/docs/03-how-to-guides/operations/manual-stack-run-clickhouse.md @@ -1,26 +1,26 @@ # Running the ASAP Stack Manually (Clickhouse) -This guide covers running the ASAP stack manually with Clickhouse for development and debugging. For Prometheus, see [manual-stack-run-prometheus.md](manual-stack-run-prometheus.md). For automated experiments, use the experiment framework in `Utilities/experiments/`. +This guide covers running the ASAP stack manually with Clickhouse for development and debugging. For Prometheus, see [manual-stack-run-prometheus.md](manual-stack-run-prometheus.md). For automated experiments, use the experiment framework in `asap-tools/experiments/`. ## Prerequisites - Kafka installed - Arroyo built at `~/code/arroyo/target/release/arroyo` -- QueryEngineRust built at `~/code/QueryEngineRust/target/release/query_engine_rust` -- PrometheusExporters built (fake_kafka_exporter) +- asap-query-engine built at `~/code/asap-query-engine/target/release/query_engine_rust` +- asap-tools/prometheus-exporters built (fake_kafka_exporter) - Clickhouse installed and accessible ## Directory Structure ``` ~/code/ -├── ArroyoSketch/ # Pipeline configuration scripts +├── asap-sketch-ingest/ # Pipeline configuration scripts │ ├── config.yaml # Arroyo cluster config │ └── run_arroyosketch.py # Creates sources, sinks, and pipelines -├── QueryEngineRust/ # Query interception layer -├── PrometheusExporters/ # Data generators +├── asap-query-engine/ # Query interception layer +├── asap-tools/prometheus-exporters/ # Data generators │ └── fake_kafka_exporter/ -└── Utilities/experiments/ # Automated experiment framework +└── asap-tools/experiments/ # Automated experiment framework ``` ## Config Files @@ -110,7 +110,7 @@ Wait for Kafka to be ready, then create topics: The fake_kafka_exporter generates synthetic data and writes directly to Kafka: ```bash -cd ~/code/PrometheusExporters/fake_kafka_exporter +cd ~/code/asap-tools/prometheus-exporters/fake_kafka_exporter ./target/release/fake_kafka_exporter \ --kafka-topic raw_data_topic \ --metadata-columns hostname,datacenter \ @@ -208,7 +208,7 @@ Wait for Kafka to be ready, then create topics: Same as baseline - exporter writes to Kafka: ```bash -cd ~/code/PrometheusExporters/fake_kafka_exporter +cd ~/code/asap-tools/prometheus-exporters/fake_kafka_exporter ./target/release/fake_kafka_exporter \ --kafka-topic raw_data_topic \ --metadata-columns hostname,datacenter \ @@ -230,7 +230,7 @@ Then create the tables as shown in the Baseline section (step 4). ```bash cd ~/code/arroyo -./target/release/arroyo --config ~/code/ArroyoSketch/config.yaml cluster +./target/release/arroyo --config ~/code/asap-sketch-ingest/config.yaml cluster ``` Arroyo API runs at `http://localhost:5115`. Verify with: @@ -243,7 +243,7 @@ curl http://localhost:5115/api/v1/pipelines Run `run_arroyosketch.py` to create Arroyo sources, sinks, and pipeline. For Clickhouse, always use Kafka source: ```bash -cd ~/code/ArroyoSketch +cd ~/code/asap-sketch-ingest python run_arroyosketch.py \ --source_type kafka \ --kafka_input_format json \ @@ -287,7 +287,7 @@ let adapter_config = AdapterConfig::clickhouse_sql( Recompile with `cargo build --release`. ```bash -cd ~/code/QueryEngineRust +cd ~/code/asap-query-engine ./target/release/query_engine_rust \ --kafka-topic sketch_topic \ --input-format json \ diff --git a/docs/03-how-to-guides/operations/manual-stack-run-prometheus.md b/docs/03-how-to-guides/operations/manual-stack-run-prometheus.md index fce3357..9066cca 100644 --- a/docs/03-how-to-guides/operations/manual-stack-run-prometheus.md +++ b/docs/03-how-to-guides/operations/manual-stack-run-prometheus.md @@ -1,26 +1,26 @@ # Running the ASAP Stack Manually (Prometheus) -This guide covers running the ASAP stack manually with Prometheus for development and debugging. For Clickhouse, see [manual-stack-run-clickhouse.md](manual-stack-run-clickhouse.md). For automated experiments, use the experiment framework in `Utilities/experiments/`. +This guide covers running the ASAP stack manually with Prometheus for development and debugging. For Clickhouse, see [manual-stack-run-clickhouse.md](manual-stack-run-clickhouse.md). For automated experiments, use the experiment framework in `asap-tools/experiments/`. ## Prerequisites - Kafka installed - Arroyo built at `~/code/arroyo/target/release/arroyo` -- QueryEngineRust built at `~/code/QueryEngineRust/target/release/query_engine_rust` -- PrometheusExporters built (fake_exporter_rust) +- asap-query-engine built at `~/code/asap-query-engine/target/release/query_engine_rust` +- asap-tools/prometheus-exporters built (fake_exporter_rust) - Prometheus installed and accessible ## Directory Structure ``` ~/code/ -├── ArroyoSketch/ # Pipeline configuration scripts +├── asap-sketch-ingest/ # Pipeline configuration scripts │ ├── config.yaml # Arroyo cluster config │ └── run_arroyosketch.py # Creates sources, sinks, and pipelines -├── QueryEngineRust/ # Query interception layer -├── PrometheusExporters/ # Data generators +├── asap-query-engine/ # Query interception layer +├── asap-tools/prometheus-exporters/ # Data generators │ └── fake_exporter/ -└── Utilities/experiments/ # Automated experiment framework +└── asap-tools/experiments/ # Automated experiment framework ``` ## Config Files @@ -80,7 +80,7 @@ Baseline mode runs queries directly against Prometheus or Clickhouse without ASA The fake exporter generates synthetic metrics exposed at `/metrics`: ```bash -cd ~/code/PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter +cd ~/code/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter ./target/release/fake_exporter \ --num-labels 3 \ --cardinality 100 \ @@ -142,7 +142,7 @@ Wait for Kafka to be ready, then create topics: Same as baseline - exporter just exposes `/metrics`: ```bash -cd ~/code/PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter +cd ~/code/asap-tools/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter ./target/release/fake_exporter \ --num-labels 3 \ --cardinality 100 \ @@ -153,7 +153,7 @@ cd ~/code/PrometheusExporters/fake_exporter/fake_exporter_rust/fake_exporter ```bash cd ~/code/arroyo -./target/release/arroyo --config ~/code/ArroyoSketch/config.yaml cluster +./target/release/arroyo --config ~/code/asap-sketch-ingest/config.yaml cluster ``` Arroyo API runs at `http://localhost:5115`. Verify with: @@ -166,7 +166,7 @@ curl http://localhost:5115/api/v1/pipelines Run `run_arroyosketch.py` to create Arroyo sources, sinks, and pipeline: ```bash -cd ~/code/ArroyoSketch +cd ~/code/asap-sketch-ingest python run_arroyosketch.py \ --source_type prometheus_remote_write \ --prometheus_bind_ip 0.0.0.0 \ @@ -202,7 +202,7 @@ curl http://localhost:5115/api/v1/pipelines ### 5. Start QueryEngineRust ```bash -cd ~/code/QueryEngineRust +cd ~/code/asap-query-engine ./target/release/query_engine_rust \ --kafka-topic sketch_topic \ --input-format json \ diff --git a/docs/README.md b/docs/README.md index 8d91680..2699e63 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,10 +13,10 @@ Deep dives into each component: - [Component Index](02-components/README.md) - Component overview & 1-liners - [Query Engine](02-components/query-engine.md) - Rust query processor - [Arroyo](02-components/arroyo.md) - Streaming engine (fork + customizations) -- [ArroyoSketch](02-components/arroyosketch.md) - Pipeline configurator -- [Controller](02-components/controller.md) - Auto-configuration service +- [asap-sketch-ingest](02-components/arroyosketch.md) - Pipeline configurator +- [asap-planner](02-components/controller.md) - Auto-configuration service - [Exporters](02-components/exporters.md) - Metric generators -- [Utilities](02-components/utilities.md) - Experiment framework +- [asap-tools](02-components/utilities.md) - Experiment framework ## 03. How-To Guides @@ -52,8 +52,8 @@ Developer practices and infrastructure: ## Component-Specific Documentation Technical details co-located with code: -- [QueryEngineRust](../QueryEngineRust/docs/README.md) - Extensibility guides -- [Utilities/Experiments](../Utilities/docs/architecture.md) - Experiment framework architecture -- [Controller](../Controller/README.md) - Controller internals -- [ArroyoSketch](../ArroyoSketch/README.md) - Pipeline configuration -- [PrometheusExporters](../PrometheusExporters/README.md) - Exporter implementations +- [asap-query-engine](../asap-query-engine/docs/README.md) - Extensibility guides +- [asap-tools/Experiments](../asap-tools/docs/architecture.md) - Experiment framework architecture +- [asap-planner](../asap-planner/README.md) - Controller internals +- [asap-sketch-ingest](../asap-sketch-ingest/README.md) - Pipeline configuration +- [asap-tools/prometheus-exporters](../asap-tools/prometheus-exporters/README.md) - Exporter implementations