From bb6bb0cfb16ecc0942416024273b4be513f52497 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 19 Mar 2026 16:35:43 -0400 Subject: [PATCH 1/4] Added Cargo.lock --- Cargo.lock | 2 +- asap-quickstart/README.md | 2 +- asap-quickstart/config/controller-config.yaml | 2 +- asap-quickstart/docker-compose.yml | 14 +++++++------- asap-tools/asap-cli/src/docker_util.rs | 2 +- asap-tools/components.conf | 2 +- asap-tools/docker/arroyo-cli-compose.yml | 2 +- asap-tools/docs/deployment.md | 12 ++++++------ asap-tools/docs/troubleshooting.md | 2 +- .../experiment_utils/services/arroyo.py | 10 +++++----- 10 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b520550..d106dfc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -376,7 +376,7 @@ version = "0.1.0" dependencies = [ "anyhow", "clap 4.5.60", - "indexmap", + "indexmap 2.13.0", "pretty_assertions", "promql-parser", "promql_utilities", diff --git a/asap-quickstart/README.md b/asap-quickstart/README.md index ec69f60..1419da9 100644 --- a/asap-quickstart/README.md +++ b/asap-quickstart/README.md @@ -14,7 +14,7 @@ This quickstart simulates a typical monitoring deployment with components you mi Then it adds ASAPQuery's components on top: - **Query Engine** - Prometheus-compatible API with sketch-based acceleration -- **[Arroyo](https://github.com/ProjectASAP/arroyo) + asap-sketch-ingest** - Streaming engine with pipelines configured for building sketches +- **[Arroyo](https://github.com/ProjectASAP/arroyo) + asap-summary-ingest** - Streaming engine with pipelines configured for building sketches - **Kafka** - Message broker for streaming data from Arroyo to the Query Engine - **asap-planner** - Automatically configures sketches from PromQL queries diff --git a/asap-quickstart/config/controller-config.yaml b/asap-quickstart/config/controller-config.yaml index 9462fd7..cacd044 100644 --- a/asap-quickstart/config/controller-config.yaml +++ b/asap-quickstart/config/controller-config.yaml @@ -1,4 +1,4 @@ -# SketchDB Configuration for Pattern-based Demo +# ASAP Configuration for Pattern-based Demo # Uses deterministic time-based patterns for visual comparison query_groups: diff --git a/asap-quickstart/docker-compose.yml b/asap-quickstart/docker-compose.yml index 4d0dcba..565a91e 100644 --- a/asap-quickstart/docker-compose.yml +++ b/asap-quickstart/docker-compose.yml @@ -137,7 +137,7 @@ services: timeout: 5s retries: 5 depends_on: - asap-sketch-ingest: + asap-summary-ingest: condition: service_completed_successfully restart: no @@ -187,10 +187,10 @@ services: - asap-planner-output:/asap-planner-output restart: "no" - asap-sketch-ingest: + asap-summary-ingest: image: ghcr.io/projectasap/asap-arroyosketch:v0.1.0 - container_name: asap-sketch-ingest - hostname: asap-sketch-ingest + container_name: asap-summary-ingest + hostname: asap-summary-ingest networks: - asap-network command: @@ -203,12 +203,12 @@ services: - "--output_kafka_topic=flink_output" - "--output_format=json" - "--pipeline_name=asap-demo" - - "--output_dir=/asap-sketch-ingest-output" + - "--output_dir=/asap-summary-ingest-output" - "--arroyo_url=http://arroyo:5115/api/v1" - "--bootstrap_servers=kafka:9092" volumes: - asap-planner-output:/asap-planner-output:ro - - ./output/asap-sketch-ingest:/asap-sketch-ingest-output + - ./output/asap-summary-ingest:/asap-summary-ingest-output depends_on: asap-planner: condition: service_completed_successfully @@ -251,7 +251,7 @@ services: - "--decompress-json" - "--forward-unsupported-queries" depends_on: - asap-sketch-ingest: + asap-summary-ingest: condition: service_completed_successfully kafka: condition: service_healthy diff --git a/asap-tools/asap-cli/src/docker_util.rs b/asap-tools/asap-cli/src/docker_util.rs index f37ad07..89f1c9b 100644 --- a/asap-tools/asap-cli/src/docker_util.rs +++ b/asap-tools/asap-cli/src/docker_util.rs @@ -583,7 +583,7 @@ pub async fn generate_arroyosketch_compose( let code_dir = project_root.to_string_lossy(); // Paths - let arroyosketch_dir = format!("{}/asap-sketch-ingest", code_dir); + let arroyosketch_dir = format!("{}/asap-summary-ingest", code_dir); let template_path = format!("{}/arroyosketch-cli-compose.yml.j2", arroyosketch_dir); let helper_script = format!("{}/asap-tools/experiments/generate_arroyosketch_compose.py", code_dir); let compose_output_path = format!("{}/asap-tools/docker/generated_compose_files/arroyosketch-compose.yml", code_dir); diff --git a/asap-tools/components.conf b/asap-tools/components.conf index 5f325d8..ae810e7 100644 --- a/asap-tools/components.conf +++ b/asap-tools/components.conf @@ -9,7 +9,7 @@ asap-common asap-query-engine asap-planner #prometheus-kafka-adapter -asap-sketch-ingest +asap-summary-ingest asap-quickstart asap-tools/data-sources/prometheus-exporters asap-tools/queriers/prometheus-client diff --git a/asap-tools/docker/arroyo-cli-compose.yml b/asap-tools/docker/arroyo-cli-compose.yml index a33f3b6..8e96b32 100644 --- a/asap-tools/docker/arroyo-cli-compose.yml +++ b/asap-tools/docker/arroyo-cli-compose.yml @@ -8,7 +8,7 @@ services: ports: - "5115:5115" volumes: - - ../../asap-sketch-ingest/config.yaml:/config.yaml + - ../../asap-summary-ingest/config.yaml:/config.yaml command: ["--config", "/config.yaml", "cluster"] environment: - ARROYO__API__RUN_HTTP_PORT=5115 diff --git a/asap-tools/docs/deployment.md b/asap-tools/docs/deployment.md index 8ec76f0..79e7cc2 100644 --- a/asap-tools/docs/deployment.md +++ b/asap-tools/docs/deployment.md @@ -66,7 +66,7 @@ The script executes these phases in order: - asap-query-engine (Rust binary + Docker image) - asap-planner (Docker image) - Arroyo (Node.js frontend + Rust binary + Docker image) - - asap-sketch-ingest (Python scripts) + - asap-summary-ingest (Python scripts) - asap-tools/queriers/prometheus-client, asap-tools/data-sources/prometheus-exporters, asap-tools/execution-utilities, asap-tools/prometheus-benchmark ### Directory Structure Created @@ -77,7 +77,7 @@ The script executes these phases in order: │ ├── asap-tools/ │ ├── asap-query-engine/ │ ├── asap-planner/ -│ ├── asap-sketch-ingest/ +│ ├── asap-summary-ingest/ │ ├── arroyo/ │ ├── asap-common/ │ └── asap-tools/prometheus-benchmark/ @@ -148,7 +148,7 @@ asap-common sketchlib-rust asap-query-engine asap-planner -asap-sketch-ingest +asap-summary-ingest asap-quickstart asap-tools/data-sources/prometheus-exporters asap-tools/queriers/prometheus-client @@ -296,11 +296,11 @@ cargo install refinery_cli - Web console (Node.js/React frontend) - Controller and workers (Rust binaries) -#### 5. asap-sketch-ingest -**What:** Python scripts for deploying asap-sketch-ingest pipelines +#### 5. asap-summary-ingest +**What:** Python scripts for deploying asap-summary-ingest pipelines **Build Process:** ```bash -cd asap-sketch-ingest +cd asap-summary-ingest pip install -r requirements.txt # jinja2 for templating ``` **Deployment:** Python scripts, no Docker image diff --git a/asap-tools/docs/troubleshooting.md b/asap-tools/docs/troubleshooting.md index 99aef08..faf7f04 100644 --- a/asap-tools/docs/troubleshooting.md +++ b/asap-tools/docs/troubleshooting.md @@ -94,7 +94,7 @@ Kafka has three places where message size limits need to be configured (see [PR 2. **Kafka topic configuration** (`asap-tools/experiments/experiment_utils/services/kafka.py`): - Update `max.message.bytes` in the topic creation command -3. **Arroyo connection profile** (`asap-sketch-ingest/templates/json/connection_profile.j2`): +3. **Arroyo connection profile** (`asap-summary-ingest/templates/json/connection_profile.j2`): - Add connection properties with `message.max.bytes` and `batch.size` **Example values:** diff --git a/asap-tools/experiments/experiment_utils/services/arroyo.py b/asap-tools/experiments/experiment_utils/services/arroyo.py index bc8148d..e329a80 100644 --- a/asap-tools/experiments/experiment_utils/services/arroyo.py +++ b/asap-tools/experiments/experiment_utils/services/arroyo.py @@ -66,7 +66,7 @@ def stop_all_jobs(self) -> None: """Stop all running Arroyo jobs.""" cmd = "python3 delete_pipeline.py --all_pipelines" cmd_dir = os.path.join( - self.provider.get_home_dir(), "code", "asap-sketch-ingest" + self.provider.get_home_dir(), "code", "asap-summary-ingest" ) self.provider.execute_command( node_idx=self.node_offset, @@ -160,7 +160,7 @@ def run_arroyosketch( if enable_optimized_remote_write: cmd += " --prometheus_remote_write_source optimized" cmd_dir = os.path.join( - constants.CLOUDLAB_HOME_DIR, "code", "asap-sketch-ingest" + constants.CLOUDLAB_HOME_DIR, "code", "asap-summary-ingest" ) if avoid_long_ssh: @@ -216,7 +216,7 @@ def stop_arroyosketch(self, pipeline_id: str) -> None: """ cmd = "python3 delete_pipeline.py --pipeline_id {}".format(pipeline_id) cmd_dir = os.path.join( - constants.CLOUDLAB_HOME_DIR, "code", "asap-sketch-ingest" + constants.CLOUDLAB_HOME_DIR, "code", "asap-summary-ingest" ) self.provider.execute_command( node_idx=self.node_offset, @@ -309,7 +309,7 @@ def is_healthy(self) -> bool: def _start_bare_metal(self, experiment_output_dir: str, **kwargs) -> None: """Start Arroyo cluster using bare metal deployment (original implementation).""" arroyo_config_file_path = os.path.join( - constants.CLOUDLAB_HOME_DIR, "code", "asap-sketch-ingest", "config.yaml" + constants.CLOUDLAB_HOME_DIR, "code", "asap-summary-ingest", "config.yaml" ) arroyo_bin_path = os.path.join( constants.CLOUDLAB_HOME_DIR, "code", "arroyo", "target", "release", "arroyo" @@ -332,7 +332,7 @@ def _start_bare_metal(self, experiment_output_dir: str, **kwargs) -> None: def _start_containerized(self, experiment_output_dir: str, **kwargs) -> None: """Start Arroyo cluster using Docker container deployment.""" arroyo_config_file_path = os.path.join( - constants.CLOUDLAB_HOME_DIR, "code", "asap-sketch-ingest", "config.yaml" + constants.CLOUDLAB_HOME_DIR, "code", "asap-summary-ingest", "config.yaml" ) arroyo_output_file = os.path.join(experiment_output_dir, "arroyo_cluster.out") From b7cdddcc254e6df85e5fc108c579beaf934ab73f Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 19 Mar 2026 17:06:57 -0400 Subject: [PATCH 2/4] Renamed folder --- .pre-commit-config.yaml | 2 +- {asap-sketch-ingest => asap-summary-ingest}/.gitignore | 0 {asap-sketch-ingest => asap-summary-ingest}/Dockerfile | 0 {asap-sketch-ingest => asap-summary-ingest}/LICENSE | 0 {asap-sketch-ingest => asap-summary-ingest}/README.md | 8 ++++---- .../arroyo-compose.yml | 0 .../arroyosketch-cli-compose.yml.j2 | 0 {asap-sketch-ingest => asap-summary-ingest}/config.yaml | 0 .../delete_pipeline.py | 0 .../examples/configs/streaming_config.yaml | 0 .../examples/configs/test_promql_streaming_config.yaml | 0 .../examples/configs/test_sql_streaming_config.yaml | 0 .../examples/inputs/fake_metric_total_10.json | 0 .../examples/inputs/fake_metric_total_10_2.json | 0 .../examples/inputs/value.json | 0 .../examples/json/connection_profile.json | 0 .../examples/json/connection_table.json | 0 .../examples/json/connection_table_sink.json | 0 .../examples/json/pipeline.json | 0 .../examples/sql/create_fake_metric.sql | 0 .../examples/sql/create_value.sql | 0 .../examples/sql/tumbling_window.sql | 0 .../installation/setup_dependencies.sh | 0 .../run_arroyosketch.py | 0 .../templates/hashed_key_udfs/countminsketch_sum.rs | 0 .../templates/hashed_key_udfs/multipleincrease_.rs | 0 .../templates/hashed_key_udfs/multipleminmax_max.rs | 0 .../templates/hashed_key_udfs/multipleminmax_min.rs | 0 .../templates/hashed_key_udfs/multiplesum_count.rs | 0 .../templates/hashed_key_udfs/multiplesum_sum.rs | 0 .../templates/hashed_key_udfs/old/countminsketch_sum.rs | 0 .../templates/hashed_key_udfs/old/string_to_hash.rs | 0 .../templates/hashed_key_udfs/string_to_hash.rs | 0 .../templates/json/connection_profile.j2 | 0 .../templates/json/connection_table_file.j2 | 0 .../templates/json/connection_table_kafka.j2 | 0 .../templates/json/connection_table_kafka_sql.j2 | 0 .../json/connection_table_prometheus_remote_write.j2 | 0 .../connection_table_prometheus_remote_write_optimized.j2 | 0 .../templates/json/connection_table_sink.j2 | 0 .../templates/json/pipeline.j2 | 0 .../templates/sql/distinct_windowed_labels.j2 | 0 .../sql/distinct_windowed_labels_deltasetaggregator.j2 | 0 .../templates/sql/single_arg_value_aggregation.j2 | 0 .../templates/sql/single_windowed_aggregation.j2 | 0 .../templates/udfs/countminsketch_count.rs.j2 | 0 .../templates/udfs/countminsketch_sum.rs.j2 | 0 .../templates/udfs/countminsketchwithheap_topk.rs.j2 | 0 .../templates/udfs/datasketcheskll_.rs.j2 | 0 .../templates/udfs/deltasetaggregator_.rs.j2 | 0 .../templates/udfs/gzip_compress.rs | 0 .../templates/udfs/hydrakll_.rs.j2 | 0 .../templates/udfs/multipleincrease_.rs | 0 .../templates/udfs/multipleminmax_max.rs | 0 .../templates/udfs/multipleminmax_min.rs | 0 .../templates/udfs/multiplesum_count.rs | 0 .../templates/udfs/multiplesum_sum.rs | 0 .../templates/udfs/setaggregator_.rs | 0 .../tests/__init__.py | 0 .../tests/test_integration.py | 0 .../tests/test_sql_schema.py | 0 .../utils/arroyo_utils.py | 0 .../utils/http_utils.py | 0 .../utils/jinja_utils.py | 0 .../validate_udfs.py | 0 65 files changed, 5 insertions(+), 5 deletions(-) rename {asap-sketch-ingest => asap-summary-ingest}/.gitignore (100%) rename {asap-sketch-ingest => asap-summary-ingest}/Dockerfile (100%) rename {asap-sketch-ingest => asap-summary-ingest}/LICENSE (100%) rename {asap-sketch-ingest => asap-summary-ingest}/README.md (91%) rename {asap-sketch-ingest => asap-summary-ingest}/arroyo-compose.yml (100%) rename {asap-sketch-ingest => asap-summary-ingest}/arroyosketch-cli-compose.yml.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/config.yaml (100%) rename {asap-sketch-ingest => asap-summary-ingest}/delete_pipeline.py (100%) rename {asap-sketch-ingest => asap-summary-ingest}/examples/configs/streaming_config.yaml (100%) rename {asap-sketch-ingest => asap-summary-ingest}/examples/configs/test_promql_streaming_config.yaml (100%) rename {asap-sketch-ingest => asap-summary-ingest}/examples/configs/test_sql_streaming_config.yaml (100%) rename {asap-sketch-ingest => asap-summary-ingest}/examples/inputs/fake_metric_total_10.json (100%) rename {asap-sketch-ingest => asap-summary-ingest}/examples/inputs/fake_metric_total_10_2.json (100%) rename {asap-sketch-ingest => asap-summary-ingest}/examples/inputs/value.json (100%) rename {asap-sketch-ingest => asap-summary-ingest}/examples/json/connection_profile.json (100%) rename {asap-sketch-ingest => asap-summary-ingest}/examples/json/connection_table.json (100%) rename {asap-sketch-ingest => asap-summary-ingest}/examples/json/connection_table_sink.json (100%) rename {asap-sketch-ingest => asap-summary-ingest}/examples/json/pipeline.json (100%) rename {asap-sketch-ingest => asap-summary-ingest}/examples/sql/create_fake_metric.sql (100%) rename {asap-sketch-ingest => asap-summary-ingest}/examples/sql/create_value.sql (100%) rename {asap-sketch-ingest => asap-summary-ingest}/examples/sql/tumbling_window.sql (100%) rename {asap-sketch-ingest => asap-summary-ingest}/installation/setup_dependencies.sh (100%) rename {asap-sketch-ingest => asap-summary-ingest}/run_arroyosketch.py (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/hashed_key_udfs/countminsketch_sum.rs (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/hashed_key_udfs/multipleincrease_.rs (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/hashed_key_udfs/multipleminmax_max.rs (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/hashed_key_udfs/multipleminmax_min.rs (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/hashed_key_udfs/multiplesum_count.rs (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/hashed_key_udfs/multiplesum_sum.rs (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/hashed_key_udfs/old/countminsketch_sum.rs (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/hashed_key_udfs/old/string_to_hash.rs (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/hashed_key_udfs/string_to_hash.rs (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/json/connection_profile.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/json/connection_table_file.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/json/connection_table_kafka.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/json/connection_table_kafka_sql.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/json/connection_table_prometheus_remote_write.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/json/connection_table_prometheus_remote_write_optimized.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/json/connection_table_sink.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/json/pipeline.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/sql/distinct_windowed_labels.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/sql/distinct_windowed_labels_deltasetaggregator.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/sql/single_arg_value_aggregation.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/sql/single_windowed_aggregation.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/udfs/countminsketch_count.rs.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/udfs/countminsketch_sum.rs.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/udfs/countminsketchwithheap_topk.rs.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/udfs/datasketcheskll_.rs.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/udfs/deltasetaggregator_.rs.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/udfs/gzip_compress.rs (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/udfs/hydrakll_.rs.j2 (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/udfs/multipleincrease_.rs (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/udfs/multipleminmax_max.rs (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/udfs/multipleminmax_min.rs (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/udfs/multiplesum_count.rs (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/udfs/multiplesum_sum.rs (100%) rename {asap-sketch-ingest => asap-summary-ingest}/templates/udfs/setaggregator_.rs (100%) rename {asap-sketch-ingest => asap-summary-ingest}/tests/__init__.py (100%) rename {asap-sketch-ingest => asap-summary-ingest}/tests/test_integration.py (100%) rename {asap-sketch-ingest => asap-summary-ingest}/tests/test_sql_schema.py (100%) rename {asap-sketch-ingest => asap-summary-ingest}/utils/arroyo_utils.py (100%) rename {asap-sketch-ingest => asap-summary-ingest}/utils/http_utils.py (100%) rename {asap-sketch-ingest => asap-summary-ingest}/utils/jinja_utils.py (100%) rename {asap-sketch-ingest => asap-summary-ingest}/validate_udfs.py (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fe877d1..b573ca4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: hooks: - id: hadolint-docker files: ^.*Dockerfile.*$ - args: ['--ignore', 'DL3008'] + args: ['--ignore', 'DL3007', '--ignore', 'DL3008', '--ignore', 'DL3013'] # Shell script linting - repo: https://github.com/shellcheck-py/shellcheck-py diff --git a/asap-sketch-ingest/.gitignore b/asap-summary-ingest/.gitignore similarity index 100% rename from asap-sketch-ingest/.gitignore rename to asap-summary-ingest/.gitignore diff --git a/asap-sketch-ingest/Dockerfile b/asap-summary-ingest/Dockerfile similarity index 100% rename from asap-sketch-ingest/Dockerfile rename to asap-summary-ingest/Dockerfile diff --git a/asap-sketch-ingest/LICENSE b/asap-summary-ingest/LICENSE similarity index 100% rename from asap-sketch-ingest/LICENSE rename to asap-summary-ingest/LICENSE diff --git a/asap-sketch-ingest/README.md b/asap-summary-ingest/README.md similarity index 91% rename from asap-sketch-ingest/README.md rename to asap-summary-ingest/README.md index cfda240..25a985f 100644 --- a/asap-sketch-ingest/README.md +++ b/asap-summary-ingest/README.md @@ -1,10 +1,10 @@ -# asap-sketch-ingest +# asap-summary-ingest -asap-sketch-ingest is the pipeline configurator that creates Arroyo streaming pipelines from configuration files. +asap-summary-ingest is the pipeline configurator that creates Arroyo streaming pipelines from configuration files. ## Purpose -Given `streaming_config.yaml` (generated by asap-planner), asap-sketch-ingest: +Given `streaming_config.yaml` (generated by asap-planner), asap-summary-ingest: 1. Renders SQL query templates using Jinja2 2. Creates Arroyo pipelines via REST API 3. Configures sketch-building UDFs with parameters @@ -89,7 +89,7 @@ cd arroyo docker compose up ``` -2. Run asap-sketch-ingest: +2. Run asap-summary-ingest: ```bash python run_arroyosketch.py --config test_config.yaml ``` diff --git a/asap-sketch-ingest/arroyo-compose.yml b/asap-summary-ingest/arroyo-compose.yml similarity index 100% rename from asap-sketch-ingest/arroyo-compose.yml rename to asap-summary-ingest/arroyo-compose.yml diff --git a/asap-sketch-ingest/arroyosketch-cli-compose.yml.j2 b/asap-summary-ingest/arroyosketch-cli-compose.yml.j2 similarity index 100% rename from asap-sketch-ingest/arroyosketch-cli-compose.yml.j2 rename to asap-summary-ingest/arroyosketch-cli-compose.yml.j2 diff --git a/asap-sketch-ingest/config.yaml b/asap-summary-ingest/config.yaml similarity index 100% rename from asap-sketch-ingest/config.yaml rename to asap-summary-ingest/config.yaml diff --git a/asap-sketch-ingest/delete_pipeline.py b/asap-summary-ingest/delete_pipeline.py similarity index 100% rename from asap-sketch-ingest/delete_pipeline.py rename to asap-summary-ingest/delete_pipeline.py diff --git a/asap-sketch-ingest/examples/configs/streaming_config.yaml b/asap-summary-ingest/examples/configs/streaming_config.yaml similarity index 100% rename from asap-sketch-ingest/examples/configs/streaming_config.yaml rename to asap-summary-ingest/examples/configs/streaming_config.yaml diff --git a/asap-sketch-ingest/examples/configs/test_promql_streaming_config.yaml b/asap-summary-ingest/examples/configs/test_promql_streaming_config.yaml similarity index 100% rename from asap-sketch-ingest/examples/configs/test_promql_streaming_config.yaml rename to asap-summary-ingest/examples/configs/test_promql_streaming_config.yaml diff --git a/asap-sketch-ingest/examples/configs/test_sql_streaming_config.yaml b/asap-summary-ingest/examples/configs/test_sql_streaming_config.yaml similarity index 100% rename from asap-sketch-ingest/examples/configs/test_sql_streaming_config.yaml rename to asap-summary-ingest/examples/configs/test_sql_streaming_config.yaml diff --git a/asap-sketch-ingest/examples/inputs/fake_metric_total_10.json b/asap-summary-ingest/examples/inputs/fake_metric_total_10.json similarity index 100% rename from asap-sketch-ingest/examples/inputs/fake_metric_total_10.json rename to asap-summary-ingest/examples/inputs/fake_metric_total_10.json diff --git a/asap-sketch-ingest/examples/inputs/fake_metric_total_10_2.json b/asap-summary-ingest/examples/inputs/fake_metric_total_10_2.json similarity index 100% rename from asap-sketch-ingest/examples/inputs/fake_metric_total_10_2.json rename to asap-summary-ingest/examples/inputs/fake_metric_total_10_2.json diff --git a/asap-sketch-ingest/examples/inputs/value.json b/asap-summary-ingest/examples/inputs/value.json similarity index 100% rename from asap-sketch-ingest/examples/inputs/value.json rename to asap-summary-ingest/examples/inputs/value.json diff --git a/asap-sketch-ingest/examples/json/connection_profile.json b/asap-summary-ingest/examples/json/connection_profile.json similarity index 100% rename from asap-sketch-ingest/examples/json/connection_profile.json rename to asap-summary-ingest/examples/json/connection_profile.json diff --git a/asap-sketch-ingest/examples/json/connection_table.json b/asap-summary-ingest/examples/json/connection_table.json similarity index 100% rename from asap-sketch-ingest/examples/json/connection_table.json rename to asap-summary-ingest/examples/json/connection_table.json diff --git a/asap-sketch-ingest/examples/json/connection_table_sink.json b/asap-summary-ingest/examples/json/connection_table_sink.json similarity index 100% rename from asap-sketch-ingest/examples/json/connection_table_sink.json rename to asap-summary-ingest/examples/json/connection_table_sink.json diff --git a/asap-sketch-ingest/examples/json/pipeline.json b/asap-summary-ingest/examples/json/pipeline.json similarity index 100% rename from asap-sketch-ingest/examples/json/pipeline.json rename to asap-summary-ingest/examples/json/pipeline.json diff --git a/asap-sketch-ingest/examples/sql/create_fake_metric.sql b/asap-summary-ingest/examples/sql/create_fake_metric.sql similarity index 100% rename from asap-sketch-ingest/examples/sql/create_fake_metric.sql rename to asap-summary-ingest/examples/sql/create_fake_metric.sql diff --git a/asap-sketch-ingest/examples/sql/create_value.sql b/asap-summary-ingest/examples/sql/create_value.sql similarity index 100% rename from asap-sketch-ingest/examples/sql/create_value.sql rename to asap-summary-ingest/examples/sql/create_value.sql diff --git a/asap-sketch-ingest/examples/sql/tumbling_window.sql b/asap-summary-ingest/examples/sql/tumbling_window.sql similarity index 100% rename from asap-sketch-ingest/examples/sql/tumbling_window.sql rename to asap-summary-ingest/examples/sql/tumbling_window.sql diff --git a/asap-sketch-ingest/installation/setup_dependencies.sh b/asap-summary-ingest/installation/setup_dependencies.sh similarity index 100% rename from asap-sketch-ingest/installation/setup_dependencies.sh rename to asap-summary-ingest/installation/setup_dependencies.sh diff --git a/asap-sketch-ingest/run_arroyosketch.py b/asap-summary-ingest/run_arroyosketch.py similarity index 100% rename from asap-sketch-ingest/run_arroyosketch.py rename to asap-summary-ingest/run_arroyosketch.py diff --git a/asap-sketch-ingest/templates/hashed_key_udfs/countminsketch_sum.rs b/asap-summary-ingest/templates/hashed_key_udfs/countminsketch_sum.rs similarity index 100% rename from asap-sketch-ingest/templates/hashed_key_udfs/countminsketch_sum.rs rename to asap-summary-ingest/templates/hashed_key_udfs/countminsketch_sum.rs diff --git a/asap-sketch-ingest/templates/hashed_key_udfs/multipleincrease_.rs b/asap-summary-ingest/templates/hashed_key_udfs/multipleincrease_.rs similarity index 100% rename from asap-sketch-ingest/templates/hashed_key_udfs/multipleincrease_.rs rename to asap-summary-ingest/templates/hashed_key_udfs/multipleincrease_.rs diff --git a/asap-sketch-ingest/templates/hashed_key_udfs/multipleminmax_max.rs b/asap-summary-ingest/templates/hashed_key_udfs/multipleminmax_max.rs similarity index 100% rename from asap-sketch-ingest/templates/hashed_key_udfs/multipleminmax_max.rs rename to asap-summary-ingest/templates/hashed_key_udfs/multipleminmax_max.rs diff --git a/asap-sketch-ingest/templates/hashed_key_udfs/multipleminmax_min.rs b/asap-summary-ingest/templates/hashed_key_udfs/multipleminmax_min.rs similarity index 100% rename from asap-sketch-ingest/templates/hashed_key_udfs/multipleminmax_min.rs rename to asap-summary-ingest/templates/hashed_key_udfs/multipleminmax_min.rs diff --git a/asap-sketch-ingest/templates/hashed_key_udfs/multiplesum_count.rs b/asap-summary-ingest/templates/hashed_key_udfs/multiplesum_count.rs similarity index 100% rename from asap-sketch-ingest/templates/hashed_key_udfs/multiplesum_count.rs rename to asap-summary-ingest/templates/hashed_key_udfs/multiplesum_count.rs diff --git a/asap-sketch-ingest/templates/hashed_key_udfs/multiplesum_sum.rs b/asap-summary-ingest/templates/hashed_key_udfs/multiplesum_sum.rs similarity index 100% rename from asap-sketch-ingest/templates/hashed_key_udfs/multiplesum_sum.rs rename to asap-summary-ingest/templates/hashed_key_udfs/multiplesum_sum.rs diff --git a/asap-sketch-ingest/templates/hashed_key_udfs/old/countminsketch_sum.rs b/asap-summary-ingest/templates/hashed_key_udfs/old/countminsketch_sum.rs similarity index 100% rename from asap-sketch-ingest/templates/hashed_key_udfs/old/countminsketch_sum.rs rename to asap-summary-ingest/templates/hashed_key_udfs/old/countminsketch_sum.rs diff --git a/asap-sketch-ingest/templates/hashed_key_udfs/old/string_to_hash.rs b/asap-summary-ingest/templates/hashed_key_udfs/old/string_to_hash.rs similarity index 100% rename from asap-sketch-ingest/templates/hashed_key_udfs/old/string_to_hash.rs rename to asap-summary-ingest/templates/hashed_key_udfs/old/string_to_hash.rs diff --git a/asap-sketch-ingest/templates/hashed_key_udfs/string_to_hash.rs b/asap-summary-ingest/templates/hashed_key_udfs/string_to_hash.rs similarity index 100% rename from asap-sketch-ingest/templates/hashed_key_udfs/string_to_hash.rs rename to asap-summary-ingest/templates/hashed_key_udfs/string_to_hash.rs diff --git a/asap-sketch-ingest/templates/json/connection_profile.j2 b/asap-summary-ingest/templates/json/connection_profile.j2 similarity index 100% rename from asap-sketch-ingest/templates/json/connection_profile.j2 rename to asap-summary-ingest/templates/json/connection_profile.j2 diff --git a/asap-sketch-ingest/templates/json/connection_table_file.j2 b/asap-summary-ingest/templates/json/connection_table_file.j2 similarity index 100% rename from asap-sketch-ingest/templates/json/connection_table_file.j2 rename to asap-summary-ingest/templates/json/connection_table_file.j2 diff --git a/asap-sketch-ingest/templates/json/connection_table_kafka.j2 b/asap-summary-ingest/templates/json/connection_table_kafka.j2 similarity index 100% rename from asap-sketch-ingest/templates/json/connection_table_kafka.j2 rename to asap-summary-ingest/templates/json/connection_table_kafka.j2 diff --git a/asap-sketch-ingest/templates/json/connection_table_kafka_sql.j2 b/asap-summary-ingest/templates/json/connection_table_kafka_sql.j2 similarity index 100% rename from asap-sketch-ingest/templates/json/connection_table_kafka_sql.j2 rename to asap-summary-ingest/templates/json/connection_table_kafka_sql.j2 diff --git a/asap-sketch-ingest/templates/json/connection_table_prometheus_remote_write.j2 b/asap-summary-ingest/templates/json/connection_table_prometheus_remote_write.j2 similarity index 100% rename from asap-sketch-ingest/templates/json/connection_table_prometheus_remote_write.j2 rename to asap-summary-ingest/templates/json/connection_table_prometheus_remote_write.j2 diff --git a/asap-sketch-ingest/templates/json/connection_table_prometheus_remote_write_optimized.j2 b/asap-summary-ingest/templates/json/connection_table_prometheus_remote_write_optimized.j2 similarity index 100% rename from asap-sketch-ingest/templates/json/connection_table_prometheus_remote_write_optimized.j2 rename to asap-summary-ingest/templates/json/connection_table_prometheus_remote_write_optimized.j2 diff --git a/asap-sketch-ingest/templates/json/connection_table_sink.j2 b/asap-summary-ingest/templates/json/connection_table_sink.j2 similarity index 100% rename from asap-sketch-ingest/templates/json/connection_table_sink.j2 rename to asap-summary-ingest/templates/json/connection_table_sink.j2 diff --git a/asap-sketch-ingest/templates/json/pipeline.j2 b/asap-summary-ingest/templates/json/pipeline.j2 similarity index 100% rename from asap-sketch-ingest/templates/json/pipeline.j2 rename to asap-summary-ingest/templates/json/pipeline.j2 diff --git a/asap-sketch-ingest/templates/sql/distinct_windowed_labels.j2 b/asap-summary-ingest/templates/sql/distinct_windowed_labels.j2 similarity index 100% rename from asap-sketch-ingest/templates/sql/distinct_windowed_labels.j2 rename to asap-summary-ingest/templates/sql/distinct_windowed_labels.j2 diff --git a/asap-sketch-ingest/templates/sql/distinct_windowed_labels_deltasetaggregator.j2 b/asap-summary-ingest/templates/sql/distinct_windowed_labels_deltasetaggregator.j2 similarity index 100% rename from asap-sketch-ingest/templates/sql/distinct_windowed_labels_deltasetaggregator.j2 rename to asap-summary-ingest/templates/sql/distinct_windowed_labels_deltasetaggregator.j2 diff --git a/asap-sketch-ingest/templates/sql/single_arg_value_aggregation.j2 b/asap-summary-ingest/templates/sql/single_arg_value_aggregation.j2 similarity index 100% rename from asap-sketch-ingest/templates/sql/single_arg_value_aggregation.j2 rename to asap-summary-ingest/templates/sql/single_arg_value_aggregation.j2 diff --git a/asap-sketch-ingest/templates/sql/single_windowed_aggregation.j2 b/asap-summary-ingest/templates/sql/single_windowed_aggregation.j2 similarity index 100% rename from asap-sketch-ingest/templates/sql/single_windowed_aggregation.j2 rename to asap-summary-ingest/templates/sql/single_windowed_aggregation.j2 diff --git a/asap-sketch-ingest/templates/udfs/countminsketch_count.rs.j2 b/asap-summary-ingest/templates/udfs/countminsketch_count.rs.j2 similarity index 100% rename from asap-sketch-ingest/templates/udfs/countminsketch_count.rs.j2 rename to asap-summary-ingest/templates/udfs/countminsketch_count.rs.j2 diff --git a/asap-sketch-ingest/templates/udfs/countminsketch_sum.rs.j2 b/asap-summary-ingest/templates/udfs/countminsketch_sum.rs.j2 similarity index 100% rename from asap-sketch-ingest/templates/udfs/countminsketch_sum.rs.j2 rename to asap-summary-ingest/templates/udfs/countminsketch_sum.rs.j2 diff --git a/asap-sketch-ingest/templates/udfs/countminsketchwithheap_topk.rs.j2 b/asap-summary-ingest/templates/udfs/countminsketchwithheap_topk.rs.j2 similarity index 100% rename from asap-sketch-ingest/templates/udfs/countminsketchwithheap_topk.rs.j2 rename to asap-summary-ingest/templates/udfs/countminsketchwithheap_topk.rs.j2 diff --git a/asap-sketch-ingest/templates/udfs/datasketcheskll_.rs.j2 b/asap-summary-ingest/templates/udfs/datasketcheskll_.rs.j2 similarity index 100% rename from asap-sketch-ingest/templates/udfs/datasketcheskll_.rs.j2 rename to asap-summary-ingest/templates/udfs/datasketcheskll_.rs.j2 diff --git a/asap-sketch-ingest/templates/udfs/deltasetaggregator_.rs.j2 b/asap-summary-ingest/templates/udfs/deltasetaggregator_.rs.j2 similarity index 100% rename from asap-sketch-ingest/templates/udfs/deltasetaggregator_.rs.j2 rename to asap-summary-ingest/templates/udfs/deltasetaggregator_.rs.j2 diff --git a/asap-sketch-ingest/templates/udfs/gzip_compress.rs b/asap-summary-ingest/templates/udfs/gzip_compress.rs similarity index 100% rename from asap-sketch-ingest/templates/udfs/gzip_compress.rs rename to asap-summary-ingest/templates/udfs/gzip_compress.rs diff --git a/asap-sketch-ingest/templates/udfs/hydrakll_.rs.j2 b/asap-summary-ingest/templates/udfs/hydrakll_.rs.j2 similarity index 100% rename from asap-sketch-ingest/templates/udfs/hydrakll_.rs.j2 rename to asap-summary-ingest/templates/udfs/hydrakll_.rs.j2 diff --git a/asap-sketch-ingest/templates/udfs/multipleincrease_.rs b/asap-summary-ingest/templates/udfs/multipleincrease_.rs similarity index 100% rename from asap-sketch-ingest/templates/udfs/multipleincrease_.rs rename to asap-summary-ingest/templates/udfs/multipleincrease_.rs diff --git a/asap-sketch-ingest/templates/udfs/multipleminmax_max.rs b/asap-summary-ingest/templates/udfs/multipleminmax_max.rs similarity index 100% rename from asap-sketch-ingest/templates/udfs/multipleminmax_max.rs rename to asap-summary-ingest/templates/udfs/multipleminmax_max.rs diff --git a/asap-sketch-ingest/templates/udfs/multipleminmax_min.rs b/asap-summary-ingest/templates/udfs/multipleminmax_min.rs similarity index 100% rename from asap-sketch-ingest/templates/udfs/multipleminmax_min.rs rename to asap-summary-ingest/templates/udfs/multipleminmax_min.rs diff --git a/asap-sketch-ingest/templates/udfs/multiplesum_count.rs b/asap-summary-ingest/templates/udfs/multiplesum_count.rs similarity index 100% rename from asap-sketch-ingest/templates/udfs/multiplesum_count.rs rename to asap-summary-ingest/templates/udfs/multiplesum_count.rs diff --git a/asap-sketch-ingest/templates/udfs/multiplesum_sum.rs b/asap-summary-ingest/templates/udfs/multiplesum_sum.rs similarity index 100% rename from asap-sketch-ingest/templates/udfs/multiplesum_sum.rs rename to asap-summary-ingest/templates/udfs/multiplesum_sum.rs diff --git a/asap-sketch-ingest/templates/udfs/setaggregator_.rs b/asap-summary-ingest/templates/udfs/setaggregator_.rs similarity index 100% rename from asap-sketch-ingest/templates/udfs/setaggregator_.rs rename to asap-summary-ingest/templates/udfs/setaggregator_.rs diff --git a/asap-sketch-ingest/tests/__init__.py b/asap-summary-ingest/tests/__init__.py similarity index 100% rename from asap-sketch-ingest/tests/__init__.py rename to asap-summary-ingest/tests/__init__.py diff --git a/asap-sketch-ingest/tests/test_integration.py b/asap-summary-ingest/tests/test_integration.py similarity index 100% rename from asap-sketch-ingest/tests/test_integration.py rename to asap-summary-ingest/tests/test_integration.py diff --git a/asap-sketch-ingest/tests/test_sql_schema.py b/asap-summary-ingest/tests/test_sql_schema.py similarity index 100% rename from asap-sketch-ingest/tests/test_sql_schema.py rename to asap-summary-ingest/tests/test_sql_schema.py diff --git a/asap-sketch-ingest/utils/arroyo_utils.py b/asap-summary-ingest/utils/arroyo_utils.py similarity index 100% rename from asap-sketch-ingest/utils/arroyo_utils.py rename to asap-summary-ingest/utils/arroyo_utils.py diff --git a/asap-sketch-ingest/utils/http_utils.py b/asap-summary-ingest/utils/http_utils.py similarity index 100% rename from asap-sketch-ingest/utils/http_utils.py rename to asap-summary-ingest/utils/http_utils.py diff --git a/asap-sketch-ingest/utils/jinja_utils.py b/asap-summary-ingest/utils/jinja_utils.py similarity index 100% rename from asap-sketch-ingest/utils/jinja_utils.py rename to asap-summary-ingest/utils/jinja_utils.py diff --git a/asap-sketch-ingest/validate_udfs.py b/asap-summary-ingest/validate_udfs.py similarity index 100% rename from asap-sketch-ingest/validate_udfs.py rename to asap-summary-ingest/validate_udfs.py From a8b426179ec89408b6aecb82a90d5cc3ad48cdb2 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 19 Mar 2026 17:09:04 -0400 Subject: [PATCH 3/4] Updated CI --- .github/workflows/docker.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3be7453..eb588cc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,7 +9,7 @@ on: paths: - 'asap-common/installation/**' - 'asap-planner/**' - - 'asap-sketch-ingest/**' + - 'asap-summary-ingest/**' - 'asap-query-engine/**' - 'asap-tools/queriers/prometheus-client/**' - '.github/workflows/docker.yml' @@ -68,21 +68,21 @@ jobs: docker push ghcr.io/projectasap/asap-planner:${{ github.ref_name }} docker push ghcr.io/projectasap/asap-planner:latest - # --- Sketch Ingest (Python, depends on base) --- - - name: Build sketch-ingest image + # --- Summary Ingest (Python, depends on base) --- + - name: Build summary-ingest image run: | docker build \ - -t asap-sketch-ingest:local \ - -f asap-sketch-ingest/Dockerfile \ - asap-sketch-ingest + -t asap-summary-ingest:local \ + -f asap-summary-ingest/Dockerfile \ + asap-summary-ingest - - name: Push sketch-ingest image + - name: Push summary-ingest image if: startsWith(github.ref, 'refs/tags/') run: | - docker tag asap-sketch-ingest:local ghcr.io/projectasap/asap-sketch-ingest:${{ github.ref_name }} - docker tag asap-sketch-ingest:local ghcr.io/projectasap/asap-sketch-ingest:latest - docker push ghcr.io/projectasap/asap-sketch-ingest:${{ github.ref_name }} - docker push ghcr.io/projectasap/asap-sketch-ingest:latest + docker tag asap-summary-ingest:local ghcr.io/projectasap/asap-summary-ingest:${{ github.ref_name }} + docker tag asap-summary-ingest:local ghcr.io/projectasap/asap-summary-ingest:latest + docker push ghcr.io/projectasap/asap-summary-ingest:${{ github.ref_name }} + docker push ghcr.io/projectasap/asap-summary-ingest:latest # --- Prometheus Client (Python, depends on base) --- - name: Build prometheus-client image From 087dbf877e40d90e5bb0a9221dad8087663d9dc7 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 19 Mar 2026 17:10:43 -0400 Subject: [PATCH 4/4] Updated CI --- .github/workflows/python.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index bd4ac21..858230c 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -4,7 +4,7 @@ on: push: branches: [ main ] paths: - - 'asap-sketch-ingest/**' + - 'asap-summary-ingest/**' - 'asap-tools/queriers/prometheus-client/**' - 'asap-planner/**' - 'asap-tools/**' @@ -15,7 +15,7 @@ on: pull_request: branches: [ main ] paths: - - 'asap-sketch-ingest/**' + - 'asap-summary-ingest/**' - 'asap-tools/queriers/prometheus-client/**' - 'asap-planner/**' - 'asap-tools/**' @@ -33,7 +33,7 @@ jobs: detect-changes: runs-on: ubuntu-latest outputs: - arroyo_sketch: ${{ steps.filter.outputs.arroyo_sketch }} + summary_ingest: ${{ steps.filter.outputs.summary_ingest }} prometheus_client: ${{ steps.filter.outputs.prometheus_client }} controller: ${{ steps.filter.outputs.controller }} utilities: ${{ steps.filter.outputs.utilities }} @@ -45,8 +45,8 @@ jobs: id: filter with: filters: | - arroyo_sketch: - - 'asap-sketch-ingest/**' + summary_ingest: + - 'asap-summary-ingest/**' - 'asap-common/dependencies/py/**' prometheus_client: - 'asap-tools/queriers/prometheus-client/**' @@ -64,9 +64,9 @@ jobs: - 'asap-tools/execution-utilities/**' - 'asap-common/dependencies/py/**' - test-arroyo-sketch: + test-summary-ingest: needs: detect-changes - if: needs.detect-changes.outputs.arroyo_sketch == 'true' + if: needs.detect-changes.outputs.summary_ingest == 'true' runs-on: ubuntu-latest strategy: matrix: @@ -81,12 +81,12 @@ jobs: run: | python -m pip install --upgrade pip pip install black==24.8.0 flake8==6.1.0 - if [ -f asap-sketch-ingest/requirements.txt ]; then pip install -r asap-sketch-ingest/requirements.txt; fi + if [ -f asap-summary-ingest/requirements.txt ]; then pip install -r asap-summary-ingest/requirements.txt; fi - name: Check formatting with Black - working-directory: asap-sketch-ingest + working-directory: asap-summary-ingest run: black --check --diff . - name: Lint with flake8 - working-directory: asap-sketch-ingest + working-directory: asap-summary-ingest run: | # Stop the build if there are Python syntax errors or undefined names flake8 . --config=../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics