Merge main and configure all sketchlib backends as default

- Merged latest main (includes backend abstraction from PR #207)
- Set all sketch backends to use sketchlib by default
- Count-Min, Count-Min-With-Heap, and KLL all use sketchlib
- UDFs correctly configured: all use sketchlib

Author: GnaneshGnani

.github/workflows/docker.yml

@@ -8,12 +8,18 @@ on:
     branches: [main]
     paths:
       - 'asap-common/installation/**'
-      - 'asap-planner/**'
-      - 'asap-sketch-ingest/**'
+      - 'asap-planner-rs/**'
+      - 'asap-summary-ingest/**'
       - 'asap-query-engine/**'
       - 'asap-tools/queriers/prometheus-client/**'
+      - 'asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/**'
       - '.github/workflows/docker.yml'
   workflow_dispatch:
+    inputs:
+      image-tag:
+        description: 'Tag to use for pushed images (e.g. v0.1.0, dev)'
+        required: true
+        default: 'dev'
 
 permissions:
   packages: write
@@ -36,6 +42,17 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Resolve image tag
+        id: tag
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "value=${{ inputs.image-tag }}" >> $GITHUB_OUTPUT
+          elif [ "${{ github.event_name }}" = "push" ]; then
+            echo "value=${{ github.ref_name }}" >> $GITHUB_OUTPUT
+          else
+            echo "value=pr-test" >> $GITHUB_OUTPUT
+          fi
+
       # --- Base image (Python, fast) ---
       - name: Build base image
         run: |
@@ -45,44 +62,42 @@ jobs:
             asap-common
 
       - name: Push base image
-        if: startsWith(github.ref, 'refs/tags/')
+        if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch'
         run: |
-          docker tag sketchdb-base:latest ghcr.io/projectasap/asap-base:${{ github.ref_name }}
+          docker tag sketchdb-base:latest ghcr.io/projectasap/asap-base:${{ steps.tag.outputs.value }}
           docker tag sketchdb-base:latest ghcr.io/projectasap/asap-base:latest
-          docker push ghcr.io/projectasap/asap-base:${{ github.ref_name }}
+          docker push ghcr.io/projectasap/asap-base:${{ steps.tag.outputs.value }}
           docker push ghcr.io/projectasap/asap-base:latest
 
-      # --- Planner (Python, depends on base) ---
-      - name: Build planner image
-        run: |
-          docker build \
-            -t asap-planner:local \
-            -f asap-planner/Dockerfile \
-            asap-planner
+      # --- Planner RS (Rust) ---
+      - name: Build and push planner-rs
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: asap-planner-rs/Dockerfile
+          push: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }}
+          tags: |
+            ghcr.io/projectasap/asap-planner-rs:${{ steps.tag.outputs.value }}
+            ghcr.io/projectasap/asap-planner-rs:latest
+          cache-from: type=registry,ref=ghcr.io/projectasap/asap-planner-rs:buildcache
+          cache-to: ${{ (startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch') && 'type=registry,ref=ghcr.io/projectasap/asap-planner-rs:buildcache,mode=max' || '' }}
 
-      - name: Push planner image
-        if: startsWith(github.ref, 'refs/tags/')
-        run: |
-          docker tag asap-planner:local ghcr.io/projectasap/asap-planner:${{ github.ref_name }}
-          docker tag asap-planner:local ghcr.io/projectasap/asap-planner:latest
-          docker push ghcr.io/projectasap/asap-planner:${{ github.ref_name }}
-          docker push ghcr.io/projectasap/asap-planner:latest
 
-      # --- Sketch Ingest (Python, depends on base) ---
-      - name: Build sketch-ingest image
+      # --- Summary Ingest (Python, depends on base) ---
+      - name: Build summary-ingest image
         run: |
           docker build \
-            -t asap-sketch-ingest:local \
-            -f asap-sketch-ingest/Dockerfile \
-            asap-sketch-ingest
+            -t asap-summary-ingest:local \
+            -f asap-summary-ingest/Dockerfile \
+            asap-summary-ingest
 
-      - name: Push sketch-ingest image
-        if: startsWith(github.ref, 'refs/tags/')
+      - name: Push summary-ingest image
+        if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch'
         run: |
-          docker tag asap-sketch-ingest:local ghcr.io/projectasap/asap-sketch-ingest:${{ github.ref_name }}
-          docker tag asap-sketch-ingest:local ghcr.io/projectasap/asap-sketch-ingest:latest
-          docker push ghcr.io/projectasap/asap-sketch-ingest:${{ github.ref_name }}
-          docker push ghcr.io/projectasap/asap-sketch-ingest:latest
+          docker tag asap-summary-ingest:local ghcr.io/projectasap/asap-summary-ingest:${{ steps.tag.outputs.value }}
+          docker tag asap-summary-ingest:local ghcr.io/projectasap/asap-summary-ingest:latest
+          docker push ghcr.io/projectasap/asap-summary-ingest:${{ steps.tag.outputs.value }}
+          docker push ghcr.io/projectasap/asap-summary-ingest:latest
 
       # --- Prometheus Client (Python, depends on base) ---
       - name: Build prometheus-client image
@@ -93,22 +108,35 @@ jobs:
             asap-tools/queriers/prometheus-client
 
       - name: Push prometheus-client image
-        if: startsWith(github.ref, 'refs/tags/')
+        if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch'
         run: |
-          docker tag asap-prometheus-client:local ghcr.io/projectasap/asap-prometheus-client:${{ github.ref_name }}
+          docker tag asap-prometheus-client:local ghcr.io/projectasap/asap-prometheus-client:${{ steps.tag.outputs.value }}
           docker tag asap-prometheus-client:local ghcr.io/projectasap/asap-prometheus-client:latest
-          docker push ghcr.io/projectasap/asap-prometheus-client:${{ github.ref_name }}
+          docker push ghcr.io/projectasap/asap-prometheus-client:${{ steps.tag.outputs.value }}
           docker push ghcr.io/projectasap/asap-prometheus-client:latest
 
+      # --- Fake Exporter (Rust) ---
+      - name: Build and push fake-exporter
+        uses: docker/build-push-action@v6
+        with:
+          context: asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter
+          file: asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Dockerfile
+          push: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }}
+          tags: |
+            ghcr.io/projectasap/asap-fake-exporter:${{ steps.tag.outputs.value }}
+            ghcr.io/projectasap/asap-fake-exporter:latest
+          cache-from: type=registry,ref=ghcr.io/projectasap/asap-fake-exporter:buildcache
+          cache-to: ${{ (startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch') && 'type=registry,ref=ghcr.io/projectasap/asap-fake-exporter:buildcache,mode=max' || '' }}
+
       # --- Query Engine (Rust, slow — uses GHCR layer cache) ---
       - name: Build and push query-engine
         uses: docker/build-push-action@v6
         with:
           context: .
           file: asap-query-engine/Dockerfile
-          push: ${{ startsWith(github.ref, 'refs/tags/') }}
+          push: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }}
           tags: |
-            ghcr.io/projectasap/asap-query-engine:${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || 'pr-test' }}
+            ghcr.io/projectasap/asap-query-engine:${{ steps.tag.outputs.value }}
             ghcr.io/projectasap/asap-query-engine:latest
           cache-from: type=registry,ref=ghcr.io/projectasap/asap-query-engine:buildcache
-          cache-to: type=registry,ref=ghcr.io/projectasap/asap-query-engine:buildcache,mode=max
+          cache-to: ${{ (startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch') && 'type=registry,ref=ghcr.io/projectasap/asap-query-engine:buildcache,mode=max' || '' }}

.github/workflows/python.yml

@@ -4,9 +4,8 @@ on:
   push:
     branches: [ main ]
     paths:
-      - 'asap-sketch-ingest/**'
+      - 'asap-summary-ingest/**'
       - 'asap-tools/queriers/prometheus-client/**'
-      - 'asap-planner/**'
       - 'asap-tools/**'
       - 'asap-tools/data-sources/prometheus-exporters/**'
       - 'asap-tools/execution-utilities/**'
@@ -15,9 +14,8 @@ on:
   pull_request:
     branches: [ main ]
     paths:
-      - 'asap-sketch-ingest/**'
+      - 'asap-summary-ingest/**'
       - 'asap-tools/queriers/prometheus-client/**'
-      - 'asap-planner/**'
       - 'asap-tools/**'
       - 'asap-tools/data-sources/prometheus-exporters/**'
       - 'asap-tools/execution-utilities/**'
@@ -33,9 +31,8 @@ jobs:
   detect-changes:
     runs-on: ubuntu-latest
     outputs:
-      arroyo_sketch: ${{ steps.filter.outputs.arroyo_sketch }}
+      summary_ingest: ${{ steps.filter.outputs.summary_ingest }}
       prometheus_client: ${{ steps.filter.outputs.prometheus_client }}
-      controller: ${{ steps.filter.outputs.controller }}
       utilities: ${{ steps.filter.outputs.utilities }}
       prometheus_exporters: ${{ steps.filter.outputs.prometheus_exporters }}
       execution_utilities: ${{ steps.filter.outputs.execution_utilities }}
@@ -45,15 +42,12 @@ jobs:
         id: filter
         with:
           filters: |
-            arroyo_sketch:
-              - 'asap-sketch-ingest/**'
+            summary_ingest:
+              - 'asap-summary-ingest/**'
               - 'asap-common/dependencies/py/**'
             prometheus_client:
               - 'asap-tools/queriers/prometheus-client/**'
               - 'asap-common/dependencies/py/**'
-            controller:
-              - 'asap-planner/**'
-              - 'asap-common/dependencies/py/**'
             utilities:
               - 'asap-tools/**'
               - 'asap-common/dependencies/py/**'
@@ -64,9 +58,9 @@ jobs:
               - 'asap-tools/execution-utilities/**'
               - 'asap-common/dependencies/py/**'
 
-  test-arroyo-sketch:
+  test-summary-ingest:
     needs: detect-changes
-    if: needs.detect-changes.outputs.arroyo_sketch == 'true'
+    if: needs.detect-changes.outputs.summary_ingest == 'true'
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -81,12 +75,12 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install black==24.8.0 flake8==6.1.0
-        if [ -f asap-sketch-ingest/requirements.txt ]; then pip install -r asap-sketch-ingest/requirements.txt; fi
+        if [ -f asap-summary-ingest/requirements.txt ]; then pip install -r asap-summary-ingest/requirements.txt; fi
     - name: Check formatting with Black
-      working-directory: asap-sketch-ingest
+      working-directory: asap-summary-ingest
       run: black --check --diff .
     - name: Lint with flake8
-      working-directory: asap-sketch-ingest
+      working-directory: asap-summary-ingest
       run: |
         # Stop the build if there are Python syntax errors or undefined names
         flake8 . --config=../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics
@@ -125,35 +119,6 @@ jobs:
       working-directory: asap-tools/queriers/prometheus-client
       run: mypy .
 
-  test-controller:
-    needs: detect-changes
-    if: needs.detect-changes.outputs.controller == 'true'
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
-    steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install black==24.8.0 flake8==6.1.0
-        if [ -f asap-planner/requirements.txt ]; then pip install -r asap-planner/requirements.txt; fi
-    - name: Check formatting with Black
-      working-directory: asap-planner
-      run: black --check --diff .
-    - name: Lint with flake8
-      working-directory: asap-planner
-      run: |
-        # Stop the build if there are Python syntax errors or undefined names
-        flake8 . --config=../.flake8 --count --select=E9,F63,F7,F82 --show-source --statistics
-        # Exit-zero treats all errors as warnings
-        flake8 . --config=../.flake8 --count --exit-zero --max-complexity=10 --statistics
-
   test-utilities:
     needs: detect-changes
     if: needs.detect-changes.outputs.utilities == 'true'

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
     hooks:
       - id: hadolint-docker
         files: ^.*Dockerfile.*$
-        args: ['--ignore', 'DL3008']
+        args: ['--ignore', 'DL3007', '--ignore', 'DL3008', '--ignore', 'DL3013']
 
   # Shell script linting
   - repo: https://github.com/shellcheck-py/shellcheck-py