[CSR-3558] chore: prepare persisted redis and clickhouse

.github/workflows/check-compose-sync.yml

@@ -5,13 +5,15 @@ on:
     paths:
       - 'on-prem/templates/**'
       - 'on-prem/scripts/generate-compose.sh'
+      - 'on-prem/scripts/sync-templates.sh'
       - 'on-prem/docker-compose.full.yml'
       - 'on-prem/docker-compose.database.yml'
       - 'on-prem/docker-compose.cache.yml'
   pull_request:
     paths:
       - 'on-prem/templates/**'
       - 'on-prem/scripts/generate-compose.sh'
+      - 'on-prem/scripts/sync-templates.sh'
       - 'on-prem/docker-compose.full.yml'
       - 'on-prem/docker-compose.database.yml'
       - 'on-prem/docker-compose.cache.yml'
@@ -23,12 +25,12 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Install yq
+        uses: mikefarah/yq@v4
+
       - name: Generate compose files
         working-directory: on-prem
-        run: |
-          ./scripts/generate-compose.sh full
-          ./scripts/generate-compose.sh database
-          ./scripts/generate-compose.sh cache
+        run: ./scripts/sync-templates.sh
 
       - name: Check for differences
         working-directory: on-prem
@@ -37,12 +39,8 @@ jobs:
             echo ""
             echo "❌ Generated compose files are out of sync with committed files!"
             echo ""
-            echo "To fix this, run the following commands and commit the changes:"
-            echo "  cd on-prem"
-            echo "  ./scripts/generate-compose.sh full"
-            echo "  ./scripts/generate-compose.sh database"
-            echo "  ./scripts/generate-compose.sh cache"
+            echo "To fix this, run and commit the changes:"
+            echo "  cd on-prem && ./scripts/sync-templates.sh"
             exit 1
           fi
           echo "✅ All compose files are in sync"
-

on-prem/.env.example

@@ -1,9 +1,17 @@
-CURRENTS_ENV=onprem
 # IMAGE_REPOSITORY=513558712013.dkr.ecr.us-east-1.amazonaws.com/currents/on-prem/  
 # IMAGE_TAG=staging
 
 GITLAB_REDIRECT_URL=http://localhost:4000/integrations/gitlab/callback
 
+
+# =============================================================================
+# Redis Configuration
+# =============================================================================
+# Default points to redis service in the compose network
+# Change these if using an external Redis instance
+REDIS_URI=redis://redis:6379
+REDIS_URI_SLAVE=redis://redis:6379
+
 # Remote Object Storage
 # For minio when using the full data environment
 MINIO_SERVER_ACCESS_KEY=minio-access-key
@@ -41,4 +49,34 @@ AUTOMATED_REPORTS_EMAIL_BCC=
 # Coralogix support
 CORALOGIX_API_ENDPOINT=
 CORALOGIX_API_KEY=
-CORALOGIX_APP_NAME="customer-name"
+CORALOGIX_APP_NAME="customer-name"
+
+# =============================================================================
+# Data Volume Configuration
+# =============================================================================
+# By default, data is stored in ./data/<service> (auto-created)
+# Set to a custom path or a pre-created Docker volume name
+#
+# Examples:
+#   REDIS_VOLUME=./data/redis          # Local path (default)
+#   REDIS_VOLUME=/mnt/ssd/redis        # Custom path
+#   REDIS_VOLUME=my_redis_volume       # Pre-created Docker volume
+#
+# REDIS_VOLUME=./data/redis
+# MONGODB_VOLUME=./data/mongodb
+# CLICKHOUSE_VOLUME=./data/clickhouse
+# MINIO_VOLUME=./data/minio
+# SCHEDULER_STARTUP_VOLUME=./data/startup
+
+# =============================================================================
+# ClickHouse Configuration
+# =============================================================================
+# Connection settings (uses service name for internal Docker network)
+CLICKHOUSE_URL=http://clickhouse:8123
+CLICKHOUSE_USERNAME=currents
+
+# Passwords for ClickHouse users (generated by setup.sh)
+# Default PASSWORD only used if using the included clickhouse
+CLICKHOUSE_DEFAULT_PASSWORD=
+# The CURRENTS USER PASSWORD is how all services connect
+CLICKHOUSE_CURRENTS_PASSWORD=

on-prem/.gitignore

@@ -1,3 +1,4 @@
 .startup
 /.env
-/.gitlab-key.pem
+/.gitlab-key.pem
+/data/

on-prem/docker-compose.cache.yml

@@ -2,170 +2,130 @@
 # Profile: cache - Cache (redis)
 
 name: currents_cache
-networks:
-  default:
-    name: currents_cache_default
+
+# This core Currents application services
+
+# Redis service configuration
+# Almost always included - provides caching and pub/sub messaging
 services:
-  api:
-    container_name: currentsapi
+  # Currents services depend on redis
+  director:
+    hostname: director
+    image: ${IMAGE_REPOSITORY:-currents-}director:${IMAGE_TAG:-dev}
+    expose:
+      - 1234
+    ports:
+      - 1234:1234
+    restart: unless-stopped
+    env_file:
+      - .env
+    networks:
+      - default
+    environment:
+      - CURRENTS_ENV=onprem
+      - API_URL="http://localhost:4000/v1"
+      - APP_BASE_URL=http://localhost:4000
+      - MONGODB_URI=mongodb://host.docker.internal:27017/?readPreference=primary&directConnection=true&ssl=false
     depends_on:
       redis:
         condition: service_started
         required: false
+  api:
+    hostname: api
+    image: ${IMAGE_REPOSITORY:-currents-}api:${IMAGE_TAG:-dev}
+    expose:
+      - 4000
+    ports:
+      - 4000:4000
+    restart: unless-stopped
+    networks:
+      - default
     env_file:
-      - path: .env
-        required: true
+      - .env
     environment:
+      - CURRENTS_ENV=onprem
       - INCLUDE_DASHBOARD=onprem
       - MONGODB_URI=mongodb://host.docker.internal:27017/?readPreference=primary&directConnection=true&ssl=false
-      - REDIS_URI=redis://host.docker.internal:6379
-      - REDIS_URI_SLAVE=redis://host.docker.internal:6379
-      - CLICKHOUSE_URL=http://host.docker.internal:8123
+      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_CURRENTS_PASSWORD}
       - AUTOMATED_REPORTS_CURRENTS_DASHBOARD_HOSTNAME=http://host.docker.internal:4000
       - CURRENTS_RECORD_API_URL=http://localhost:1234
-    expose:
-      - 4000
-    hostname: api
-    image: ${IMAGE_REPOSITORY:-currents-}api:${IMAGE_TAG:-dev}
-    networks:
-      default: null
-    ports:
-      - mode: ingress
-        protocol: tcp
-        published: "4000"
-        target: 4000
-    restart: unless-stopped
-  changestreams-worker:
-    container_name: currrents-changestreams-worker
     depends_on:
       redis:
         condition: service_started
         required: false
+  changestreams-worker:
+    image: ${IMAGE_REPOSITORY:-currents-}change-streams:${IMAGE_TAG:-dev}
+    restart: unless-stopped
+    networks:
+      - default
     env_file:
-      - path: .env
-        required: true
+      - .env
     environment:
+      - CURRENTS_ENV=onprem
       - MONGODB_URI=mongodb://host.docker.internal:27017/?readPreference=primary&directConnection=true&ssl=false
-      - REDIS_URI=redis://host.docker.internal:6379
-      - REDIS_URI_SLAVE=redis://host.docker.internal:6379
-      - CLICKHOUSE_URL=http://host.docker.internal:8123
-    image: ${IMAGE_REPOSITORY:-currents-}change-streams:${IMAGE_TAG:-dev}
-    networks:
-      default: null
-    restart: unless-stopped
-  director:
-    container_name: currentsdirector
+      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_CURRENTS_PASSWORD}
     depends_on:
       redis:
         condition: service_started
         required: false
+  write-worker:
+    image: ${IMAGE_REPOSITORY:-currents-}writer:${IMAGE_TAG:-dev}
+    restart: unless-stopped
+    networks:
+      - default
     env_file:
-      - path: .env
-        required: true
+      - .env
     environment:
-      - API_URL="http://localhost:4000/v1"
-      - APP_BASE_URL=http://localhost:4000
-      - REDIS_URI=redis://host.docker.internal:6379
-      - REDIS_URI_SLAVE=redis://host.docker.internal:6379
+      - CURRENTS_ENV=onprem
       - MONGODB_URI=mongodb://host.docker.internal:27017/?readPreference=primary&directConnection=true&ssl=false
-    expose:
-      - 1234
-    hostname: director
-    image: ${IMAGE_REPOSITORY:-currents-}director:${IMAGE_TAG:-dev}
-    networks:
-      default: null
-    ports:
-      - mode: ingress
-        protocol: tcp
-        published: "1234"
-        target: 1234
-    restart: unless-stopped
-  redis:
-    expose:
-      - 6379
-      - 8001
-    hostname: redis
-    image: redis/redis-stack
-    networks:
-      default: null
-    ports:
-      - mode: ingress
-        protocol: tcp
-        published: "6379"
-        target: 6379
-      - mode: ingress
-        protocol: tcp
-        published: "8001"
-        target: 8001
-    restart: unless-stopped
-    volumes:
-      - source: redisdata
-        target: /data
-        type: volume
-        volume: {}
-  scheduler:
-    container_name: currents-scheduler
+      - AUTOMATED_REPORTS_CURRENTS_DASHBOARD_HOSTNAME=http://host.docker.internal:4000
+      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_CURRENTS_PASSWORD}
     depends_on:
       redis:
         condition: service_started
         required: false
+  scheduler:
+    image: ${IMAGE_REPOSITORY:-currents-}scheduler:${IMAGE_TAG:-dev}
+    restart: unless-stopped
+    networks:
+      - default
     env_file:
-      - path: .env
-        required: true
+      - .env
     environment:
+      - CURRENTS_ENV=onprem
       - MONGODB_URI=mongodb://host.docker.internal:27017/?readPreference=primary&directConnection=true&ssl=false
-      - REDIS_URI=redis://host.docker.internal:6379
-      - REDIS_URI_SLAVE=redis://host.docker.internal:6379
-      - CLICKHOUSE_URL=http://host.docker.internal:8123
+      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_CURRENTS_PASSWORD}
       - RUN_STARTUP_TASKS=true
-    image: ${IMAGE_REPOSITORY:-currents-}scheduler:${IMAGE_TAG:-dev}
-    networks:
-      default: null
-    restart: unless-stopped
     volumes:
-      - bind:
-          create_host_path: true
-        read_only: true
-        source: ./.startup
-        target: /app/packages/scheduler/dist/.startup
-        type: bind
-  webhooks:
-    container_name: currents-webhooks
+      - ${SCHEDULER_STARTUP_VOLUME:-./data/startup}:/app/packages/scheduler/dist/.startup
     depends_on:
       redis:
         condition: service_started
         required: false
+  webhooks:
+    image: ${IMAGE_REPOSITORY:-currents-}webhooks:${IMAGE_TAG:-dev}
+    restart: unless-stopped
+    networks:
+      - default
     env_file:
-      - path: .env
-        required: true
+      - .env
     environment:
+      - CURRENTS_ENV=onprem
       - MONGODB_URI=mongodb://host.docker.internal:27017/?readPreference=primary&directConnection=true&ssl=false
-      - REDIS_URI=redis://host.docker.internal:6379
-      - REDIS_URI_SLAVE=redis://host.docker.internal:6379
       - DASHBOARD_URL=http://localhost:4000
-    image: ${IMAGE_REPOSITORY:-currents-}webhooks:${IMAGE_TAG:-dev}
-    networks:
-      default: null
-    restart: unless-stopped
-  write-worker:
-    container_name: currents-write-worker
     depends_on:
       redis:
         condition: service_started
         required: false
-    env_file:
-      - path: .env
-        required: true
-    environment:
-      - MONGODB_URI=mongodb://host.docker.internal:27017/?readPreference=primary&directConnection=true&ssl=false
-      - REDIS_URI=redis://host.docker.internal:6379
-      - REDIS_URI_SLAVE=redis://host.docker.internal:6379
-      - AUTOMATED_REPORTS_CURRENTS_DASHBOARD_HOSTNAME=http://host.docker.internal:4000
-      - CLICKHOUSE_URL=http://host.docker.internal:8123
-    image: ${IMAGE_REPOSITORY:-currents-}writer:${IMAGE_TAG:-dev}
-    networks:
-      default: null
+  redis:
+    hostname: redis
+    image: redis/redis-stack-server:7.4.0-v8
+    ports:
+      - 6379:6379
     restart: unless-stopped
-volumes:
-  redisdata:
-    name: currents_cache_redisdata
+    networks:
+      - default
+    volumes:
+      - ${REDIS_VOLUME:-./data/redis}:/data
+networks:
+  default: null