diff --git a/on-prem/.env.example b/on-prem/.env.example index 8e3898d..bdbcbc5 100644 --- a/on-prem/.env.example +++ b/on-prem/.env.example @@ -1,8 +1,26 @@ -# IMAGE_REPOSITORY=513558712013.dkr.ecr.us-east-1.amazonaws.com/currents/on-prem/ -# IMAGE_TAG=staging - -GITLAB_REDIRECT_URL=http://localhost:4000/integrations/gitlab/callback +# ============================================================================= +# Docker Compose Configuration +# ============================================================================= +# These variables are used by docker-compose only (not passed to containers) +# DC_IMAGE_REPOSITORY=513558712013.dkr.ecr.us-east-1.amazonaws.com/currents/on-prem/ +# DC_IMAGE_TAG=staging +# ============================================================================= +# Data Volume Configuration +# ============================================================================= +# By default, data is stored in ./data/ (auto-created) +# Set to a custom path or a pre-created Docker volume name +# +# Examples: +# DC_REDIS_VOLUME=./data/redis # Local path (default) +# DC_REDIS_VOLUME=/mnt/ssd/redis # Custom path +# DC_REDIS_VOLUME=my_redis_volume # Pre-created Docker volume +# +# DC_REDIS_VOLUME=./data/redis +# DC_MONGODB_VOLUME=./data/mongodb +# DC_CLICKHOUSE_VOLUME=./data/clickhouse +# DC_RUSTFS_VOLUME=./data/rustfs +# DC_SCHEDULER_STARTUP_VOLUME=./data/startup # ============================================================================= # Redis Configuration @@ -12,29 +30,51 @@ GITLAB_REDIRECT_URL=http://localhost:4000/integrations/gitlab/callback REDIS_URI=redis://redis:6379 REDIS_URI_SLAVE=redis://redis:6379 -# Remote Object Storage -# For minio when using the full data environment -MINIO_SERVER_ACCESS_KEY=minio-access-key +# ============================================================================= +# Object Storage (RustFS / S3) +# ============================================================================= +# For rustfs when using the full data environment +RUSTFS_ACCESS_KEY=rustfs-access-key # Run `./scripts/generate-secrets.sh token` from the root of the project to generate a new secret -MINIO_SERVER_SECRET_KEY=minio-secret-key +RUSTFS_SECRET_KEY=rustfs-secret-key FILE_STORAGE_ENDPOINT="http://localhost:9000" FILE_STORAGE_INTERNAL_ENDPOINT="http://host.docker.internal:9000" -FILE_STORAGE_ACCESS_KEY_ID=${MINIO_SERVER_ACCESS_KEY} -FILE_STORAGE_SECRET_ACCESS_KEY=${MINIO_SERVER_SECRET_KEY} +FILE_STORAGE_ACCESS_KEY_ID=${RUSTFS_ACCESS_KEY} +FILE_STORAGE_SECRET_ACCESS_KEY=${RUSTFS_SECRET_KEY} FILE_STORAGE_BUCKET="currents" FILE_STORAGE_FORCE_PATH_STYLE=true -# ON_PREM_EMAIL=admin@currents.local +# ============================================================================= +# ClickHouse Configuration +# ============================================================================= +# Connection settings (uses service name for internal Docker network) +CLICKHOUSE_URL=http://clickhouse:8123 +CLICKHOUSE_USERNAME=currents +# Passwords for ClickHouse users (generated by setup.sh) +# Default PASSWORD only used if using the included clickhouse +CLICKHOUSE_DEFAULT_PASSWORD= +# The CURRENTS USER PASSWORD is how all services connect +CLICKHOUSE_CURRENTS_PASSWORD= + +# ============================================================================= +# Authentication +# ============================================================================= # Run `./scripts/generate-secrets.sh token` from the root of the project to generate a new secret JWT_SECRET= JWT_SECRET_EXPIRY=10m +# ============================================================================= +# GitLab Integration +# ============================================================================= +GITLAB_REDIRECT_URL=http://localhost:4000/integrations/gitlab/callback # Run `./scripts/generate-secrets key gitlab-key.pem` # Then `base64 < "gitlab-key.pem" | tr -d '\n'` to get the base64 secret to set here GITLAB_STATE_SECRET= -# Email Configuration +# ============================================================================= +# Email Configuration +# ============================================================================= EMAIL_TRANSPORTER=smtp SMTP_HOST=localhost SMTP_PORT=1025 @@ -43,40 +83,15 @@ SMTP_USER=test SMTP_PASS=test # Automated Reports Configuration -# ------------------------------- AUTOMATED_REPORTS_EMAIL_FROM="Currents Report " AUTOMATED_REPORTS_EMAIL_BCC= + +# ============================================================================= +# Observability (Optional) +# ============================================================================= # Coralogix support CORALOGIX_API_ENDPOINT= CORALOGIX_API_KEY= CORALOGIX_APP_NAME="customer-name" -# ============================================================================= -# Data Volume Configuration -# ============================================================================= -# By default, data is stored in ./data/ (auto-created) -# Set to a custom path or a pre-created Docker volume name -# -# Examples: -# REDIS_VOLUME=./data/redis # Local path (default) -# REDIS_VOLUME=/mnt/ssd/redis # Custom path -# REDIS_VOLUME=my_redis_volume # Pre-created Docker volume -# -# REDIS_VOLUME=./data/redis -# MONGODB_VOLUME=./data/mongodb -# CLICKHOUSE_VOLUME=./data/clickhouse -# MINIO_VOLUME=./data/minio -# SCHEDULER_STARTUP_VOLUME=./data/startup - -# ============================================================================= -# ClickHouse Configuration -# ============================================================================= -# Connection settings (uses service name for internal Docker network) -CLICKHOUSE_URL=http://clickhouse:8123 -CLICKHOUSE_USERNAME=currents - -# Passwords for ClickHouse users (generated by setup.sh) -# Default PASSWORD only used if using the included clickhouse -CLICKHOUSE_DEFAULT_PASSWORD= -# The CURRENTS USER PASSWORD is how all services connect -CLICKHOUSE_CURRENTS_PASSWORD= +# ON_PREM_EMAIL=admin@currents.local diff --git a/on-prem/README.md b/on-prem/README.md index 0c90176..c5a1425 100644 --- a/on-prem/README.md +++ b/on-prem/README.md @@ -24,7 +24,7 @@ docker compose up -d | Profile | Services Included | Use Case | |---------|-------------------|----------| -| `full` | Redis, MongoDB, ClickHouse, MinIO | Running everything locally | +| `full` | Redis, MongoDB, ClickHouse, RustFS | Running everything locally | | `database` | Redis, MongoDB, ClickHouse | External S3-compatible storage | | `cache` | Redis | External MongoDB, ClickHouse, and S3 | @@ -55,7 +55,7 @@ Generates a docker-compose file for a specific profile. Used by `setup.sh` but c ./scripts/generate-compose.sh redis ./scripts/generate-compose.sh mongodb ./scripts/generate-compose.sh clickhouse -./scripts/generate-compose.sh minio +./scripts/generate-compose.sh rustfs # Combine multiple profiles/services ./scripts/generate-compose.sh database storage @@ -65,15 +65,15 @@ Generates a docker-compose file for a specific profile. Used by `setup.sh` but c | Profile | Description | |---------|-------------| -| `full` | All services (redis, mongodb, clickhouse, minio) | +| `full` | All services (redis, mongodb, clickhouse, rustfs) | | `database` | Database services (redis, mongodb, clickhouse) | | `cache` | Cache (redis) | | `analytics` | Analytics (clickhouse) | -| `storage` | Object storage (minio) | +| `storage` | Object storage (rustfs) | | `redis` | Redis only | | `mongodb` | MongoDB only | | `clickhouse` | ClickHouse only | -| `minio` | MinIO only | +| `rustfs` | RustFS only | ### `scripts/generate-secrets.sh` @@ -106,11 +106,24 @@ Utility for generating secrets and keys. Copy `.env.example` to `.env` and configure as needed. The `setup.sh` script does this automatically and generates required secrets. -Key variables: +### Application Variables + - `JWT_SECRET` - Authentication token secret -- `MINIO_SERVER_ACCESS_KEY` / `MINIO_SERVER_SECRET_KEY` - MinIO credentials +- `RUSTFS_ACCESS_KEY` / `RUSTFS_SECRET_KEY` - RustFS credentials - `GITLAB_STATE_SECRET` - GitLab integration key (base64-encoded PEM) +### Compose-Only Variables + +Variables prefixed with `DC_` are used only by docker-compose (not passed to containers): + +- `DC_IMAGE_REPOSITORY` - Docker image repository prefix +- `DC_IMAGE_TAG` - Docker image tag +- `DC_REDIS_VOLUME` - Redis data volume path +- `DC_MONGODB_VOLUME` - MongoDB data volume path +- `DC_CLICKHOUSE_VOLUME` - ClickHouse data volume path +- `DC_RUSTFS_VOLUME` - RustFS data volume path +- `DC_SCHEDULER_STARTUP_VOLUME` - Scheduler startup data volume path + ## Services ### Currents Application @@ -125,7 +138,7 @@ Key variables: - **redis** (port 6379, 8001) - Cache and pub/sub - **mongodb** (port 27017) - Primary database - **clickhouse** (port 8123, 9123) - Analytics database -- **minio** (port 9000, 9001) - S3-compatible object storage +- **rustfs** (port 9000, 9001) - S3-compatible object storage ## Development diff --git a/on-prem/docker-compose.cache.yml b/on-prem/docker-compose.cache.yml index 72bf9bb..3568e12 100644 --- a/on-prem/docker-compose.cache.yml +++ b/on-prem/docker-compose.cache.yml @@ -11,7 +11,7 @@ services: # Currents services depend on redis director: hostname: director - image: ${IMAGE_REPOSITORY:-currents-}director:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}director:${DC_IMAGE_TAG:-dev} expose: - 1234 ports: @@ -32,7 +32,7 @@ services: required: false api: hostname: api - image: ${IMAGE_REPOSITORY:-currents-}api:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}api:${DC_IMAGE_TAG:-dev} expose: - 4000 ports: @@ -54,7 +54,7 @@ services: condition: service_started required: false changestreams-worker: - image: ${IMAGE_REPOSITORY:-currents-}change-streams:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}change-streams:${DC_IMAGE_TAG:-dev} restart: unless-stopped networks: - default @@ -69,7 +69,7 @@ services: condition: service_started required: false write-worker: - image: ${IMAGE_REPOSITORY:-currents-}writer:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}writer:${DC_IMAGE_TAG:-dev} restart: unless-stopped networks: - default @@ -85,7 +85,7 @@ services: condition: service_started required: false scheduler: - image: ${IMAGE_REPOSITORY:-currents-}scheduler:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}scheduler:${DC_IMAGE_TAG:-dev} restart: unless-stopped networks: - default @@ -97,13 +97,13 @@ services: - CLICKHOUSE_PASSWORD=${CLICKHOUSE_CURRENTS_PASSWORD} - RUN_STARTUP_TASKS=true volumes: - - ${SCHEDULER_STARTUP_VOLUME:-./data/startup}:/app/packages/scheduler/dist/.startup + - ${DC_SCHEDULER_STARTUP_VOLUME:-./data/startup}:/app/packages/scheduler/dist/.startup depends_on: redis: condition: service_started required: false webhooks: - image: ${IMAGE_REPOSITORY:-currents-}webhooks:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}webhooks:${DC_IMAGE_TAG:-dev} restart: unless-stopped networks: - default @@ -126,6 +126,6 @@ services: networks: - default volumes: - - ${REDIS_VOLUME:-./data/redis}:/data + - ${DC_REDIS_VOLUME:-./data/redis}:/data networks: default: null diff --git a/on-prem/docker-compose.database.yml b/on-prem/docker-compose.database.yml index 1ab2aad..e34d300 100644 --- a/on-prem/docker-compose.database.yml +++ b/on-prem/docker-compose.database.yml @@ -11,7 +11,7 @@ services: # Currents services depend on redis director: hostname: director - image: ${IMAGE_REPOSITORY:-currents-}director:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}director:${DC_IMAGE_TAG:-dev} expose: - 1234 ports: @@ -38,7 +38,7 @@ services: required: false api: hostname: api - image: ${IMAGE_REPOSITORY:-currents-}api:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}api:${DC_IMAGE_TAG:-dev} expose: - 4000 ports: @@ -66,7 +66,7 @@ services: condition: service_started required: false changestreams-worker: - image: ${IMAGE_REPOSITORY:-currents-}change-streams:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}change-streams:${DC_IMAGE_TAG:-dev} restart: unless-stopped networks: - default @@ -87,7 +87,7 @@ services: condition: service_started required: false write-worker: - image: ${IMAGE_REPOSITORY:-currents-}writer:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}writer:${DC_IMAGE_TAG:-dev} restart: unless-stopped networks: - default @@ -109,7 +109,7 @@ services: condition: service_started required: false scheduler: - image: ${IMAGE_REPOSITORY:-currents-}scheduler:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}scheduler:${DC_IMAGE_TAG:-dev} restart: unless-stopped networks: - default @@ -121,7 +121,7 @@ services: - CLICKHOUSE_PASSWORD=${CLICKHOUSE_CURRENTS_PASSWORD} - RUN_STARTUP_TASKS=true volumes: - - ${SCHEDULER_STARTUP_VOLUME:-./data/startup}:/app/packages/scheduler/dist/.startup + - ${DC_SCHEDULER_STARTUP_VOLUME:-./data/startup}:/app/packages/scheduler/dist/.startup depends_on: clickhouse: condition: service_healthy @@ -133,7 +133,7 @@ services: condition: service_started required: false webhooks: - image: ${IMAGE_REPOSITORY:-currents-}webhooks:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}webhooks:${DC_IMAGE_TAG:-dev} restart: unless-stopped networks: - default @@ -177,7 +177,7 @@ services: - source: clickhouse_users target: /etc/clickhouse-server/users.d/users.xml volumes: - - ${CLICKHOUSE_VOLUME:-./data/clickhouse}:/var/lib/clickhouse + - ${DC_CLICKHOUSE_VOLUME:-./data/clickhouse}:/var/lib/clickhouse mongodb: hostname: mongodb image: mongo:7.0.0 @@ -197,7 +197,7 @@ services: timeout: 5s retries: 10 volumes: - - ${MONGODB_VOLUME:-./data/mongodb}:/data/db + - ${DC_MONGODB_VOLUME:-./data/mongodb}:/data/db redis: hostname: redis image: redis/redis-stack-server:7.4.0-v8 @@ -207,7 +207,7 @@ services: networks: - default volumes: - - ${REDIS_VOLUME:-./data/redis}:/data + - ${DC_REDIS_VOLUME:-./data/redis}:/data networks: default: null configs: diff --git a/on-prem/docker-compose.full.yml b/on-prem/docker-compose.full.yml index 7a521b2..3556012 100644 --- a/on-prem/docker-compose.full.yml +++ b/on-prem/docker-compose.full.yml @@ -1,17 +1,17 @@ # Generated by: ./scripts/generate-compose.sh -# Profile: full - All services (redis, mongodb, clickhouse, minio) +# Profile: full - All services (redis, mongodb, clickhouse, rustfs) name: currents_full # This core Currents application services -# Redis service configuration -# Almost always included - provides caching and pub/sub messaging +# RustFS service configuration +# S3-compatible object storage - usually excluded when using external S3/cloud storage services: # Currents services depend on redis director: hostname: director - image: ${IMAGE_REPOSITORY:-currents-}director:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}director:${DC_IMAGE_TAG:-dev} expose: - 1234 ports: @@ -38,7 +38,7 @@ services: required: false api: hostname: api - image: ${IMAGE_REPOSITORY:-currents-}api:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}api:${DC_IMAGE_TAG:-dev} expose: - 4000 ports: @@ -66,7 +66,7 @@ services: condition: service_started required: false changestreams-worker: - image: ${IMAGE_REPOSITORY:-currents-}change-streams:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}change-streams:${DC_IMAGE_TAG:-dev} restart: unless-stopped networks: - default @@ -87,7 +87,7 @@ services: condition: service_started required: false write-worker: - image: ${IMAGE_REPOSITORY:-currents-}writer:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}writer:${DC_IMAGE_TAG:-dev} restart: unless-stopped networks: - default @@ -109,7 +109,7 @@ services: condition: service_started required: false scheduler: - image: ${IMAGE_REPOSITORY:-currents-}scheduler:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}scheduler:${DC_IMAGE_TAG:-dev} restart: unless-stopped networks: - default @@ -121,7 +121,7 @@ services: - CLICKHOUSE_PASSWORD=${CLICKHOUSE_CURRENTS_PASSWORD} - RUN_STARTUP_TASKS=true volumes: - - ${SCHEDULER_STARTUP_VOLUME:-./data/startup}:/app/packages/scheduler/dist/.startup + - ${DC_SCHEDULER_STARTUP_VOLUME:-./data/startup}:/app/packages/scheduler/dist/.startup depends_on: clickhouse: condition: service_healthy @@ -133,7 +133,7 @@ services: condition: service_started required: false webhooks: - image: ${IMAGE_REPOSITORY:-currents-}webhooks:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}webhooks:${DC_IMAGE_TAG:-dev} restart: unless-stopped networks: - default @@ -177,20 +177,7 @@ services: - source: clickhouse_users target: /etc/clickhouse-server/users.d/users.xml volumes: - - ${CLICKHOUSE_VOLUME:-./data/clickhouse}:/var/lib/clickhouse - minio: - image: bitnamilegacy/minio:2025.2.28 - volumes: - - ${MINIO_VOLUME:-./data/minio}:/bitnami/minio/data - ports: - - 9000:9000 - - 9001:9001 - env_file: - - .env - environment: - MINIO_DEFAULT_BUCKETS: currents - MINIO_ROOT_USER: ${MINIO_SERVER_ACCESS_KEY} - MINIO_ROOT_PASSWORD: ${MINIO_SERVER_SECRET_KEY} + - ${DC_CLICKHOUSE_VOLUME:-./data/clickhouse}:/var/lib/clickhouse mongodb: hostname: mongodb image: mongo:7.0.0 @@ -210,7 +197,7 @@ services: timeout: 5s retries: 10 volumes: - - ${MONGODB_VOLUME:-./data/mongodb}:/data/db + - ${DC_MONGODB_VOLUME:-./data/mongodb}:/data/db redis: hostname: redis image: redis/redis-stack-server:7.4.0-v8 @@ -220,7 +207,42 @@ services: networks: - default volumes: - - ${REDIS_VOLUME:-./data/redis}:/data + - ${DC_REDIS_VOLUME:-./data/redis}:/data + rustfs: + hostname: rustfs + image: rustfs/rustfs:1.0.0-alpha.79 + ports: + - 9000:9000 # S3 API port + - 9001:9001 # Web console port + environment: + RUSTFS_CONSOLE_ENABLE: "true" + RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY} + RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY} + env_file: + - .env + networks: + - default + volumes: + - ${DC_RUSTFS_VOLUME:-./data/rustfs}:/data # /data is RustFS's default storage location + healthcheck: + test: ['CMD-SHELL', 'curl -f http://localhost:9000/health || exit 1'] + interval: 2s + timeout: 5s + retries: 10 + rustfs-init: + image: amazon/aws-cli:latest + depends_on: + rustfs: + condition: service_healthy + environment: + AWS_ACCESS_KEY_ID: ${RUSTFS_ACCESS_KEY} + AWS_SECRET_ACCESS_KEY: ${RUSTFS_SECRET_KEY} + AWS_DEFAULT_REGION: us-east-1 + entrypoint: > + sh -c "aws --endpoint-url=http://rustfs:9000 s3 mb s3://currents || true" + + networks: + - default networks: default: null configs: diff --git a/on-prem/scripts/generate-compose.sh b/on-prem/scripts/generate-compose.sh index 79e61bd..3c533c7 100755 --- a/on-prem/scripts/generate-compose.sh +++ b/on-prem/scripts/generate-compose.sh @@ -9,7 +9,7 @@ # ./generate-compose.sh full # All services # ./generate-compose.sh database # redis, mongodb, clickhouse # ./generate-compose.sh cache # redis only -# ./generate-compose.sh database storage # All data services including minio +# ./generate-compose.sh database storage # All data services including rustfs # # Requirements: # - yq (https://github.com/mikefarah/yq) @@ -30,15 +30,15 @@ fi # Profile descriptions get_profile_desc() { case "$1" in - full) echo "All services (redis, mongodb, clickhouse, minio)" ;; + full) echo "All services (redis, mongodb, clickhouse, rustfs)" ;; database) echo "Database services (redis, mongodb, clickhouse)" ;; cache) echo "Cache (redis)" ;; analytics) echo "Analytics (clickhouse)" ;; - storage) echo "Object storage (minio)" ;; + storage) echo "Object storage (rustfs)" ;; redis) echo "Redis" ;; mongodb) echo "MongoDB" ;; clickhouse) echo "ClickHouse" ;; - minio) echo "MinIO" ;; + rustfs) echo "RustFS" ;; *) echo "$1" ;; esac } @@ -46,15 +46,15 @@ get_profile_desc() { # Profile to services mapping get_profile_services() { case "$1" in - full) echo "redis mongodb clickhouse minio" ;; + full) echo "redis mongodb clickhouse rustfs" ;; database) echo "redis mongodb clickhouse" ;; cache) echo "redis" ;; analytics) echo "clickhouse" ;; - storage) echo "minio" ;; + storage) echo "rustfs" ;; redis) echo "redis" ;; mongodb) echo "mongodb" ;; clickhouse) echo "clickhouse" ;; - minio) echo "minio" ;; + rustfs) echo "rustfs" ;; *) echo "" ;; esac } @@ -64,7 +64,7 @@ show_usage() { echo "Usage: $0 [profile...]" echo "" echo "Available profiles (all include the currents services):" - for profile in full database cache analytics storage redis mongodb clickhouse minio; do + for profile in full database cache analytics storage redis mongodb clickhouse rustfs; do printf " %-12s - %s\n" "$profile" "$(get_profile_desc $profile)" done } diff --git a/on-prem/scripts/setup.sh b/on-prem/scripts/setup.sh index b155c0a..29ef614 100755 --- a/on-prem/scripts/setup.sh +++ b/on-prem/scripts/setup.sh @@ -53,10 +53,10 @@ setup_env_file() { sed -i.bak "s|^JWT_SECRET=.*|JWT_SECRET=$JWT_SECRET|" "$ENV_FILE" fi - # Generate MINIO_SERVER_SECRET_KEY - MINIO_SECRET=$("$SCRIPT_DIR/generate-secrets.sh" token 32) - if grep -q "^MINIO_SERVER_SECRET_KEY=" "$ENV_FILE"; then - sed -i.bak "s|^MINIO_SERVER_SECRET_KEY=.*|MINIO_SERVER_SECRET_KEY=$MINIO_SECRET|" "$ENV_FILE" + # Generate RUSTFS_SECRET_KEY + RUSTFS_SECRET=$("$SCRIPT_DIR/generate-secrets.sh" token 32) + if grep -q "^RUSTFS_SECRET_KEY=" "$ENV_FILE"; then + sed -i.bak "s|^RUSTFS_SECRET_KEY=.*|RUSTFS_SECRET_KEY=$RUSTFS_SECRET|" "$ENV_FILE" fi # Generate CLICKHOUSE_DEFAULT_PASSWORD @@ -92,7 +92,7 @@ setup_env_file() { # ============================================================================= echo -e "${YELLOW}Select a configuration profile:${NC}" echo "" -echo " 1) full - All services (redis, mongodb, clickhouse, minio)" +echo " 1) full - All services (redis, mongodb, clickhouse, rustfs)" echo " Use when running everything locally" echo "" echo " 2) database - Database services (redis, mongodb, clickhouse)" @@ -142,10 +142,10 @@ case $choice in SERVICES="$SERVICES clickhouse" fi - read -p "Include MinIO (S3-compatible storage)? [y/N]: " include_minio - include_minio=${include_minio:-N} - if [[ $include_minio =~ ^[Yy] ]]; then - SERVICES="$SERVICES minio" + read -p "Include RustFS (S3-compatible storage)? [y/N]: " include_rustfs + include_rustfs=${include_rustfs:-N} + if [[ $include_rustfs =~ ^[Yy] ]]; then + SERVICES="$SERVICES rustfs" fi if [ -z "$SERVICES" ]; then diff --git a/on-prem/templates/compose.clickhouse.yml b/on-prem/templates/compose.clickhouse.yml index afe5af1..7d1e591 100644 --- a/on-prem/templates/compose.clickhouse.yml +++ b/on-prem/templates/compose.clickhouse.yml @@ -55,7 +55,7 @@ services: - source: clickhouse_users target: /etc/clickhouse-server/users.d/users.xml volumes: - - ${CLICKHOUSE_VOLUME:-./data/clickhouse}:/var/lib/clickhouse + - ${DC_CLICKHOUSE_VOLUME:-./data/clickhouse}:/var/lib/clickhouse # Currents services depend on clickhouse director: diff --git a/on-prem/templates/compose.currents.yml b/on-prem/templates/compose.currents.yml index b227671..14eb2c0 100644 --- a/on-prem/templates/compose.currents.yml +++ b/on-prem/templates/compose.currents.yml @@ -3,7 +3,7 @@ services: director: hostname: director - image: ${IMAGE_REPOSITORY:-currents-}director:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}director:${DC_IMAGE_TAG:-dev} expose: - 1234 ports: @@ -21,7 +21,7 @@ services: api: hostname: api - image: ${IMAGE_REPOSITORY:-currents-}api:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}api:${DC_IMAGE_TAG:-dev} expose: - 4000 ports: @@ -40,7 +40,7 @@ services: - CURRENTS_RECORD_API_URL=http://localhost:1234 changestreams-worker: - image: ${IMAGE_REPOSITORY:-currents-}change-streams:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}change-streams:${DC_IMAGE_TAG:-dev} restart: unless-stopped networks: - default @@ -52,7 +52,7 @@ services: - CLICKHOUSE_PASSWORD=${CLICKHOUSE_CURRENTS_PASSWORD} write-worker: - image: ${IMAGE_REPOSITORY:-currents-}writer:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}writer:${DC_IMAGE_TAG:-dev} restart: unless-stopped networks: - default @@ -65,7 +65,7 @@ services: - CLICKHOUSE_PASSWORD=${CLICKHOUSE_CURRENTS_PASSWORD} scheduler: - image: ${IMAGE_REPOSITORY:-currents-}scheduler:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}scheduler:${DC_IMAGE_TAG:-dev} restart: unless-stopped networks: - default @@ -77,10 +77,10 @@ services: - CLICKHOUSE_PASSWORD=${CLICKHOUSE_CURRENTS_PASSWORD} - RUN_STARTUP_TASKS=true volumes: - - ${SCHEDULER_STARTUP_VOLUME:-./data/startup}:/app/packages/scheduler/dist/.startup + - ${DC_SCHEDULER_STARTUP_VOLUME:-./data/startup}:/app/packages/scheduler/dist/.startup webhooks: - image: ${IMAGE_REPOSITORY:-currents-}webhooks:${IMAGE_TAG:-dev} + image: ${DC_IMAGE_REPOSITORY:-currents-}webhooks:${DC_IMAGE_TAG:-dev} restart: unless-stopped networks: - default diff --git a/on-prem/templates/compose.minio.yml b/on-prem/templates/compose.minio.yml deleted file mode 100644 index e342a10..0000000 --- a/on-prem/templates/compose.minio.yml +++ /dev/null @@ -1,17 +0,0 @@ -# MinIO service configuration -# S3-compatible object storage - usually excluded when using external S3/cloud storage - -services: - minio: - image: bitnamilegacy/minio:2025.2.28 - volumes: - - ${MINIO_VOLUME:-./data/minio}:/bitnami/minio/data - ports: - - 9000:9000 - - 9001:9001 - env_file: - - .env - environment: - MINIO_DEFAULT_BUCKETS: currents - MINIO_ROOT_USER: ${MINIO_SERVER_ACCESS_KEY} - MINIO_ROOT_PASSWORD: ${MINIO_SERVER_SECRET_KEY} diff --git a/on-prem/templates/compose.mongodb.yml b/on-prem/templates/compose.mongodb.yml index e7970aa..4ac23f9 100644 --- a/on-prem/templates/compose.mongodb.yml +++ b/on-prem/templates/compose.mongodb.yml @@ -21,7 +21,7 @@ services: timeout: 5s retries: 10 volumes: - - ${MONGODB_VOLUME:-./data/mongodb}:/data/db + - ${DC_MONGODB_VOLUME:-./data/mongodb}:/data/db # Currents services depend on mongodb director: diff --git a/on-prem/templates/compose.redis.yml b/on-prem/templates/compose.redis.yml index 838e37a..4f40ac0 100644 --- a/on-prem/templates/compose.redis.yml +++ b/on-prem/templates/compose.redis.yml @@ -11,7 +11,7 @@ services: networks: - default volumes: - - ${REDIS_VOLUME:-./data/redis}:/data + - ${DC_REDIS_VOLUME:-./data/redis}:/data # Currents services depend on redis director: diff --git a/on-prem/templates/compose.rustfs.yml b/on-prem/templates/compose.rustfs.yml new file mode 100644 index 0000000..54e39d5 --- /dev/null +++ b/on-prem/templates/compose.rustfs.yml @@ -0,0 +1,39 @@ +# RustFS service configuration +# S3-compatible object storage - usually excluded when using external S3/cloud storage + +services: + rustfs: + hostname: rustfs + image: rustfs/rustfs:1.0.0-alpha.79 + ports: + - 9000:9000 # S3 API port + - 9001:9001 # Web console port + environment: + RUSTFS_CONSOLE_ENABLE: "true" + RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY} + RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY} + env_file: + - .env + networks: + - default + volumes: + - ${DC_RUSTFS_VOLUME:-./data/rustfs}:/data # /data is RustFS's default storage location + healthcheck: + test: ['CMD-SHELL', 'curl -f http://localhost:9000/health || exit 1'] + interval: 2s + timeout: 5s + retries: 10 + + rustfs-init: + image: amazon/aws-cli:latest + depends_on: + rustfs: + condition: service_healthy + environment: + AWS_ACCESS_KEY_ID: ${RUSTFS_ACCESS_KEY} + AWS_SECRET_ACCESS_KEY: ${RUSTFS_SECRET_KEY} + AWS_DEFAULT_REGION: us-east-1 + entrypoint: > + sh -c "aws --endpoint-url=http://rustfs:9000 s3 mb s3://currents || true" + networks: + - default