Титаев Андрей и Сафин Руслан

.gitignore

@@ -1,3 +1,5 @@
+shared
+
 # Binaries for programs and plugins
 *.exe
 *.exe~

docker-compose.yml

@@ -1,11 +1,12 @@
-version: '3.8'
-
 # Включаем все сервисы из отдельных файлов
 include:
   - docker-compose/mysql.yml
   - docker-compose/event-ingest.yml
   - docker-compose/shard-splitter.yml
   - docker-compose/shard-workers.yml
-  - docker-compose/balance-daemon.yml
-  - docker-compose/cache-service.yml
-  - docker-compose/tests.yml
+  - docker-compose/nginx.yml
+  - docker-compose/pipeline-api.yml
+  - docker-compose/mysql-sender.yml
+#  - docker-compose/balance-daemon.yml
+#  - docker-compose/cache-service.yml
+#  - docker-compose/tests.yml

docker-compose/.gitignore

@@ -0,0 +1 @@
+logs

docker-compose/event-ingest.yml

@@ -8,3 +8,5 @@ services:
       - "8080:8080"
     volumes:
       - ../shared:/shared
+    networks:
+      - network

docker-compose/mysql-sender.yml

@@ -0,0 +1,22 @@
+services:
+  mysql-sender:
+    build:
+      context: ../services/mysql-sender
+      dockerfile: Dockerfile
+    container_name: shop-mysql-sender
+    environment:
+      - PIPELINE_API_URL=http://pipeline-apid:8082/api/v1/pipeline
+      - MYSQL_SHARD_0_HOST=mysql-shard-0
+      - MYSQL_SHARD_1_HOST=mysql-shard-1
+      - MYSQL_PORT=3306
+      - MYSQL_USER=shop_user
+      - MYSQL_PASSWORD=shop_password
+      - MYSQL_DATABASE_PREFIX=shop_shard_
+      - POLL_INTERVAL=5
+    volumes:
+      - ../shared:/shared
+    networks:
+      - network
+    depends_on:
+      - pipeline-apid-impl
+    restart: unless-stopped

docker-compose/mysql.yml

@@ -13,6 +13,8 @@ services:
       - ../db/ddl.sql:/docker-entrypoint-initdb.d/01-ddl.sql
       - mysql_shard_0_data:/var/lib/mysql
     command: --default-authentication-plugin=mysql_native_password
+    networks:
+      - network
 
   mysql-shard-1:
     image: mysql:8.0
@@ -28,6 +30,8 @@ services:
       - ../db/ddl.sql:/docker-entrypoint-initdb.d/01-ddl.sql
       - mysql_shard_1_data:/var/lib/mysql
     command: --default-authentication-plugin=mysql_native_password
+    networks:
+      - network
 
   mysql-pipeline:
     image: mysql:8.0
@@ -43,6 +47,8 @@ services:
       - ../db/pipeline_ddl.sql:/docker-entrypoint-initdb.d/01-pipeline-ddl.sql
       - mysql_pipeline_data:/var/lib/mysql
     command: --default-authentication-plugin=mysql_native_password
+    networks:
+      - network
 volumes:
   mysql_shard_0_data:
   mysql_shard_1_data:

docker-compose/nginx.conf

@@ -0,0 +1,80 @@
+worker_processes auto;
+error_log /var/log/nginx/error.log warn;
+pid /var/run/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log /var/log/nginx/access.log main;
+
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 65;
+    types_hash_max_size 2048;
+
+    upstream event_ingest {
+        server event-ingest:8080;
+    }
+
+    upstream pipeline_apid_impl {
+        server pipeline-apid-impl:8083;
+    }
+
+    server {
+        listen 8082;
+        server_name _;
+        location /health {
+            proxy_pass http://event_ingest;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_read_timeout 300;
+            proxy_connect_timeout 300;
+            proxy_send_timeout 300;
+        }
+
+        location /ready {
+            proxy_pass http://event_ingest;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_read_timeout 300;
+            proxy_connect_timeout 300;
+            proxy_send_timeout 300;
+        }
+
+        location /api/v1/events/ {
+            proxy_pass http://event_ingest;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_read_timeout 300;
+            proxy_connect_timeout 300;
+            proxy_send_timeout 300;
+        }
+
+        location /api/v1/pipeline/ {
+            proxy_pass http://pipeline_apid_impl;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_read_timeout 300;
+            proxy_connect_timeout 300;
+            proxy_send_timeout 300;
+        }
+    }
+}

docker-compose/nginx.yml

@@ -0,0 +1,20 @@
+services:
+  pipeline-apid:
+    image: nginx:1.23-alpine
+    container_name: shop-nginx
+    ports:
+      - "8082:8082"
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+      - ./logs:/var/log/nginx
+      - ../shared:/shared:ro
+    networks:
+      - network
+    depends_on:
+      - event-ingest
+      - pipeline-apid-impl
+      - shard-splitter
+      - mysql-pipeline
+      - mysql-shard-0
+      - mysql-shard-1
+    restart: unless-stopped

docker-compose/pipeline-api.yml

@@ -0,0 +1,23 @@
+services:
+  pipeline-apid-impl:
+    build:
+      context: ../services/pipeline-apid
+      dockerfile: Dockerfile
+    container_name: shop-pipeline-apid-impl
+    ports:
+      - "8083:8083"
+    environment:
+      - MYSQL_HOST=mysql-pipeline
+      - MYSQL_PORT=3306
+      - MYSQL_USER=shop_user
+      - MYSQL_PASSWORD=shop_password
+      - MYSQL_DATABASE=pipeline_db
+    volumes:
+      - ../shared:/shared
+    networks:
+      - network
+    restart: unless-stopped
+
+networks:
+  network:
+    driver: bridge

docker-compose/shard-splitter.yml

@@ -6,3 +6,6 @@ services:
     container_name: shop-shard-splitter
     volumes:
       - ../shared:/shared
+    networks:
+      - network
+

docker-compose/shard-workers.yml

@@ -12,7 +12,8 @@ services:
       - ../shared:/shared
     depends_on:
       - mysql-shard-0
-      - balance-daemon
+    networks:
+      - network
 
   shard-worker-1:
     build:
@@ -27,4 +28,5 @@ services:
       - ../shared:/shared
     depends_on:
       - mysql-shard-1
-      - balance-daemon
+    networks:
+      - network

services/event-ingest/Dockerfile

@@ -2,33 +2,35 @@ FROM golang:1.21-alpine AS builder
 
 WORKDIR /app
 
-# Копируем go.mod и go.sum
 COPY go.mod go.sum ./
 RUN go mod download
 
-# Копируем исходный код
 COPY . .
-
-# Синхронизируем зависимости и go.sum
 RUN go mod tidy
 
-# Собираем приложение
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o main ./cmd/event-ingest
 
 FROM alpine:latest
 
-RUN apk --no-cache add ca-certificates bash
+RUN apk --no-cache add ca-certificates
 WORKDIR /root/
 
-# Копируем бинарный файл
 COPY --from=builder /app/main .
-
-# Копируем конфигурацию
 COPY --from=builder /app/config ./config
 
-# Обертка для подготовки директории и очистки при остановке
-RUN echo '#!/usr/bin/env bash\n\nset -e\nSERVICE_DIR="/shared/events"\nmkdir -p "$SERVICE_DIR"\ncleanup() {\n  echo "Cleaning $SERVICE_DIR ..."\n  rm -rf "$SERVICE_DIR"/* || true\n}\ntrap cleanup EXIT TERM INT\nexec /root/main\n' > /root/entrypoint.sh && chmod +x /root/entrypoint.sh
+# Создаем скрипт построчно
+RUN printf '#!/bin/sh\n' > /root/entrypoint.sh && \
+    printf 'set -e\n' >> /root/entrypoint.sh && \
+    printf 'SERVICE_DIR="/shared/events"\n' >> /root/entrypoint.sh && \
+    printf 'mkdir -p "$SERVICE_DIR"\n' >> /root/entrypoint.sh && \
+    printf 'cleanup() {\n' >> /root/entrypoint.sh && \
+    printf '  echo "Cleaning $SERVICE_DIR ..."\n' >> /root/entrypoint.sh && \
+    printf '  rm -rf "$SERVICE_DIR"/* || true\n' >> /root/entrypoint.sh && \
+    printf '}\n' >> /root/entrypoint.sh && \
+    printf 'trap cleanup EXIT TERM INT\n' >> /root/entrypoint.sh && \
+    printf 'exec /root/main\n' >> /root/entrypoint.sh && \
+    chmod +x /root/entrypoint.sh
 
 EXPOSE 8080
 
-CMD ["/root/entrypoint.sh"]
+CMD ["/bin/sh", "/root/entrypoint.sh"]

services/event-ingest/config/app.toml

@@ -9,7 +9,7 @@ idle_timeout = 120
 max_payload_size = 1048576
 validation_timeout = 5
 max_batch_size = 1000
-generation_interval = 10  # секунд между генерациями
+generation_interval = 5  # секунд между генерациями
 events_per_batch = 500    # максимальное количество событий за раз
 
 [logging]

services/event-ingest/internal/models/event.go

@@ -2,7 +2,6 @@ package models
 
 import (
 	"encoding/json"
-	"strconv"
 	"time"
 )
 

services/mysql-sender/Dockerfile

@@ -0,0 +1,25 @@
+# Используем официальный Python образ с нужной версией
+FROM python:3.11-slim
+
+# Устанавливаем системные зависимости (если нужны)
+RUN apt-get update && apt-get install -y \
+    gcc \
+    && rm -rf /var/lib/apt/lists/*
+
+# Создаем рабочую директорию
+WORKDIR /app
+
+# Копируем зависимости
+COPY requirements.txt .
+
+# Устанавливаем Python-зависимости
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Копируем исходный код
+COPY . .
+
+# Опционально: запускаем тесты
+# RUN pytest tests/ -v
+
+# Указываем команду запуска
+CMD ["python", "main.py"]