From 42f4b644c9aba01dccc17e47701bd75b4024b11b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Thu, 30 Apr 2026 14:54:42 -0300 Subject: [PATCH 1/2] fix: endpoint fixes to work on production & dev at the same time --- docker-compose.yaml | 8 ++++---- env.tpl | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index f3a44f2..2c26b50 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -48,12 +48,12 @@ x-airflow-common: AIRFLOW__CORE__AUTH_MANAGER: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager AIRFLOW__CORE__LOAD_EXAMPLES: false AIRFLOW__CORE__DEFAULT_TIMEZONE: America/Sao_Paulo - AIRFLOW__CORE__INTERNAL_API_URL: http://localhost:${AIRFLOW_PORT} + AIRFLOW__CORE__INTERNAL_API_URL: http://alertflow_webserver:${AIRFLOW_PORT}/alertflow AIRFLOW__CORE__INTERNAL_API_SECRET_KEY: ${AIRFLOW__CORE__INTERNAL_API_SECRET_KEY} - AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:${AIRFLOW_PORT}/execution/' + AIRFLOW__CORE__EXECUTION_API_SERVER_URL: http://alertflow_webserver:${AIRFLOW_PORT}/alertflow/execution AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: ${AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION:-true} - AIRFLOW__CLI__ENDPOINT_URL: http://localhost:${AIRFLOW_PORT}/alertflow - AIRFLOW__WEBSERVER__BASE_URL: http://localhost:${AIRFLOW_PORT}/alertflow + AIRFLOW__CLI__ENDPOINT_URL: http://alertflow_webserver:${AIRFLOW_PORT}/alertflow + AIRFLOW__WEBSERVER__BASE_URL: ${AIRFLOW__WEBSERVER__BASE_URL} AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY} AIRFLOW__WEBSERVER__WEB_SERVER_HOST: 0.0.0.0 AIRFLOW__WEBSERVER__WEB_SERVER_PORT: ${AIRFLOW_PORT} diff --git a/env.tpl b/env.tpl index 1ea4175..7a6e532 100644 --- a/env.tpl +++ b/env.tpl @@ -39,6 +39,7 @@ AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=${AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_C AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK=${AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK:-false} AIRFLOW__API_AUTH__JWT_SECRET=${AIRFLOW__API_AUTH__JWT_SECRET} AIRFLOW__API_AUTH__JWT_ISSUER=${AIRFLOW__API_AUTH__JWT_ISSUER:-airflow} +AIRFLOW__WEBSERVER__BASE_URL=${AIRFLOW__WEBSERVER__BASE_URL:-http://localhost:8081/alertflow} AIRFLOW__WEBSERVER__SECRET_KEY=${AIRFLOW__WEBSERVER__SECRET_KEY} AIRFLOW__LOGGING__LOGGING_LEVEL=${AIRFLOW__LOGGING__LOGGING_LEVEL:-INFO} AIRFLOW__SENTRY__SENTRY_ON=${AIRFLOW__SENTRY__SENTRY_ON:-true} From 0d132bb39d3b5a0a4dae00f5de386b2b4b9a0637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Thu, 30 Apr 2026 15:11:43 -0300 Subject: [PATCH 2/2] chore: add README.md --- README.md | 204 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) diff --git a/README.md b/README.md index e69de29..5da55d3 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,204 @@ +# AlertFlow - InfoDengue ETL Pipeline + +Airflow-based ETL system for the InfoDengue epidemiological surveillance project. + +## Prerequisites + +- Docker & Docker Compose +- GNU Make +- Python 3.14 (for local development) +- Conda/Mamba + +## Quick Start + +### 1. Environment Setup + +Create `.env` file in the project root and populate the variables: + +```bash +envsubst < .env.tpl > .env +``` + +Generate Fernet key: +```bash +python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" +``` + +### 2. Build Containers + +```bash +docker compose build +``` + +The build process: +- Uses multi-stage Dockerfile +- Installs system dependencies (git, postgres-client, build tools) +- Installs Python dependencies via Poetry +- Configures Airflow with Celery executor + +### 3. Start Services + +```bash +docker compose up -d +``` + +Services started: +- **postgres** - Metadata database (port 5432) +- **redis** - Message broker (port 6379) +- **airflow-webserver** - API server (port 8080) +- **airflow-scheduler** - DAG scheduler +- **airflow-worker** - Celery worker +- **airflow-dag-processor** - DAG file processor +- **airflow-triggerer** - Deferrable task triggerer + +### 4. Initialize Airflow + +On first run, the `airflow-init` service: +1. Creates admin user +2. Runs database migrations +3. Sets up connections and variables + +Check initialization: +```bash +docker compose logs airflow-init +``` + +### 5. Access Airflow UI + +Open: http://localhost:${AIRFLOW_PORT}/alertflow + +Default credentials (set in `.env`): +- Username: (set via `_AIRFLOW_WWW_USER_USERNAME`) +- Password: (set via `_AIRFLOW_WWW_USER_PASSWORD`) + +## Development Workflow + +### Project Structure + +``` +AlertFlow/ +├── alertflow/ +│ ├── dags/ # Airflow DAG definitions +│ ├── plugins/ # Custom plugins +│ └── logs/ # Task logs +├── docker/ +│ ├── compose.yaml # Main compose file +│ ├── compose-dev.yaml # Development overrides +│ └── Dockerfile # Container image definition +├── pyproject.toml # Python dependencies (Poetry) +├── Makefile # Build/test automation +└── .env # Environment variables (not committed) +``` + +### Adding DAGs + +Place DAG files in `alertflow/dags/`: +- Changes detected automatically (30s interval) +- No container restart needed +- View parsed DAGs at http://localhost:${AIRFLOW_PORT}/alertflow + +### Running CLI Commands + +```bash +./airflow.sh dags list +``` + +### Viewing Logs + +```bash +# All services +docker compose logs -f + +# Specific service +docker compose logs -f airflow-webserver +``` + +### Restarting Services + +```bash +# All services +docker compose restart +# or: +docker compose down && docker compose up -d +``` + +## Configuration + +### Airflow Config + +Override any config via environment variables in `docker-compose.yaml`: + +```yaml +environment: + AIRFLOW__CORE__EXECUTOR: CeleryExecutor + AIRFLOW__CORE__LOAD_EXAMPLES: 'false' +``` + +## Common Tasks + +### Database Migrations + +```bash +docker compose run --rm airflow-cli db migrate +``` + +### Create Admin User + +```bash +docker compose run --rm airflow-cli users create \ + --username admin \ + --firstname Admin \ + --lastname User \ + --role Admin \ + --email admin@example.com \ + --password admin +``` + +## Troubleshooting + +### Port Already in Use + +```bash +# Change port in .env +echo "AIRFLOW_PORT=8081" >> .env +docker compose -f docker-compose.yaml down +docker compose -f docker-compose.yaml up -d +``` + +### Database Connection Issues + +```bash +# Check postgres health +docker compose exec postgres pg_isready -U airflow + +# Reset database (WARNING: destroys data) +docker compose down -v +docker volume rm alertflow_postgres-db-volume +docker compose up -d +``` + +### View Container Resource Usage + +```bash +docker stats +``` + +## Make Targets + +| Target | Description | +|--------|-------------| +| `make build` | Build all container images | +| `make up` | Start all services in background | +| `make down` | Stop all services | +| `make restart` | Restart all services | +| `make logs` | Tail logs from all services | + +## InfoDengue Specific + +This setup is tailored for the InfoDengue ETL pipeline: +- Processes epidemiological data for dengue, zika, chikungunya +- Integrates with climate data APIs (COPERNICUS) +- Uses geospatial analysis for disease mapping +- Outputs feed the InfoDengue dashboard + +For more details on the ETL pipeline, see `alertflow/dags/`.