From 8cb0579d34900bf900e4bce8923f60146e4e38c4 Mon Sep 17 00:00:00 2001 From: Mark Vrijlandt Date: Tue, 7 Apr 2026 20:46:39 +0200 Subject: [PATCH 1/8] use postgres for time series storage --- .env.template | 4 ++ .vscode/launch.json | 17 ++++++ docker-compose.override.dev.yml | 22 ++++---- docker-compose.override.setup.yml | 2 +- docker-compose.yml | 19 +++---- example_sdk_client/optimizer.py | 14 ++--- postgres/postgres-init.sql | 69 +++++++++++++++++++++--- scripts/setup.ps1 | 26 ++++----- scripts/setup.sh | 16 +++--- scripts/start-dev.sh | 2 +- scripts/start.sh | 2 +- scripts/start_postgres_in_dev_mode.sh | 4 +- system_tests/docker-compose.override.yml | 4 +- track_memory.sh | 4 ++ 14 files changed, 142 insertions(+), 63 deletions(-) create mode 100644 .vscode/launch.json create mode 100755 track_memory.sh diff --git a/.env.template b/.env.template index 8478243..3659dc5 100644 --- a/.env.template +++ b/.env.template @@ -3,6 +3,10 @@ POSTGRES_ROOT_PASSWORD=1234 POSTGRES_DEV_PORT=6432 POSTGRES_ORCHESTRATOR_USER_NAME=omotes_orchestrator POSTGRES_ORCHESTRATOR_USER_PASSWORD=somepass3 +POSTGRES_TIMESERIES_RW_USER_NAME=omotes_timeseries_rw +POSTGRES_TIMESERIES_RW_USER_PASSWORD=somepass4 +POSTGRES_TIMESERIES_RO_USER_NAME=omotes_timeseries_ro +POSTGRES_TIMESERIES_RO_USER_PASSWORD=somepass5 RABBITMQ_PORT=5672 RABBITMQ_MANAGEMENT_PORT=15672 diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..92f667f --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Run optimizer.py", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/example_sdk_client/optimizer.py", + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/example_sdk_client", + "justMyCode": true, + "env": { + "PYTHONPATH": "${workspaceFolder}/example_sdk_client" + } + } + ] +} \ No newline at end of file diff --git a/docker-compose.override.dev.yml b/docker-compose.override.dev.yml index 301455f..0bc04ad 100644 --- a/docker-compose.override.dev.yml +++ b/docker-compose.override.dev.yml @@ -1,10 +1,10 @@ services: - orchestrator: - image: omotes-orchestrator-dev - build: - context: .. - dockerfile: orchestrator/dev.Dockerfile +# orchestrator: +# image: omotes-orchestrator-dev +# build: +# context: .. +# dockerfile: orchestrator/dev.Dockerfile grow_worker_optimizer: &grow_worker image: omotes-worker-mesido-dev @@ -27,12 +27,12 @@ services: grow_worker_gurobi: <<: *grow_worker - omotes_simulator_worker: - image: omotes-worker-simulator-dev - build: - context: .. - dockerfile: simulator-worker/dev.Dockerfile +# omotes_simulator_worker: +# image: omotes-worker-simulator-dev +# build: +# context: .. +# dockerfile: simulator-worker/dev.Dockerfile - orchestrator_postgres_db: + omotes_postgres_db: ports: - "${POSTGRES_DEV_PORT}:5432" diff --git a/docker-compose.override.setup.yml b/docker-compose.override.setup.yml index 2d2e023..03f2702 100644 --- a/docker-compose.override.setup.yml +++ b/docker-compose.override.setup.yml @@ -18,6 +18,6 @@ services: INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD: ${INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD} INFLUXDB_PORT: ${INFLUXDB_PORT} - orchestrator_postgres_db: + omotes_postgres_db: volumes: - "${POSTGRES_INIT_FILE:-./postgres/postgres-init.sql}:/setup/init.sql:ro" diff --git a/docker-compose.yml b/docker-compose.yml index df64bb4..511641a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,7 +60,7 @@ services: - ./influxdb/influxdb.conf:/etc/influxdb/influxdb.conf:ro - influxdb_storage:/var/lib/influxdb - orchestrator_postgres_db: &orchestrator_postgres_db + omotes_postgres_db: &omotes_postgres_db image: postgres:15.4 restart: unless-stopped volumes: @@ -69,7 +69,6 @@ services: - omotes environment: PGDATA: /var/lib/postgresql/data - POSTGRES_DB: omotes_jobs POSTGRES_USER: ${POSTGRES_ROOT_USER} POSTGRES_PASSWORD: ${POSTGRES_ROOT_PASSWORD} @@ -80,18 +79,16 @@ services: "CMD", "pg_isready", "-U", - "${POSTGRES_ROOT_USER}", - "-d", - "omotes_jobs", + "${POSTGRES_ROOT_USER}" ] interval: 5s timeout: 5s retries: 10 # Skipped in 'docker-compose up', only used to develop database revisions. - orchestrator_postgres_db_dev: - <<: *orchestrator_postgres_db - hostname: orchestrator_postgres_db + omotes_postgres_db_dev: + <<: *omotes_postgres_db + hostname: omotes_postgres_db ports: - "${POSTGRES_DEV_PORT}:5432" profiles: @@ -123,7 +120,7 @@ services: TASK_RABBITMQ_PORT: 5672 TASK_RABBITMQ_VIRTUALHOST: omotes_celery - POSTGRESQL_HOST: orchestrator_postgres_db + POSTGRESQL_HOST: omotes_postgres_db POSTGRESQL_PORT: 5432 POSTGRESQL_DATABASE: omotes_jobs POSTGRESQL_USERNAME: ${POSTGRES_ORCHESTRATOR_USER_NAME} @@ -143,13 +140,13 @@ services: depends_on: rabbitmq: condition: service_healthy - orchestrator_postgres_db: + omotes_postgres_db: condition: service_healthy omotes_influxdb: condition: service_healthy grow_worker_optimizer: &grow_worker - image: ghcr.io/project-omotes/omotes-grow-optimizer-worker:2.1.3 + image: ghcr.io/project-omotes/omotes-grow-optimizer-worker:2.1.7 restart: unless-stopped deploy: &grow_worker_deploy replicas: 2 diff --git a/example_sdk_client/optimizer.py b/example_sdk_client/optimizer.py index b3f692a..134679e 100644 --- a/example_sdk_client/optimizer.py +++ b/example_sdk_client/optimizer.py @@ -11,17 +11,17 @@ JobStatusUpdate, ) -rabbitmq_config = RabbitMQConfig(username="omotes", password="somepass1", virtual_host="omotes") +rabbitmq_config = RabbitMQConfig(port=5673,username="omotes", password="somepass1", virtual_host="omotes") STOP_EVENT = threading.Event() def handle_on_finished(job: Job, result: JobResult): - print("--------------Logs:") - print(result.logs) - print() - print("--------------ESDL:") - print(result.output_esdl) - print() + # print("--------------Logs:") + # print(result.logs) + # print() + # print("--------------ESDL:") + # print(result.output_esdl) + # print() print("--------------ESDL messages:") print(result.esdl_messages) print("--------------Result") diff --git a/postgres/postgres-init.sql b/postgres/postgres-init.sql index dc3ebbb..f80fa34 100644 --- a/postgres/postgres-init.sql +++ b/postgres/postgres-init.sql @@ -1,8 +1,61 @@ -CREATE USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD'; -ALTER USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD'; -GRANT ALL PRIVILEGES ON DATABASE :PG_DB TO :PG_USERNAME; -GRANT ALL PRIVILEGES ON SCHEMA public TO :PG_USERNAME; -GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO :PG_USERNAME; -GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO :PG_USERNAME; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :PG_USERNAME; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :PG_USERNAME; +-- CREATE USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD'; +-- ALTER USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD'; +-- GRANT ALL PRIVILEGES ON DATABASE :PG_DB TO :PG_USERNAME; +-- GRANT ALL PRIVILEGES ON SCHEMA public TO :PG_USERNAME; +-- GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO :PG_USERNAME; +-- GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO :PG_USERNAME; +-- ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :PG_USERNAME; +-- ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :PG_USERNAME; + + +-- create databases +CREATE DATABASE :PG_DB_JOBS; +CREATE DATABASE :PG_DB_TIMESERIES; + +-- create roles +CREATE ROLE :JOBS_RW_USER_NAME LOGIN PASSWORD :'JOBS_RW_USER_PASSWORD'; +CREATE ROLE :TIMESERIES_RW_USER_NAME LOGIN PASSWORD :'TIMESERIES_RW_USER_PASSWORD'; +CREATE ROLE :TIMESERIES_RO_USER_NAME LOGIN PASSWORD :'TIMESERIES_RO_USER_PASSWORD'; + +-- DB1 +\connect :PG_DB_JOBS + +-- Make RW user owner +ALTER DATABASE :PG_DB_JOBS OWNER TO :JOBS_RW_USER_NAME; +ALTER SCHEMA public OWNER TO :JOBS_RW_USER_NAME; + +-- Ensure RW can create objects +GRANT ALL ON SCHEMA public TO :JOBS_RW_USER_NAME; + +-- Existing objects +GRANT ALL ON ALL TABLES IN SCHEMA public TO :JOBS_RW_USER_NAME; +GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO :JOBS_RW_USER_NAME; +GRANT ALL ON ALL FUNCTIONS IN SCHEMA public TO :JOBS_RW_USER_NAME; + +-- Default privileges +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :JOBS_RW_USER_NAME; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :JOBS_RW_USER_NAME; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON FUNCTIONS TO :JOBS_RW_USER_NAME; + +-- DB2 +\connect :PG_DB_TIMESERIES + +-- Make RW owner +ALTER DATABASE :PG_DB_TIMESERIES OWNER TO :TIMESERIES_RW_USER_NAME; +ALTER SCHEMA public OWNER TO :TIMESERIES_RW_USER_NAME; + +-- Schema privileges +GRANT ALL ON SCHEMA public TO :TIMESERIES_RW_USER_NAME; +GRANT USAGE ON SCHEMA public TO :TIMESERIES_RO_USER_NAME; + +-- Existing objects +GRANT ALL ON ALL TABLES IN SCHEMA public TO :TIMESERIES_RW_USER_NAME; +GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO :TIMESERIES_RW_USER_NAME; +GRANT ALL ON ALL FUNCTIONS IN SCHEMA public TO :TIMESERIES_RW_USER_NAME; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO :TIMESERIES_RO_USER_NAME; + +-- Default privileges +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :TIMESERIES_RW_USER_NAME; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :TIMESERIES_RW_USER_NAME; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON FUNCTIONS TO :TIMESERIES_RW_USER_NAME; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO :TIMESERIES_RO_USER_NAME; diff --git a/scripts/setup.ps1 b/scripts/setup.ps1 index 023f568..21776fd 100644 --- a/scripts/setup.ps1 +++ b/scripts/setup.ps1 @@ -13,21 +13,21 @@ get-content .env | foreach { # docker compose stop # Deploy postgres omotes schema -docker compose --profile=manual_dev down orchestrator_postgres_db_dev -docker compose up -d --wait orchestrator_postgres_db -docker compose exec orchestrator_postgres_db psql -d postgres -c 'CREATE DATABASE omotes_jobs;' -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "CREATE USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "ALTER USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON DATABASE omotes_jobs TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose --profile=manual_dev down omotes_postgres_db_dev +docker compose up -d --wait postgres_db +docker compose exec omotes_postgres_db psql -d postgres -c 'CREATE DATABASE omotes_jobs;' +docker compose exec omotes_postgres_db psql -d omotes_jobs -c "CREATE USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" +docker compose exec omotes_postgres_db psql -d omotes_jobs -c "ALTER USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" +docker compose exec omotes_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON DATABASE omotes_jobs TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" # Upgrade omotes tables -docker compose build orchestrator_postgres_db_upgrade -docker compose run --rm orchestrator_postgres_db_upgrade +docker compose build omotes_postgres_db_upgrade +docker compose run --rm omotes_postgres_db_upgrade # Setup rabbitmq docker compose up -d --wait rabbitmq diff --git a/scripts/setup.sh b/scripts/setup.sh index 57ca0cf..cd86471 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -13,12 +13,16 @@ echo "Using docker compose file at: $DOCKER_COMPOSE_FILE" $DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE --profile=manual_dev down # add postgres user with privileges -$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE up --wait orchestrator_postgres_db -$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec orchestrator_postgres_db psql \ - -d omotes_jobs \ - -v PG_USERNAME="$POSTGRES_ORCHESTRATOR_USER_NAME" \ - -v PG_PASSWORD="$POSTGRES_ORCHESTRATOR_USER_PASSWORD" \ - -v PG_DB=omotes_jobs \ +$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE up --wait omotes_postgres_db +$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec omotes_postgres_db psql \ + -v PG_DB_JOBS=omotes_jobs \ + -v JOBS_RW_USER_NAME="$POSTGRES_ORCHESTRATOR_USER_NAME" \ + -v JOBS_RW_USER_PASSWORD="$POSTGRES_ORCHESTRATOR_USER_PASSWORD" \ + -v PG_DB_TIMESERIES=omotes_timeseries \ + -v TIMESERIES_RW_USER_NAME="$POSTGRES_TIMESERIES_RW_USER_NAME" \ + -v TIMESERIES_RW_USER_PASSWORD="$POSTGRES_TIMESERIES_RW_USER_PASSWORD" \ + -v TIMESERIES_RO_USER_NAME="$POSTGRES_TIMESERIES_RO_USER_NAME" \ + -v TIMESERIES_RO_USER_PASSWORD="$POSTGRES_TIMESERIES_RO_USER_PASSWORD" \ -f /setup/init.sql # add rabbitmq 'omotes' and 'celery' vhosts and users diff --git a/scripts/start-dev.sh b/scripts/start-dev.sh index 8966e7c..ce10249 100755 --- a/scripts/start-dev.sh +++ b/scripts/start-dev.sh @@ -2,6 +2,6 @@ . scripts/_select_docker_compose.sh -$DOCKER_COMPOSE --profile=manual_dev down orchestrator_postgres_db_dev +$DOCKER_COMPOSE --profile=manual_dev down omotes_postgres_db_dev $DOCKER_COMPOSE -f docker-compose.yml -f docker-compose.override.dev.yml build $DOCKER_COMPOSE -f docker-compose.yml -f docker-compose.override.dev.yml up --wait diff --git a/scripts/start.sh b/scripts/start.sh index 8592058..a3ef662 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -2,5 +2,5 @@ . scripts/_select_docker_compose.sh -$DOCKER_COMPOSE --profile=manual_dev down orchestrator_postgres_db_dev +$DOCKER_COMPOSE --profile=manual_dev down omotes_postgres_db_dev $DOCKER_COMPOSE up --wait diff --git a/scripts/start_postgres_in_dev_mode.sh b/scripts/start_postgres_in_dev_mode.sh index d899b06..13da87f 100755 --- a/scripts/start_postgres_in_dev_mode.sh +++ b/scripts/start_postgres_in_dev_mode.sh @@ -1,4 +1,4 @@ #!/bin/bash -docker compose -f docker-compose.yml down orchestrator_postgres_db -docker compose -f docker-compose.yml up --wait orchestrator_postgres_db_dev +docker compose -f docker-compose.yml down omotes_postgres_db +docker compose -f docker-compose.yml up --wait omotes_postgres_db_dev diff --git a/system_tests/docker-compose.override.yml b/system_tests/docker-compose.override.yml index ba603c6..3f6f8a4 100644 --- a/system_tests/docker-compose.override.yml +++ b/system_tests/docker-compose.override.yml @@ -17,7 +17,7 @@ services: RABBITMQ_HOST: rabbitmq RABBITMQ_PORT: 5672 - POSTGRES_HOST: orchestrator_postgres_db + POSTGRES_HOST: omotes_postgres_db POSTGRES_PORT: 5432 POSTGRES_ORCHESTRATOR_USER_NAME: omotes_orchestrator POSTGRES_ORCHESTRATOR_USER_PASSWORD: somepass3 @@ -32,6 +32,6 @@ services: TIMEOUT_JOB_HANDLER_RERUN_SEC: 5 JOB_RETENTION_SEC: 20 - orchestrator_postgres_db: + omotes_postgres_db: ports: - "${POSTGRES_DEV_PORT}:5432" diff --git a/track_memory.sh b/track_memory.sh new file mode 100755 index 0000000..46fdc1d --- /dev/null +++ b/track_memory.sh @@ -0,0 +1,4 @@ +while true; do + date +"%Y-%m-%d %H:%M:%S" + docker stats 79e4e234711e --no-stream --format "{{.Name}} {{.MemUsage}}" +done >> container_mem.log \ No newline at end of file From da3e913868bd49d2adad5e8985175aa5ced01f59 Mon Sep 17 00:00:00 2001 From: Mark Vrijlandt Date: Fri, 5 Jun 2026 11:47:18 +0200 Subject: [PATCH 2/8] timeseries in postgres enabled --- .env.template | 2 +- .vscode/launch.json | 18 ++++++++++++++++ docker-compose.override.dev.yml | 4 ---- docker-compose.override.setup.yml | 2 +- docker-compose.yml | 26 +++++++++++------------- postgres/postgres-init.sql | 1 + scripts/setup.ps1 | 24 +++++++++++----------- scripts/setup.sh | 9 +++++--- scripts/start-dev.sh | 2 +- scripts/start.sh | 2 +- scripts/start_postgres_in_dev_mode.sh | 4 ---- scripts/stop.sh | 2 +- system_tests/docker-compose.override.yml | 4 ++-- 13 files changed, 56 insertions(+), 44 deletions(-) delete mode 100755 scripts/start_postgres_in_dev_mode.sh diff --git a/.env.template b/.env.template index 3659dc5..706df58 100644 --- a/.env.template +++ b/.env.template @@ -1,6 +1,6 @@ POSTGRES_ROOT_USER=root POSTGRES_ROOT_PASSWORD=1234 -POSTGRES_DEV_PORT=6432 +POSTGRES_PORT=6432 POSTGRES_ORCHESTRATOR_USER_NAME=omotes_orchestrator POSTGRES_ORCHESTRATOR_USER_PASSWORD=somepass3 POSTGRES_TIMESERIES_RW_USER_NAME=omotes_timeseries_rw diff --git a/.vscode/launch.json b/.vscode/launch.json index 92f667f..792f947 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,6 +12,24 @@ "env": { "PYTHONPATH": "${workspaceFolder}/example_sdk_client" } + }, + { + "name": "test__simulator__multiple_ates_run", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "args": [ + "test_workflows_steps.py::TestWorkflows::test__simulator__multiple_ates_run", + "-v", + "-s" + ], + "jinja": true, + "justMyCode": true, + "cwd": "${workspaceFolder}/system_tests/src", + "envFile": "${workspaceFolder}/.env.test", + "env": { + "PYTHONPATH": "${workspaceFolder}/system_tests" + } } ] } \ No newline at end of file diff --git a/docker-compose.override.dev.yml b/docker-compose.override.dev.yml index 0bc04ad..4a6b573 100644 --- a/docker-compose.override.dev.yml +++ b/docker-compose.override.dev.yml @@ -32,7 +32,3 @@ services: # build: # context: .. # dockerfile: simulator-worker/dev.Dockerfile - - omotes_postgres_db: - ports: - - "${POSTGRES_DEV_PORT}:5432" diff --git a/docker-compose.override.setup.yml b/docker-compose.override.setup.yml index 03f2702..8b85c7f 100644 --- a/docker-compose.override.setup.yml +++ b/docker-compose.override.setup.yml @@ -18,6 +18,6 @@ services: INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD: ${INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD} INFLUXDB_PORT: ${INFLUXDB_PORT} - omotes_postgres_db: + omotes_postgres: volumes: - "${POSTGRES_INIT_FILE:-./postgres/postgres-init.sql}:/setup/init.sql:ro" diff --git a/docker-compose.yml b/docker-compose.yml index 511641a..f613d7d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,11 +60,16 @@ services: - ./influxdb/influxdb.conf:/etc/influxdb/influxdb.conf:ro - influxdb_storage:/var/lib/influxdb - omotes_postgres_db: &omotes_postgres_db + omotes_postgres: &omotes_postgres image: postgres:15.4 restart: unless-stopped + command: ["postgres", "-p", "${POSTGRES_PORT}"] volumes: - "db-data:/var/lib/postgresql/data/" + expose: + - "${POSTGRES_PORT}" + ports: + - "${POSTGRES_PORT}:${POSTGRES_PORT}" networks: - omotes environment: @@ -79,21 +84,14 @@ services: "CMD", "pg_isready", "-U", - "${POSTGRES_ROOT_USER}" + "${POSTGRES_ROOT_USER}", + "-p", + "${POSTGRES_PORT}" ] interval: 5s timeout: 5s retries: 10 - # Skipped in 'docker-compose up', only used to develop database revisions. - omotes_postgres_db_dev: - <<: *omotes_postgres_db - hostname: omotes_postgres_db - ports: - - "${POSTGRES_DEV_PORT}:5432" - profiles: - - manual_dev - orchestrator: image: ghcr.io/project-omotes/omotes_orchestrator:1.2.6 restart: unless-stopped @@ -120,8 +118,8 @@ services: TASK_RABBITMQ_PORT: 5672 TASK_RABBITMQ_VIRTUALHOST: omotes_celery - POSTGRESQL_HOST: omotes_postgres_db - POSTGRESQL_PORT: 5432 + POSTGRESQL_HOST: omotes_postgres + POSTGRESQL_PORT: ${POSTGRES_PORT} POSTGRESQL_DATABASE: omotes_jobs POSTGRESQL_USERNAME: ${POSTGRES_ORCHESTRATOR_USER_NAME} POSTGRESQL_PASSWORD: ${POSTGRES_ORCHESTRATOR_USER_PASSWORD} @@ -140,7 +138,7 @@ services: depends_on: rabbitmq: condition: service_healthy - omotes_postgres_db: + omotes_postgres: condition: service_healthy omotes_influxdb: condition: service_healthy diff --git a/postgres/postgres-init.sql b/postgres/postgres-init.sql index f80fa34..010e4c0 100644 --- a/postgres/postgres-init.sql +++ b/postgres/postgres-init.sql @@ -15,6 +15,7 @@ CREATE DATABASE :PG_DB_TIMESERIES; -- create roles CREATE ROLE :JOBS_RW_USER_NAME LOGIN PASSWORD :'JOBS_RW_USER_PASSWORD'; CREATE ROLE :TIMESERIES_RW_USER_NAME LOGIN PASSWORD :'TIMESERIES_RW_USER_PASSWORD'; +ALTER ROLE :TIMESERIES_RW_USER_NAME CREATEDB; CREATE ROLE :TIMESERIES_RO_USER_NAME LOGIN PASSWORD :'TIMESERIES_RO_USER_PASSWORD'; -- DB1 diff --git a/scripts/setup.ps1 b/scripts/setup.ps1 index 21776fd..e2e4e62 100644 --- a/scripts/setup.ps1 +++ b/scripts/setup.ps1 @@ -13,21 +13,21 @@ get-content .env | foreach { # docker compose stop # Deploy postgres omotes schema -docker compose --profile=manual_dev down omotes_postgres_db_dev +docker compose down docker compose up -d --wait postgres_db -docker compose exec omotes_postgres_db psql -d postgres -c 'CREATE DATABASE omotes_jobs;' -docker compose exec omotes_postgres_db psql -d omotes_jobs -c "CREATE USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" -docker compose exec omotes_postgres_db psql -d omotes_jobs -c "ALTER USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" -docker compose exec omotes_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON DATABASE omotes_jobs TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec omotes_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec omotes_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec omotes_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec omotes_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec omotes_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d postgres -c 'CREATE DATABASE omotes_jobs;' +docker compose exec omotes_postgres psql -d omotes_jobs -c "CREATE USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" +docker compose exec omotes_postgres psql -d omotes_jobs -c "ALTER USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" +docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON DATABASE omotes_jobs TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" # Upgrade omotes tables -docker compose build omotes_postgres_db_upgrade -docker compose run --rm omotes_postgres_db_upgrade +docker compose build omotes_postgres_upgrade +docker compose run --rm omotes_postgres_upgrade # Setup rabbitmq docker compose up -d --wait rabbitmq diff --git a/scripts/setup.sh b/scripts/setup.sh index cd86471..a6c90f0 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -10,11 +10,14 @@ DOCKER_COMPOSE_FILE=${2:-"./docker-compose.yml -f ./docker-compose.override.setu echo "Using docker compose file at: $DOCKER_COMPOSE_FILE" # stop system before setup -$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE --profile=manual_dev down +$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE down # add postgres user with privileges -$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE up --wait omotes_postgres_db -$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec omotes_postgres_db psql \ +$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE up --wait omotes_postgres +$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec omotes_postgres psql \ + -U "$POSTGRES_ROOT_USER" \ + -p "$POSTGRES_PORT" \ + -d postgres \ -v PG_DB_JOBS=omotes_jobs \ -v JOBS_RW_USER_NAME="$POSTGRES_ORCHESTRATOR_USER_NAME" \ -v JOBS_RW_USER_PASSWORD="$POSTGRES_ORCHESTRATOR_USER_PASSWORD" \ diff --git a/scripts/start-dev.sh b/scripts/start-dev.sh index ce10249..9f7c31d 100755 --- a/scripts/start-dev.sh +++ b/scripts/start-dev.sh @@ -2,6 +2,6 @@ . scripts/_select_docker_compose.sh -$DOCKER_COMPOSE --profile=manual_dev down omotes_postgres_db_dev +$DOCKER_COMPOSE down $DOCKER_COMPOSE -f docker-compose.yml -f docker-compose.override.dev.yml build $DOCKER_COMPOSE -f docker-compose.yml -f docker-compose.override.dev.yml up --wait diff --git a/scripts/start.sh b/scripts/start.sh index a3ef662..d963b44 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -2,5 +2,5 @@ . scripts/_select_docker_compose.sh -$DOCKER_COMPOSE --profile=manual_dev down omotes_postgres_db_dev +$DOCKER_COMPOSE down $DOCKER_COMPOSE up --wait diff --git a/scripts/start_postgres_in_dev_mode.sh b/scripts/start_postgres_in_dev_mode.sh deleted file mode 100755 index 13da87f..0000000 --- a/scripts/start_postgres_in_dev_mode.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -docker compose -f docker-compose.yml down omotes_postgres_db -docker compose -f docker-compose.yml up --wait omotes_postgres_db_dev diff --git a/scripts/stop.sh b/scripts/stop.sh index b23ebf4..cfa898d 100755 --- a/scripts/stop.sh +++ b/scripts/stop.sh @@ -1,4 +1,4 @@ #!/bin/bash . scripts/_select_docker_compose.sh -$DOCKER_COMPOSE --profile=manual_dev down +$DOCKER_COMPOSE down diff --git a/system_tests/docker-compose.override.yml b/system_tests/docker-compose.override.yml index 3f6f8a4..0cd1a94 100644 --- a/system_tests/docker-compose.override.yml +++ b/system_tests/docker-compose.override.yml @@ -17,7 +17,7 @@ services: RABBITMQ_HOST: rabbitmq RABBITMQ_PORT: 5672 - POSTGRES_HOST: omotes_postgres_db + POSTGRES_HOST: omotes_postgres POSTGRES_PORT: 5432 POSTGRES_ORCHESTRATOR_USER_NAME: omotes_orchestrator POSTGRES_ORCHESTRATOR_USER_PASSWORD: somepass3 @@ -32,6 +32,6 @@ services: TIMEOUT_JOB_HANDLER_RERUN_SEC: 5 JOB_RETENTION_SEC: 20 - omotes_postgres_db: + omotes_postgres: ports: - "${POSTGRES_DEV_PORT}:5432" From c5c775732759beee0614969d54afeac9004bd6c2 Mon Sep 17 00:00:00 2001 From: Mark Vrijlandt Date: Thu, 16 Jul 2026 15:00:18 +0200 Subject: [PATCH 3/8] update optimizer worker env vars --- docker-compose.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f613d7d..499d4f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,7 +93,7 @@ services: retries: 10 orchestrator: - image: ghcr.io/project-omotes/omotes_orchestrator:1.2.6 + image: ghcr.io/project-omotes/omotes_orchestrator:1.2.8 restart: unless-stopped networks: - omotes @@ -157,10 +157,11 @@ services: - omotes # - mapeditor-net environment: &grow_worker_env - INFLUXDB_HOSTNAME: omotes_influxdb - INFLUXDB_PORT: ${INFLUXDB_PORT} - INFLUXDB_USERNAME: ${INFLUXDB_FRONTEND_ADMIN_USER} - INFLUXDB_PASSWORD: ${INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD} + ESDL_OUTPUT_PROFILES_TYPE: POSTGRESQL + DB_HOSTNAME: omotes_postgres + DB_PORT: 6432 + DB_USERNAME: omotes_timeseries_rw + DB_PASSWORD: somepass4 RABBITMQ_HOSTNAME: rabbitmq-nwn RABBITMQ_PORT: 5672 @@ -169,6 +170,7 @@ services: RABBITMQ_VIRTUALHOST: omotes_celery GROW_TASK_TYPE: grow_optimizer_default + MAX_TASKS_BEFORE_RESTART: 1 LOG_LEVEL: ${LOG_LEVEL} depends_on: rabbitmq: From 644c0d6ffebc477ad2edc317eb5fa7caf19de275 Mon Sep 17 00:00:00 2001 From: Mark Vrijlandt Date: Thu, 16 Jul 2026 15:24:14 +0200 Subject: [PATCH 4/8] small fixes --- .env.template | 6 +++--- docker-compose.override.dev.yml | 20 ++++++++++---------- postgres/postgres-init.sql | 10 ---------- 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/.env.template b/.env.template index e593fd8..af71596 100644 --- a/.env.template +++ b/.env.template @@ -8,8 +8,8 @@ POSTGRES_TIMESERIES_RW_USER_PASSWORD=somepass4 POSTGRES_TIMESERIES_RO_USER_NAME=omotes_timeseries_ro POSTGRES_TIMESERIES_RO_USER_PASSWORD=somepass5 -RABBITMQ_PORT=5672 -RABBITMQ_MANAGEMENT_PORT=15672 +RABBITMQ_PORT=5673 +RABBITMQ_MANAGEMENT_PORT=15673 RABBITMQ_ROOT_USER=root RABBITMQ_ROOT_PASSWORD=5678 RABBITMQ_OMOTES_USER_NAME=omotes @@ -32,7 +32,7 @@ TIMEOUT_JOB_MANAGER_START_BUFFER_SEC=60 TIMEOUT_JOB_HANDLER_RERUN_SEC=30 WORKFLOW_SETTINGS_FILE=./config/workflow_config_nwn_no_gurobi.json - MAX_TASKS_BEFORE_RESTART=1 +ESDL_OUTPUT_PROFILES_TYPE=POSTGRESQL LOG_LEVEL=INFO diff --git a/docker-compose.override.dev.yml b/docker-compose.override.dev.yml index 4a6b573..d20aad2 100644 --- a/docker-compose.override.dev.yml +++ b/docker-compose.override.dev.yml @@ -1,10 +1,10 @@ services: -# orchestrator: -# image: omotes-orchestrator-dev -# build: -# context: .. -# dockerfile: orchestrator/dev.Dockerfile + orchestrator: + image: omotes-orchestrator-dev + build: + context: .. + dockerfile: orchestrator/dev.Dockerfile grow_worker_optimizer: &grow_worker image: omotes-worker-mesido-dev @@ -27,8 +27,8 @@ services: grow_worker_gurobi: <<: *grow_worker -# omotes_simulator_worker: -# image: omotes-worker-simulator-dev -# build: -# context: .. -# dockerfile: simulator-worker/dev.Dockerfile + omotes_simulator_worker: + image: omotes-worker-simulator-dev + build: + context: .. + dockerfile: simulator-worker/dev.Dockerfile diff --git a/postgres/postgres-init.sql b/postgres/postgres-init.sql index 010e4c0..148d29d 100644 --- a/postgres/postgres-init.sql +++ b/postgres/postgres-init.sql @@ -1,13 +1,3 @@ --- CREATE USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD'; --- ALTER USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD'; --- GRANT ALL PRIVILEGES ON DATABASE :PG_DB TO :PG_USERNAME; --- GRANT ALL PRIVILEGES ON SCHEMA public TO :PG_USERNAME; --- GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO :PG_USERNAME; --- GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO :PG_USERNAME; --- ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :PG_USERNAME; --- ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :PG_USERNAME; - - -- create databases CREATE DATABASE :PG_DB_JOBS; CREATE DATABASE :PG_DB_TIMESERIES; From 1f499ea0553d0fb148ca9194c9db3ad485de286a Mon Sep 17 00:00:00 2001 From: Mark Vrijlandt Date: Thu, 16 Jul 2026 17:44:46 +0200 Subject: [PATCH 5/8] fix optimizer tests --- .vscode/launch.json | 22 +- system_tests/docker-compose.override.yml | 6 +- ...t__grow_optimizer_default__happy_path.esdl | 286 +++++++++--------- ...optimizer_default__happy_path_1source.esdl | 286 +++++++++--------- ...timizer_default__happy_path_2ndsource.esdl | 286 +++++++++--------- ...py_path_2ndsource_merit_order_swapped.esdl | 286 +++++++++--------- ..._optimizer_no_heat_losses__happy_path.esdl | 286 +++++++++--------- system_tests/src/test_workflows_steps.py | 11 +- 8 files changed, 758 insertions(+), 711 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 792f947..aa7e3e0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -14,12 +14,30 @@ } }, { - "name": "test__simulator__multiple_ates_run", + "name": "test__grow_optimizer_default__happy_path", "type": "debugpy", "request": "launch", "module": "pytest", "args": [ - "test_workflows_steps.py::TestWorkflows::test__simulator__multiple_ates_run", + "test_workflows_steps.py::TestWorkflows::test__grow_optimizer_default__happy_path", + "-v", + "-s" + ], + "jinja": true, + "justMyCode": true, + "cwd": "${workspaceFolder}/system_tests/src", + "envFile": "${workspaceFolder}/.env.test", + "env": { + "PYTHONPATH": "${workspaceFolder}/system_tests" + } + }, + { + "name": "test__all", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "args": [ + "test_workflows_steps.py::TestWorkflows", "-v", "-s" ], diff --git a/system_tests/docker-compose.override.yml b/system_tests/docker-compose.override.yml index c30a0aa..a2ab9b9 100644 --- a/system_tests/docker-compose.override.yml +++ b/system_tests/docker-compose.override.yml @@ -20,7 +20,7 @@ services: RABBITMQ_PORT: 5672 POSTGRES_HOST: omotes_postgres - POSTGRES_PORT: 5432 + POSTGRESQL_PORT: ${POSTGRES_PORT} POSTGRES_ORCHESTRATOR_USER_NAME: omotes_orchestrator POSTGRES_ORCHESTRATOR_USER_PASSWORD: somepass3 @@ -33,7 +33,3 @@ services: TIMEOUT_JOB_MANAGER_START_BUFFER_SEC: 2 TIMEOUT_JOB_HANDLER_RERUN_SEC: 5 JOB_RETENTION_SEC: 20 - - omotes_postgres: - ports: - - "${POSTGRES_DEV_PORT}:5432" diff --git a/system_tests/src/test_esdl/output/test__grow_optimizer_default__happy_path.esdl b/system_tests/src/test_esdl/output/test__grow_optimizer_default__happy_path.esdl index 5868b1b..2b3340d 100644 --- a/system_tests/src/test_esdl/output/test__grow_optimizer_default__happy_path.esdl +++ b/system_tests/src/test_esdl/output/test__grow_optimizer_default__happy_path.esdl @@ -1,5 +1,5 @@ - + @@ -7,20 +7,26 @@ + + + + + + - - - + + + - - - + + + @@ -61,17 +67,17 @@ - - - + + + - - - + + + - - - + + + @@ -104,17 +110,17 @@ - - - + + + - - - + + + - - - + + + @@ -143,17 +149,17 @@ - - - + + + - - - + + + - - - + + + @@ -186,17 +192,17 @@ - - - + + + - - - + + + - - - + + + @@ -224,17 +230,17 @@ - - - + + + - - - + + + - - - + + + @@ -262,17 +268,17 @@ - - - + + + - - - + + + - - - + + + @@ -294,17 +300,17 @@ - - - + + + - - - + + + - - - + + + @@ -316,17 +322,17 @@ - - - + + + - - - + + + - - - + + + @@ -337,17 +343,17 @@ - - - + + + - - - + + + - - - + + + @@ -358,17 +364,17 @@ - - - + + + - - - + + + - - - + + + @@ -381,13 +387,13 @@ - - - + + + - - - + + + @@ -411,7 +417,7 @@ - + @@ -424,13 +430,13 @@ - + - + - + @@ -444,13 +450,13 @@ - - - + + + - - - + + + @@ -472,7 +478,7 @@ - + @@ -485,13 +491,13 @@ - + - + - + @@ -505,13 +511,13 @@ - - - + + + - - - + + + @@ -531,7 +537,7 @@ - + @@ -544,33 +550,33 @@ - + - + - + - - + + - + - + @@ -579,7 +585,7 @@ - + @@ -588,7 +594,7 @@ - + @@ -596,39 +602,39 @@ - + - + - + - + - + - + - + diff --git a/system_tests/src/test_esdl/output/test__grow_optimizer_default__happy_path_1source.esdl b/system_tests/src/test_esdl/output/test__grow_optimizer_default__happy_path_1source.esdl index 73aa539..436385f 100644 --- a/system_tests/src/test_esdl/output/test__grow_optimizer_default__happy_path_1source.esdl +++ b/system_tests/src/test_esdl/output/test__grow_optimizer_default__happy_path_1source.esdl @@ -1,5 +1,5 @@ - + @@ -7,20 +7,26 @@ + + + + + + - - - + + + - - - + + + @@ -50,17 +56,17 @@ - - - + + + - - - + + + - - - + + + @@ -93,17 +99,17 @@ - - - + + + - - - + + + - - - + + + @@ -132,17 +138,17 @@ - - - + + + - - - + + + - - - + + + @@ -175,17 +181,17 @@ - - - + + + - - - + + + - - - + + + @@ -213,17 +219,17 @@ - - - + + + - - - + + + - - - + + + @@ -262,17 +268,17 @@ - - - + + + - - - + + + - - - + + + @@ -283,17 +289,17 @@ - - - + + + - - - + + + - - - + + + @@ -305,17 +311,17 @@ - - - + + + - - - + + + - - - + + + @@ -326,17 +332,17 @@ - - - + + + - - - + + + - - - + + + @@ -347,17 +353,17 @@ - - - + + + - - - + + + - - - + + + @@ -370,13 +376,13 @@ - - - + + + - - - + + + @@ -400,7 +406,7 @@ - + @@ -413,13 +419,13 @@ - + - + - + @@ -433,13 +439,13 @@ - - - + + + - - - + + + @@ -461,7 +467,7 @@ - + @@ -474,13 +480,13 @@ - + - + - + @@ -494,13 +500,13 @@ - - - + + + - - - + + + @@ -520,7 +526,7 @@ - + @@ -533,33 +539,33 @@ - + - + - + - - + + - + - + @@ -568,7 +574,7 @@ - + @@ -577,7 +583,7 @@ - + @@ -585,39 +591,39 @@ - + - + - + - + - + - + - + diff --git a/system_tests/src/test_esdl/output/test__grow_optimizer_default__happy_path_2ndsource.esdl b/system_tests/src/test_esdl/output/test__grow_optimizer_default__happy_path_2ndsource.esdl index 2db3289..fb43739 100644 --- a/system_tests/src/test_esdl/output/test__grow_optimizer_default__happy_path_2ndsource.esdl +++ b/system_tests/src/test_esdl/output/test__grow_optimizer_default__happy_path_2ndsource.esdl @@ -1,5 +1,5 @@ - + @@ -7,20 +7,26 @@ + + + + + + - - - + + + - - - + + + @@ -61,17 +67,17 @@ - - - + + + - - - + + + - - - + + + @@ -104,17 +110,17 @@ - - - + + + - - - + + + - - - + + + @@ -143,17 +149,17 @@ - - - + + + - - - + + + - - - + + + @@ -186,17 +192,17 @@ - - - + + + - - - + + + - - - + + + @@ -224,17 +230,17 @@ - - - + + + - - - + + + - - - + + + @@ -262,17 +268,17 @@ - - - + + + - - - + + + - - - + + + @@ -294,17 +300,17 @@ - - - + + + - - - + + + - - - + + + @@ -316,17 +322,17 @@ - - - + + + - - - + + + - - - + + + @@ -337,17 +343,17 @@ - - - + + + - - - + + + - - - + + + @@ -358,17 +364,17 @@ - - - + + + - - - + + + - - - + + + @@ -381,13 +387,13 @@ - - - + + + - - - + + + @@ -411,7 +417,7 @@ - + @@ -424,13 +430,13 @@ - + - + - + @@ -444,13 +450,13 @@ - - - + + + - - - + + + @@ -472,7 +478,7 @@ - + @@ -485,13 +491,13 @@ - + - + - + @@ -505,13 +511,13 @@ - - - + + + - - - + + + @@ -531,7 +537,7 @@ - + @@ -544,33 +550,33 @@ - + - + - + - - + + - + - + @@ -579,7 +585,7 @@ - + @@ -588,7 +594,7 @@ - + @@ -596,39 +602,39 @@ - + - + - + - + - + - + - + diff --git a/system_tests/src/test_esdl/output/test__grow_optimizer_default__happy_path_2ndsource_merit_order_swapped.esdl b/system_tests/src/test_esdl/output/test__grow_optimizer_default__happy_path_2ndsource_merit_order_swapped.esdl index d0c9bc9..d8b6543 100644 --- a/system_tests/src/test_esdl/output/test__grow_optimizer_default__happy_path_2ndsource_merit_order_swapped.esdl +++ b/system_tests/src/test_esdl/output/test__grow_optimizer_default__happy_path_2ndsource_merit_order_swapped.esdl @@ -1,5 +1,5 @@ - + @@ -7,20 +7,26 @@ + + + + + + - - - + + + - - - + + + @@ -61,17 +67,17 @@ - - - + + + - - - + + + - - - + + + @@ -104,17 +110,17 @@ - - - + + + - - - + + + - - - + + + @@ -143,17 +149,17 @@ - - - + + + - - - + + + - - - + + + @@ -186,17 +192,17 @@ - - - + + + - - - + + + - - - + + + @@ -224,17 +230,17 @@ - - - + + + - - - + + + - - - + + + @@ -262,17 +268,17 @@ - - - + + + - - - + + + - - - + + + @@ -294,17 +300,17 @@ - - - + + + - - - + + + - - - + + + @@ -316,17 +322,17 @@ - - - + + + - - - + + + - - - + + + @@ -337,17 +343,17 @@ - - - + + + - - - + + + - - - + + + @@ -358,17 +364,17 @@ - - - + + + - - - + + + - - - + + + @@ -381,13 +387,13 @@ - - - + + + - - - + + + @@ -411,7 +417,7 @@ - + @@ -424,13 +430,13 @@ - + - + - + @@ -444,13 +450,13 @@ - - - + + + - - - + + + @@ -472,7 +478,7 @@ - + @@ -485,13 +491,13 @@ - + - + - + @@ -505,13 +511,13 @@ - - - + + + - - - + + + @@ -531,7 +537,7 @@ - + @@ -544,33 +550,33 @@ - + - + - + - - + + - + - + @@ -579,7 +585,7 @@ - + @@ -588,7 +594,7 @@ - + @@ -596,39 +602,39 @@ - + - + - + - + - + - + - + diff --git a/system_tests/src/test_esdl/output/test__grow_optimizer_no_heat_losses__happy_path.esdl b/system_tests/src/test_esdl/output/test__grow_optimizer_no_heat_losses__happy_path.esdl index 9a8f3f9..9aef6de 100644 --- a/system_tests/src/test_esdl/output/test__grow_optimizer_no_heat_losses__happy_path.esdl +++ b/system_tests/src/test_esdl/output/test__grow_optimizer_no_heat_losses__happy_path.esdl @@ -1,5 +1,5 @@ - + @@ -7,20 +7,26 @@ + + + + + + - - - + + + - - - + + + @@ -61,17 +67,17 @@ - - - + + + - - - + + + - - - + + + @@ -104,17 +110,17 @@ - - - + + + - - - + + + - - - + + + @@ -143,17 +149,17 @@ - - - + + + - - - + + + - - - + + + @@ -186,17 +192,17 @@ - - - + + + - - - + + + - - - + + + @@ -224,17 +230,17 @@ - - - + + + - - - + + + - - - + + + @@ -262,17 +268,17 @@ - - - + + + - - - + + + - - - + + + @@ -294,17 +300,17 @@ - - - + + + - - - + + + - - - + + + @@ -316,17 +322,17 @@ - - - + + + - - - + + + - - - + + + @@ -337,17 +343,17 @@ - - - + + + - - - + + + - - - + + + @@ -358,17 +364,17 @@ - - - + + + - - - + + + - - - + + + @@ -381,13 +387,13 @@ - - - + + + - - - + + + @@ -411,7 +417,7 @@ - + @@ -424,13 +430,13 @@ - + - + - + @@ -444,13 +450,13 @@ - - - + + + - - - + + + @@ -472,7 +478,7 @@ - + @@ -485,13 +491,13 @@ - + - + - + @@ -505,13 +511,13 @@ - - - + + + - - - + + + @@ -531,7 +537,7 @@ - + @@ -544,33 +550,33 @@ - + - + - + - - + + - + - + @@ -579,7 +585,7 @@ - + @@ -588,7 +594,7 @@ - + @@ -596,39 +602,39 @@ - + - + - + - + - + - + - + diff --git a/system_tests/src/test_workflows_steps.py b/system_tests/src/test_workflows_steps.py index c598037..092f504 100644 --- a/system_tests/src/test_workflows_steps.py +++ b/system_tests/src/test_workflows_steps.py @@ -122,11 +122,14 @@ def retrieve_esdl_file(path_str: str) -> str: return esdl_file + ATTRIBUTE_REGEX_TO_IGNORE = dict( - id="[a-z0-9-]+", # uuid - database="[a-z0-9-]+", # uuid - reference="[a-z0-9-]+", # uuid - releaseDate=".*", # any format + id="[a-z0-9-]+", # uuid + database="[a-z0-9-]+", # uuid + reference="[a-z0-9-]+", # uuid + tableName="[a-z0-9-]+", # uuid + configuration="[a-z0-9-]+", # uuid + releaseDate=".*", # any format ) NUMBER_REGEX = re.compile(r"^-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?$") From d7dc2aeb6537e51240d86b8b72d1880019a63ad3 Mon Sep 17 00:00:00 2001 From: Mark Vrijlandt Date: Fri, 17 Jul 2026 13:42:30 +0200 Subject: [PATCH 6/8] to optimzer-worker 2.1.15 and orchestrator 1.2.9 --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 70ac1ce..587b530 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,7 +93,7 @@ services: retries: 10 orchestrator: - image: ghcr.io/project-omotes/omotes_orchestrator:1.2.8 + image: ghcr.io/project-omotes/omotes_orchestrator:1.2.9 restart: unless-stopped networks: - omotes @@ -144,7 +144,7 @@ services: condition: service_healthy grow_worker_optimizer: &grow_worker - image: ghcr.io/project-omotes/omotes-grow-optimizer-worker:2.1.10 + image: ghcr.io/project-omotes/omotes-grow-optimizer-worker:2.1.15 restart: unless-stopped deploy: &grow_worker_deploy replicas: 2 From 4bf746d29abc3b6ba39baa39cf5bbca649ffe74e Mon Sep 17 00:00:00 2001 From: Mark Vrijlandt Date: Fri, 17 Jul 2026 15:58:18 +0200 Subject: [PATCH 7/8] add mapeditor network to postgres --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 587b530..cc541d0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,6 +72,7 @@ services: - "${POSTGRES_PORT}:${POSTGRES_PORT}" networks: - omotes +# - mapeditor-net environment: PGDATA: /var/lib/postgresql/data POSTGRES_USER: ${POSTGRES_ROOT_USER} From 6f5f0fa19ddfaaf27ca74133a0bf5d238ba81c75 Mon Sep 17 00:00:00 2001 From: Mark Vrijlandt Date: Fri, 17 Jul 2026 16:26:50 +0200 Subject: [PATCH 8/8] give proper permissions for new pg dbs --- postgres/postgres-init.sql | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/postgres/postgres-init.sql b/postgres/postgres-init.sql index 148d29d..81d1153 100644 --- a/postgres/postgres-init.sql +++ b/postgres/postgres-init.sql @@ -8,6 +8,13 @@ CREATE ROLE :TIMESERIES_RW_USER_NAME LOGIN PASSWORD :'TIMESERIES_RW_USER_PASSWOR ALTER ROLE :TIMESERIES_RW_USER_NAME CREATEDB; CREATE ROLE :TIMESERIES_RO_USER_NAME LOGIN PASSWORD :'TIMESERIES_RO_USER_PASSWORD'; +-- Template privileges for databases created later by TIMESERIES_RW_USER_NAME. +-- New databases created with default CREATE DATABASE settings inherit from template1. +\connect template1 +GRANT USAGE ON SCHEMA public TO :TIMESERIES_RO_USER_NAME; +ALTER DEFAULT PRIVILEGES FOR ROLE :TIMESERIES_RW_USER_NAME IN SCHEMA public + GRANT SELECT ON TABLES TO :TIMESERIES_RO_USER_NAME; + -- DB1 \connect :PG_DB_JOBS @@ -36,6 +43,7 @@ ALTER DATABASE :PG_DB_TIMESERIES OWNER TO :TIMESERIES_RW_USER_NAME; ALTER SCHEMA public OWNER TO :TIMESERIES_RW_USER_NAME; -- Schema privileges +GRANT CONNECT ON DATABASE :PG_DB_TIMESERIES TO :TIMESERIES_RO_USER_NAME; GRANT ALL ON SCHEMA public TO :TIMESERIES_RW_USER_NAME; GRANT USAGE ON SCHEMA public TO :TIMESERIES_RO_USER_NAME;