From 15ba918bbea9849c9193a76a5b50d8d8876cf765 Mon Sep 17 00:00:00 2001 From: hud Date: Fri, 6 Dec 2019 18:49:51 +0100 Subject: [PATCH 01/11] Set prod config --- api/app/__init__.py | 5 ++++- api/app/constants.py | 2 +- api/config/production.py | 15 ++++++++------- api/run.py | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/api/app/__init__.py b/api/app/__init__.py index 7409ed7a..64b4b8d3 100644 --- a/api/app/__init__.py +++ b/api/app/__init__.py @@ -145,7 +145,10 @@ def create_app(config_name): "http://hotmapsdev.hevs.ch", "http://maps.googleapis.com/maps/api/geocode/", "http://maps.googleapis.com/maps/api/", - "http://maps.googleapis.com/*" + "http://maps.googleapis.com/*", + "https://hotmaps.eurac.edu", + "https://www.hotmaps.eurac.edu", + "https://geoserver.hotmaps.eurac.edu" } }}) return app diff --git a/api/app/constants.py b/api/app/constants.py index 60d1f0cf..775207b1 100644 --- a/api/app/constants.py +++ b/api/app/constants.py @@ -26,7 +26,7 @@ #TODO ********************setup this URL depending on which version you are running*************************** CELERY_BROKER_URL = CELERY_BROKER_URL_DOCKER -CLIENT_URL = CLIENT_URL_DEV +CLIENT_URL = CLIENT_URL_PROD PORT = PORT_DOCKER diff --git a/api/config/production.py b/api/config/production.py index 9a7f56de..904415ee 100644 --- a/api/config/production.py +++ b/api/config/production.py @@ -1,18 +1,18 @@ -DEBUG = False +# Flask settings import os basedir = os.path.abspath(os.path.dirname(__file__)) -db_path = os.path.join(basedir, '../data.sqlite') +db_path = os.path.join(basedir, '../data-dev.sqlite') + from app import secrets -SECRET_KEY = 'top-secret!' -# Flask settings -FLASK_SECRET_KEY = 'paPTvnNME5NBHHuIOlFqG6zS77vHadbo' +DEBUG = False +FLASK_SECRET_KEY = 'paPTvnNME5NBHHuIOlFqG6zS77vHadbo' SQLALCHEMY_DATABASE_URI = secrets.SQLALCHEMY_DATABASE_URI_PRODUCTION SQLALCHEMY_BINDS = { - 'cm_db': 'sqlite:///' + db_path + 'cm_db': os.environ.get('DATABASE_URL') or \ + 'sqlite:///' + db_path } -#flask_app.config['SERVER_NAME'] = settings.FLASK_SERVER_NAME SECRET_KEY = 'paPTvnNME5NBHHuIOlFqG6zS77vHadbo' SQLALCHEMY_TRACK_MODIFICATIONS = True SWAGGER_UI_DOC_EXPANSION = 'list' @@ -23,6 +23,7 @@ 'separators': (',', ':') } CELERY_BROKER_URL = 'amqp://admin:mypass@rabbit:5672/' + CELERY_RESULT_BACKEND = 'redis://localhost:6379/0' UPLOAD_FOLDER = 'uploaded_file' diff --git a/api/run.py b/api/run.py index 471bbfa0..657a89f1 100644 --- a/api/run.py +++ b/api/run.py @@ -2,7 +2,7 @@ from app import create_app, log from app.model import init_sqlite_caculation_module_database -application = create_app(os.environ.get('FLASK_CONFIG', 'development')) +application = create_app(os.environ.get('FLASK_CONFIG', 'production')) if __name__ == '__main__': with application.app_context(): From ab96b0b57e853b504db513ef54f8b1449e29a0a5 Mon Sep 17 00:00:00 2001 From: Malik Bougacha Date: Thu, 3 Sep 2020 11:22:24 +0200 Subject: [PATCH 02/11] allow supervisorctl to access supervisor in the docker image for inspecting the state of process in the docker image --- api/supervisord.conf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api/supervisord.conf b/api/supervisord.conf index 05cfb6b9..c8e100ca 100644 --- a/api/supervisord.conf +++ b/api/supervisord.conf @@ -1,7 +1,15 @@ [supervisord] nodaemon=true +#the three section below allow us to access supervisorctl +[unix_http_server] +file=/var/run/supervisord.sock +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisord.sock [program:mainwebservice] command = gunicorn run:application --config gunicorn-config.py --timeout 9000 From 276626a4e25182bfd82a14eb35008a5d8bc51606 Mon Sep 17 00:00:00 2001 From: Malik Bougacha Date: Thu, 3 Sep 2020 11:30:40 +0200 Subject: [PATCH 03/11] unify ini file for supervisor --- api/supervisord.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/supervisord.conf b/api/supervisord.conf index c8e100ca..8572c272 100644 --- a/api/supervisord.conf +++ b/api/supervisord.conf @@ -6,20 +6,20 @@ nodaemon=true file=/var/run/supervisord.sock [rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface +supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface [supervisorctl] serverurl=unix:///var/run/supervisord.sock [program:mainwebservice] -command = gunicorn run:application --config gunicorn-config.py --timeout 9000 -directory = /api +command=gunicorn run:application --config gunicorn-config.py --timeout 9000 +directory=/api autostart=true autorestart=true startretries=30000 stderr_logfile=/var/log/mainwebservice.err.log stdout_logfile=/var/log/mainwebservice.out.log -user = root +user=root [program:producer_cm_alive] From 89abcd9a8c29272747da6bead1908986c10ec128 Mon Sep 17 00:00:00 2001 From: Malik Bougacha Date: Thu, 3 Sep 2020 11:46:17 +0200 Subject: [PATCH 04/11] fix environment for the dev docker compose --- docker-compose-dev.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 1a4ca4b8..a66db79a 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -25,8 +25,10 @@ services: networks: - cm-net volumes: - - /var/tmp:/var/tmp - - /var/hotmaps/repositories:/var/hotmaps/repositories + - /var/tmp:/var/tmp + - /var/hotmaps/repositories:/var/hotmaps/repositories + environment: + - ENVIRONMENT=development redis: From f6673892981c50ecc19e0d9bfb1ac8c022868f31 Mon Sep 17 00:00:00 2001 From: Malik Bougacha Date: Thu, 3 Sep 2020 11:47:33 +0200 Subject: [PATCH 05/11] add dev env file --- .env.dev | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .env.dev diff --git a/.env.dev b/.env.dev new file mode 100644 index 00000000..d1a1a246 --- /dev/null +++ b/.env.dev @@ -0,0 +1,46 @@ +# This is an example of a .env file. +# This file contains all environment variables needed to run the service +# Duplicate this file, fill it with your configuration and rename it to '.env' + +# Set the environment of the service +ENVIRONMENT=development # development|production + +# Set the backend and frontend url + port +API_URL=https://localhost # host used to access the backend +API_PORT=5000 # port used to access the backend (ie. docker-compose external port) +CLIENT_URL=http://localhost:4200 # url used to access the frontend +GEOSERVER_URL=https://geo.wxyz # base url to access Geoserver (not REST) + + +# Flask configuration +FLASK_SECRET_KEY=MyFlaskSecretKey +FLASK_SALT=MyFlaskSalt +FLASK_SERVER_NAME=0.0.0.0:5556 + +# Celery configuration +CELERY_BROKER_URL=amqp://admin:mypass@rabbit:5672/ +CELERY_RESULT_BACKEND=redis://redis:6379 +# CELERYD_SOFT_TIME_LIMIT=10 # does not work +USER_UPLOAD_FOLDER=/var/hotmaps/users/ # directory where user uploads are stored on the 'api' container + # must match users path in docker-compose.yml volumes + +# Database configuration +DB_USER=root +DB_PASSWORD=password +DB_HOST=localhost +DB_PORT=5432 +DB_DATABASE=toolboxdb + +# Geoserver configuration +GEOSERVER_API_URL=https://geo.wxyz/geoserver/rest/ # full REST url to access geoserver (display layers on the map) +GEOSERVER_USER=admin +GEOSERVER_PASSWORD=password + +# SMTP Mail configuration (used to send emails to the users (register, recover, feedback...)) +MAIL_USERNAME=support@example.com +MAIL_PASSWORD=password +MAIL_SERVER=mail.server.com +MAIL_PORT=587 + +# Testing configuration +PYTEST_BASE_URL=http://127.0.0.1:5000/api # url used to run the tests From 1f6aa6018a5899c83dd4e219a9dd9fdfbfce5891 Mon Sep 17 00:00:00 2001 From: Malik Bougacha Date: Thu, 3 Sep 2020 12:06:39 +0200 Subject: [PATCH 06/11] add a wait time between two pass on the production alive check --- api/producer_cm_alive.py | 49 +++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/api/producer_cm_alive.py b/api/producer_cm_alive.py index ffaaeeec..dab54d03 100644 --- a/api/producer_cm_alive.py +++ b/api/producer_cm_alive.py @@ -13,7 +13,6 @@ '-35s %(lineno) -5d: %(message)s') LOGGER = logging.getLogger(__name__) - class HeartBeatCalculationModuleProducer(object): def __init__(self): parameters = pika.URLParameters(constants.CELERY_BROKER_URL) @@ -48,30 +47,28 @@ def call(self, queue_name): break return self.response - - - - - -while True : - listofCM = getCMList() - start = time.time() - if len(listofCM)>0: - for value in enumerate(listofCM): - time.sleep(constants.TIMEOUT_ALIVE_CM) - end = time. time() - print(end - start) - - - heart_cm = HeartBeatCalculationModuleProducer() - cm_id = value[1]['cm_id'] - print(" [HTAPI] Requesting cm_id = ",cm_id) - response = heart_cm.call(constants.RPC_CM_ALIVE + str(cm_id)) - if response is not None: - print("[HTAPI] is connected to the Calculation module with id: %s ", str(cm_id)) - LOGGER.info("[HTAPI] is connected to the Calculation module with id: %s ", str(cm_id)) - else: - LOGGER.info("[HTAPI] is going to delete: %s ",str(cm_id)) - delete_cm(str(cm_id)) +if __name__ == "__main__": + while True: + listofCM = getCMList() + start = time.time() + if listofCM: + for value in enumerate(listofCM): + time.sleep(constants.TIMEOUT_ALIVE_CM) + end = time. time() + print(end - start) + + + heart_cm = HeartBeatCalculationModuleProducer() + cm_id = value[1]['cm_id'] + print(" [HTAPI] Requesting cm_id = ", cm_id) + response = heart_cm.call(constants.RPC_CM_ALIVE + str(cm_id)) + if response is not None: + print("[HTAPI] is connected to the Calculation module with id: %s ", str(cm_id)) + LOGGER.info("[HTAPI] is connected to the Calculation module with id: %s ", str(cm_id)) + else: + LOGGER.info("[HTAPI] is going to delete: %s ",str(cm_id)) + delete_cm(str(cm_id)) + #sleep a bit between two alive checks + time.sleep(30) From 021db6f7ff6c253d3ce6cc518a411f234082acae Mon Sep 17 00:00:00 2001 From: Malik Bougacha Date: Thu, 3 Sep 2020 13:09:20 +0200 Subject: [PATCH 07/11] add locale, we should at least use utf-8 --- api/dockerfiles/api/Dockerfile | 8 +++++++- docker-compose-dev.yml | 5 ++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/api/dockerfiles/api/Dockerfile b/api/dockerfiles/api/Dockerfile index 0f7f2752..621464b2 100644 --- a/api/dockerfiles/api/Dockerfile +++ b/api/dockerfiles/api/Dockerfile @@ -6,6 +6,12 @@ ENV C_FORCE_ROOT true LABEL maintainers="lesly Houndole , Daniel Hunacek " +RUN apt-get clean && apt-get update && apt-get install -y locales +RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ + locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 # setup volume RUN mkdir -p /data @@ -33,4 +39,4 @@ RUN mkdir -p /var/log/supervisor COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf # Start processes -CMD ["/usr/bin/supervisord"] \ No newline at end of file +CMD ["/usr/bin/supervisord"] diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index a66db79a..6c9cdfef 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -27,9 +27,8 @@ services: volumes: - /var/tmp:/var/tmp - /var/hotmaps/repositories:/var/hotmaps/repositories - environment: - - ENVIRONMENT=development - + env_file: + - ./.env.dev redis: image: redis From 074ff5f0bb0c847cee42b885d886a1d91010c15f Mon Sep 17 00:00:00 2001 From: Malik Bougacha Date: Thu, 3 Sep 2020 15:51:43 +0200 Subject: [PATCH 08/11] not sure how port were mapped before, now mapped to localhost:8080->80 --- docker-compose-dev.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 6c9cdfef..1cc34b27 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -10,18 +10,36 @@ services: - "5672:5672" networks: - cm-net + healthcheck: + test: [ "CMD", "nc", "-z", "localhost", "5672" ] + interval: 10s + timeout: 10s + retries: 5 + db: + image: postgres + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 api: build: context: ./api dockerfile: dockerfiles/api/Dockerfile restart: always ports: - - "5000:5000" + - "127.0.0.1:8080:80" links: - rabbit - redis depends_on: - redis + - db + - rabbit networks: - cm-net volumes: From 42da5c4b7bb09e6e1b416f9abae4b8ce593e3064 Mon Sep 17 00:00:00 2001 From: Malik Bougacha Date: Thu, 3 Sep 2020 16:00:01 +0200 Subject: [PATCH 09/11] add postgres db in dev --- docker-compose-dev.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 1cc34b27..bc06de51 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -18,9 +18,9 @@ services: db: image: postgres environment: - - POSTGRES_DB=postgres - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=toolboxdb + - POSTGRES_USER=root + - POSTGRES_PASSWORD=password healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 10s From a16c421f1a6ae5863530d7eae4e03592cdf933d0 Mon Sep 17 00:00:00 2001 From: Malik Bougacha Date: Thu, 3 Sep 2020 16:02:42 +0200 Subject: [PATCH 10/11] cleanup the port for rabbitmq, as we dont expose rabbitmq externally --- docker-compose-dev.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index bc06de51..71f76781 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -6,8 +6,6 @@ services: environment: - RABBITMQ_DEFAULT_USER=admin - RABBITMQ_DEFAULT_PASS=mypass - ports: - - "5672:5672" networks: - cm-net healthcheck: @@ -21,6 +19,8 @@ services: - POSTGRES_DB=toolboxdb - POSTGRES_USER=root - POSTGRES_PASSWORD=password + networks: + - cm-net healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 10s From d7fd921b26f0c7fec6f2a82b768e2b0b791c3c32 Mon Sep 17 00:00:00 2001 From: Malik Bougacha Date: Thu, 3 Sep 2020 16:03:20 +0200 Subject: [PATCH 11/11] also fix environment now that we have a db --- .env.dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.dev b/.env.dev index d1a1a246..76bb53e0 100644 --- a/.env.dev +++ b/.env.dev @@ -27,7 +27,7 @@ USER_UPLOAD_FOLDER=/var/hotmaps/users/ # directory where user uploa # Database configuration DB_USER=root DB_PASSWORD=password -DB_HOST=localhost +DB_HOST=postgres DB_PORT=5432 DB_DATABASE=toolboxdb