diff --git a/assets/share/freva/deployment/config/inventory.toml b/assets/share/freva/deployment/config/inventory.toml index 193f7191..978c8566 100644 --- a/assets/share/freva/deployment/config/inventory.toml +++ b/assets/share/freva/deployment/config/inventory.toml @@ -8,6 +8,12 @@ project_name = "freva" ## Choose between: "docker", "podman", "conda", "k8s" deployment_method = "podman" +## The `master_password` is used as the admin password for the web admin +## interface and the mysql root password. It *can* be set in the config file +## but it is reocmmended to leave this key blank and set enter it during +## the setup. +master_password = "" + ## Kubernetes deployment settings [kubernetes] ## Set the group ID for all volmes used by pods diff --git a/assets/share/freva/deployment/k8s-deployment/templates/01-secrets.yaml.j2 b/assets/share/freva/deployment/k8s-deployment/templates/01-secrets.yaml.j2 index 92d3a312..e3d55a82 100644 --- a/assets/share/freva/deployment/k8s-deployment/templates/01-secrets.yaml.j2 +++ b/assets/share/freva/deployment/k8s-deployment/templates/01-secrets.yaml.j2 @@ -17,7 +17,14 @@ stringData: MYSQL_PASSWORD: "{{ db_passwd }}" REDIS_USERNAME: "{{ redis_username }}" REDIS_PASSWORD: "{{ redis_password }}" + OIDC_DISCOVERY_URL: "{{ freva_rest_oidc_url }}" OIDC_CLIENT_SECRET: "{{ freva_rest_oidc_client_secret }}" + OIDC_CLIENT_ID : "{{ freva_rest_oidc_client }}" + OIDC_DISCOVERY_URL: "{{ freva_rest_oidc_client }}" + OIDC_TOKEN_CLAIMS: "{{ freva_rest_oidc_token_claims | join(',')}}" + OIDC_ADMIN_CLAIMS: "{{ freva_rest_oidc_admin_claims | join(",") }}" + OIDC_SYSTEM_USER_CLAIM: "{{ freva_rest_oidc_systemuser_claim }}" + OIDC_SCOPES: "{{ freva_rest_oidc_scopes }}" FREVA_REST_DB_USER: "{{ freva_rest_db_user }}" FREVA_REST_DB_PASSWORD: "{{ freva_rest_db_passwd }}" MONGO_APP_USER: "{{ mongodb_server_db_user }}" diff --git a/assets/share/freva/deployment/k8s-deployment/templates/10-mysql.yaml.j2 b/assets/share/freva/deployment/k8s-deployment/templates/10-mysql.yaml.j2 index 5aab77d4..3bd574e5 100644 --- a/assets/share/freva/deployment/k8s-deployment/templates/10-mysql.yaml.j2 +++ b/assets/share/freva/deployment/k8s-deployment/templates/10-mysql.yaml.j2 @@ -11,9 +11,9 @@ metadata: app.kubernetes.io/instance: database-server app.kubernetes.io/component: web-app app.kubernetes.io/part-of: freva - app.kubernetes.io/version: {{ db_version }} + app.kubernetes.io/version: "{{ db_version }}" app.kubernetes.io/managed-by: ansible - freva.org/stateful: 'true' + freva.org/stateful: "true" freva.org/tier: database spec: type: ClusterIP @@ -36,54 +36,71 @@ metadata: app.kubernetes.io/instance: database-server app.kubernetes.io/component: web-app app.kubernetes.io/part-of: freva - app.kubernetes.io/version: {{ db_version }} + app.kubernetes.io/version: "{{ db_version }}" app.kubernetes.io/managed-by: ansible - freva.org/stateful: 'true' + freva.org/stateful: "true" freva.org/tier: database spec: serviceName: database-server replicas: 1 - strategy: + revisionHistoryLimit: 1 + updateStrategy: type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - selector: { matchLabels: { app: database-server } } + selector: + matchLabels: + app: database-server template: metadata: - name: database-server - namespace: {{ ns }} labels: app: database-server app.kubernetes.io/name: freva app.kubernetes.io/instance: database-server app.kubernetes.io/component: web-app app.kubernetes.io/part-of: freva - app.kubernetes.io/version: {{ db_version }} + app.kubernetes.io/version: "{{ db_version }}" app.kubernetes.io/managed-by: ansible - freva.org/stateful: 'true' + freva.org/stateful: "true" freva.org/tier: database spec: - securityContext: { fsGroup: {{ fs_group }} } + securityContext: + fsGroup: {{ fs_group }} containers: - name: mysql image: ghcr.io/freva-org/freva-mysql:{{ db_version }} imagePullPolicy: {{ image_pull_policy }} - ports: [{ containerPort: 3306, name: mysql }] - resources: {{ resources['database-server'] }} + ports: + - containerPort: 3306 + name: mysql + resources: +{{ resources['database-server'] | to_nice_yaml | indent(12, true) }} env: - name: ROOT_PW - valueFrom: { secretKeyRef: { name: freva-secrets, key: MYSQL_ROOT_PASSWORD } } + valueFrom: + secretKeyRef: + name: freva-secrets + key: MYSQL_ROOT_PASSWORD - name: MYSQL_ROOT_PASSWORD - valueFrom: { secretKeyRef: { name: freva-secrets, key: MYSQL_ROOT_PASSWORD } } - - { name: MYSQL_USER, value: "{{ db_user }}" } + valueFrom: + secretKeyRef: + name: freva-secrets + key: MYSQL_ROOT_PASSWORD + - name: MYSQL_USER + value: "{{ db_user }}" - name: MYSQL_PASSWORD - valueFrom: { secretKeyRef: { name: freva-secrets, key: MYSQL_PASSWORD } } - - { name: MYSQL_DATABASE, value: "{{ db }}" } - - { name: HOST, value: "{{ db_host }}" } - - { name: PROJECT, value: "{{ project_name }}" } + valueFrom: + secretKeyRef: + name: freva-secrets + key: MYSQL_PASSWORD + - name: MYSQL_DATABASE + value: "{{ db }}" + - name: HOST + value: "{{ db_host }}" + - name: PROJECT + value: "{{ project_name }}" volumeMounts: - - { name: db-data, mountPath: /data/db } + - name: db-data + mountPath: /data/db volumes: - name: db-data - persistentVolumeClaim: { claimName: db-data } + persistentVolumeClaim: + claimName: db-data diff --git a/assets/share/freva/deployment/k8s-deployment/templates/11-redis.yaml.j2 b/assets/share/freva/deployment/k8s-deployment/templates/11-redis.yaml.j2 index a663f55c..19910a73 100644 --- a/assets/share/freva/deployment/k8s-deployment/templates/11-redis.yaml.j2 +++ b/assets/share/freva/deployment/k8s-deployment/templates/11-redis.yaml.j2 @@ -37,6 +37,7 @@ metadata: freva.org/tier: database spec: replicas: 1 + revisionHistoryLimit: 1 strategy: type: RollingUpdate rollingUpdate: diff --git a/assets/share/freva/deployment/k8s-deployment/templates/12-solr.yaml.j2 b/assets/share/freva/deployment/k8s-deployment/templates/12-solr.yaml.j2 index 5233c441..f59a77d6 100644 --- a/assets/share/freva/deployment/k8s-deployment/templates/12-solr.yaml.j2 +++ b/assets/share/freva/deployment/k8s-deployment/templates/12-solr.yaml.j2 @@ -11,15 +11,19 @@ metadata: app.kubernetes.io/instance: search-server app.kubernetes.io/component: backend app.kubernetes.io/part-of: freva - app.kubernetes.io/version: {{ solr_version }} + app.kubernetes.io/version: "{{ solr_version }}" app.kubernetes.io/managed-by: ansible - freva.org/stateful: 'true' + freva.org/stateful: "true" freva.org/tier: database - spec: type: ClusterIP - selector: { app: search-server } - ports: [{ port: 8983, targetPort: 8983, name: http }] + selector: + app: search-server + ports: + - port: 8983 + targetPort: 8983 + name: http + --- apiVersion: apps/v1 kind: StatefulSet @@ -32,46 +36,52 @@ metadata: app.kubernetes.io/instance: search-server app.kubernetes.io/component: backend app.kubernetes.io/part-of: freva - app.kubernetes.io/version: {{ solr_version }} + app.kubernetes.io/version: "{{ solr_version }}" app.kubernetes.io/managed-by: ansible - freva.org/stateful: 'true' + freva.org/stateful: "true" freva.org/tier: database spec: serviceName: search-server replicas: 1 - strategy: + revisionHistoryLimit: 1 + updateStrategy: type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - selector: { matchLabels: { app: search-server } } + selector: + matchLabels: + app: search-server template: metadata: - name: search-server - namespace: {{ ns }} labels: app: search-server app.kubernetes.io/name: freva app.kubernetes.io/instance: search-server app.kubernetes.io/component: backend app.kubernetes.io/part-of: freva - app.kubernetes.io/version: {{ solr_version }} + app.kubernetes.io/version: "{{ solr_version }}" app.kubernetes.io/managed-by: ansible - freva.org/stateful: 'true' + freva.org/stateful: "true" freva.org/tier: database spec: - securityContext: { fsGroup: {{ fs_group }} } + securityContext: + fsGroup: {{ fs_group }} containers: - name: solr image: ghcr.io/freva-org/freva-solr:{{ solr_version }} imagePullPolicy: {{ image_pull_policy }} - ports: [{ containerPort: 8983, name: http }] - resources: {{ resources['search-server'] }} + ports: + - containerPort: 8983 + name: http + resources: +{{ resources['search-server'] | to_nice_yaml | indent(12, true) }} env: - - { name: API_SOLR_PORT, value: "8983" } - - { name: API_SOLR_HEAP, value: "{{ search_server_solr_mem }}" } + - name: API_SOLR_PORT + value: "8983" + - name: API_SOLR_HEAP + value: "{{ search_server_solr_mem }}" volumeMounts: - - { name: solr-data, mountPath: /data/db } + - name: solr-data + mountPath: /data/db volumes: - name: solr-data - persistentVolumeClaim: { claimName: solr-data } + persistentVolumeClaim: + claimName: solr-data diff --git a/assets/share/freva/deployment/k8s-deployment/templates/13-vault.yaml.j2 b/assets/share/freva/deployment/k8s-deployment/templates/13-vault.yaml.j2 index ababd184..96f9a77e 100644 --- a/assets/share/freva/deployment/k8s-deployment/templates/13-vault.yaml.j2 +++ b/assets/share/freva/deployment/k8s-deployment/templates/13-vault.yaml.j2 @@ -11,14 +11,19 @@ metadata: app.kubernetes.io/instance: vault-server app.kubernetes.io/component: web-app app.kubernetes.io/part-of: freva - app.kubernetes.io/version: {{ vault_version }} + app.kubernetes.io/version: "{{ vault_version }}" app.kubernetes.io/managed-by: ansible - freva.org/stateful: 'true' + freva.org/stateful: "true" freva.org/tier: backend spec: type: ClusterIP - selector: { app: vault-server } - ports: [{ port: 5002, targetPort: 5002, name: vault }] + selector: + app: vault-server + ports: + - port: 5002 + targetPort: 5002 + name: vault + --- apiVersion: apps/v1 kind: StatefulSet @@ -31,47 +36,55 @@ metadata: app.kubernetes.io/instance: vault-server app.kubernetes.io/component: web-app app.kubernetes.io/part-of: freva - app.kubernetes.io/version: {{ vault_version }} + app.kubernetes.io/version: "{{ vault_version }}" app.kubernetes.io/managed-by: ansible - freva.org/stateful: 'true' + freva.org/stateful: "true" freva.org/tier: backend spec: serviceName: vault-server replicas: 1 - strategy: + revisionHistoryLimit: 1 + updateStrategy: type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - selector: { matchLabels: { app: vault-server } } + selector: + matchLabels: + app: vault-server template: metadata: - name: vault-server - namespace: {{ ns }} labels: app: vault-server app.kubernetes.io/name: freva app.kubernetes.io/instance: vault-server app.kubernetes.io/component: web-app app.kubernetes.io/part-of: freva - app.kubernetes.io/version: {{ vault_version }} + app.kubernetes.io/version: "{{ vault_version }}" app.kubernetes.io/managed-by: ansible - freva.org/stateful: 'true' + freva.org/stateful: "true" freva.org/tier: backend spec: - securityContext: { fsGroup: {{ fs_group }} } + securityContext: + fsGroup: {{ fs_group }} containers: - name: vault - image: ghcr.io/freva-org/freva-vault:{{vault_version}} + image: ghcr.io/freva-org/freva-vault:{{ vault_version }} imagePullPolicy: {{ image_pull_policy }} - ports: [{ containerPort: 5002, name: vault }] - resources: {{ resources['vault-server'] }} + ports: + - containerPort: 5002 + name: vault + resources: +{{ resources['vault-server'] | to_nice_yaml | indent(12, true) }} env: - name: ROOT_PWD - valueFrom: { secretKeyRef: { name: freva-secrets, key: ADMIN_PASSWORD } } - - { name: KEY_FILE, value: "/vault/file/keys" } + valueFrom: + secretKeyRef: + name: freva-secrets + key: ADMIN_PASSWORD + - name: KEY_FILE + value: "/vault/file/keys" volumeMounts: - - { name: vault-data, mountPath: /vault/file } + - name: vault-data + mountPath: /vault/file volumes: - name: vault-data - persistentVolumeClaim: { claimName: vault-data } + persistentVolumeClaim: + claimName: vault-data diff --git a/assets/share/freva/deployment/k8s-deployment/templates/14-mongo.yaml.j2 b/assets/share/freva/deployment/k8s-deployment/templates/14-mongo.yaml.j2 index 7704dd67..30068a8a 100644 --- a/assets/share/freva/deployment/k8s-deployment/templates/14-mongo.yaml.j2 +++ b/assets/share/freva/deployment/k8s-deployment/templates/14-mongo.yaml.j2 @@ -11,14 +11,19 @@ metadata: app.kubernetes.io/instance: mongo-server app.kubernetes.io/component: rest-api app.kubernetes.io/part-of: freva - app.kubernetes.io/version: {{ mongodb_server_version }} + app.kubernetes.io/version: "{{ mongodb_server_version }}" app.kubernetes.io/managed-by: ansible - freva.org/stateful: 'true' + freva.org/stateful: "true" freva.org/tier: database spec: type: ClusterIP - selector: { app: mongo-server } - ports: [{ port: 27017, targetPort: 27017, name: mongo }] + selector: + app: mongo-server + ports: + - port: 27017 + targetPort: 27017 + name: mongo + --- apiVersion: apps/v1 kind: StatefulSet @@ -31,49 +36,60 @@ metadata: app.kubernetes.io/instance: mongo-server app.kubernetes.io/component: rest-api app.kubernetes.io/part-of: freva - app.kubernetes.io/version: {{ mongodb_server_version }} + app.kubernetes.io/version: "{{ mongodb_server_version }}" app.kubernetes.io/managed-by: ansible - freva.org/stateful: 'true' + freva.org/stateful: "true" freva.org/tier: database spec: serviceName: mongo-server replicas: 1 - strategy: + revisionHistoryLimit: 1 + updateStrategy: type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - selector: { matchLabels: { app: mongo-server } } + selector: + matchLabels: + app: mongo-server template: metadata: - name: mongo-server - namespace: {{ ns }} labels: app: mongo-server app.kubernetes.io/name: freva app.kubernetes.io/instance: mongo-server app.kubernetes.io/component: rest-api app.kubernetes.io/part-of: freva - app.kubernetes.io/version: {{ mongodb_server_version }} + app.kubernetes.io/version: "{{ mongodb_server_version }}" app.kubernetes.io/managed-by: ansible - freva.org/stateful: 'true' + freva.org/stateful: "true" freva.org/tier: database spec: - securityContext: { fsGroup: {{ fs_group }} } + securityContext: + fsGroup: {{ fs_group }} containers: - name: mongodb image: ghcr.io/freva-org/freva-mongo:{{ mongodb_server_version }} imagePullPolicy: {{ image_pull_policy }} - ports: [{ containerPort: 27017, name: mongo }] - resources: {{ resources['mongo-server'] }} + ports: + - containerPort: 27017 + name: mongo + resources: +{{ resources['mongo-server'] | to_nice_yaml | indent(12, true) }} env: - name: API_MONGO_USER - valueFrom: { secretKeyRef: { name: freva-secrets, key: MONGO_APP_USER } } + valueFrom: + secretKeyRef: + name: freva-secrets + key: MONGO_APP_USER - name: API_MONGO_PASSWORD - valueFrom: { secretKeyRef: { name: freva-secrets, key: MONGO_APP_PASSWORD } } - - { name: API_MONGO_DB, value: "search_stats" } + valueFrom: + secretKeyRef: + name: freva-secrets + key: MONGO_APP_PASSWORD + - name: API_MONGO_DB + value: "search_stats" volumeMounts: - - { name: mongo-data, mountPath: /data/db } + - name: mongo-data + mountPath: /data/db volumes: - name: mongo-data - persistentVolumeClaim: { claimName: mongo-data } + persistentVolumeClaim: + claimName: mongo-data diff --git a/assets/share/freva/deployment/k8s-deployment/templates/20-rest.yaml.j2 b/assets/share/freva/deployment/k8s-deployment/templates/20-rest.yaml.j2 index 45d64fe3..dbde1eee 100644 --- a/assets/share/freva/deployment/k8s-deployment/templates/20-rest.yaml.j2 +++ b/assets/share/freva/deployment/k8s-deployment/templates/20-rest.yaml.j2 @@ -37,6 +37,7 @@ metadata: freva.org/tier: backend spec: replicas: 1 + revisionHistoryLimit: 1 strategy: type: RollingUpdate rollingUpdate: @@ -70,13 +71,20 @@ spec: - {name: API_PORT, value: "{{ freva_rest_port }}"} - {name: API_PROXY, value: "{{ freva_rest_proxy_url }}"} - {name: COLUMNS, value: "140"} - - {name: API_OIDC_CLIENT_ID, value: "{{ freva_rest_oidc_client }}"} - - {name: API_OIDC_DISCOVERY_URL, value: "{{ freva_rest_oidc_url }}"} + - name: API_OIDC_CLIENT_ID + valueFrom: { secretKeyRef: {name: freva-secrets, key: OIDC_CLIENT_ID } } - name: API_OIDC_CLIENT_SECRET valueFrom: { secretKeyRef: {name: freva-secrets, key: OIDC_CLIENT_SECRET } } - - {name: API_OIDC_TOKEN_CLAIMS, value: "{{ freva_rest_oidc_token_claims }}"} - - {name: API_ADMIN_CLAIMS, value: "{{ freva_rest_oidc_admin_claims }}"} - - {name: API_OIDC_SYSTEMUSER_CLAIM, value: "{{ freva_rest_oidc_systemuser_claim }}"} + - name: API_OIDC_DISCOVERY_URL + valueFrom: { secretKeyRef: {name: freva-secrets, key: OIDC_DISCOVERY_URL } } + - name: API_OIDC_TOKEN_CLAIMS + valueFrom: { secretKeyRef: {name: freva-secrets, key: OIDC_TOKEN_CLAIMS } } + - name: API_ADMIN_CLAIMS + valueFrom: { secretKeyRef: {name: freva-secrets, key: OIDC_ADMIN_CLAIMS } } + - name: API_OIDC_SYSTEMUSER_CLAIM + valueFrom: { secretKeyRef: {name: freva-secrets, key: OIDC_SYSTEM_USER_CLAIM } } + - name: API_OIDC_SCOPES + valueFrom: { secretKeyRef: {name: freva-secrets, key: OIDC_SCOPES } } - {name: API_OIDC_TRUSTED_ISSUERS, value: "{{ freva_rest_oidc_trusted_issuers | join(',') }}"} - name: API_REDIS_USER valueFrom: { secretKeyRef: {name: freva-secrets, key: REDIS_USERNAME } } diff --git a/assets/share/freva/deployment/k8s-deployment/templates/22-data-loader.yaml.j2 b/assets/share/freva/deployment/k8s-deployment/templates/22-data-loader.yaml.j2 index f885afff..d14de403 100644 --- a/assets/share/freva/deployment/k8s-deployment/templates/22-data-loader.yaml.j2 +++ b/assets/share/freva/deployment/k8s-deployment/templates/22-data-loader.yaml.j2 @@ -18,7 +18,8 @@ metadata: freva.org/tier: backend spec: - replicas: 2 + replicas: 1 + revisionHistoryLimit: 1 selector: { matchLabels: { app: data-loader-server } } template: metadata: @@ -58,6 +59,8 @@ spec: key: REDIS_PASSWORD - name: API_REDIS_HOST value: cache-server + - name: API_LOGLEVEL + value: INFO {% if volumes %} volumeMounts: {% for vol in volumes %} diff --git a/assets/share/freva/deployment/k8s-deployment/templates/32-web-app.yaml.j2 b/assets/share/freva/deployment/k8s-deployment/templates/32-web-app.yaml.j2 index 1d06f66c..dd89fb57 100644 --- a/assets/share/freva/deployment/k8s-deployment/templates/32-web-app.yaml.j2 +++ b/assets/share/freva/deployment/k8s-deployment/templates/32-web-app.yaml.j2 @@ -68,6 +68,7 @@ metadata: freva.org/tier: frontend spec: replicas: 1 + revisionHistoryLimit: 1 strategy: type: RollingUpdate rollingUpdate: @@ -180,15 +181,6 @@ spec: secretKeyRef: name: freva-secrets key: REDIS_USERNAME - - name: OIDC_DISCOVERY_URL - value: "{{ web_oidc_url }}" - - name: OIDC_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: freva-secrets - key: OIDC_CLIENT_SECRET - - name: OIDC_CLIENT_ID - value: "{{ web_oidc_client }}" - name: CHATBOT_HOST value: "{{ web_chatbot_host.replace('http://','') }}" - name: STAC_BROWSER diff --git a/assets/share/freva/deployment/playbooks/templates/service-compose.yml.j2 b/assets/share/freva/deployment/playbooks/templates/service-compose.yml.j2 index 5238ccf8..366a96d9 100644 --- a/assets/share/freva/deployment/playbooks/templates/service-compose.yml.j2 +++ b/assets/share/freva/deployment/playbooks/templates/service-compose.yml.j2 @@ -167,7 +167,7 @@ services: - API_OIDC_CLIENT_ID={{freva_rest_oidc_client}} - API_OIDC_DISCOVERY_URL={{freva_rest_oidc_url}} - API_OIDC_CLIENT_SECRET={{freva_rest_oidc_client_secret}} - - API_OIDC_TOKEN_CLAIMS={{ freva_rest_oidc_token_claims }} + - API_OIDC_TOKEN_CLAIMS={{ freva_rest_oidc_token_claims | join(',') }} - API_ADMIN_CLAIMS={{ freva_rest_oidc_admin_claims | join(',')}} - API_OIDC_SYSTEMUSER_CLAIM={{ freva_rest_oidc_systemuser_claim }} - API_OIDC_TRUSTED_ISSUERS={{ freva_rest_oidc_trusted_issuers | join(',') }} diff --git a/docs/deployment/Configure.md b/docs/deployment/Configure.md index a25f66b6..2534dcd8 100644 --- a/docs/deployment/Configure.md +++ b/docs/deployment/Configure.md @@ -114,7 +114,27 @@ issue the `deploy-freva cmd` command: The `--steps` flags can be used if not all services should be deployed. -## Setting the python +## Keeping secrets out of version control + +Some configuration variables are sensitive and must not be shared publicly. +[OpenID Connect](https://openid.net) credentials are a typical example +- client IDs, client secrets, and token endpoints should never end up +in a public repository. + +Starting with version `2506.2.0`, you can split your configuration across +two files: a main configuration file that is safe to commit, and a separate +secrets file that you keep out of version control entirely. +Pass the secrets file via the `--secrets-file` flag: + +```console + --secrets-file secrets.toml +``` + +The secrets file follows the same structure as the main configuration file. +Any values it defines override those in the main file, so you only need to +include the keys you want to keep private. + +## Setting the python environment Some systems do not have access to python3.4+ (/usr/bin/python3) or git by default. In such cases you can overwrite the `ansible_python_interpreter` in the inventory settings of the server section to point ansible to a custom `python3` binary. For example diff --git a/docs/whatsnew.rst b/docs/whatsnew.rst index 848336fa..383e77bc 100644 --- a/docs/whatsnew.rst +++ b/docs/whatsnew.rst @@ -7,6 +7,10 @@ What's new :maxdepth: 0 :titlesonly: +v2606.3.0 +~~~~~~~~ +* Add external secrets file. + v2606.2.0 ~~~~~~~~ * Bumped version of freva-web to 2606.0.2 diff --git a/mypy.ini b/mypy.ini index 177ca6ac..c3e5c6d3 100644 --- a/mypy.ini +++ b/mypy.ini @@ -2,7 +2,7 @@ files = src/freva_deployment strict = False -warn_unused_ignores = True +warn_unused_ignores = False warn_unreachable = True show_error_codes = True ignore_missing_imports = True diff --git a/src/freva_deployment/__init__.py b/src/freva_deployment/__init__.py index 195be1f7..7c996786 100644 --- a/src/freva_deployment/__init__.py +++ b/src/freva_deployment/__init__.py @@ -1,7 +1,7 @@ import argparse from urllib.request import urlretrieve -__version__ = "2606.2.0" +__version__ = "2606.3.0" FREVA_PYTHON_VERSION = "3.13" AVAILABLE_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12", "3.13"] diff --git a/src/freva_deployment/cli/_compose.py b/src/freva_deployment/cli/_compose.py index a0670e64..4b94e424 100644 --- a/src/freva_deployment/cli/_compose.py +++ b/src/freva_deployment/cli/_compose.py @@ -89,21 +89,24 @@ def create_compose(args: argparse.Namespace) -> None: with DeployFactory( steps=None, config_file=args.config_file, + secrets_file=args.secrets_file, local_debug=False, gen_keys=True, ) as DF: eval_conf_enc = b64encode(DF.create_eval_config().read_text().encode()).decode() extra = { - "eval_config_content": eval_conf_enc, - "use_core": args.no_plugins is False, - "uid": args.user, - "redis_password": DF._create_random_passwd(30, 10), - "redis_username": petname.generate(), - "redis_version": get_versions()["redis"], - "current_nameservers": " ".join(args.dns_nameservers or []), - "ansible_python_interpreter": sys.executable, - "data_loader_volumes": DF.cfg["freva_rest"].get("data_loader_volumes") - or [], + **{ + "eval_config_content": eval_conf_enc, + "use_core": args.no_plugins is False, + "uid": args.user, + "redis_password": DF._create_random_passwd(30, 10), + "redis_username": petname.generate(), + "redis_version": get_versions()["redis"], + "current_nameservers": " ".join(args.dns_nameservers or []), + "ansible_python_interpreter": sys.executable, + "data_loader_volumes": DF.cfg["freva_rest"].get("data_loader_volumes") + or [], + }, } level = logger.getEffectiveLevel() logger.info("Parsing configurations") @@ -250,6 +253,14 @@ def compose_parser( action="store_true", help="Do not setup core library to use plugins.", ) + parser.add_argument( + "--secrets-file", + "--secrets_file", + "--secrets", + type=Path, + default=None, + help="Set a secrets file to read sensitive variables from.", + ) parser.add_argument( "-e", "--container-engine", diff --git a/src/freva_deployment/cli/_config.py b/src/freva_deployment/cli/_config.py index fe703a16..822e5dd5 100644 --- a/src/freva_deployment/cli/_config.py +++ b/src/freva_deployment/cli/_config.py @@ -4,15 +4,16 @@ import argparse from pathlib import Path -from typing import Any, Optional +from typing import Any, Optional, cast import tomlkit from rich.console import Console from rich_argparse import ArgumentDefaultsRichHelpFormatter +from tomlkit.items import Table from freva_deployment import __version__ -from ..utils import config_dir, load_config +from ..utils import config_dir, load_config, merge_toml_documents def _get_config(parser: argparse.Namespace) -> None: @@ -20,7 +21,7 @@ def _get_config(parser: argparse.Namespace) -> None: error_console = Console(markup=True, force_terminal=True, stderr=True) std_console = Console(markup=True, force_terminal=True) try: - cfg = load_config(parser.config_file) + cfg = _merge_config(parser.config_file, parser.secrets_file) except Exception as error: if parser.verbose > 0: raise @@ -28,31 +29,42 @@ def _get_config(parser: argparse.Namespace) -> None: raise SystemExit(1) if not parser.keys: if parser.raw: - print(parser.config_file.read_text()) + print(tomlkit.dumps(cfg)) else: - std_console.print(parser.config_file.read_text()) + std_console.print(tomlkit.dumps(cfg)) return for key in parser.keys: section, _, value = key.partition(".") try: sec = cfg[section] if value: - disp = sec[value] + disp = sec[value] # type: ignore else: disp = sec except Exception as error: error_console.print(f"[b red]:warning: {error}[/b red]") continue try: - disp_dump = tomlkit.dumps(disp) + disp_dump = tomlkit.dumps(disp) # type: ignore except Exception: - disp_dump = disp + disp_dump = disp # type: ignore if parser.raw: print(disp_dump) else: std_console.print(disp_dump) +def _merge_config( + config_file: Path | str, + secrets_file: None | str | Path = None, +) -> tomlkit.TOMLDocument: + if secrets_file is not None: + secrets = tomlkit.loads(Path(secrets_file).read_text()) + else: + secrets = None + return merge_toml_documents(load_config(config_file), secrets) + + def _set_config(parser: argparse.Namespace) -> None: """Set config.""" default_config_file = config_dir / "config" / "inventory.toml" @@ -64,7 +76,7 @@ def _set_config(parser: argparse.Namespace) -> None: "on update.[/b]" ) try: - cfg = load_config(parser.config_file) + cfg = _merge_config(parser.config_file, parser.secrets_file) except Exception as error: if parser.verbose > 0: raise @@ -93,7 +105,7 @@ def _set_config(parser: argparse.Namespace) -> None: else: cfg_value = tomlkit.loads(f"foo={value}")["foo"] if key: - cfg[section][key] = cfg_value + cast(Table, cfg[section])[key] = cfg_value else: cfg[section] = cfg_value except Exception as error: @@ -150,6 +162,13 @@ def config_parser( help="Path to ansible inventory file.", default=config_dir / "config" / "inventory.toml", ) + get_parser.add_argument( + "--secrets-file", + "--secrets_file", + type=Path, + default=None, + help="Set a secrets file to read sensitive variables from.", + ) get_parser.add_argument( "-r", "--raw", help="Raw output", action="store_true", default=False ) @@ -160,9 +179,7 @@ def config_parser( set_parser = subparsers.add_parser( "set", description="Inspect configuration values.", - help=( - "Use this command to set/override values " "of the deployment config." - ), + help=("Use this command to set/override values of the deployment config."), epilog=epilog, formatter_class=ArgumentDefaultsRichHelpFormatter, ) @@ -173,6 +190,14 @@ def config_parser( help="Path to ansible inventory file.", default=config_dir / "config" / "inventory.toml", ) + set_parser.add_argument( + "--secrets-file", + "--secrets_file", + type=Path, + default=None, + help="Set a secrets file to read sensitive variables from.", + ) + set_parser.add_argument( "values", nargs=2, diff --git a/src/freva_deployment/cli/_deploy.py b/src/freva_deployment/cli/_deploy.py index a7e92961..cbd4f8db 100644 --- a/src/freva_deployment/cli/_deploy.py +++ b/src/freva_deployment/cli/_deploy.py @@ -127,6 +127,13 @@ def __init__( "set those ansible tasks ([i]tags[/i]) to be deployed." ), ) + self.parser.add_argument( + "--secrets-file", + "--secrets_file", + type=Path, + default=None, + help="Set a secrets file to read sensitive variables from.", + ) self.parser.add_argument( "-e", "--extra", @@ -163,6 +170,7 @@ def run_cli(args: argparse.Namespace) -> None: with DeployFactory( steps=steps, config_file=args.config, + secrets_file=args.secrets_file, local_debug=args.debug, gen_keys=args.gen_keys, _cowsay=args.cowsay, diff --git a/src/freva_deployment/cli/_kubernets.py b/src/freva_deployment/cli/_kubernets.py index 0dfd471d..ce45ce77 100644 --- a/src/freva_deployment/cli/_kubernets.py +++ b/src/freva_deployment/cli/_kubernets.py @@ -201,9 +201,13 @@ def create_manifest(args: argparse.Namespace) -> None: """Create the k8s manifests.""" set_log_level(args.verbose) services = args.services + override = { + **{k: v for (k, v) in args.extra or {}}, + } with DeployFactory( steps=None, config_file=args.config_file, + secrets_file=args.secrets_file, local_debug=False, gen_keys=True, ) as DF: @@ -234,6 +238,7 @@ def create_manifest(args: argparse.Namespace) -> None: ), }, **DF.cfg["kubernetes"], + **override, } logger.info("Parsing configurations") logger.debug("Extra args for playbooks:\n%s", extra) @@ -268,6 +273,10 @@ def create_manifest(args: argparse.Namespace) -> None: ) RichConsole.rule("") + if args.secrets_file: + add = f"--secrets-file {args.secrets_file} " + else: + add = "" RichConsole.print( ( f"The k8s manifests have been created in [b]{out_dir}[/b].\n" @@ -276,7 +285,7 @@ def create_manifest(args: argparse.Namespace) -> None: "library installed and prepared the web-directory structure." " on the HPC.\n" "You can do this by running the following command:\n\n" - f" [b]deploy-freva cmd -c {args.config_file} " + f" [b]deploy-freva cmd -c {args.config_file} {add}" "-t core pre-web --skip-version-check -g" ) ) @@ -317,6 +326,22 @@ def kubernetes_parser( choices=["web", "db", "freva-rest", "data-loader"], help="The services to be deployed.", ) + parser.add_argument( + "--secrets-file", + "--secrets_file", + type=Path, + default=None, + help="Set a secrets file to read sensitive variables from.", + ) + + parser.add_argument( + "-e", + "--extra", + type=str, + nargs=2, + action="append", + help="Add/Override inventory settings.", + ) parser.add_argument( "--secrets", "--with-secrets", diff --git a/src/freva_deployment/deploy.py b/src/freva_deployment/deploy.py index 3e5ba140..e680d4df 100644 --- a/src/freva_deployment/deploy.py +++ b/src/freva_deployment/deploy.py @@ -42,6 +42,7 @@ get_cache_information, get_passwd, load_config, + merge_toml_documents, ) from .versions import get_steps_from_versions, get_versions @@ -131,6 +132,7 @@ class DeployFactory: The components that are going to be deployed. config_file: os.PathLike, default: None Path to any existing deployment configuration file. + secrets_file: os.PathLike, default: None local_debug: bool, default: False Run deployment only on local machine, debug mode. gen_keys: bool, default: False @@ -157,6 +159,7 @@ def __init__( self, steps: list[str] | None = None, config_file: Path | str | None = None, + secrets_file: Path | str | None = None, local_debug: bool = False, gen_keys: bool = False, _cowsay: bool = False, @@ -175,6 +178,7 @@ def __init__( if self._steps in (["auto"], "auto"): self._steps = [] self._inv_tmpl = Path(config_file or config_dir / "inventory.toml") + self._secrets_file = secrets_file self._cfg_tmpl = self.aux_dir / "evaluation_system.conf.tmpl" self.cfg = self._read_cfg() self.project_name = self.cfg.pop("project_name", None) @@ -678,7 +682,14 @@ def _read_cfg(self) -> dict[str, Any]: "search_server": "freva_rest", } try: - config = dict(load_config(self._inv_tmpl, convert=True).items()) + if self._secrets_file: + secrets = tomlkit.loads(Path(self._secrets_file).read_text()) + else: + secrets = None + config: Dict[str, Any] = dict( + merge_toml_documents(load_config(self._inv_tmpl, convert=True), secrets) + ) + self._master_pass = cast(str, config.pop("master_password", "")) for dest, source in mapper.items(): host = ( config[source].get(f"{dest}_host") diff --git a/src/freva_deployment/ui/deployment_tui/deploy_forms.py b/src/freva_deployment/ui/deployment_tui/deploy_forms.py index 4957adbe..aca0a8da 100644 --- a/src/freva_deployment/ui/deployment_tui/deploy_forms.py +++ b/src/freva_deployment/ui/deployment_tui/deploy_forms.py @@ -772,7 +772,7 @@ def _add_widgets(self) -> None: name=(f"{self.num}Config url of the OIDC service"), value=cast(str, cfg.get("oidc_url", "")), ), - True, + False, ), oidc_client=( self.add_widget_intelligent( @@ -782,7 +782,7 @@ def _add_widgets(self) -> None: name=(f"{self.num}Name of the OIDC client (app name)"), value=cast(str, cfg.get("oidc_client", "freva")), ), - True, + False, ), oidc_client_secret=( self.add_widget_intelligent( @@ -811,7 +811,8 @@ def _add_widgets(self) -> None: key="oidc_scopes", name=f"{self.num}OIDC scopes to request from the IDP", value=cast(str, cfg.get("oidc_scopes", "profile email")), - ) + ), + False, ), oidc_admin_claims=( self.add_widget_intelligent( @@ -956,6 +957,7 @@ def on_ok(self) -> None: "skip_version_check": bool(self.skip_version_check.value), "local_debug": bool(self.local_debug.value), "gen_keys": bool(gen_keys), + "secrets_file": self.secrets_file.value or None, } self.parentApp.thread_stop.set() self.parentApp.exit_application( @@ -1039,6 +1041,11 @@ def _add_widgets(self) -> None: value=self.parentApp._read_cache("gen_keys", False), name=f"{self.num}Generate a pair web certificates, debugging", ) + self.secrets_file = self.add_widget_intelligent( + npyscreen.TitleFilename, + name=f"{self.num}A secrets file to read sensitive variables from.", + value=self.parentApp._read_cache("secrets_file") or None, + ) self.k8s_deploy_host = self.add_widget_intelligent( TextInfo, key="deploy_host", diff --git a/src/freva_deployment/ui/deployment_tui/main_window.py b/src/freva_deployment/ui/deployment_tui/main_window.py index e7ef0fa1..b9cc5da8 100644 --- a/src/freva_deployment/ui/deployment_tui/main_window.py +++ b/src/freva_deployment/ui/deployment_tui/main_window.py @@ -90,18 +90,12 @@ def _add_froms(self) -> None: CoreScreen, name="Core deployment", ) - self._forms["web"] = self.addForm( - "SECOND", WebScreen, name="Web deployment" - ) - self._forms["db"] = self.addForm( - "THIRD", DBScreen, name="Database deployment" - ) + self._forms["web"] = self.addForm("SECOND", WebScreen, name="Web deployment") + self._forms["db"] = self.addForm("THIRD", DBScreen, name="Database deployment") self._forms["freva_rest"] = self.addForm( "FOURTH", FrevaRestScreen, name="Freva Rest deployment" ) - self._setup_form = self.addForm( - "SETUP", RunForm, name="Apply the Deployment" - ) + self._setup_form = self.addForm("SETUP", RunForm, name="Apply the Deployment") def exit_application(self, *args, **kwargs) -> None: value = npyscreen.notify_ok_cancel( @@ -134,7 +128,7 @@ def check_missing_config(self, stop_at_missing: bool = True) -> str | None: return self._steps_lookup[step] try: self.config[step] = cfg - except Exception as error: + except Exception: raise ValueError((step, cfg)) from None return None @@ -163,9 +157,7 @@ def save_dialog(self, *args, **kwargs) -> None: the_selected_file = str(the_selected_file.expanduser().absolute()) self.check_missing_config(stop_at_missing=False) self._setup_form.inventory_file.value = the_selected_file - self.save_config_to_file( - save_file=the_selected_file, write_toml_file=True - ) + self.save_config_to_file(save_file=the_selected_file, write_toml_file=True) def _update_config(self, config_file: Path | str) -> None: """Update the main window after a new configuration has been loaded.""" @@ -173,9 +165,7 @@ def _update_config(self, config_file: Path | str) -> None: try: self.config = load_config(config_file) except Exception as error: - npyscreen.notify_confirm( - str(error), title=f"Error loading {config_file}" - ) + npyscreen.notify_confirm(str(error), title=f"Error loading {config_file}") return self.resetHistory() self.editing = True @@ -260,6 +250,7 @@ def _save_config_to_file( "steps": self.steps, "ssh_port": ssh_port, "config": self.config, + "secrets_file": self._setup_form.secrets_file.value, }, } with open(self.cache_dir / "freva_deployment.json", "w") as f: @@ -268,10 +259,12 @@ def _save_config_to_file( return None try: - config_tmpl = load_config(asset_dir / "config" / "inventory.toml") + config_tmpl: Dict[str, Any] = dict( + load_config(asset_dir / "config" / "inventory.toml") + ) except Exception as error: npyscreen.notify_confirm(error) - config_tmpl = self.config + config_tmpl = dict(self.config) config_tmpl["certificates"] = cert_files config_tmpl["project_name"] = project_name config_tmpl["deployment_method"] = deployment_method diff --git a/src/freva_deployment/utils.py b/src/freva_deployment/utils.py index 4f6a9fb3..ad2dceff 100644 --- a/src/freva_deployment/utils.py +++ b/src/freva_deployment/utils.py @@ -20,6 +20,8 @@ import tomlkit from rich.console import Console from rich.prompt import Prompt +from tomlkit.container import OutOfOrderTableProxy +from tomlkit.items import Table from .error import ConfigurationError from .keys import RandomKeys @@ -238,7 +240,7 @@ def get_current_file_dir(inp_dir: str | Path, value: str) -> str: def _convert_dict( - inp_dict: dict[str, str | dict[str, Any]], + inp_dict: dict[str, Any], variables: dict[str, str], cfd: Path, ) -> None: @@ -344,7 +346,7 @@ def _create_new_config(inp_file: Path) -> Path: return inp_file -def load_config(inp_file: str | Path, convert: bool = False) -> dict[str, Any]: +def load_config(inp_file: str | Path, convert: bool = False) -> tomlkit.TOMLDocument: """Load the inventory toml file and replace all environment variables.""" inp_file = _create_new_config(Path(inp_file).expanduser().absolute()) variables = cast( @@ -352,10 +354,46 @@ def load_config(inp_file: str | Path, convert: bool = False) -> dict[str, Any]: ) config = tomlkit.loads(inp_file.read_text(encoding="utf-8")) if convert: - _convert_dict(config, variables, inp_file.parent) + _convert_dict(cast(Dict[str, Any], config), variables, inp_file.parent) return config +def merge_toml_documents( + *documents: tomlkit.TOMLDocument | None, +) -> tomlkit.TOMLDocument: + """Merge multiple TOML documents into one. + + Later documents take precedence over earlier ones for duplicate keys. + + Parameters + ---------- + documents: + List of tomlkit TOMLDocuments to merge. + + Returns + ------- + tomlkit.TOMLDocument: + The merged TOML document. + """ + DocT = (dict, Table, tomlkit.TOMLDocument, OutOfOrderTableProxy) + + def deep_merge( + base: Dict[str, Any] | Table | tomlkit.TOMLDocument | OutOfOrderTableProxy, + override: Dict[str, Any] | Table | tomlkit.TOMLDocument | OutOfOrderTableProxy, + ) -> None: + for key, value in override.items(): + if key in base and isinstance(base[key], DocT) and isinstance(value, DocT): + deep_merge(cast(Table, base[key]), value) + else: + base[key] = value + + merged = tomlkit.document() + for doc in documents: + if doc: + deep_merge(merged, doc) + return merged + + def get_setup_for_service(service: str, setups: list[ServiceInfo]) -> tuple[str, str]: """Get the setup of a service configuration.""" for setup in setups: