diff --git a/helm/arborist/Chart.yaml b/helm/arborist/Chart.yaml index 396968729..f892dbcef 100644 --- a/helm/arborist/Chart.yaml +++ b/helm/arborist/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.11 +version: 0.1.12 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/arborist/README.md b/helm/arborist/README.md index 74cb57d68..10f0953e9 100644 --- a/helm/arborist/README.md +++ b/helm/arborist/README.md @@ -60,11 +60,14 @@ A Helm chart for gen3 arborist | global.publicDataSets | bool | `true` | Whether public datasets are enabled. | | global.revproxyArn | string | `"arn:aws:acm:us-east-1:123456:certificate"` | ARN of the reverse proxy certificate. | | global.tierAccessLevel | string | `"libre"` | Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` | -| image | map | `{"pullPolicy":"IfNotPresent","repository":"quay.io/cdis/arborist","tag":""}` | Docker image information. | +| image | map | `{"pullPolicy":"IfNotPresent","repository":"quay.io/ohsu-comp-bio/arborist","tag":""}` | Docker image information. | | image.pullPolicy | string | `"IfNotPresent"` | Docker pull policy. | -| image.repository | string | `"quay.io/cdis/arborist"` | Docker repository. | +| image.repository | string | `"quay.io/ohsu-comp-bio/arborist"` | Docker repository. | | image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | | imagePullSecrets | list | `[]` | Docker image pull secrets. | +| migrations | map | `{"command":"/app/migrations/latest","enabled":true}` | Database migration settings. Arborist migrations are packaged in the application image; deploying a new image with new migration files is enough for this chart to apply them before starting Arborist. | +| migrations.command | string | `"/app/migrations/latest"` | Migration command inside the Arborist image. | +| migrations.enabled | bool | `true` | Whether to run Arborist database migrations at container startup. | | nameOverride | string | `""` | Override the name of the chart. | | nodeSelector | map | `{}` | Node selector to apply to the pod | | partOf | string | `"Authentication"` | Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. | @@ -104,4 +107,3 @@ A Helm chart for gen3 arborist | tolerations | list | `[]` | Tolerations to apply to the pod | | volumeMounts | list | `[]` | Volume mounts to attach to the container | | volumes | list | `[]` | Volumes to attach to the pod | - diff --git a/helm/arborist/templates/deployment.yaml b/helm/arborist/templates/deployment.yaml index 3fb7963eb..45da13adb 100644 --- a/helm/arborist/templates/deployment.yaml +++ b/helm/arborist/templates/deployment.yaml @@ -37,6 +37,27 @@ spec: serviceAccountName: {{ include "arborist.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if .Values.authzSnapshotCache.waitForRedis.enabled }} + initContainers: + - name: wait-for-authz-cache + image: "{{ .Values.authzSnapshotCache.waitForRedis.image }}" + imagePullPolicy: IfNotPresent + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.authzSnapshotCache.waitForRedis.secretName }} + key: {{ .Values.authzSnapshotCache.waitForRedis.passwordKey }} + optional: false + command: + - /bin/sh + - -c + - | + until redis-cli -h {{ .Values.authzSnapshotCache.waitForRedis.host }} -p {{ .Values.authzSnapshotCache.waitForRedis.port }} -a "$REDIS_PASSWORD" --no-auth-warning ping | grep -q PONG; do + echo "waiting for authz cache redis at {{ .Values.authzSnapshotCache.waitForRedis.host }}:{{ .Values.authzSnapshotCache.waitForRedis.port }}" + sleep 2 + done + {{- end }} containers: - name: {{ .Chart.Name }} securityContext: @@ -68,11 +89,13 @@ spec: # set env vars export PGSSLMODE="disable" + {{- if .Values.migrations.enabled }} # bring the database schema up to the latest version - /go/src/github.com/uc-cdis/arborist/migrations/latest + {{ .Values.migrations.command }} + {{- end }} # run arborist - /go/src/github.com/uc-cdis/arborist/bin/arborist + /usr/local/bin/arborist env: {{- if .Values.global.ddEnabled }} {{- include "common.datadogEnvVar" . | nindent 12 }} @@ -132,4 +155,4 @@ spec: {{- with .Values.tolerations }} tolerations: {{- toYaml . | nindent 8 }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/helm/arborist/values.yaml b/helm/arborist/values.yaml index c472742f4..3a0341a6a 100644 --- a/helm/arborist/values.yaml +++ b/helm/arborist/values.yaml @@ -109,12 +109,21 @@ replicaCount: 1 # -- (map) Docker image information. image: # -- (string) Docker repository. - repository: quay.io/cdis/arborist + repository: quay.io/ohsu-comp-bio/arborist # -- (string) Docker pull policy. - pullPolicy: IfNotPresent + pullPolicy: Always # -- (string) Overrides the image tag whose default is the chart appVersion. tag: "" +# -- (map) Database migration settings. Arborist migrations are packaged in the +# application image; deploying a new image with new migration files is enough for +# this chart to apply them before starting Arborist. +migrations: + # -- (bool) Whether to run Arborist database migrations at container startup. + enabled: true + # -- (string) Migration command inside the Arborist image. + command: /app/migrations/latest + # -- (list) Docker image pull secrets. imagePullSecrets: [] @@ -214,7 +223,23 @@ env: # -- (string) The URL of the JSON Web Key Set (JWKS) endpoint for authentication - name: JWKS_ENDPOINT value: "http://fence-service/.well-known/jwks" - + - name: AUTHZ_SNAPSHOT_CACHE_REDIS_URL + value: "redis://authz-cache-service:6379/0" + - name: AUTHZ_SNAPSHOT_CACHE_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: authz-cache-credentials + key: redis-password + optional: false + +authzSnapshotCache: + waitForRedis: + enabled: true + image: "redis:7.2-alpine" + host: "authz-cache-service" + port: 6379 + secretName: "authz-cache-credentials" + passwordKey: "redis-password" # Values to determine the labels that are used for the deployment, pod, etc. # -- (string) Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". diff --git a/helm/fence/README.md b/helm/fence/README.md index 7f318071a..297e309c1 100644 --- a/helm/fence/README.md +++ b/helm/fence/README.md @@ -15,7 +15,7 @@ A Helm chart for gen3 Fence | Key | Type | Default | Description | |-----|------|---------|-------------| -| FENCE_CONFIG | map | `{"ACCESS_TOKEN_COOKIE_NAME":"access_token","ACCESS_TOKEN_EXPIRES_IN":1200,"ALLOWED_USER_SERVICE_ACCOUNT_DOMAINS":["developer.gserviceaccount.com","appspot.gserviceaccount.com","iam.gserviceaccount.com"],"ALLOW_GOOGLE_LINKING":true,"APPLICATION_ROOT":"/user","APP_NAME":"Gen3 Data Commons","ARBORIST":"http://arborist-service","ASSUME_ROLE_CACHE_SECONDS":1800,"AUDIT_SERVICE":"http://audit-service","AUTHLIB_INSECURE_TRANSPORT":true,"AWS_CREDENTIALS":{},"AZ_BLOB_CONTAINER_URL":"https://myfakeblob.blob.core.windows.net/my-fake-container/","AZ_BLOB_CREDENTIALS":null,"BILLING_PROJECT_FOR_SA_CREDS":null,"BILLING_PROJECT_FOR_SIGNED_URLS":null,"CIRRUS_CFG":{"GOOGLE_ADMIN_EMAIL":"","GOOGLE_API_KEY":"","GOOGLE_APPLICATION_CREDENTIALS":"","GOOGLE_CLOUD_IDENTITY_ADMIN_EMAIL":"","GOOGLE_IDENTITY_DOMAIN":"","GOOGLE_PROJECT_ID":"","GOOGLE_STORAGE_CREDS":""},"CLIENT_ALLOWED_SCOPES":["openid","user","data","google_credentials","google_service_account","google_link","ga4gh_passport_v1"],"DATA_UPLOAD_BUCKET":"bucket1","DBGAP_ACCESSION_WITH_CONSENT_REGEX":"(?Pphs[0-9]+)(.(?Pv[0-9]+)){0,1}(.(?Pp[0-9]+)){0,1}.(?Pc[0-9]+)","DEBUG":false,"DEFAULT_LOGIN_IDP":"google","DEFAULT_LOGIN_URL":"{{BASE_URL}}/login/google","DEV_LOGIN_COOKIE_NAME":"dev_login","DREAM_CHALLENGE_GROUP":"DREAM","DREAM_CHALLENGE_TEAM":"DREAM","EMAIL_SERVER":"localhost","ENABLED_IDENTITY_PROVIDERS":{},"ENABLE_AUDIT_LOGS":{"login":false,"presigned_url":false},"ENABLE_AUTOMATIC_BILLING_PERMISSION_SA_CREDS":false,"ENABLE_AUTOMATIC_BILLING_PERMISSION_SIGNED_URLS":false,"ENABLE_CSRF_PROTECTION":true,"ENABLE_DB_MIGRATION":true,"ENABLE_PROMETHEUS_METRICS":false,"ENCRYPTION_KEY":"REPLACEME","GA4GH_VISA_ISSUER_ALLOWLIST":["{{BASE_URL}}","https://sts.nih.gov","https://stsstg.nih.gov"],"GEN3_PASSPORT_EXPIRES_IN":43200,"GLOBAL_PARSE_VISAS_ON_LOGIN":false,"GOOGLE_ACCOUNT_ACCESS_EXPIRES_IN":86400,"GOOGLE_BULK_UPDATES":false,"GOOGLE_GROUP_PREFIX":"","GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS":["dataflow-service-producer-prod.iam.gserviceaccount.com","cloudbuild.gserviceaccount.com","cloud-ml.google.com.iam.gserviceaccount.com","container-engine-robot.iam.gserviceaccount.com","dataflow-service-producer-prod.iam.gserviceaccount.com","sourcerepo-service-accounts.iam.gserviceaccount.com","dataproc-accounts.iam.gserviceaccount.com","gae-api-prod.google.com.iam.gserviceaccount.com","genomics-api.google.com.iam.gserviceaccount.com","containerregistry.iam.gserviceaccount.com","container-analysis.iam.gserviceaccount.com","cloudservices.gserviceaccount.com","stackdriver-service.iam.gserviceaccount.com","appspot.gserviceaccount.com","partnercontent.gserviceaccount.com","trifacta-gcloud-prod.iam.gserviceaccount.com","gcf-admin-robot.iam.gserviceaccount.com","compute-system.iam.gserviceaccount.com","gcp-sa-websecurityscanner.iam.gserviceaccount.com","storage-transfer-service.iam.gserviceaccount.com","firebase-sa-management.iam.gserviceaccount.com","firebase-rules.iam.gserviceaccount.com","gcp-sa-cloudbuild.iam.gserviceaccount.com","gcp-sa-automl.iam.gserviceaccount.com","gcp-sa-datalabeling.iam.gserviceaccount.com","gcp-sa-cloudscheduler.iam.gserviceaccount.com"],"GOOGLE_SERVICE_ACCOUNT_KEY_FOR_URL_SIGNING_EXPIRES_IN":2592000,"GOOGLE_SERVICE_ACCOUNT_PREFIX":"","GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN":604800,"GUN_MAIL":{"datacommons.io":{"api_key":"","api_url":"https://api.mailgun.net/v3/mailgun.example.com","default_login":"postmaster@mailgun.example.com","smtp_hostname":"smtp.mailgun.org","smtp_password":""}},"HTTP_PROXY":{"host":null,"port":3128},"INDEXD":"http://indexd-service","INDEXD_PASSWORD":"","INDEXD_USERNAME":"fence","ITRUST_GLOBAL_LOGOUT":"https://auth.nih.gov/siteminderagent/smlogout.asp?mode=nih&AppReturnUrl=","LOGIN_OPTIONS":[{"desc":"description","idp":"google","name":"Login from Google"}],"LOGIN_REDIRECT_WHITELIST":[],"MAX_ACCESS_TOKEN_TTL":3600,"MAX_API_KEY_TTL":2592000,"MAX_PRESIGNED_URL_TTL":3600,"MAX_ROLE_SESSION_INCREASE":false,"MOCK_AUTH":false,"MOCK_GOOGLE_AUTH":false,"MOCK_STORAGE":false,"OAUTH2_JWT_ALG":"RS256","OAUTH2_JWT_ENABLED":true,"OAUTH2_JWT_ISS":"{{BASE_URL}}","OAUTH2_PROVIDER_ERROR_URI":"/api/oauth2/errors","OAUTH2_TOKEN_EXPIRES_IN":{"authorization_code":1200,"implicit":1200},"OPENID_CONNECT":{"cilogon":{"client_id":"","client_secret":"","discovery_url":"https://cilogon.org/.well-known/openid-configuration","mock":false,"mock_default_user":"http://cilogon.org/serverT/users/64703","redirect_url":"{{BASE_URL}}/login/cilogon/login/","scope":"openid email profile"},"cognito":{"client_id":"","client_secret":"","discovery_url":"https://cognito-idp.{REGION}.amazonaws.com/{USER-POOL-ID}/.well-known/openid-configuration","redirect_url":"{{BASE_URL}}/login/cognito/login/","scope":"openid email"},"fence":{"access_token_url":"{{api_base_url}}/oauth2/token","api_base_url":"","authorize_url":"{{api_base_url}}/oauth2/authorize","client_id":"","client_kwargs":{"redirect_uri":"{{BASE_URL}}/login/fence/login","scope":"openid"},"client_secret":"","mock":false,"mock_default_user":"test@example.com","name":"","refresh_token_url":"{{api_base_url}}/oauth2/token","shibboleth_discovery_url":"https://login.bionimbus.org/Shibboleth.sso/DiscoFeed"},"generic_oidc_idp":{"client_id":"","client_secret":"","discovery":{"authorization_endpoint":"","jwks_uri":"","token_endpoint":""},"discovery_url":"https://server.com/.well-known/openid-configuration","email_field":"","name":"some_idp","redirect_url":"{{BASE_URL}}/login/some_idp/login","scope":"","user_id_field":""},"google":{"client_id":"","client_secret":"","discovery_url":"https://accounts.google.com/.well-known/openid-configuration","mock":"","mock_default_user":"test@example.com","redirect_url":"{{BASE_URL}}/login/google/login/","scope":"openid email"},"microsoft":{"client_id":"","client_secret":"","discovery_url":"https://login.microsoftonline.com/organizations/v2.0/.well-known/openid-configuration","mock":false,"mock_default_user":"test@example.com","redirect_url":"{{BASE_URL}}/login/microsoft/login/","scope":"openid email"},"okta":{"client_id":"","client_secret":"","discovery_url":"","redirect_url":"{{BASE_URL}}/login/okta/login/","scope":"openid email"},"orcid":{"client_id":"","client_secret":"","discovery_url":"https://orcid.org/.well-known/openid-configuration","mock":false,"mock_default_user":"0000-0002-2601-8132","redirect_url":"{{BASE_URL}}/login/orcid/login/","scope":"openid"},"ras":{"client_id":"","client_secret":"","discovery_url":"https://sts.nih.gov/.well-known/openid-configuration","mock":false,"mock_default_user":"test@example.com","redirect_url":"{{BASE_URL}}/login/ras/callback","scope":"openid email profile ga4gh_passport_v1"},"shibboleth":{"client_id":"","client_secret":"","redirect_url":"{{BASE_URL}}/login/shib/login"},"synapse":{"client_id":"","client_secret":"","discovery_url":"","redirect_url":"","scope":"openid"}},"OVERRIDE_NGINX_RATE_LIMIT":18,"PRIVACY_POLICY_URL":null,"PROBLEM_USER_EMAIL_NOTIFICATION":{"admin":["admin@example.edu"],"content":"The Data Commons Framework utilizes dbGaP for data access authorization. Another member of a Google project you belong to ({}) is attempting to register a service account to the following additional datasets ({}). Please contact dbGaP to request access.\n","domain":"example.com","from":"do-not-reply@example.com","subject":"Account access error notification"},"PUSH_AUDIT_LOGS_CONFIG":{"aws_sqs_config":{"aws_cred":null,"region":null,"sqs_url":null},"type":"aws_sqs"},"RAS_REFRESH_EXPIRATION":1296000,"RAS_USERINFO_ENDPOINT":"/openid/connect/v1.1/userinfo","REFRESH_TOKEN_EXPIRES_IN":2592000,"REGISTERED_USERS_GROUP":"","REGISTER_USERS_ON":false,"REMOVE_SERVICE_ACCOUNT_EMAIL_NOTIFICATION":{"admin":["admin@example.edu"],"content":"Service accounts were removed from access control data because some users or service accounts of GCP Project {} are not authorized to access the data sets associated to the service accounts, or do not adhere to the security policies.\n","domain":"example.com","enable":false,"from":"do-not-reply@example.com","subject":"User service account removal notification"},"RENEW_ACCESS_TOKEN_BEFORE_EXPIRATION":false,"S3_BUCKETS":{},"SEND_FROM":"example@gmail.com","SEND_TO":"example@gmail.com","SERVICE_ACCOUNT_LIMIT":6,"SESSION_ALLOWED_SCOPES":["openid","user","credentials","data","admin","google_credentials","google_service_account","google_link","ga4gh_passport_v1"],"SESSION_COOKIE_DOMAIN":null,"SESSION_COOKIE_NAME":"fence","SESSION_COOKIE_SECURE":true,"SESSION_LIFETIME":28800,"SESSION_TIMEOUT":1800,"SHIBBOLETH_HEADER":"persistent_id","SSO_URL":"https://auth.nih.gov/affwebservices/public/saml2sso?SPID={{BASE_URL}}/shibboleth&RelayState=","STORAGE_CREDENTIALS":{},"SUPPORT_EMAIL_FOR_ERRORS":null,"SYNAPSE_AUTHZ_TTL":86400,"SYNAPSE_DISCOVERY_URL":null,"SYNAPSE_JWKS_URI":null,"SYNAPSE_URI":"https://repo-prod.prod.sagebase.org/auth/v1","TOKEN_PROJECTS_CUTOFF":10,"USERSYNC":{"fallback_to_dbgap_sftp":false,"sync_from_visas":false,"visa_types":{"ras":["https://ras.nih.gov/visas/v1","https://ras.nih.gov/visas/v1.1"]}},"USER_ALLOWED_SCOPES":["fence","openid","user","data","admin","google_credentials","google_service_account","google_link","ga4gh_passport_v1"],"WHITE_LISTED_GOOGLE_PARENT_ORGS":[],"WHITE_LISTED_SERVICE_ACCOUNT_EMAILS":[],"WTF_CSRF_SECRET_KEY":"{{ENCRYPTION_KEY}}","dbGaP":[{"decrypt_key":"","enable_common_exchange_area_access":false,"info":{"host":"","password":"","port":22,"proxy":"","username":""},"parse_consent_code":true,"protocol":"sftp","study_common_exchange_areas":{"example":"test_common_exchange_area"},"study_to_resource_namespaces":{"_default":["/"],"test_common_exchange_area":["/dbgap/"]}}]}` | Configuration settings for Fence app | +| FENCE_CONFIG | map | `{"ACCESS_TOKEN_COOKIE_NAME":"access_token","ACCESS_TOKEN_EXPIRES_IN":1200,"ALLOWED_USER_SERVICE_ACCOUNT_DOMAINS":["developer.gserviceaccount.com","appspot.gserviceaccount.com","iam.gserviceaccount.com"],"ALLOW_GOOGLE_LINKING":true,"APPLICATION_ROOT":"/user","APP_NAME":"Gen3 Data Commons","ARBORIST":"http://arborist-service","ASSUME_ROLE_CACHE_SECONDS":1800,"AUDIT_SERVICE":"http://audit-service","AUTHLIB_INSECURE_TRANSPORT":true,"AWS_CREDENTIALS":{},"AZ_BLOB_CONTAINER_URL":"https://myfakeblob.blob.core.windows.net/my-fake-container/","AZ_BLOB_CREDENTIALS":null,"BILLING_PROJECT_FOR_SA_CREDS":null,"BILLING_PROJECT_FOR_SIGNED_URLS":null,"CIRRUS_CFG":{"GOOGLE_ADMIN_EMAIL":"","GOOGLE_API_KEY":"","GOOGLE_APPLICATION_CREDENTIALS":"","GOOGLE_CLOUD_IDENTITY_ADMIN_EMAIL":"","GOOGLE_IDENTITY_DOMAIN":"","GOOGLE_PROJECT_ID":"","GOOGLE_STORAGE_CREDS":""},"CLIENT_ALLOWED_SCOPES":["openid","user","data","google_credentials","google_service_account","google_link","ga4gh_passport_v1"],"DATA_UPLOAD_BUCKET":"bucket1","DBGAP_ACCESSION_WITH_CONSENT_REGEX":"(?Pphs[0-9]+)(.(?Pv[0-9]+)){0,1}(.(?Pp[0-9]+)){0,1}.(?Pc[0-9]+)","DEBUG":false,"DEFAULT_LOGIN_IDP":"google","DEFAULT_LOGIN_URL":"{{BASE_URL}}/login/google","DEV_LOGIN_COOKIE_NAME":"dev_login","DREAM_CHALLENGE_GROUP":"DREAM","DREAM_CHALLENGE_TEAM":"DREAM","EMAIL_SERVER":"localhost","ENABLED_IDENTITY_PROVIDERS":{},"ENABLE_AUDIT_LOGS":{"login":false,"presigned_url":false},"ENABLE_AUTOMATIC_BILLING_PERMISSION_SA_CREDS":false,"ENABLE_AUTOMATIC_BILLING_PERMISSION_SIGNED_URLS":false,"ENABLE_CSRF_PROTECTION":true,"ENABLE_DB_MIGRATION":true,"ENABLE_PROMETHEUS_METRICS":false,"ENCRYPTION_KEY":"REPLACEME","GA4GH_VISA_ISSUER_ALLOWLIST":["{{BASE_URL}}","https://sts.nih.gov","https://stsstg.nih.gov"],"GEN3_PASSPORT_EXPIRES_IN":43200,"GLOBAL_PARSE_VISAS_ON_LOGIN":false,"GOOGLE_ACCOUNT_ACCESS_EXPIRES_IN":86400,"GOOGLE_BULK_UPDATES":false,"GOOGLE_GROUP_PREFIX":"","GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS":["dataflow-service-producer-prod.iam.gserviceaccount.com","cloudbuild.gserviceaccount.com","cloud-ml.google.com.iam.gserviceaccount.com","container-engine-robot.iam.gserviceaccount.com","dataflow-service-producer-prod.iam.gserviceaccount.com","sourcerepo-service-accounts.iam.gserviceaccount.com","dataproc-accounts.iam.gserviceaccount.com","gae-api-prod.google.com.iam.gserviceaccount.com","genomics-api.google.com.iam.gserviceaccount.com","containerregistry.iam.gserviceaccount.com","container-analysis.iam.gserviceaccount.com","cloudservices.gserviceaccount.com","stackdriver-service.iam.gserviceaccount.com","appspot.gserviceaccount.com","partnercontent.gserviceaccount.com","trifacta-gcloud-prod.iam.gserviceaccount.com","gcf-admin-robot.iam.gserviceaccount.com","compute-system.iam.gserviceaccount.com","gcp-sa-websecurityscanner.iam.gserviceaccount.com","storage-transfer-service.iam.gserviceaccount.com","firebase-sa-management.iam.gserviceaccount.com","firebase-rules.iam.gserviceaccount.com","gcp-sa-cloudbuild.iam.gserviceaccount.com","gcp-sa-automl.iam.gserviceaccount.com","gcp-sa-datalabeling.iam.gserviceaccount.com","gcp-sa-cloudscheduler.iam.gserviceaccount.com"],"GOOGLE_SERVICE_ACCOUNT_KEY_FOR_URL_SIGNING_EXPIRES_IN":2592000,"GOOGLE_SERVICE_ACCOUNT_PREFIX":"","GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN":604800,"GUN_MAIL":{"datacommons.io":{"api_key":"","api_url":"https://api.mailgun.net/v3/mailgun.example.com","default_login":"postmaster@mailgun.example.com","smtp_hostname":"smtp.mailgun.org","smtp_password":""}},"HTTP_PROXY":{"host":null,"port":3128},"ITRUST_GLOBAL_LOGOUT":"https://auth.nih.gov/siteminderagent/smlogout.asp?mode=nih&AppReturnUrl=","LOGIN_OPTIONS":[{"desc":"description","idp":"google","name":"Login from Google"}],"LOGIN_REDIRECT_WHITELIST":[],"MAX_ACCESS_TOKEN_TTL":3600,"MAX_API_KEY_TTL":2592000,"MAX_PRESIGNED_URL_TTL":3600,"MAX_ROLE_SESSION_INCREASE":false,"MOCK_AUTH":false,"MOCK_GOOGLE_AUTH":false,"MOCK_STORAGE":false,"OAUTH2_JWT_ALG":"RS256","OAUTH2_JWT_ENABLED":true,"OAUTH2_JWT_ISS":"{{BASE_URL}}","OAUTH2_PROVIDER_ERROR_URI":"/api/oauth2/errors","OAUTH2_TOKEN_EXPIRES_IN":{"authorization_code":1200,"implicit":1200},"OPENID_CONNECT":{"cilogon":{"client_id":"","client_secret":"","discovery_url":"https://cilogon.org/.well-known/openid-configuration","mock":false,"mock_default_user":"http://cilogon.org/serverT/users/64703","redirect_url":"{{BASE_URL}}/login/cilogon/login/","scope":"openid email profile"},"cognito":{"client_id":"","client_secret":"","discovery_url":"https://cognito-idp.{REGION}.amazonaws.com/{USER-POOL-ID}/.well-known/openid-configuration","redirect_url":"{{BASE_URL}}/login/cognito/login/","scope":"openid email"},"fence":{"access_token_url":"{{api_base_url}}/oauth2/token","api_base_url":"","authorize_url":"{{api_base_url}}/oauth2/authorize","client_id":"","client_kwargs":{"redirect_uri":"{{BASE_URL}}/login/fence/login","scope":"openid"},"client_secret":"","mock":false,"mock_default_user":"test@example.com","name":"","refresh_token_url":"{{api_base_url}}/oauth2/token","shibboleth_discovery_url":"https://login.bionimbus.org/Shibboleth.sso/DiscoFeed"},"generic_oidc_idp":{"client_id":"","client_secret":"","discovery":{"authorization_endpoint":"","jwks_uri":"","token_endpoint":""},"discovery_url":"https://server.com/.well-known/openid-configuration","email_field":"","name":"some_idp","redirect_url":"{{BASE_URL}}/login/some_idp/login","scope":"","user_id_field":""},"google":{"client_id":"","client_secret":"","discovery_url":"https://accounts.google.com/.well-known/openid-configuration","mock":"","mock_default_user":"test@example.com","redirect_url":"{{BASE_URL}}/login/google/login/","scope":"openid email"},"microsoft":{"client_id":"","client_secret":"","discovery_url":"https://login.microsoftonline.com/organizations/v2.0/.well-known/openid-configuration","mock":false,"mock_default_user":"test@example.com","redirect_url":"{{BASE_URL}}/login/microsoft/login/","scope":"openid email"},"okta":{"client_id":"","client_secret":"","discovery_url":"","redirect_url":"{{BASE_URL}}/login/okta/login/","scope":"openid email"},"orcid":{"client_id":"","client_secret":"","discovery_url":"https://orcid.org/.well-known/openid-configuration","mock":false,"mock_default_user":"0000-0002-2601-8132","redirect_url":"{{BASE_URL}}/login/orcid/login/","scope":"openid"},"ras":{"client_id":"","client_secret":"","discovery_url":"https://sts.nih.gov/.well-known/openid-configuration","mock":false,"mock_default_user":"test@example.com","redirect_url":"{{BASE_URL}}/login/ras/callback","scope":"openid email profile ga4gh_passport_v1"},"shibboleth":{"client_id":"","client_secret":"","redirect_url":"{{BASE_URL}}/login/shib/login"},"synapse":{"client_id":"","client_secret":"","discovery_url":"","redirect_url":"","scope":"openid"}},"OVERRIDE_NGINX_RATE_LIMIT":18,"PRIVACY_POLICY_URL":null,"PROBLEM_USER_EMAIL_NOTIFICATION":{"admin":["admin@example.edu"],"content":"The Data Commons Framework utilizes dbGaP for data access authorization. Another member of a Google project you belong to ({}) is attempting to register a service account to the following additional datasets ({}). Please contact dbGaP to request access.\n","domain":"example.com","from":"do-not-reply@example.com","subject":"Account access error notification"},"PUSH_AUDIT_LOGS_CONFIG":{"aws_sqs_config":{"aws_cred":null,"region":null,"sqs_url":null},"type":"aws_sqs"},"RAS_REFRESH_EXPIRATION":1296000,"RAS_USERINFO_ENDPOINT":"/openid/connect/v1.1/userinfo","REFRESH_TOKEN_EXPIRES_IN":2592000,"REGISTERED_USERS_GROUP":"","REGISTER_USERS_ON":false,"REMOVE_SERVICE_ACCOUNT_EMAIL_NOTIFICATION":{"admin":["admin@example.edu"],"content":"Service accounts were removed from access control data because some users or service accounts of GCP Project {} are not authorized to access the data sets associated to the service accounts, or do not adhere to the security policies.\n","domain":"example.com","enable":false,"from":"do-not-reply@example.com","subject":"User service account removal notification"},"RENEW_ACCESS_TOKEN_BEFORE_EXPIRATION":false,"S3_BUCKETS":{},"SEND_FROM":"example@gmail.com","SEND_TO":"example@gmail.com","SERVICE_ACCOUNT_LIMIT":6,"SESSION_ALLOWED_SCOPES":["openid","user","credentials","data","admin","google_credentials","google_service_account","google_link","ga4gh_passport_v1"],"SESSION_COOKIE_DOMAIN":null,"SESSION_COOKIE_NAME":"fence","SESSION_COOKIE_SECURE":true,"SESSION_LIFETIME":28800,"SESSION_TIMEOUT":1800,"SHIBBOLETH_HEADER":"persistent_id","SSO_URL":"https://auth.nih.gov/affwebservices/public/saml2sso?SPID={{BASE_URL}}/shibboleth&RelayState=","STORAGE_CREDENTIALS":{},"SUPPORT_EMAIL_FOR_ERRORS":null,"SYNAPSE_AUTHZ_TTL":86400,"SYNAPSE_DISCOVERY_URL":null,"SYNAPSE_JWKS_URI":null,"SYNAPSE_URI":"https://repo-prod.prod.sagebase.org/auth/v1","TOKEN_PROJECTS_CUTOFF":10,"USERSYNC":{"fallback_to_dbgap_sftp":false,"sync_from_visas":false,"visa_types":{"ras":["https://ras.nih.gov/visas/v1","https://ras.nih.gov/visas/v1.1"]}},"USER_ALLOWED_SCOPES":["fence","openid","user","data","admin","google_credentials","google_service_account","google_link","ga4gh_passport_v1"],"WHITE_LISTED_GOOGLE_PARENT_ORGS":[],"WHITE_LISTED_SERVICE_ACCOUNT_EMAILS":[],"WTF_CSRF_SECRET_KEY":"{{ENCRYPTION_KEY}}","dbGaP":[{"decrypt_key":"","enable_common_exchange_area_access":false,"info":{"host":"","password":"","port":22,"proxy":"","username":""},"parse_consent_code":true,"protocol":"sftp","study_common_exchange_areas":{"example":"test_common_exchange_area"},"study_to_resource_namespaces":{"_default":["/"],"test_common_exchange_area":["/dbgap/"]}}]}` | Configuration settings for Fence app | | FENCE_CONFIG.APP_NAME | string | `"Gen3 Data Commons"` | Name of the Fence app | | FENCE_CONFIG.AUTHLIB_INSECURE_TRANSPORT | bool | `true` | allow OIDC traffic on http for development. By default it requires https. WARNING: ONLY set to true when fence will be deployed in such a way that it will ONLY receive traffic from internal clients and can safely use HTTP. | | FENCE_CONFIG.CLIENT_ALLOWED_SCOPES | list | `["openid","user","data","google_credentials","google_service_account","google_link","ga4gh_passport_v1"]` | These are the *possible* scopes a client can be given, NOT scopes that are given to all clients. You can be more restrictive during client creation | @@ -88,7 +88,7 @@ A Helm chart for gen3 Fence | datadogLogsInjection | bool | `true` | If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. | | datadogProfilingEnabled | bool | `true` | If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. | | datadogTraceSampleRate | int | `1` | A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. | -| env | list | `[{"name":"GEN3_UWSGI_TIMEOUT","valueFrom":{"configMapKeyRef":{"key":"uwsgi-timeout","name":"manifest-global","optional":true}}},{"name":"DD_AGENT_HOST","valueFrom":{"fieldRef":{"fieldPath":"status.hostIP"}}},{"name":"AWS_STS_REGIONAL_ENDPOINTS","value":"regional"},{"name":"PYTHONPATH","value":"/var/www/fence"},{"name":"GEN3_DEBUG","value":"False"},{"name":"FENCE_PUBLIC_CONFIG","valueFrom":{"configMapKeyRef":{"key":"fence-config-public.yaml","name":"manifest-fence","optional":true}}},{"name":"PGHOST","valueFrom":{"secretKeyRef":{"key":"host","name":"fence-dbcreds","optional":false}}},{"name":"PGUSER","valueFrom":{"secretKeyRef":{"key":"username","name":"fence-dbcreds","optional":false}}},{"name":"PGPASSWORD","valueFrom":{"secretKeyRef":{"key":"password","name":"fence-dbcreds","optional":false}}},{"name":"PGDB","valueFrom":{"secretKeyRef":{"key":"database","name":"fence-dbcreds","optional":false}}},{"name":"DBREADY","valueFrom":{"secretKeyRef":{"key":"dbcreated","name":"fence-dbcreds","optional":false}}},{"name":"DB","value":"postgresql://$(PGUSER):$(PGPASSWORD)@$(PGHOST):5432/$(PGDB)"},{"name":"INDEXD_PASSWORD","valueFrom":{"secretKeyRef":{"key":"fence","name":"indexd-service-creds"}}},{"name":"gen3Env","valueFrom":{"configMapKeyRef":{"key":"hostname","name":"manifest-global"}}}]` | Environment variables to pass to the container | +| env | list | `[{"name":"GEN3_UWSGI_TIMEOUT","valueFrom":{"configMapKeyRef":{"key":"uwsgi-timeout","name":"manifest-global","optional":true}}},{"name":"DD_AGENT_HOST","valueFrom":{"fieldRef":{"fieldPath":"status.hostIP"}}},{"name":"AWS_STS_REGIONAL_ENDPOINTS","value":"regional"},{"name":"PYTHONPATH","value":"/var/www/fence"},{"name":"GEN3_DEBUG","value":"False"},{"name":"FENCE_PUBLIC_CONFIG","valueFrom":{"configMapKeyRef":{"key":"fence-config-public.yaml","name":"manifest-fence","optional":true}}},{"name":"PGHOST","valueFrom":{"secretKeyRef":{"key":"host","name":"fence-dbcreds","optional":false}}},{"name":"PGUSER","valueFrom":{"secretKeyRef":{"key":"username","name":"fence-dbcreds","optional":false}}},{"name":"PGPASSWORD","valueFrom":{"secretKeyRef":{"key":"password","name":"fence-dbcreds","optional":false}}},{"name":"PGDB","valueFrom":{"secretKeyRef":{"key":"database","name":"fence-dbcreds","optional":false}}},{"name":"DBREADY","valueFrom":{"secretKeyRef":{"key":"dbcreated","name":"fence-dbcreds","optional":false}}},{"name":"DB","value":"postgresql://$(PGUSER):$(PGPASSWORD)@$(PGHOST):5432/$(PGDB)"},{"name":"gen3Env","valueFrom":{"configMapKeyRef":{"key":"hostname","name":"manifest-global"}}}]` | Environment variables to pass to the container | | externalSecrets | map | `{"createK8sFenceConfigSecret":false,"createK8sGoogleAppSecrets":false,"createK8sJwtKeysSecret":false,"dbcreds":null,"fenceConfig":null,"fenceGoogleAppCredsSecret":null,"fenceGoogleStorageCredsSecret":null,"fenceJwtKeys":null}` | External Secrets settings. | | externalSecrets.createK8sFenceConfigSecret | string | `false` | Will create the Helm "fence-config" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. | | externalSecrets.createK8sGoogleAppSecrets | string | `false` | Will create the Helm "fence-google-app-creds-secret" and "fence-google-storage-creds-secret" secrets even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. | @@ -198,4 +198,3 @@ A Helm chart for gen3 Fence | usersync.usersync | bool | `true` | Whether to run Fence usersync or not. | | volumeMounts | list | `[{"mountPath":"/var/www/fence/local_settings.py","name":"old-config-volume","readOnly":true,"subPath":"local_settings.py"},{"mountPath":"/var/www/fence/fence_credentials.json","name":"json-secret-volume","readOnly":true,"subPath":"fence_credentials.json"},{"mountPath":"/var/www/fence/creds.json","name":"creds-volume","readOnly":true,"subPath":"creds.json"},{"mountPath":"/var/www/fence/config_helper.py","name":"config-helper","readOnly":true,"subPath":"config_helper.py"},{"mountPath":"/fence/fence/static/img/logo.svg","name":"logo-volume","readOnly":true,"subPath":"logo.svg"},{"mountPath":"/fence/fence/static/privacy_policy.md","name":"privacy-policy","readOnly":true,"subPath":"privacy_policy.md"},{"mountPath":"/var/www/fence/fence-config.yaml","name":"config-volume","readOnly":true,"subPath":"fence-config.yaml"},{"mountPath":"/var/www/fence/yaml_merge.py","name":"yaml-merge","readOnly":true,"subPath":"yaml_merge.py"},{"mountPath":"/var/www/fence/fence_google_app_creds_secret.json","name":"fence-google-app-creds-secret-volume","readOnly":true,"subPath":"fence_google_app_creds_secret.json"},{"mountPath":"/var/www/fence/fence_google_storage_creds_secret.json","name":"fence-google-storage-creds-secret-volume","readOnly":true,"subPath":"fence_google_storage_creds_secret.json"},{"mountPath":"/fence/keys/key/jwt_private_key.pem","name":"fence-jwt-keys","readOnly":true,"subPath":"jwt_private_key.pem"}]` | Volumes to mount to the container. | | volumes | list | `[{"name":"old-config-volume","secret":{"secretName":"fence-secret"}},{"name":"json-secret-volume","secret":{"optional":true,"secretName":"fence-json-secret"}},{"name":"creds-volume","secret":{"secretName":"fence-creds"}},{"configMap":{"name":"config-helper","optional":true},"name":"config-helper"},{"configMap":{"name":"logo-config"},"name":"logo-volume"},{"name":"config-volume","secret":{"secretName":"fence-config"}},{"name":"fence-google-app-creds-secret-volume","secret":{"secretName":"fence-google-app-creds-secret"}},{"name":"fence-google-storage-creds-secret-volume","secret":{"secretName":"fence-google-storage-creds-secret"}},{"name":"fence-jwt-keys","secret":{"secretName":"fence-jwt-keys"}},{"configMap":{"name":"privacy-policy"},"name":"privacy-policy"},{"configMap":{"name":"fence-yaml-merge","optional":true},"name":"yaml-merge"}]` | Volumes to attach to the container. | - diff --git a/helm/fence/fence-secret/config_helper.py b/helm/fence/fence-secret/config_helper.py index 6b303beac..0515c8c9b 100644 --- a/helm/fence/fence-secret/config_helper.py +++ b/helm/fence/fence-secret/config_helper.py @@ -59,7 +59,6 @@ def inject_creds_into_fence_config(creds_file_path, config_file_path): db_password = _get_nested_value(creds, "db_password") db_database = _get_nested_value(creds, "db_database") hostname = _get_nested_value(creds, "hostname") - indexd_password = _get_nested_value(creds, "indexd_password") google_client_secret = _get_nested_value(creds, "google_client_secret") google_client_id = _get_nested_value(creds, "google_client_id") hmac_key = _get_nested_value(creds, "hmac_key") @@ -75,10 +74,6 @@ def inject_creds_into_fence_config(creds_file_path, config_file_path): print(" BASE_URL injected with value(s) from creds.json") config_file = _replace(config_file, "BASE_URL", "https://{}/user".format(hostname)) - print(" INDEXD_PASSWORD injected with value(s) from creds.json") - config_file = _replace(config_file, "INDEXD_PASSWORD", indexd_password) - config_file = _replace(config_file, "INDEXD_USERNAME", "fence") - print(" ENCRYPTION_KEY injected with value(s) from creds.json") config_file = _replace(config_file, "ENCRYPTION_KEY", hmac_key) @@ -121,9 +116,6 @@ def set_prod_defaults(config_file_path): "/var/www/fence/fence_google_storage_creds_secret.json", ) - print(" INDEXD set as http://indexd-service/") - config_file = _replace(config_file, "INDEXD", "http://indexd-service/") - print(" ARBORIST set as http://arborist-service/") config_file = _replace(config_file, "ARBORIST", "http://arborist-service/") diff --git a/helm/fence/fence-secret/fence_settings.py b/helm/fence/fence-secret/fence_settings.py index eb2cf818d..1685e4e13 100644 --- a/helm/fence/fence-secret/fence_settings.py +++ b/helm/fence/fence-secret/fence_settings.py @@ -165,6 +165,4 @@ def get_from_dict(dictionary, key, default=""): DEFAULT_LOGIN_URL_REDIRECT_PARAM = "redirect" -INDEXD = "http://indexd-service/" - ARBORIST = "http://arborist-service/" diff --git a/helm/fence/templates/fence-creds.yaml b/helm/fence/templates/fence-creds.yaml index 24cfb7adc..8ec771127 100644 --- a/helm/fence/templates/fence-creds.yaml +++ b/helm/fence/templates/fence-creds.yaml @@ -11,9 +11,7 @@ stringData: "db_password": "{{include "gen3.service-postgres" (dict "key" "password" "service" $.Chart.Name "context" $) }}", "db_database": "{{ include "gen3.service-postgres" (dict "key" "database" "service" $.Chart.Name "context" $)}}", "hostname": "{{ .Values.global.hostname }}", - "indexd_password": "", "google_client_secret": "YOUR.GOOGLE.SECRET", "google_client_id": "YOUR.GOOGLE.CLIENT", "hmac_key": "" } - diff --git a/helm/fence/templates/fence-deployment.yaml b/helm/fence/templates/fence-deployment.yaml index 49ee32d49..211871564 100644 --- a/helm/fence/templates/fence-deployment.yaml +++ b/helm/fence/templates/fence-deployment.yaml @@ -74,6 +74,10 @@ spec: cp /var/run/fence-secrets/fence-config-secret.yaml /var/www/fence/fence-config.yaml fi + if [[ -n "${AUTHZ_SNAPSHOT_CACHE_REDIS_PASSWORD:-""}" ]]; then + python -c $'import os, yaml\npath = "/var/www/fence/fence-config.yaml"\nwith open(path) as f:\n data = yaml.safe_load(f) or {}\nurl = data.get("AUTHZ_SNAPSHOT_CACHE_REDIS_URL")\npassword = os.environ.get("AUTHZ_SNAPSHOT_CACHE_REDIS_PASSWORD", "")\nif url and password and url.startswith("redis://") and "@" not in url:\n data["AUTHZ_SNAPSHOT_CACHE_REDIS_URL"] = f"redis://:{password}@{url[8:]}"\n with open(path, "w") as f:\n yaml.safe_dump(data, f, sort_keys=False)' + fi + if [[ -f /var/run/fence-secrets/jwt_private_key.pem ]]; then mkdir -p /fence/keys/key cp /var/run/fence-secrets/jwt_private_key.pem /fence/keys/key/jwt_private_key.pem @@ -90,6 +94,26 @@ spec: volumeMounts: {{- toYaml .Values.volumeMounts | nindent 12 }} initContainers: + {{- if .Values.authzSnapshotCache.waitForRedis.enabled }} + - name: wait-for-authz-cache + image: "{{ .Values.authzSnapshotCache.waitForRedis.image }}" + imagePullPolicy: IfNotPresent + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.authzSnapshotCache.waitForRedis.secretName }} + key: {{ .Values.authzSnapshotCache.waitForRedis.passwordKey }} + optional: false + command: + - /bin/sh + - -c + - | + until redis-cli -h {{ .Values.authzSnapshotCache.waitForRedis.host }} -p {{ .Values.authzSnapshotCache.waitForRedis.port }} -a "$REDIS_PASSWORD" --no-auth-warning ping | grep -q PONG; do + echo "waiting for authz cache redis at {{ .Values.authzSnapshotCache.waitForRedis.host }}:{{ .Values.authzSnapshotCache.waitForRedis.port }}" + sleep 2 + done + {{- end }} - name: fence-init image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} diff --git a/helm/fence/templates/usersync-cron.yaml b/helm/fence/templates/usersync-cron.yaml index 7bf9d8b85..dfb18c7bc 100644 --- a/helm/fence/templates/usersync-cron.yaml +++ b/helm/fence/templates/usersync-cron.yaml @@ -124,7 +124,7 @@ spec: containers: - name: usersync image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: Always + imagePullPolicy: {{ .Values.image.pullPolicy }} env: - name: SYNC_FROM_DBGAP value: {{ .Values.usersync.syncFromDbgap | quote }} @@ -211,4 +211,4 @@ spec: echo "Exit code: $exitcode" exit "$exitcode" restartPolicy: "Never" -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/fence/templates/useryaml-job.yaml b/helm/fence/templates/useryaml-job.yaml index c6375af47..7b1de16d4 100644 --- a/helm/fence/templates/useryaml-job.yaml +++ b/helm/fence/templates/useryaml-job.yaml @@ -27,7 +27,7 @@ spec: containers: - name: fence image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: Always + imagePullPolicy: {{ .Values.image.pullPolicy }} env: {{- toYaml .Values.env | nindent 10 }} volumeMounts: @@ -44,5 +44,7 @@ spec: # Script always succeeds if it runs (echo exits with 0) - | # can be removed once this is merged: https://github.com/uc-cdis/fence/pull/1096 - fence-create sync --arborist http://arborist-service --yaml /var/www/fence/user.yaml + cd /fence + export PYTHONPATH=/fence${PYTHONPATH:+:$PYTHONPATH} + /fence/.venv/bin/python -m bin.fence_create sync --arborist http://arborist-service --yaml /var/www/fence/user.yaml restartPolicy: OnFailure diff --git a/helm/fence/values.yaml b/helm/fence/values.yaml index aab50de27..335806b55 100644 --- a/helm/fence/values.yaml +++ b/helm/fence/values.yaml @@ -297,6 +297,12 @@ env: name: manifest-fence key: fence-config-public.yaml optional: true + - name: AUTHZ_SNAPSHOT_CACHE_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: authz-cache-credentials + key: redis-password + optional: false - name: PGHOST valueFrom: secretKeyRef: @@ -329,17 +335,21 @@ env: optional: false - name: DB value: postgresql://$(PGUSER):$(PGPASSWORD)@$(PGHOST):5432/$(PGDB) - - name: INDEXD_PASSWORD - valueFrom: - secretKeyRef: - name: indexd-service-creds - key: fence - name: gen3Env valueFrom: configMapKeyRef: name: manifest-global key: hostname +authzSnapshotCache: + waitForRedis: + enabled: true + image: "redis:7.2-alpine" + host: "authz-cache-service" + port: 6379 + secretName: "authz-cache-credentials" + passwordKey: "redis-password" + # -- (list) Volumes to attach to the container. volumes: - name: old-config-volume @@ -1677,6 +1687,7 @@ FENCE_CONFIG: - "openid" - "user" - "data" + - "github_credentials" - "google_credentials" - "google_service_account" - "google_link" @@ -1690,6 +1701,7 @@ FENCE_CONFIG: - "user" - "data" - "admin" + - "github_credentials" - "google_credentials" - "google_service_account" - "google_link" @@ -1708,6 +1720,19 @@ FENCE_CONFIG: - "google_link" - "ga4gh_passport_v1" + # -- (map) GitHub App configuration for Fence-managed installation token brokering + GITHUB_APP: + # -- (string) GitHub App ID + app_id: '' + # -- (string) PEM-encoded GitHub App private key + private_key: '' + # -- (string) Optional path to a PEM file mounted in the container. Mutually exclusive with private_key. + private_key_file: '' + # -- (string) GitHub API base URL. Leave default for github.com, override for GitHub Enterprise. + api_base_url: 'https://api.github.com' + # -- (int) Timeout in seconds for outbound GitHub API requests + timeout_seconds: 30 + # ////////////////////////////////////////////////////////////////////////////////////// # LOGIN # - Modify based on which OIDC provider(s) you configured above @@ -2018,20 +2043,6 @@ FENCE_CONFIG: host: null port: 3128 - # ////////////////////////////////////////////////////////////////////////////////////// - # MICROSERVICE PATHS - # - Support `/data` endpoints & authz functionality - # ////////////////////////////////////////////////////////////////////////////////////// - # url where indexd microservice is running (for signed urls primarily) - # NOTE: Leaving as null will force fence to default to {{BASE_URL}}/index - # example value: 'https://example.com/index' - INDEXD: http://indexd-service - - # this is the username which fence uses to make authenticated requests to indexd - INDEXD_USERNAME: 'fence' - # this is the password which fence uses to make authenticated requests to indexd - INDEXD_PASSWORD: '' - # ////////////////////////////////////////////////////////////////////////////////////// # AZURE STORAGE BLOB CONFIGURATION # - Support Azure Blob Data Access Methods @@ -2048,6 +2059,9 @@ FENCE_CONFIG: # url where authz microservice is running ARBORIST: http://arborist-service + AUTHZ_SNAPSHOT_CACHE_ENABLED: true + AUTHZ_SNAPSHOT_CACHE_REDIS_URL: redis://authz-cache-service:6379/0 + AUTHZ_SNAPSHOT_CACHE_TTL_SECONDS: 3600 # url where the audit-service is running AUDIT_SERVICE: 'http://audit-service' diff --git a/helm/funnel/Chart.yaml b/helm/funnel/Chart.yaml index 5d48fb078..3df83f5e8 100644 --- a/helm/funnel/Chart.yaml +++ b/helm/funnel/Chart.yaml @@ -17,7 +17,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.76 +version: 0.1.77 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -26,12 +26,6 @@ version: 0.1.76 appVersion: "2025-12-22" dependencies: - - name: postgresql - version: 18.1.15 - repository: https://charts.bitnami.com/bitnami - condition: postgresql.enabled - - - name: mongodb - version: 13.9.4 - repository: https://charts.bitnami.com/bitnami - condition: mongodb.enabled + - name: common + version: 0.1.10 + repository: file://../common diff --git a/helm/funnel/README.md b/helm/funnel/README.md index e37281279..6c4611f7c 100644 --- a/helm/funnel/README.md +++ b/helm/funnel/README.md @@ -1,182 +1,37 @@ # funnel -![Version: 0.1.75](https://img.shields.io/badge/Version-0.1.75-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2025-12-22](https://img.shields.io/badge/AppVersion-2025--12--22-informational?style=flat-square) +Funnel is source-owned in `gen3-helm` and is wired into the Gen3 umbrella chart +as a local subchart through `file://../funnel`. -A toolkit for distributed task execution ⚙️ +## Database -## Requirements +This chart is configured to use Gen3-managed PostgreSQL for Funnel task and +event storage. The previous bundled MongoDB dependency was removed because we +believe the MongoDB event writer is no longer used by this deployment path. -| Repository | Name | Version | -|------------|------|---------| -| https://charts.bitnami.com/bitnami | mongodb | 13.9.4 | -| https://charts.bitnami.com/bitnami | postgresql | 18.1.15 | +The Helm render verifies the generated Funnel config points at PostgreSQL and no +MongoDB resources are created. A deployed-cluster smoke test is still required +to prove the Funnel server binary successfully connects to and migrates/uses the +PostgreSQL database in a real environment. -## Values +Deployments that still need MongoDB event writing must reintroduce explicit +external MongoDB configuration. -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| AWSBatch.DisableReconciler | bool | `true` | | -| AWSBatch.JobDefinition | string | `"funnel-job-def"` | | -| AWSBatch.JobQueue | string | `"funnel-job-queue"` | | -| AWSBatch.Key | string | `""` | | -| AWSBatch.ReconcileRate | string | `"10s"` | | -| AWSBatch.Region | string | `""` | | -| AWSBatch.Secret | string | `""` | | -| AmazonS3.AWSConfig.Key | string | `""` | | -| AmazonS3.AWSConfig.MaxRetries | int | `10` | | -| AmazonS3.AWSConfig.Secret | string | `""` | | -| AmazonS3.Disabled | bool | `false` | | -| AmazonS3.SSE.CustomerKeyFile | string | `""` | | -| AmazonS3.SSE.KMSKey | string | `""` | | -| BoltDB.Path | string | `"./funnel-work-dir/funnel.db"` | | -| Compute | string | `"kubernetes"` | | -| Database | string | `"postgres"` | | -| Datastore.CredentialsFile | string | `""` | | -| Datastore.Project | string | `""` | | -| DynamoDB.AWSConfig.Key | string | `""` | | -| DynamoDB.AWSConfig.Region | string | `""` | | -| DynamoDB.AWSConfig.Secret | string | `""` | | -| DynamoDB.TableBasename | string | `"funnel"` | | -| Elastic.IndexPrefix | string | `"funnel"` | | -| Elastic.URL | string | `"http://localhost:9200"` | | -| EventWriters[0] | string | `"postgres"` | | -| EventWriters[1] | string | `"log"` | | -| FTPStorage.Disabled | bool | `false` | | -| FTPStorage.Password | string | `"anonymous"` | | -| FTPStorage.Timeout | string | `"10s"` | | -| FTPStorage.User | string | `"anonymous"` | | -| GoogleStorage.CredentialsFile | string | `""` | | -| GoogleStorage.Disabled | bool | `false` | | -| GridEngine.Template | string | `"#!bin/bash\n#$ -N {{.TaskId}}\n#$ -o {{.WorkDir}}/funnel-stdout\n#$ -e {{.WorkDir}}/funnel-stderr\n#$ -l nodes=1\n{{if ne .Cpus 0 -}}\n{{printf \"#$ -pe mpi %d\" .Cpus}}\n{{- end}}\n{{if ne .RamGb 0.0 -}}\n{{printf \"#$ -l h_vmem=%.0fG\" .RamGb}}\n{{- end}}\n{{if ne .DiskGb 0.0 -}}\n{{printf \"#$ -l h_fsize=%.0fG\" .DiskGb}}\n{{- end}}\n\n{{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}}\n"` | | -| GridEngine.TemplateFile | string | `""` | | -| HTCondor.DisableReconciler | bool | `true` | | -| HTCondor.ReconcileRate | string | `"10s"` | | -| HTCondor.Template | string | `"universe = vanilla\ngetenv = True\nexecutable = {{.Executable}}\narguments = worker run --config {{.Config}} --task-id {{.TaskId}}\nlog = {{.WorkDir}}/condor-event-log\nerror = {{.WorkDir}}/funnel-stderr\noutput = {{.WorkDir}}/funnel-stdout\nshould_transfer_files = YES\nwhen_to_transfer_output = ON_EXIT_OR_EVICT\n{{if ne .Cpus 0 -}}\n{{printf \"request_cpus = %d\" .Cpus}}\n{{- end}}\n{{if ne .RamGb 0.0 -}}\n{{printf \"request_memory = %.0f GB\" .RamGb}}\n{{- end}}\n{{if ne .DiskGb 0.0 -}}\n{{printf \"request_disk = %.0f GB\" .DiskGb}}\n{{- end}}\n\nqueue\n"` | | -| HTCondor.TemplateFile | string | `""` | | -| HTTPStorage.Timeout | string | `"30s"` | | -| Kafka.Topic | string | `"funnel"` | | -| Kubernetes.DisableJobCleanup | bool | `false` | | -| Kubernetes.DisableReconciler | bool | `false` | | -| Kubernetes.Executor | string | `"kubernetes"` | | -| Kubernetes.ExecutorTemplate | string | `""` | | -| Kubernetes.JobsNamespace | string | `""` | | -| Kubernetes.Namespace | string | `""` | | -| Kubernetes.NodeSelector | object | `{}` | | -| Kubernetes.PVCTemplate | string | `""` | | -| Kubernetes.PVTemplate | string | `""` | | -| Kubernetes.ReconcileRate | string | `"10s"` | | -| Kubernetes.ServiceAccount | string | `""` | | -| Kubernetes.Tolerations | list | `[]` | | -| Kubernetes.WorkerTemplate | string | `""` | | -| LocalStorage.AllowedDirs[0] | string | `"./"` | | -| Logger.level | string | `"debug"` | | -| Logger.outputFile | string | `""` | | -| MongoDB.Addrs | list | `[]` | | -| MongoDB.Database | string | `"funnel"` | | -| MongoDB.Timeout.duration | string | `"300s"` | | -| MongoDB.Username | string | `"example"` | | -| Node.ID | string | `""` | | -| Node.Resources.Cpus | int | `0` | | -| Node.Resources.DiskGb | float | `0` | | -| Node.Resources.RamGb | float | `0` | | -| Node.Timeout.disabled | bool | `true` | | -| Node.UpdateRate | string | `"5s"` | | -| PBS.DisableReconciler | bool | `true` | | -| PBS.ReconcileRate | string | `"10s"` | | -| PBS.Template | string | `"#!bin/bash\n#PBS -N {{.TaskId}}\n#PBS -o {{.WorkDir}}/funnel-stdout\n#PBS -e {{.WorkDir}}/funnel-stderr\n{{if ne .Cpus 0 -}}\n{{printf \"#PBS -l nodes=1:ppn=%d\" .Cpus}}\n{{- end}}\n{{if ne .RamGb 0.0 -}}\n{{printf \"#PBS -l mem=%.0fgb\" .RamGb}}\n{{- end}}\n{{if ne .DiskGb 0.0 -}}\n{{printf \"#PBS -l file=%.0fgb\" .DiskGb}}\n{{- end}}\n\n{{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}}\n"` | | -| PBS.TemplateFile | string | `""` | | -| Postgres.AdminPassword | string | `"example"` | | -| Postgres.AdminUser | string | `"postgres"` | | -| Postgres.Database | string | `"funnel"` | | -| Postgres.Host | string | `"funnel-postgresql.default.svc.cluster.local"` | | -| Postgres.Password | string | `"example"` | | -| Postgres.Timeout.duration | string | `"300s"` | | -| Postgres.User | string | `"funnel"` | | -| RPCClient.MaxRetries | int | `10` | | -| RPCClient.ServerAddress | string | `"localhost:9090"` | | -| RPCClient.Timeout.duration | string | `"60s"` | | -| Scheduler.NodeInitTimeout.duration | string | `"300s"` | | -| Scheduler.NodePingTimeout.duration | string | `"60s"` | | -| Scheduler.ScheduleChunk | int | `10` | | -| Scheduler.ScheduleRate | string | `"1s"` | | -| Server.DisableHTTPCache | bool | `true` | | -| Server.HTTPPort | string | `"8000"` | | -| Server.HostName | string | `"funnel"` | | -| Server.RPCPort | string | `"9090"` | | -| Slurm.DisableReconciler | bool | `true` | | -| Slurm.ReconcileRate | string | `"10s"` | | -| Slurm.Template | string | `"#!/bin/bash\n#SBATCH --job-name {{.TaskId}}\n#SBATCH --ntasks 1\n#SBATCH --error {{.WorkDir}}/funnel-stderr\n#SBATCH --output {{.WorkDir}}/funnel-stdout\n{{if ne .Cpus 0 -}}\n{{printf \"#SBATCH --cpus-per-task %d\" .Cpus}}\n{{- end}}\n{{if ne .RamGb 0.0 -}}\n{{printf \"#SBATCH --mem %.0fGB\" .RamGb}}\n{{- end}}\n{{if ne .DiskGb 0.0 -}}\n{{printf \"#SBATCH --tmp %.0fGB\" .DiskGb}}\n{{- end}}\n\n{{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}}\n"` | | -| Slurm.TemplateFile | string | `""` | | -| Swift.AuthURL | string | `""` | | -| Swift.ChunkSizeBytes | int | `500000000` | | -| Swift.Disabled | bool | `false` | | -| Swift.Password | string | `""` | | -| Swift.RegionName | string | `""` | | -| Swift.TenantID | string | `""` | | -| Swift.TenantName | string | `""` | | -| Swift.UserName | string | `""` | | -| Worker.LeaveWorkDir | bool | `false` | | -| Worker.LogTailSize | int | `10000` | | -| Worker.LogUpdateRate | string | `"5s"` | | -| Worker.MaxParallelTransfers | int | `10` | | -| Worker.PollingRate | string | `"5s"` | | -| Worker.WorkDir | string | `"./funnel-work-dir"` | | -| backoffLimit | int | `1` | | -| completions | int | `1` | | -| image.initContainers[0].command[0] | string | `"cp"` | | -| image.initContainers[0].command[1] | string | `"/app/build/plugins/authorizer"` | | -| image.initContainers[0].command[2] | string | `"/opt/funnel/plugin-binaries/auth-plugin"` | | -| image.initContainers[0].image | string | `"quay.io/ohsu-comp-bio/funnel-plugins"` | | -| image.initContainers[0].name | string | `"plugins"` | | -| image.initContainers[0].pullPolicy | string | `"Always"` | | -| image.initContainers[0].tag | string | `"pr-1"` | | -| image.initContainers[0].volumeMounts[0].mountPath | string | `"/opt/funnel/plugin-binaries"` | | -| image.initContainers[0].volumeMounts[0].name | string | `"plugin-volume"` | | -| image.pullPolicy | string | `"Always"` | | -| image.repository | string | `"quay.io/ohsu-comp-bio/funnel"` | | -| labels.app | string | `"funnel"` | | -| mongodb.app | string | `"funnel-mongodb"` | | -| mongodb.architecture | string | `"standalone"` | | -| mongodb.auth.enabled | bool | `true` | | -| mongodb.auth.rootPassword | string | `"example"` | | -| mongodb.auth.rootUser | string | `"example"` | | -| mongodb.commonLabels.app | string | `"funnel-mongodb"` | | -| mongodb.enabled | bool | `false` | | -| mongodb.image.registry | string | `"public.ecr.aws"` | | -| mongodb.persistence.enabled | bool | `false` | | -| mongodb.persistence.size | string | `"1Gi"` | | -| postgresql.enabled | bool | `true` | | -| postgresql.global.postgresql.auth.database | string | `"funnel"` | | -| postgresql.global.postgresql.auth.password | string | `"example"` | | -| postgresql.global.postgresql.auth.postgresPassword | string | `"example"` | | -| postgresql.global.postgresql.auth.username | string | `"funnel"` | | -| postgresql.primary.persistence.enabled | bool | `false` | | -| rbac.create | bool | `true` | | -| replicaCount | int | `1` | | -| resources.limits.cpu | string | `"1000m"` | | -| resources.limits.ephemeral_storage | string | `"2048Mi"` | | -| resources.limits.memory | string | `"2048Mi"` | | -| resources.requests.cpu | string | `"100m"` | | -| resources.requests.ephemeral_storage | string | `"512Mi"` | | -| resources.requests.memory | string | `"512Mi"` | | -| service.httpPort | int | `8000` | | -| service.rpcPort | int | `9090` | | -| service.type | string | `"ClusterIP"` | | -| storage.accessMode | string | `"ReadWriteMany"` | | -| storage.className | string | `"s3-csi-sc"` | | -| storage.createStorageClass | bool | `true` | | -| storage.driver | string | `"aws-s3"` | | -| storage.provisioner | string | `"s3.csi.aws.com"` | | -| storage.size | string | `"10Mi"` | | -| volumeMounts[0].mountPath | string | `"/etc/config/funnel-server.yaml"` | | -| volumeMounts[0].name | string | `"funnel-server-config-volume"` | | -| volumeMounts[0].subPath | string | `"funnel-server.yaml"` | | -| volumeMounts[1].mountPath | string | `"/opt/funnel/plugin-binaries"` | | -| volumeMounts[1].name | string | `"plugin-volume"` | | -| volumes[0].configMap.name | string | `"funnel-server-config"` | | -| volumes[0].name | string | `"funnel-server-config-volume"` | | -| volumes[1].emptyDir | object | `{}` | | -| volumes[1].name | string | `"plugin-volume"` | | +## Cleanup CronJob ----------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) +The optional cleanup CronJob runs: + +```bash +funnel kubernetes cleanup --config /etc/config/funnel-server.yaml +``` + +It is intentionally disabled by default. Enable it with: + +```yaml +cleanup: + enabled: true +``` + +When `cleanup.schedule` is empty, the chart derives the CronJob schedule from +`Kubernetes.ReconcileRate`. Set `cleanup.schedule` to a standard 5-field cron +expression to override it. diff --git a/helm/funnel/files/executor-job.yaml b/helm/funnel/files/executor-job.yaml index ff57e5a0e..f65ef175d 100644 --- a/helm/funnel/files/executor-job.yaml +++ b/helm/funnel/files/executor-job.yaml @@ -8,10 +8,37 @@ metadata: app: funnel-executor job-name: {{`{{.TaskId}}-{{.JobId}}`}} spec: - backoffLimit: {{ .Values.backoffLimit}} - completions: {{ .Values.completions }} + backoffLimit: {{ .Values.Kubernetes.Executor.backoffLimit}} + completions: {{ .Values.Kubernetes.Executor.completions }} template: + metadata: + + # Annotations + {{- with .Values.Kubernetes.Executor.Annotations }} + annotations: + {{ toYaml . | indent 2 }} + {{- end }} + + labels: + app: funnel-executor + job-name: {{`{{.TaskId}}`}}-{{`{{.JobId}}`}} spec: + + # If priorityClassName is set for the Executor, use that. + # Otherwise, if it's set for the Worker, use that. + # This allows users to set a default priority class for all jobs by setting it for the Worker, + # but also override it for the Executor if needed. + {{- $pc := ""}} + {{- if .Values.Kubernetes.Executor.PriorityClassName }} + {{- $pc = .Values.Kubernetes.Executor.PriorityClassName }} + {{- else if .Values.Kubernetes.Worker.PriorityClassName }} + {{- $pc = .Values.Kubernetes.Worker.PriorityClassName }} + {{- end }} + + {{- if $pc }} + priorityClassName: {{ $pc }} + {{- end }} + # NodeSelectors # https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes/#create-a-pod-that-gets-scheduled-to-your-chosen-node {{` @@ -38,40 +65,72 @@ spec: {{- end }} `}} - # SecurityContext - # https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod - {{` - {{- if .securityContext }} securityContext: - fsGroup: {{ .securityContext.fsGroup }} - runAsUser: {{ .securityContext.runAsUser }} - runAsGroup: {{ .securityContext.runAsGroup }} - supplementalGroups: {{ .securityContext.supplementalGroups }} - {{- end }} - `}} - - restartPolicy: OnFailure + # Use the default seccomp profile, which should block several dangerous syscalls that could be used for privilege escalation or container escape + seccompProfile: + type: RuntimeDefault + + # Kubernetes service account token is not needed for the Executor, and disabling it can help reduce the risk of token theft and misuse. + automountServiceAccountToken: false + restartPolicy: {{ .Values.Kubernetes.Executor.restartPolicy }} serviceAccountName: {{`{{.ServiceAccountName}}`}} containers: - - name: funnel-worker-{{`{{.TaskId}}`}} + - name: funnel-executor-{{`{{.TaskId}}`}} image: {{`{{.Image}}`}} imagePullPolicy: Always - command: {{`{{.Command}}`}} + securityContext: + # Block the ability to gain more privileges + allowPrivilegeEscalation: false + privileged: false + + # Block all capabilities and then add back only the ones we need + capabilities: + drop: ["ALL"] + add: ["CHOWN", "DAC_OVERRIDE", "SETUID", "SETGID"] + # Command + {{` + {{- if .UseShell}} + command: + - "/bin/sh" + - "-c" + args: + - {{printf "%q" (index .Command 0)}} + {{- else}} + command: + {{- range .Command}} + - {{printf "%q" .}} + {{- end}} + {{- end}} + `}} + workingDir: {{`{{.Workdir}}`}} + + env: + {{` + {{- range $key, $value := .Env }} + - name: {{ $key }} + value: "{{ $value }}" + {{- end }} + `}} + resources: requests: - cpu: {{ .Values.resources.requests.cpu }} - memory: {{ .Values.resources.requests.memory }} - ephemeral-storage: {{ .Values.resources.requests.ephemeral_storage }} + cpu: {{`{{ .Cpus }}`}} + memory: {{`{{ .RamGb }}`}}Gi + ephemeral-storage: {{`{{ .DiskGb }}`}}Gi + limits: + cpu: {{`{{ .CpusLimit }}`}} + memory: {{`{{ .RamGbLimit }}`}}Gi + ephemeral-storage: {{`{{ .DiskGbLimit }}`}}Gi volumeMounts: {{` {{- if .NeedsPVC }} - {{range $idx, $item := .Volumes}} + {{- range $idx, $item := .Volumes}} - name: funnel-storage-{{$.TaskId}} mountPath: {{$item.ContainerPath}} subPath: {{$.TaskId}}{{$item.ContainerPath}} - {{end}} + {{- end}} {{- end }} `}} @@ -82,4 +141,4 @@ spec: persistentVolumeClaim: claimName: funnel-worker-pvc-{{.TaskId}} {{- end }} - `}} \ No newline at end of file + `}} diff --git a/helm/funnel/files/server-config.yaml b/helm/funnel/files/server-config.yaml index db30ba2f5..fd92bb4f7 100644 --- a/helm/funnel/files/server-config.yaml +++ b/helm/funnel/files/server-config.yaml @@ -1,13 +1,23 @@ Compute: {{ .Values.Compute }} Kubernetes: - Executor: {{ .Values.Kubernetes.Executor }} DisableReconciler: {{ .Values.Kubernetes.DisableReconciler }} DisableJobCleanup: {{ .Values.Kubernetes.DisableJobCleanup }} ReconcileRate: {{ .Values.Kubernetes.ReconcileRate }} Namespace: {{ .Release.Namespace }} JobsNamespace: {{ .Values.Kubernetes.JobsNamespace }} ServiceAccount: {{ .Values.Kubernetes.ServiceAccount }} + Resources: + Defaults: + Cpus: {{ .Values.Kubernetes.Resources.Defaults.Cpus }} + RamGb: {{ .Values.Kubernetes.Resources.Defaults.RamGb }} + DiskGb: {{ .Values.Kubernetes.Resources.Defaults.DiskGb }} + Limits: + Cpus: {{ .Values.Kubernetes.Resources.Limits.Cpus }} + RamGb: {{ .Values.Kubernetes.Resources.Limits.RamGb }} + DiskGb: {{ .Values.Kubernetes.Resources.Limits.DiskGb }} + Timeout: + duration: {{ .Values.Kubernetes.Timeout.duration }} NodeSelector: {{- if .Values.Kubernetes.NodeSelector }} @@ -33,11 +43,11 @@ Kubernetes: # Worker Job WorkerTemplate: | -{{ tpl (.Files.Get "files/worker-job.yaml") . | indent 4 }} +{{ tpl ((.Values.Kubernetes.WorkerTemplate | default (.Files.Get "files/worker-job.yaml"))) . | indent 4 }} # Executor Job ExecutorTemplate: | -{{ tpl (.Files.Get "files/executor-job.yaml") . | indent 4 }} +{{ tpl ((.Values.Kubernetes.ExecutorTemplate | default (.Files.Get "files/executor-job.yaml"))) . | indent 4 }} # PV template PVTemplate: | @@ -72,7 +82,7 @@ Server: HostName: "{{ .Values.Server.HostName }}" HTTPPort: "{{ .Values.Server.HTTPPort }}" RPCPort: "{{ .Values.Server.RPCPort }}" - DisableHTTPCache: {{ .Values.Server.DisableHttpCache }} + DisableHTTPCache: {{ .Values.Server.DisableHTTPCache }} RPCClient: ServerAddress: {{ .Values.RPCClient.ServerAddress }} @@ -89,14 +99,14 @@ Scheduler: duration: {{ .Values.Scheduler.NodeInitTimeout.duration }} Node: - ID: {{ .Values.Node.Id }} + ID: {{ .Values.Node.ID }} Timeout: disabled: {{ .Values.Node.Timeout.disabled }} UpdateRate: {{ .Values.Node.UpdateRate }} Resources: - Cpus: {{ .Values.Node.Resources.cpus }} - RamGb: {{ .Values.Node.Resources.ramGb }} - DiskGb: {{ .Values.Node.Resources.diskGb }} + Cpus: {{ .Values.Node.Resources.Cpus }} + RamGb: {{ .Values.Node.Resources.RamGb }} + DiskGb: {{ .Values.Node.Resources.DiskGb }} Worker: WorkDir: {{ .Values.Worker.WorkDir }} @@ -128,32 +138,23 @@ DynamoDB: Elastic: IndexPrefix: {{ .Values.Elastic.IndexPrefix }} - URL: {{ .Values.Elastic.Url }} + URL: {{ .Values.Elastic.URL }} Datastore: Project: {{ .Values.Datastore.Project }} CredentialsFile: {{ .Values.Datastore.CredentialsFile }} -MongoDB: - Addrs: - {{- if .Values.MongoDB.Addrs }} - {{- range .Values.MongoDB.Addrs }} - - {{ . }} - {{- end }} - {{- else }} - - {{ .Release.Name }}-mongodb.{{ .Release.Namespace }}.svc.cluster.local - {{- end }} - Database: {{ .Values.MongoDB.Database }} - Timeout: - duration: {{ .Values.MongoDB.Timeout.duration }} - Username: {{ .Values.MongoDB.Username }} - Password: {{ .Values.MongoDB.Password }} - Postgres: - Host: {{ .Values.Postgres.Host }} - Database: {{ .Values.Postgres.Database }} - User: {{ .Values.Postgres.User }} - Password: {{ .Values.Postgres.Password }} + {{- if .Values.postgres.host }} + Host: {{ .Values.postgres.host }} + {{- else if .Values.global.dev }} + Host: {{ .Release.Name }}-postgresql + {{- else }} + Host: {{ .Values.global.postgres.master.host }} + {{- end }} + Database: {{ include "gen3.service-postgres" (dict "key" "database" "service" .Chart.Name "context" $) }} + User: {{ include "gen3.service-postgres" (dict "key" "username" "service" .Chart.Name "context" $) }} + Password: {{ include "gen3.service-postgres" (dict "key" "password" "service" .Chart.Name "context" $) }} AdminUser: {{ .Values.Postgres.AdminUser }} AdminPassword: {{ .Values.Postgres.AdminPassword }} Timeout: diff --git a/helm/funnel/files/worker-job.yaml b/helm/funnel/files/worker-job.yaml index f8c51c2e7..7bf13da87 100644 --- a/helm/funnel/files/worker-job.yaml +++ b/helm/funnel/files/worker-job.yaml @@ -16,14 +16,25 @@ metadata: {{- end }} `}} spec: - backoffLimit: {{ .Values.backoffLimit}} - completions: {{ .Values.completions }} + backoffLimit: {{ .Values.Kubernetes.Worker.backoffLimit}} + completions: {{ .Values.Kubernetes.Worker.completions }} template: metadata: labels: app: funnel-worker task-id: {{`{{.TaskId}}`}} + + # Annotations + {{- with .Values.Kubernetes.Worker.Annotations }} + annotations: + {{ toYaml . | indent 2 }} + {{- end }} + spec: + + {{ if .Values.Kubernetes.Worker.PriorityClassName }} + priorityClassName: {{ .Values.Kubernetes.Worker.PriorityClassName }} + {{ end }} # NodeSelectors # https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes/#create-a-pod-that-gets-scheduled-to-your-chosen-node {{` @@ -63,24 +74,26 @@ spec: `}} serviceAccountName: {{`{{.ServiceAccountName}}`}} - restartPolicy: OnFailure + restartPolicy: {{ .Values.Kubernetes.Worker.restartPolicy }} containers: - name: funnel-worker-{{`{{.TaskId}}`}} - # TODO: Make the image + tag configurable image: {{`{{.Image}}`}} imagePullPolicy: Always + command: + - /bin/sh + - -c args: - - "worker" - - "run" - - "--config" - - "/etc/config/funnel-worker.yaml" - - "--taskID" - - {{`{{.TaskId}}`}} + - | + funnel worker run --config /etc/config/funnel-worker.yaml --taskID {{`{{.TaskId}}`}} resources: requests: cpu: {{ .Values.resources.requests.cpu }} memory: {{ .Values.resources.requests.memory }} ephemeral-storage: {{ .Values.resources.requests.ephemeral_storage }} + limits: + cpu: {{ .Values.resources.limits.cpu }} + memory: {{ .Values.resources.limits.memory }} + ephemeral-storage: {{ .Values.resources.limits.ephemeral_storage }} volumeMounts: - name: config-volume mountPath: /etc/config diff --git a/helm/funnel/templates/_helpers.tpl b/helm/funnel/templates/_helpers.tpl index a16e840ba..1a2b8e634 100644 --- a/helm/funnel/templates/_helpers.tpl +++ b/helm/funnel/templates/_helpers.tpl @@ -65,3 +65,46 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} + +{{/* +Parse a Go duration string (e.g. "10s", "90s", "5m", "1h") into whole seconds. +Only the s/m/h suffixes are supported; an unsuffixed value is treated as seconds. +Returns 0 when the input is empty or unparseable. +*/}} +{{- define "funnel.durationSeconds" -}} +{{- $d := . | toString | trim -}} +{{- $seconds := 0 -}} +{{- if hasSuffix "h" $d -}} +{{- $seconds = mul (trimSuffix "h" $d | int) 3600 -}} +{{- else if hasSuffix "m" $d -}} +{{- $seconds = mul (trimSuffix "m" $d | int) 60 -}} +{{- else if hasSuffix "s" $d -}} +{{- $seconds = trimSuffix "s" $d | int -}} +{{- else if $d -}} +{{- $seconds = $d | int -}} +{{- end -}} +{{- $seconds -}} +{{- end }} + +{{/* +Derive the cleanup CronJob schedule from Kubernetes.ReconcileRate unless +cleanup.schedule is set explicitly. +*/}} +{{- define "funnel.cleanupSchedule" -}} +{{- if .Values.cleanup.schedule -}} +{{- .Values.cleanup.schedule -}} +{{- else -}} +{{- $seconds := include "funnel.durationSeconds" .Values.Kubernetes.ReconcileRate | int -}} +{{- $minutes := div (add $seconds 59) 60 -}} +{{- if lt $minutes 1 }}{{- $minutes = 1 -}}{{- end -}} +{{- $offset := .Values.cleanup.scheduleOffsetMinutes | default 0 | int -}} +{{- if and (le $minutes 59) (eq (mod 60 $minutes) 0) -}} +{{- printf "%d-59/%d * * * *" $offset $minutes -}} +{{- else if le $minutes 59 -}} +{{- printf "%d/%d * * * *" $offset $minutes -}} +{{- else -}} +{{- $hours := div (add $minutes 59) 60 -}} +{{- printf "%d */%d * * *" $offset $hours -}} +{{- end -}} +{{- end -}} +{{- end }} diff --git a/helm/funnel/templates/cronjob.yaml b/helm/funnel/templates/cronjob.yaml new file mode 100644 index 000000000..2c92e72d2 --- /dev/null +++ b/helm/funnel/templates/cronjob.yaml @@ -0,0 +1,45 @@ +{{- if .Values.cleanup.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: funnel-cleanup-{{ .Release.Namespace }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "funnel.labels" . | nindent 4 }} +spec: + # Schedule is derived from Kubernetes.ReconcileRate unless cleanup.schedule is set explicitly. + schedule: {{ include "funnel.cleanupSchedule" . | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + metadata: + labels: + {{- include "funnel.labels" . | nindent 12 }} + spec: + serviceAccountName: funnel-sa-{{ .Release.Namespace }} + restartPolicy: OnFailure + containers: + - name: funnel-cleanup + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: Always + command: + - funnel + - kubernetes + - cleanup + - --config + - /etc/config/funnel-server.yaml + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "256Mi" + volumeMounts: + {{- toYaml .Values.volumeMounts | nindent 14 }} + volumes: + {{- toYaml .Values.volumes | nindent 10 }} +{{- end }} diff --git a/helm/funnel/templates/db-init.yaml b/helm/funnel/templates/db-init.yaml new file mode 100644 index 000000000..3e29add84 --- /dev/null +++ b/helm/funnel/templates/db-init.yaml @@ -0,0 +1,5 @@ +{{ include "common.db-secret" . }} +--- +{{ include "common.db_setup_job" . }} +--- +{{ include "common.db_setup_sa" . }} diff --git a/helm/funnel/values.yaml b/helm/funnel/values.yaml index 2ef288bc2..66c4834ee 100644 --- a/helm/funnel/values.yaml +++ b/helm/funnel/values.yaml @@ -1,6 +1,21 @@ -# Kubernetes-specifc Settings +# Kubernetes-specific settings replicaCount: 1 +# Global configuration shared with the Gen3 umbrella chart. +global: + dev: true + postgres: + dbCreate: true + externalSecret: "" + master: + host: + username: postgres + password: + port: "5432" + externalSecrets: + deploy: false + dbCreate: false + image: repository: quay.io/ohsu-comp-bio/funnel pullPolicy: Always @@ -21,45 +36,27 @@ image: labels: app: funnel -# Default Funnel Server (Task Database) -# Note: This is a non-persistent database for testing/demo purposes. -postgresql: - enabled: true - global: - postgresql: - auth: - database: funnel - username: funnel - password: example - postgresPassword: example - primary: - persistence: - enabled: false - -mongodb: - enabled: false - app: funnel-mongodb - commonLabels: - app: funnel-mongodb - image: - registry: public.ecr.aws - architecture: standalone - auth: - enabled: true - rootUser: example - rootPassword: example - persistence: - enabled: false - size: 1Gi +# Funnel is configured to use Gen3-managed PostgreSQL. The previous MongoDB +# chart dependency was removed because MongoDB appears unused in this deployment +# path; runtime PostgreSQL connectivity should be validated in-cluster. +postgres: + # -- (bool) Whether the database should be created. Defaults to global.postgres.dbCreate. + dbCreate: + # -- (string) Hostname for Postgres. Defaults to global.postgres.master.host. + host: + # -- (string) Database name for Funnel. + database: funnel + # -- (string) Username for Funnel. + username: funnel + # -- (string) Port for Postgres. + port: "5432" + # -- (string) Password for Postgres. Override this for production deployments. + password: example rbac: create: true -# Backoff/Completions for Worker + Executor Jobs -backoffLimit: 1 -completions: 1 - -# Resource Requests/Limits for Worker + Executor Jobs +# Resource Requests/Limits for worker jobs and server pods. resources: requests: cpu: 100m @@ -89,14 +86,14 @@ storage: provisioner: s3.csi.aws.com createStorageClass: true -# Funnel Default Settings (with BoltDB replaced with MongoDB) +# Funnel default settings configured for Gen3-managed PostgreSQL. # # These are passed into `files/server-config.yaml` and made available to the deployment via `server-configmap.yaml` # # - Ref: https://github.com/ohsu-comp-bio/funnel/blob/master/config/default-config.yaml # The name of the active server database backend -# Available backends: boltdb, badger, datastore, dynamodb, elastic, mongodb +# Available backends: boltdb, badger, datastore, dynamodb, elastic, mongodb, postgres Database: postgres # The name of the active compute backend @@ -104,7 +101,7 @@ Database: postgres Compute: kubernetes # The name of the active event writer backend(s). -# Available backends: log, boltdb, badger, datastore, dynamodb, elastic, mongodb, kafka +# Available backends: log, boltdb, badger, datastore, dynamodb, elastic, mongodb, postgres, kafka EventWriters: - postgres - log @@ -252,23 +249,6 @@ Datastore: # from the environment. CredentialsFile: "" -MongoDB: - # Addrs holds the addresses for the seed servers. - # K8s Note: These are computed dynamically in templates (in `server-configmap.yaml`) so that users - # don't have to manually set them. - Addrs: [] - # Database is the database name used within MongoDB to store funnel data. - Database: funnel - # Timeout is the amount of time to wait for a server to respond when - # first connecting and on follow up operations in the session. If - # timeout is zero, the call may block forever waiting for a connection - # to be established. - Timeout: - duration: 300s - # Username and Password inform the credentials for the initial authentication - # done on the database defined by the Database field. - Username: example - Postgres: Host: funnel-postgresql.default.svc.cluster.local Database: funnel @@ -409,8 +389,22 @@ AWSBatch: # Kubernetes describes the configuration for the Kubernetes compute backend. Kubernetes: - # The executor used to execute tasks. Available executors: docker, kubernetes - Executor: "kubernetes" + Executor: + Annotations: {} + PriorityClassName: "" + restartPolicy: OnFailure + # Setting backoffLimit to 0 means no retries. + backoffLimit: 0 + # Setting completions to 1 means the job completes after one successful execution. + completions: 1 + + Worker: + Annotations: {} + PriorityClassName: "" + restartPolicy: Never + backoffLimit: 0 + completions: 1 + # Turn off task state reconciler. When enabled, Funnel communicates with Kubernetes # to find tasks that are stuck in a queued state or errored and # updates the task state accordingly. @@ -425,6 +419,8 @@ Kubernetes: JobsNamespace: "" # Kubernetes ServiceAccount to use for the job ServiceAccount: "" + Timeout: + duration: 30s # Master batch job template. See: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#job-v1-batch WorkerTemplate: "" # NodeSelector (scheduling) @@ -437,6 +433,28 @@ Kubernetes: PVTemplate: "" PVCTemplate: "" + Resources: + Defaults: + Cpus: 1000m + RamGb: 512Mi + DiskGb: 512Mi + Limits: + Cpus: 8000m + RamGb: 4096Mi + DiskGb: 4096Mi + +# cleanup configures an optional CronJob that runs `funnel kubernetes cleanup` +# to delete orphaned Funnel-managed Kubernetes resources. It is intentionally +# turned off by default; set cleanup.enabled=true to enable it. +cleanup: + enabled: false + # Leave empty to derive the schedule from Kubernetes.ReconcileRate. Set a + # standard 5-field cron expression to override, e.g. "*/15 * * * *". + schedule: "" + # Offset the start minute so cleanup jobs in different namespaces do not all + # fire at the same instant. Must be 0-59. + scheduleOffsetMinutes: 0 + # ------------------------------------------------------------------------------- # Storage # ------------------------------------------------------------------------------- diff --git a/helm/gecko/files/init-data/nav.json b/helm/gecko/files/init-data/nav.json index dc71508e0..69d3af835 100644 --- a/helm/gecko/files/init-data/nav.json +++ b/helm/gecko/files/init-data/nav.json @@ -52,20 +52,6 @@ "href": "/Apps", "perms": null }, - { - "title": "Upload", - "description": "Upload local files into authorized storage.", - "icon": "/icons/apps/Upload.svg", - "href": "/upload", - "perms": "" - }, - { - "title": "Directory Structure", - "description": "Search for files via a tree based interactive search", - "icon": "/icons/binary-tree.svg", - "href": "/Browser", - "perms": null - }, { "title": "File Summary", "description": "Overview of file system usage", diff --git a/helm/gecko/templates/_helpers.tpl b/helm/gecko/templates/_helpers.tpl index e45c2387f..bcb318a0c 100644 --- a/helm/gecko/templates/_helpers.tpl +++ b/helm/gecko/templates/_helpers.tpl @@ -67,6 +67,17 @@ Create the name of the service account to use {{- end }} {{- end }} +{{/* +Resolve the Gecko container image reference. +*/}} +{{- define "gecko.image" -}} +{{- if .Values.image.digest -}} +{{- printf "%s@%s" .Values.image.repository .Values.image.digest -}} +{{- else -}} +{{- printf "%s:%s" .Values.image.repository (.Values.image.tag | default .Chart.AppVersion) -}} +{{- end -}} +{{- end }} + {{/* Postgres Password lookup */}} @@ -78,4 +89,3 @@ Create the name of the service account to use {{- default .Values.postgres.password }} {{- end }} {{- end }} - diff --git a/helm/gecko/templates/db-init-job.yaml b/helm/gecko/templates/db-init-job.yaml index c5e562b5f..3a15cb46e 100644 --- a/helm/gecko/templates/db-init-job.yaml +++ b/helm/gecko/templates/db-init-job.yaml @@ -1,7 +1,8 @@ +{{- if .Values.postgres.initJob.enabled }} apiVersion: batch/v1 kind: Job metadata: - name: {{ include "gecko.fullname" . }}-db-init + name: {{ printf "%s-db-init-%s" (include "gecko.fullname" . | trunc 45 | trimSuffix "-") (include (print $.Template.BasePath "/job-db-init.yaml") . | sha256sum | trunc 8) }} spec: backoffLimit: 10 template: @@ -12,13 +13,12 @@ spec: restartPolicy: OnFailure containers: - name: db-init - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: ["/bin/bash", "-c"] + image: {{ .Values.postgres.initJob.image | quote }} + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-c"] args: - | - #!/bin/bash - set -e + set -eu INIT_DATA_PATH="/mnt/db-init-data" @@ -31,8 +31,6 @@ spec: echo "Database ready, initializing..." psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" <>'org_title' IS NULL + OR project->>'org_title' = '' + OR project->>'project_title' IS NULL + OR project->>'project_title' = '' + ) THEN + RAISE EXCEPTION 'Every Gecko project config in .Values.projects must set org_title and project_title'; + END IF; + END + \$do\$; + + INSERT INTO config_schema.projects (name, content) + SELECT concat(project->>'org_title', '/', project->>'project_title'), project + FROM jsonb_array_elements(\$json\$ + $(cat "${INIT_DATA_PATH}/projects.json") + \$json\$::jsonb) AS project + ON CONFLICT (name) DO UPDATE SET content = EXCLUDED.content; + SQL echo "Database initialization complete." env: @@ -140,4 +156,5 @@ spec: {{- with .Values.tolerations }} tolerations: {{- toYaml . | nindent 8 }} - {{- end }} \ No newline at end of file + {{- end }} +{{- end }} diff --git a/helm/gecko/templates/deployment.yaml b/helm/gecko/templates/deployment.yaml index 15b065d06..b6c753f37 100644 --- a/helm/gecko/templates/deployment.yaml +++ b/helm/gecko/templates/deployment.yaml @@ -17,9 +17,16 @@ spec: labels: {{- include "gecko.selectorLabels" . | nindent 8 }} spec: - {{- with .Values.volumes }} + {{- if or .Values.gitMirrorStorage.enabled .Values.volumes }} volumes: + {{- if .Values.gitMirrorStorage.enabled }} + - name: git-mirror-storage + persistentVolumeClaim: + claimName: {{ default (printf "%s-git-mirror" (include "gecko.fullname" .)) .Values.gitMirrorStorage.existingClaim }} + {{- end }} + {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} {{- with .Values.imagePullSecrets }} imagePullSecrets: @@ -28,40 +35,84 @@ spec: serviceAccountName: {{ include "gecko.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if and .Values.storageChainAuditCache.enabled .Values.storageChainAuditCache.waitForRedis.enabled }} + initContainers: + - name: wait-for-audit-cache + image: {{ .Values.storageChainAuditCache.waitForRedis.image | quote }} + imagePullPolicy: IfNotPresent + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.storageChainAuditCache.passwordSecretName | quote }} + key: {{ .Values.storageChainAuditCache.passwordKey | quote }} + optional: false + command: + - /bin/sh + - -c + - | + until redis-cli -h {{ .Values.storageChainAuditCache.waitForRedis.host }} -p {{ .Values.storageChainAuditCache.waitForRedis.port }} -a "$REDIS_PASSWORD" --no-auth-warning ping | grep -q PONG; do + echo "waiting for storage-chain audit cache redis at {{ .Values.storageChainAuditCache.waitForRedis.host }}:{{ .Values.storageChainAuditCache.waitForRedis.port }}" + sleep 2 + done + {{- end }} containers: - name: {{ .Chart.Name }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + image: {{ include "gecko.image" . | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} ports: - name: http - containerPort: 80 + containerPort: 8080 protocol: TCP livenessProbe: httpGet: path: /health - port: 80 + port: http readinessProbe: httpGet: path: /health - port: 80 + port: http resources: {{- toYaml .Values.resources | nindent 12 }} - command: ["sh"] - args: - - "-c" - - | - set -e - # set env vars - #until false; do - # sleep 5 - # done - ./bin/gecko - env: - {{- toYaml .Values.env | nindent 12 }} + {{- if .Values.goRuntime.enabled }} + - name: GOMEMLIMIT + value: {{ .Values.goRuntime.memoryLimit | quote }} + - name: GOGC + value: {{ .Values.goRuntime.gcPercent | quote }} + {{- end }} + {{- range .Values.env }} + - name: {{ .name | quote }} + value: {{- if eq .name "JWKS_ENDPOINT" }} + {{ default (printf "https://%s/user/.well-known/jwks" $.Values.global.hostname) .value | quote }} + {{- else if eq .name "FENCE_BASE_URL" }} + {{ default (printf "https://%s/user" $.Values.global.hostname) .value | quote }} + {{- else if eq .name "SYFON_DATA_API_BASE_URL" }} + {{ default (printf "https://%s/data" $.Values.global.hostname) .value | quote }} + {{- else }} + {{ .value | quote }} + {{- end }} + {{- end }} + {{- if .Values.storageChainAuditCache.enabled }} + - name: STORAGE_CHAIN_AUDIT_CACHE_ENABLED + value: "true" + - name: STORAGE_CHAIN_AUDIT_CACHE_REDIS_URL + value: {{ .Values.storageChainAuditCache.redisUrl | quote }} + - name: STORAGE_CHAIN_AUDIT_CACHE_TTL_SECONDS + value: {{ .Values.storageChainAuditCache.ttlSeconds | quote }} + - name: STORAGE_CHAIN_AUDIT_CACHE_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.storageChainAuditCache.passwordSecretName | quote }} + key: {{ .Values.storageChainAuditCache.passwordKey | quote }} + optional: false + {{- else }} + - name: STORAGE_CHAIN_AUDIT_CACHE_ENABLED + value: "false" + {{- end }} - name: GRIP_GRAPH valueFrom: configMapKeyRef: @@ -124,10 +175,18 @@ spec: name: gecko-dbcreds key: dbcreated optional: false + - name: GIT_DATA_DIR + value: {{ if .Values.gitMirrorStorage.enabled }}{{ .Values.gitMirrorStorage.mountPath | quote }}{{ else }}"/tmp/gecko-git"{{ end }} - {{- with .Values.volumeMounts }} + {{- if or .Values.gitMirrorStorage.enabled .Values.volumeMounts }} volumeMounts: - {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- if .Values.gitMirrorStorage.enabled }} + - name: git-mirror-storage + mountPath: {{ .Values.gitMirrorStorage.mountPath | quote }} + {{- end }} + {{- with .Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} {{- end }} {{- with .Values.nodeSelector }} nodeSelector: diff --git a/helm/gecko/templates/git-mirror-pv.yaml b/helm/gecko/templates/git-mirror-pv.yaml new file mode 100644 index 000000000..e0e71ca65 --- /dev/null +++ b/helm/gecko/templates/git-mirror-pv.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.gitMirrorStorage.enabled .Values.gitMirrorStorage.persistence.enabled (not .Values.gitMirrorStorage.existingClaim) }} +apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ include "gecko.fullname" . }}-git-mirror-pv + labels: + app.kubernetes.io/managed-by: "Helm" + annotations: + meta.helm.sh/release-name: {{ .Release.Name }} + meta.helm.sh/release-namespace: "default" +spec: + capacity: + storage: {{ .Values.gitMirrorStorage.size | quote }} + accessModes: + {{- toYaml .Values.gitMirrorStorage.accessModes | nindent 4 }} + storageClassName: {{ .Values.gitMirrorStorage.storageClass | quote }} + persistentVolumeReclaimPolicy: Retain + hostPath: + path: {{ .Values.gitMirrorStorage.persistence.hostPath | quote }} +{{- end }} diff --git a/helm/gecko/templates/job-db-init.yaml b/helm/gecko/templates/job-db-init.yaml index 1853c9073..7a8d199c1 100644 --- a/helm/gecko/templates/job-db-init.yaml +++ b/helm/gecko/templates/job-db-init.yaml @@ -13,4 +13,8 @@ data: # Reads content from the chart filesystem at files/init-data/apps_page.json apps_page.json: | -{{ .Files.Get "files/init-data/apps_page.json" | trim | nindent 4 }} \ No newline at end of file +{{ .Files.Get "files/init-data/apps_page.json" | trim | nindent 4 }} + + # Renders project configs from .Values.projects so deployment-specific/private project definitions stay out of the chart. + projects.json: | +{{ .Values.projects | toPrettyJson | nindent 4 }} diff --git a/helm/gecko/templates/pvc-git-mirror.yaml b/helm/gecko/templates/pvc-git-mirror.yaml new file mode 100644 index 000000000..6518b8e9f --- /dev/null +++ b/helm/gecko/templates/pvc-git-mirror.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.gitMirrorStorage.enabled (not .Values.gitMirrorStorage.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "gecko.fullname" . }}-git-mirror + labels: + {{- include "gecko.labels" . | nindent 4 }} + {{- with .Values.gitMirrorStorage.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.gitMirrorStorage.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: + {{- toYaml .Values.gitMirrorStorage.accessModes | nindent 4 }} + resources: + requests: + storage: {{ .Values.gitMirrorStorage.size | quote }} + {{- with .Values.gitMirrorStorage.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} +{{- end }} diff --git a/helm/gecko/values.yaml b/helm/gecko/values.yaml index 4e3bc0d89..bec91b2af 100644 --- a/helm/gecko/values.yaml +++ b/helm/gecko/values.yaml @@ -93,6 +93,11 @@ postgres: password: # -- (string) Will create a Database for the individual service to help with developing it. separate: false + initJob: + # -- (bool) Whether to run the Gecko DB init job. + enabled: true + # -- (string) Image used for DB initialization. Must include psql. + image: postgres:16 # -- (map) Postgresql subchart settings if deployed separately option is set to "true". # Disable persistence by default so we can spin up and down ephemeral environments @@ -121,6 +126,42 @@ qdrant: # -- (string) Size of the local Qdrant PersistentVolume. size: 26Gi +gitMirrorStorage: + # -- (bool) Whether to persist Gecko's local git mirrors in a PVC. + enabled: false + # -- (string) Directory inside the Gecko container where git mirrors are stored. + mountPath: /data/gecko-git + # -- (string) Existing PVC to use for git mirror storage. If empty and enabled, a PVC is created. + existingClaim: "" + # -- (list) Access modes for the git mirror PVC. + accessModes: + - ReadWriteOnce + # -- (string) StorageClass for the git mirror PV/PVC. + storageClass: gecko-git-manual-storage + # -- (string) Requested size for the git mirror PVC. + size: 10Gi + persistence: + # -- (bool) Whether to create a local PersistentVolume for git mirror storage. + enabled: true + # -- (string) Local path used by the git mirror PersistentVolume. + hostPath: /mnt/data/gecko-git + # -- (map) Annotations to add to the git mirror PVC. + annotations: {} + # -- (map) Labels to add to the git mirror PVC. + labels: {} + +# -- (list) Gecko project configs. These seed config_schema.projects keyed as "/". +projects: [] +# Example: +# projects: +# - title: BForePC +# contact_email: support@example.org +# src_repo: HTAN_INT-BForePC +# org_title: HTAN +# description: Private project description +# project_title: INT-BForePC +# icon_name: binoculars + # -- (int) Number of replicas for the deployment. replicaCount: 1 @@ -133,6 +174,8 @@ image: pullPolicy: IfNotPresent # -- (string) Overrides the image tag whose default is the chart appVersion. tag: "latest" + # -- (string) Optional image digest. When set, takes precedence over tag. + digest: "" # -- (list) Docker image pull secrets. imagePullSecrets: [] @@ -190,15 +233,24 @@ resources: # -- (map) The amount of resources that the container requests requests: # -- (string) The amount of CPU requested - cpu: 0.1 + cpu: 200m # -- (string) The amount of memory requested - memory: 12Mi + memory: 256Mi # -- (map) The maximum amount of resources that the container is allowed to use limits: # -- (string) The maximum amount of CPU the container can use - cpu: 1.0 + cpu: 2 # -- (string) The maximum amount of memory the container can use - memory: 512Mi + memory: 4Gi + +# -- (map) Go runtime memory controls for Gecko. Keep memoryLimit below resources.limits.memory so Go GC starts before Kubernetes OOM kills the pod. +goRuntime: + # -- (bool) Whether to inject Go runtime memory control environment variables. + enabled: true + # -- (string) Soft Go memory limit. For the default 4Gi pod limit this leaves headroom for non-Go heap memory and JSON/socket buffers. + memoryLimit: 3400MiB + # -- (string) Go GC target percentage. Lower values trade CPU for lower peak memory during large audits. + gcPercent: "75" # -- (map) Configuration for autoscaling the number of replicas autoscaling: @@ -232,7 +284,34 @@ volumeMounts: [] env: # -- (string) The URL of the JSON Web Key Set (JWKS) endpoint for authentication - name: JWKS_ENDPOINT - value: "http://fence-service/.well-known/jwks" + value: "" + # -- (string) The base URL of Fence for GitHub App token exchange + - name: FENCE_BASE_URL + value: "" + # -- (string) Syfon Data API base URL routed via revproxy (for example https:///data) + - name: SYFON_DATA_API_BASE_URL + value: "" + +storageChainAuditCache: + # -- (bool) Whether Gecko should cache completed storage-chain audit responses. + enabled: true + # -- (string) Redis URL for completed storage-chain audit response cache. + redisUrl: redis://authz-cache-service:6379/0 + # -- (int) Completed storage-chain audit response cache TTL in seconds. + ttlSeconds: 86400 + # -- (string) Secret containing the Redis password. + passwordSecretName: authz-cache-credentials + # -- (string) Key in the Redis password secret. + passwordKey: redis-password + waitForRedis: + # -- (bool) Whether to wait for Redis before starting Gecko. + enabled: true + # -- (string) Redis CLI image used by the wait init container. + image: "redis:7.2-alpine" + # -- (string) Redis host to wait for. + host: authz-cache-service + # -- (int) Redis port to wait for. + port: 6379 # Values to determine the labels that are used for the deployment, pod, etc. diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index f7a75c5ad..2b3b4c5a9 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -9,7 +9,7 @@ dependencies: repository: "file://../ambassador" condition: ambassador.enabled - name: arborist - version: 0.1.11 + version: 0.1.12 repository: "file://../arborist" condition: arborist.enabled - name: argo-wrapper @@ -43,6 +43,10 @@ dependencies: version: 0.1.0 repository: "file://../fhir-server" condition: fhir-server.enabled +- name: funnel + version: 0.1.77 + repository: "file://../funnel" + condition: funnel.enabled - name: gecko version: "0.1.0" repository: "file://../gecko" @@ -79,6 +83,10 @@ dependencies: version: 0.1.16 repository: "file://../requestor" condition: requestor.enabled +- name: redis + version: 0.1.0 + repository: "file://../redis" + condition: redis.enabled - name: revproxy version: 0.1.14 repository: "file://../revproxy" @@ -111,14 +119,13 @@ dependencies: version: 0.1.0 repository: "file://../viv" condition: viv.enabled -- name: qdrant - version: 1.15.4 - repository: "https://qdrant.github.io/qdrant-helm" - condition: qdrant.enabled - name: syfon version: 0.1.0 repository: "file://../syfon" condition: syfon.enabled + #- name: qdrant + #version: 1.15.4 + #repository: "https://qdrant.github.io/qdrant-helm" # A chart can be either an 'application' or a 'library' chart. # diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml index 3a6d3d9c4..6aa840aa2 100644 --- a/helm/gen3/values.yaml +++ b/helm/gen3/values.yaml @@ -145,6 +145,11 @@ frontend-framework: # -- (string) Overrides the image tag whose default is the chart appVersion. tag: "develop" +# -- (map) Configurations for Funnel chart. +funnel: + # -- (bool) Whether to deploy the Funnel subchart. + enabled: false + # -- (map) Configurations for guppy chart. guppy: # -- (bool) Whether to deploy the guppy subchart. @@ -255,6 +260,10 @@ requestor: # -- (string) Overrides the image tag whose default is the chart appVersion. tag: +redis: + # -- (bool) Whether to deploy the shared authz cache subchart. + enabled: true + revproxy: # -- (bool) Whether to deploy the revproxy subchart. enabled: true diff --git a/helm/redis/Chart.yaml b/helm/redis/Chart.yaml new file mode 100644 index 000000000..0c93f3be2 --- /dev/null +++ b/helm/redis/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +name: redis +description: A Helm chart for in-cluster Redis used by shared authz snapshot caching +type: application +version: 0.1.0 +appVersion: "7.2" + +dependencies: +- name: common + version: 0.1.10 + repository: file://../common diff --git a/helm/redis/README.md b/helm/redis/README.md new file mode 100644 index 000000000..423df2fc0 --- /dev/null +++ b/helm/redis/README.md @@ -0,0 +1,26 @@ +# redis + +In-cluster Redis used as the shared Fence authz snapshot cache backend. + +Primary consumer: + +- `AUTHZ_SNAPSHOT_CACHE_REDIS_URL=redis://authz-cache-service:6379/0` + +Deployment notes: + +- enabled through the umbrella `gen3` chart with `redis.enabled` +- exposed only as an internal Kubernetes `Service` +- intended for local/dev Kubernetes use first + +Current defaults: + +- single replica +- password auth enabled through a Kubernetes secret +- no persistence +- ingress restricted to Fence pods when network policies are enabled + +Future hardening can add: + +- persistence +- StatefulSet semantics +- egress policy diff --git a/helm/redis/templates/_helpers.tpl b/helm/redis/templates/_helpers.tpl new file mode 100644 index 000000000..8ba58a448 --- /dev/null +++ b/helm/redis/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{- define "redis.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "redis.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{- define "redis.labels" -}} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +app.kubernetes.io/name: {{ include "redis.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{- define "redis.selectorLabels" -}} +app.kubernetes.io/name: {{ include "redis.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{- define "redis.authSecretName" -}} +{{- default "authz-cache-credentials" .Values.auth.secretName -}} +{{- end }} + +{{- define "redis.password" -}} +{{- $secretName := include "redis.authSecretName" . -}} +{{- $existingSecret := (lookup "v1" "Secret" .Release.Namespace $secretName) -}} +{{- if .Values.auth.password -}} +{{- .Values.auth.password | b64enc -}} +{{- else if and $existingSecret (index $existingSecret.data "redis-password") -}} +{{- index $existingSecret.data "redis-password" -}} +{{- else -}} +{{- randAlphaNum 32 | b64enc -}} +{{- end -}} +{{- end }} diff --git a/helm/redis/templates/deployment.yaml b/helm/redis/templates/deployment.yaml new file mode 100644 index 000000000..4cc54e91b --- /dev/null +++ b/helm/redis/templates/deployment.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "redis.fullname" . }}-deployment + labels: + {{- include "redis.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "redis.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "redis.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: redis + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "redis.authSecretName" . }} + key: redis-password + optional: false + ports: + - name: redis + containerPort: 6379 + protocol: TCP + command: + - /bin/sh + - -c + - | + if [ "{{ .Values.auth.enabled }}" = "true" ]; then + exec redis-server --save "" --appendonly no --requirepass "$REDIS_PASSWORD" + fi + exec redis-server --save "" --appendonly no + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/redis/templates/networkpolicy.yaml b/helm/redis/templates/networkpolicy.yaml new file mode 100644 index 000000000..5c3b1587b --- /dev/null +++ b/helm/redis/templates/networkpolicy.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.global.netPolicy .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "redis.fullname" . }}-ingress + labels: + {{- include "redis.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "redis.selectorLabels" . | nindent 6 }} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + app: fence + - podSelector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + app: arborist + - podSelector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + app: gecko + ports: + - protocol: TCP + port: {{ .Values.service.port }} +{{- end }} diff --git a/helm/redis/templates/secret.yaml b/helm/redis/templates/secret.yaml new file mode 100644 index 000000000..b52294c8b --- /dev/null +++ b/helm/redis/templates/secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "redis.authSecretName" . }} + labels: + {{- include "redis.labels" . | nindent 4 }} +type: Opaque +data: + redis-password: {{ include "redis.password" . }} diff --git a/helm/redis/templates/service.yaml b/helm/redis/templates/service.yaml new file mode 100644 index 000000000..a2bcd6af6 --- /dev/null +++ b/helm/redis/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.service.name }} + labels: + {{- include "redis.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: redis + protocol: TCP + name: redis + selector: + {{- include "redis.selectorLabels" . | nindent 4 }} diff --git a/helm/redis/values.yaml b/helm/redis/values.yaml new file mode 100644 index 000000000..be43e37ab --- /dev/null +++ b/helm/redis/values.yaml @@ -0,0 +1,49 @@ +# Default values for redis. + +global: + dev: true + environment: default + hostname: localhost + revproxyArn: arn:aws:acm:us-east-1:123456:certificate + netPolicy: true + ddEnabled: false + +replicaCount: 1 + +imagePullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +image: + repository: redis + pullPolicy: IfNotPresent + tag: "7.2-alpine" + +service: + name: authz-cache-service + type: ClusterIP + port: 6379 + +networkPolicy: + enabled: true + +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + +persistence: + enabled: false + +auth: + enabled: true + secretName: authz-cache-credentials + password: "" + +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/helm/revproxy/gen3.nginx.conf/arborist-service.conf b/helm/revproxy/gen3.nginx.conf/arborist-service.conf index f68f6dd04..1a4ec739a 100644 --- a/helm/revproxy/gen3.nginx.conf/arborist-service.conf +++ b/helm/revproxy/gen3.nginx.conf/arborist-service.conf @@ -53,6 +53,34 @@ location = /gen3-authz { # authorization endpoint # https://hostname/authz?resource=programs/blah&method=acb&service=xyz # +location ^~ /authz/ownership/ { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + + set $proxy_service "arborist"; + set $upstream http://arborist-service.$namespace.svc.cluster.local; + + include /etc/nginx/snippets/gen3-authenticated-proxy-headers.conf; + rewrite ^/authz/ownership/(.*)$ /ownership/$1 break; + proxy_pass $upstream; + proxy_redirect http://$host/ https://$host/authz/; +} + +location ^~ /authz/access/ { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + + set $proxy_service "arborist"; + set $upstream http://arborist-service.$namespace.svc.cluster.local; + + include /etc/nginx/snippets/gen3-authenticated-proxy-headers.conf; + rewrite ^/authz/access/(.*)$ /access/$1 break; + proxy_pass $upstream; + proxy_redirect http://$host/ https://$host/authz/; +} + location ~ /authz/? { if ($csrf_check !~ ^ok-\S.+$) { return 403 "failed csrf check"; diff --git a/helm/revproxy/gen3.nginx.conf/gecko-service.conf b/helm/revproxy/gen3.nginx.conf/gecko-service.conf index 3fa3168b2..26b7cc318 100644 --- a/helm/revproxy/gen3.nginx.conf/gecko-service.conf +++ b/helm/revproxy/gen3.nginx.conf/gecko-service.conf @@ -1,61 +1,77 @@ -location /ExplorerConfig/health { +location /gecko/health { proxy_connect_timeout 600s; proxy_send_timeout 600s; proxy_read_timeout 600s; send_timeout 600s; - set $proxy_service "gecko-health"; # You can name this differently for clarity in logs + set $proxy_service "gecko-health"; set $upstream http://gecko-service.$namespace.svc.cluster.local; - rewrite ^/ExplorerConfig/health$ /health break; + rewrite ^/gecko/health$ /health break; proxy_pass $upstream; } - -location /ExplorerConfig/list { +location = /gecko/projects/list { proxy_connect_timeout 600s; proxy_send_timeout 600s; proxy_read_timeout 600s; send_timeout 600s; - set $proxy_service "gecko-list"; - set $upstream http://gecko-service.$namespace.svc.cluster.local; + set $proxy_service "gecko-project-list"; - rewrite ^/ExplorerConfig/list /config/list break; - proxy_pass $upstream$is_args$args; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + include /etc/nginx/snippets/gen3-authenticated-proxy-headers.conf; + + proxy_pass http://gecko-service.$namespace.svc.cluster.local/config/projects/list$is_args$args; client_max_body_size 0; } - -location /ExplorerConfig/ { +location = /gecko/git/github/callback { proxy_connect_timeout 600s; proxy_send_timeout 600s; proxy_read_timeout 600s; send_timeout 600s; - set $proxy_service "gecko"; + set $proxy_service "gecko-git-callback"; set $upstream http://gecko-service.$namespace.svc.cluster.local; - rewrite ^/ExplorerConfig/(.*)$ /config/$1 break; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + include /etc/nginx/snippets/gen3-authenticated-proxy-headers.conf; + + rewrite ^/gecko/git/github/callback$ /git/github/callback break; proxy_pass $upstream$is_args$args; client_max_body_size 0; } -location ~* ^/(Vector|vector)/ { +location ^~ /gecko/git/ { proxy_connect_timeout 600s; proxy_send_timeout 600s; proxy_read_timeout 600s; send_timeout 600s; - set $proxy_service "gecko"; - set $upstream http://gecko-service.$namespace.svc.cluster.local; + set $proxy_service "gecko-git"; + + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + include /etc/nginx/snippets/gen3-authenticated-proxy-headers.conf; - rewrite ^/(Vector|vector)/(.*)$ /vector/$2 break; + rewrite ^/gecko/git/(.*)$ /git/$1 break; proxy_pass $upstream; client_max_body_size 0; } -location ~* ^/Dir { +location /gecko/ { proxy_connect_timeout 600s; proxy_send_timeout 600s; proxy_read_timeout 600s; @@ -64,9 +80,35 @@ location ~* ^/Dir { set $proxy_service "gecko"; set $upstream http://gecko-service.$namespace.svc.cluster.local; - rewrite ^/Dir(.*)$ /dir$1 break; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + include /etc/nginx/snippets/gen3-authenticated-proxy-headers.conf; + + rewrite ^/gecko/(.*)$ /config/$1 break; proxy_pass $upstream; client_max_body_size 0; +} - -} \ No newline at end of file +#location ~* ^/(Vector|vector)/ { +# proxy_connect_timeout 600s; +# proxy_send_timeout 600s; +# proxy_read_timeout 600s; +# send_timeout 600s; +# +# set $proxy_service "gecko"; +# set $upstream http://gecko-service.$namespace.svc.cluster.local; +# +# proxy_http_version 1.1; +# proxy_set_header Connection ""; +# proxy_set_header Host $host; +# proxy_set_header X-Real-IP $remote_addr; +# proxy_set_header X-Forwarded-Proto $scheme; +# include /etc/nginx/snippets/gen3-authenticated-proxy-headers.conf; +# +# rewrite ^/(Vector|vector)/(.*)$ /vector/$2 break; +# proxy_pass $upstream; +# client_max_body_size 0; +#} diff --git a/helm/revproxy/gen3.nginx.conf/syfon-service.conf b/helm/revproxy/gen3.nginx.conf/syfon-service.conf index d69ee6d5c..c52831321 100644 --- a/helm/revproxy/gen3.nginx.conf/syfon-service.conf +++ b/helm/revproxy/gen3.nginx.conf/syfon-service.conf @@ -8,6 +8,7 @@ location ^~ /ga4gh/ { } set $proxy_service "syfon"; + include /etc/nginx/snippets/gen3-authenticated-proxy-headers.conf; proxy_pass $drs_upstream; proxy_redirect http://$host/ https://$host/; } @@ -25,6 +26,7 @@ location ^~ /index/ { } set $proxy_service "syfon"; + include /etc/nginx/snippets/gen3-authenticated-proxy-headers.conf; proxy_pass $drs_upstream; proxy_redirect http://$host/ https://$host/; } @@ -50,6 +52,7 @@ location ^~ /download/ { } set $proxy_service "syfon"; + include /etc/nginx/snippets/gen3-authenticated-proxy-headers.conf; proxy_pass $drs_upstream; proxy_redirect http://$host/ https://$host/; } @@ -61,6 +64,7 @@ location ^~ /info/lfs/ { } set $proxy_service "syfon"; + include /etc/nginx/snippets/gen3-authenticated-proxy-headers.conf; proxy_pass $drs_upstream; proxy_redirect http://$host/ https://$host/; @@ -77,6 +81,7 @@ location ^~ /data/ { } set $proxy_service "syfon"; + include /etc/nginx/snippets/gen3-authenticated-proxy-headers.conf; proxy_pass http://syfon$des_domain:8080; proxy_redirect http://$host/ https://$host/; diff --git a/helm/revproxy/nginx-snippets/gen3-authenticated-proxy-headers.conf b/helm/revproxy/nginx-snippets/gen3-authenticated-proxy-headers.conf new file mode 100644 index 000000000..a97785046 --- /dev/null +++ b/helm/revproxy/nginx-snippets/gen3-authenticated-proxy-headers.conf @@ -0,0 +1,7 @@ +proxy_set_header Authorization "$access_token"; +proxy_set_header X-Forwarded-For "$proxy_add_x_forwarded_for"; +proxy_set_header X-UserId "$userid"; +proxy_set_header X-ReqId "$request_id"; +proxy_set_header X-SessionId "$session_id"; +proxy_set_header X-VisitorId "$visitor_id"; +proxy_set_header X-Original-URI $request_uri; diff --git a/helm/revproxy/nginx/nginx.conf b/helm/revproxy/nginx/nginx.conf index b0a142724..1c0a03fe4 100644 --- a/helm/revproxy/nginx/nginx.conf +++ b/helm/revproxy/nginx/nginx.conf @@ -23,7 +23,7 @@ env FRONTEND_ROOT; env DOCUMENT_URL; events { - worker_connections 768; + worker_connections 4096; } http { @@ -44,6 +44,10 @@ http { '' close; } + map $request_uri $gecko_git_upstream_uri { + ~^/gecko/git(?/.*)?$ /git$rest; + default /git; + } map $proxy_protocol_addr $initialip { "" $http_x_forwarded_for; diff --git a/helm/revproxy/templates/configMaps.yaml b/helm/revproxy/templates/configMaps.yaml index 85f0869a0..8f39fb73a 100644 --- a/helm/revproxy/templates/configMaps.yaml +++ b/helm/revproxy/templates/configMaps.yaml @@ -31,10 +31,20 @@ data: --- apiVersion: v1 kind: ConfigMap +metadata: + name: revproxy-nginx-snippets +data: +{{- range $path, $bytes := .Files.Glob "nginx-snippets/*" }} + {{ ($a := split "/" $path)._1 }}: | + {{- $bytes | toString | nindent 4 }} +{{- end}} +--- +apiVersion: v1 +kind: ConfigMap metadata: name: revproxy-nginx-ssl data: {{- range $path, $bytes := .Files.Glob "ssl/*" }} {{ ($a := split "/" $path)._1 }}: | {{- $bytes | toString | nindent 4 }} -{{- end}} \ No newline at end of file +{{- end}} diff --git a/helm/revproxy/templates/deployment.yaml b/helm/revproxy/templates/deployment.yaml index c0d1eb5cd..7880fc4cc 100644 --- a/helm/revproxy/templates/deployment.yaml +++ b/helm/revproxy/templates/deployment.yaml @@ -56,6 +56,9 @@ spec: - name: revproxy-subconf configMap: name: revproxy-nginx-subconf + - name: revproxy-snippets + configMap: + name: revproxy-nginx-snippets - name: revproxy-ssl configMap: name: revproxy-nginx-ssl @@ -147,6 +150,9 @@ spec: - name: "revproxy-subconf" readOnly: true mountPath: "/etc/nginx/gen3.conf" + - name: "revproxy-snippets" + readOnly: true + mountPath: "/etc/nginx/snippets" - name: "revproxy-ssl" readOnly: true mountPath: "/etc/nginx/ssl/service.crt" diff --git a/helm/revproxy/templates/ingress_dev.yaml b/helm/revproxy/templates/ingress_dev.yaml index 1ebeac551..248be3ca1 100644 --- a/helm/revproxy/templates/ingress_dev.yaml +++ b/helm/revproxy/templates/ingress_dev.yaml @@ -3,6 +3,10 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: revproxy-dev + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: {{- if .Values.global.dev }} tls: @@ -21,4 +25,4 @@ spec: name: revproxy-service port: number: 80 -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/revproxy/values.yaml b/helm/revproxy/values.yaml index bc0342de7..8697e03f3 100644 --- a/helm/revproxy/values.yaml +++ b/helm/revproxy/values.yaml @@ -147,7 +147,11 @@ ingress: # -- (string) The ingress class name. className: "" # -- (map) Annotations to add to the ingress. - annotations: {} + annotations: + nginx.ingress.kubernetes.io/proxy-buffer-size: "256k" + nginx.ingress.kubernetes.io/proxy-buffers-number: "8" + nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "512k" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" # -- (list) Where to route the traffic. diff --git a/helm/syfon/README.md b/helm/syfon/README.md index 0bd1c63a8..8b6f334f6 100644 --- a/helm/syfon/README.md +++ b/helm/syfon/README.md @@ -5,6 +5,7 @@ This chart deploys `syfon` with: - Syfon config mounted into the pod at `/etc/drs/config.yaml` from `config` using a Kubernetes Secret - DB credentials injected via secret env vars (`DRS_DB_*`) +- A compatibility service-creds secret for legacy Fence/Sheepdog consumers - Optional PostgreSQL init job that mirrors indexd-style setup: - creates app DB user - creates app database @@ -39,7 +40,7 @@ config: default_expiry_seconds: 900 credential_encryption: master_key: base64-or-hex-or-32-byte-key - s3_credentials: + buckets: - bucket: cbds provider: s3 region: us-east-1 @@ -48,37 +49,100 @@ config: secret_key: secret-key resources: - organization: cbds - project: training org_path: programs/cbds - project_path: projects/training + projects: + - project_id: training + project_path: projects/training + bucket_scopes: + - organization: cbds + project_id: training + bucket: cbds + path_prefix: programs/cbds/projects/training +``` + +Legacy bucket-keyed config remains valid: + +```yaml +config: + s3_credentials: + - bucket: cbds + provider: s3 + region: us-east-1 + access_key: access-key + secret_key: secret-key bucket_scopes: - organization: cbds project_id: training bucket: cbds - org_path: programs/cbds - project_path: projects/training ``` -Configured `s3_credentials` are loaded by the Syfon server on startup. Syfon -requires a credential encryption key for non-empty bucket credentials; set +Configured `buckets` are loaded by the Syfon server on startup. Operators use +the physical `bucket` name in config and API requests. Syfon handles any +internal database keying itself from the non-secret credential fields, and +`secret_key` rotation does not change that internal key. + +Syfon requires a credential encryption key for non-empty bucket credentials; set `credential_encryption.master_key` in the same config block. The key may be a 32-byte raw string, a 64-character hex string, or base64-encoded 32-byte key. Configured `bucket_scopes` are loaded on startup too. `organization` and -`project_id` are the Gen3 authz labels. `organization_sub_path` / -`project_sub_path` are storage-layout prefixes, and the chart also accepts the -shorter aliases `org_path` / `project_path` and normalizes them in the rendered -config. You can define these scopes either as top-level `bucket_scopes` or -inline under each `s3_credentials[*].resources` entry. Inline resource entries -use `organization`, `project`, `org_path`, and `project_path`; the chart -attaches the parent bucket and renders the final server config as normal -`bucket_scopes`. Syfon stores the full scope prefix and prepends that prefix -when signing imported record URLs that are relative to the bucket root. You can -also set a complete `path` or explicit `bucket` plus `path_prefix`. +`project_id` are the Gen3 authz labels. Scopes reference the physical `bucket` +and set `path_prefix` when the project should write below a specific key prefix. +`organization_sub_path` / `project_sub_path` are storage-layout prefixes, and +inline `buckets[*].resources` entries can still derive normal `bucket_scopes`. +If the same physical bucket is configured more than once with different +credentials, define scopes inline under the intended `buckets[*].resources` +entry so the scope is unambiguous without exposing internal identity fields. + +Migration rule: leave existing `bucket` references in place. Do not add any +extra credential identifier values to values.yaml. + +## Legacy Credential Cleanup + +Older Syfon deployments used the physical bucket name as the database +`s3_credential.credential_id`. Current Syfon derives a stable internal +credential ID from non-secret credential fields. After switching to the derived +ID model, a database can temporarily contain both rows: + +- legacy row: `credential_id = bucket` +- current row: derived credential ID for the same physical bucket + +If the legacy row was encrypted with an old `credential_encryption.master_key`, +`GET /data/buckets` will fail while listing credentials. Use the chart helper +script to clean this up. + +Dry-run first: + +```bash +KUBECTL=kc ./helm/syfon/scripts/cleanup-legacy-credential-ids.sh \ + --namespace default \ + --db-host local-postgresql \ + --db-name syfon_db \ + --secret syfon-db-admin +``` + +Apply: + +```bash +KUBECTL=kc ./helm/syfon/scripts/cleanup-legacy-credential-ids.sh \ + --namespace default \ + --db-host local-postgresql \ + --db-name syfon_db \ + --secret syfon-db-admin \ + --apply +``` + +The script is intentionally conservative. It only remaps `bucket_scope` rows and +deletes legacy credential rows when exactly one replacement credential exists +for the same physical bucket. Ambiguous buckets are skipped and must be reviewed +manually. ## Key Compatibility Notes - Secret keys mirror indexd credentials naming (`db_host`, `db_username`, `db_password`, `db_database`) with additional `db_port` and `db_sslmode`. +- By default the chart also creates `indexd-service-creds` so existing Fence and + Sheepdog deployments can keep reading `fence` / `sheepdog` service passwords + after migrating the backend service to Syfon. - In `gen3` mode, `syfon` requires PostgreSQL. - The rendered Syfon config is stored as a Kubernetes Secret because it can contain bucket credentials and `credential_encryption.master_key`. diff --git a/helm/syfon/scripts/cleanup-legacy-credential-ids.sh b/helm/syfon/scripts/cleanup-legacy-credential-ids.sh new file mode 100755 index 000000000..db7ec3d30 --- /dev/null +++ b/helm/syfon/scripts/cleanup-legacy-credential-ids.sh @@ -0,0 +1,252 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'USAGE' +cleanup-legacy-credential-ids.sh + +Safely remap Syfon bucket scopes away from legacy bucket-name credential IDs and +delete stale legacy credential rows. + +Default mode is a dry-run. Use --apply to mutate the database. + +Options: + --namespace Kubernetes namespace. Default: default + --db-host PostgreSQL service host. Default: local-postgresql + --db-name Database name. Default: syfon_db + --db-user Database user. Default: postgres + --secret Kubernetes secret with DB password. Default: syfon-db-admin + --secret-key Secret key to read. Default: auto-detect password/db_password + --image psql client image. Default: postgres:16 + --kubectl kubectl-compatible binary. Default: kubectl, then kc + --apply Apply changes. Without this, only prints candidates. + -h, --help Show help. + +Example: + KUBECTL=kc ./cleanup-legacy-credential-ids.sh --namespace default --apply + +This script only remaps/deletes legacy rows when exactly one replacement +credential exists for the same physical bucket. Ambiguous buckets are skipped. +USAGE +} + +namespace="default" +db_host="local-postgresql" +db_name="syfon_db" +db_user="postgres" +secret_name="syfon-db-admin" +secret_key="" +image="postgres:16" +kubectl_bin="${KUBECTL:-}" +apply="false" + +while [[ $# -gt 0 ]]; do + case "$1" in + --namespace) + namespace="$2" + shift 2 + ;; + --db-host) + db_host="$2" + shift 2 + ;; + --db-name) + db_name="$2" + shift 2 + ;; + --db-user) + db_user="$2" + shift 2 + ;; + --secret) + secret_name="$2" + shift 2 + ;; + --secret-key) + secret_key="$2" + shift 2 + ;; + --image) + image="$2" + shift 2 + ;; + --kubectl) + kubectl_bin="$2" + shift 2 + ;; + --apply) + apply="true" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "$kubectl_bin" ]]; then + if command -v kubectl >/dev/null 2>&1; then + kubectl_bin="kubectl" + elif command -v kc >/dev/null 2>&1; then + kubectl_bin="kc" + else + echo "kubectl-compatible binary not found; set KUBECTL=kc or pass --kubectl" >&2 + exit 1 + fi +fi + +read_secret_key() { + local key="$1" + local encoded + encoded="$("$kubectl_bin" get secret "$secret_name" -n "$namespace" -o "jsonpath={.data.${key}}" 2>/dev/null || true)" + if [[ -n "$encoded" ]]; then + printf '%s' "$encoded" | base64 -d + return 0 + fi + return 1 +} + +if [[ -n "$secret_key" ]]; then + if ! pgpassword="$(read_secret_key "$secret_key")"; then + echo "secret $secret_name does not contain key $secret_key" >&2 + exit 1 + fi +else + if pgpassword="$(read_secret_key "password")"; then + : + elif pgpassword="$(read_secret_key "db_password")"; then + : + else + echo "secret $secret_name does not contain password or db_password" >&2 + exit 1 + fi +fi + +pod_name="syfon-psql-debug-$(date +%s)" + +run_psql() { + "$kubectl_bin" run "$pod_name" \ + -n "$namespace" \ + --rm -i \ + --restart=Never \ + --image="$image" \ + --env "PGPASSWORD=$pgpassword" \ + -- psql -h "$db_host" -U "$db_user" -d "$db_name" -v ON_ERROR_STOP=1 +} + +dry_run_sql=' +\pset pager off +\echo Safe legacy credential replacements: +WITH legacy AS ( + SELECT credential_id AS old_id, bucket + FROM s3_credential + WHERE credential_id = bucket +), +replacement_counts AS ( + SELECT l.old_id, l.bucket, count(d.credential_id) AS replacement_count, max(d.credential_id) AS new_id + FROM legacy l + JOIN s3_credential d ON d.bucket = l.bucket AND d.credential_id <> l.old_id + GROUP BY l.old_id, l.bucket +), +safe_replacements AS ( + SELECT old_id, new_id, bucket + FROM replacement_counts + WHERE replacement_count = 1 +) +SELECT old_id, new_id, bucket, + (SELECT count(*) FROM bucket_scope s WHERE s.credential_id = safe_replacements.old_id) AS scopes_to_remap +FROM safe_replacements +ORDER BY bucket, old_id; + +\echo Ambiguous legacy credentials skipped: +WITH legacy AS ( + SELECT credential_id AS old_id, bucket + FROM s3_credential + WHERE credential_id = bucket +), +replacement_counts AS ( + SELECT l.old_id, l.bucket, count(d.credential_id) AS replacement_count + FROM legacy l + LEFT JOIN s3_credential d ON d.bucket = l.bucket AND d.credential_id <> l.old_id + GROUP BY l.old_id, l.bucket +) +SELECT old_id, bucket, replacement_count +FROM replacement_counts +WHERE replacement_count <> 1 +ORDER BY bucket, old_id; + +\echo Existing dangling bucket scopes: +SELECT s.organization, s.project_id, s.credential_id, s.bucket +FROM bucket_scope s +LEFT JOIN s3_credential c ON c.credential_id = s.credential_id +WHERE c.credential_id IS NULL +ORDER BY s.organization, s.project_id, s.credential_id; +' + +apply_sql=' +\pset pager off +BEGIN; + +CREATE TEMP TABLE syfon_legacy_credential_replacements AS +WITH legacy AS ( + SELECT credential_id AS old_id, bucket + FROM s3_credential + WHERE credential_id = bucket +), +replacement_counts AS ( + SELECT l.old_id, l.bucket, count(d.credential_id) AS replacement_count, max(d.credential_id) AS new_id + FROM legacy l + JOIN s3_credential d ON d.bucket = l.bucket AND d.credential_id <> l.old_id + GROUP BY l.old_id, l.bucket +) +SELECT old_id, new_id, bucket +FROM replacement_counts +WHERE replacement_count = 1; + +\echo Remapping bucket scopes: +UPDATE bucket_scope s +SET credential_id = r.new_id +FROM syfon_legacy_credential_replacements r +WHERE s.credential_id = r.old_id; + +\echo Deleting stale legacy credential rows: +DELETE FROM s3_credential c +USING syfon_legacy_credential_replacements r +WHERE c.credential_id = r.old_id; + +COMMIT; + +\echo Remaining dangling bucket scopes: +SELECT s.organization, s.project_id, s.credential_id, s.bucket +FROM bucket_scope s +LEFT JOIN s3_credential c ON c.credential_id = s.credential_id +WHERE c.credential_id IS NULL +ORDER BY s.organization, s.project_id, s.credential_id; + +\echo Remaining legacy credentials with derived replacements: +WITH legacy AS ( + SELECT credential_id AS old_id, bucket + FROM s3_credential + WHERE credential_id = bucket +) +SELECT l.old_id, l.bucket, count(d.credential_id) AS replacement_count +FROM legacy l +JOIN s3_credential d ON d.bucket = l.bucket AND d.credential_id <> l.old_id +GROUP BY l.old_id, l.bucket +ORDER BY l.bucket, l.old_id; +' + +if [[ "$apply" == "true" ]]; then + echo "Applying Syfon legacy credential cleanup in namespace $namespace against $db_host/$db_name" + printf '%s\n' "$apply_sql" | run_psql +else + echo "Dry-run Syfon legacy credential cleanup in namespace $namespace against $db_host/$db_name" + echo "Pass --apply to perform the remap/delete transaction." + printf '%s\n' "$dry_run_sql" | run_psql +fi diff --git a/helm/syfon/templates/_helpers.tpl b/helm/syfon/templates/_helpers.tpl index 8c1ffb930..62cefbbde 100644 --- a/helm/syfon/templates/_helpers.tpl +++ b/helm/syfon/templates/_helpers.tpl @@ -59,3 +59,27 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} {{- end -}} +{{/* +Generate or reuse a secret value for compatibility credentials. +*/}} +{{- define "syfon.getOrGenSecret" -}} +{{- $value := index . 0 -}} +{{- $secretName := index . 1 -}} +{{- $secretKey := index . 2 -}} +{{- $secretLength := index . 3 -}} +{{- $namespace := index . 4 -}} +{{- if $value -}} +{{- $value = $value | b64enc -}} +{{- end -}} +{{- if not $value -}} + {{- if $secret := lookup "v1" "Secret" $namespace $secretName -}} + {{- if hasKey $secret.data $secretKey -}} + {{- $value = index $secret.data $secretKey -}} + {{- end -}} + {{- end -}} + {{- if not $value -}} + {{- $value = randAlphaNum $secretLength | b64enc -}} + {{- end -}} +{{- end -}} +{{- $value -}} +{{- end -}} diff --git a/helm/syfon/templates/config-secret.yaml b/helm/syfon/templates/config-secret.yaml index 1164bd200..b35ddba22 100644 --- a/helm/syfon/templates/config-secret.yaml +++ b/helm/syfon/templates/config-secret.yaml @@ -1,4 +1,13 @@ {{- $cfg := deepCopy (.Values.config | default dict) -}} +{{- if .Values.credential_encryption -}} + {{- $_ := set $cfg "credential_encryption" (deepCopy .Values.credential_encryption) -}} +{{- end -}} +{{- $credentialEncryption := (get $cfg "credential_encryption" | default dict) -}} +{{- $masterKey := (get $credentialEncryption "master_key" | default "" | toString | trim) -}} +{{- $localKeyFile := (get $credentialEncryption "local_key_file" | default "" | toString | trim) -}} +{{- if and (.Values.credentialEncryption.requireStableKey | default true) (not $masterKey) (not $localKeyFile) -}} + {{- fail "syfon.config.credential_encryption.master_key or syfon.config.credential_encryption.local_key_file is required; refusing to deploy with pod-local generated credential encryption key" -}} +{{- end -}} {{- $inputBuckets := default (list) (index $cfg "buckets") -}} {{- if and (eq (len $inputBuckets) 0) (hasKey $cfg "s3_credentials") -}} {{- $inputBuckets = default (list) (index $cfg "s3_credentials") -}} diff --git a/helm/syfon/templates/deployment.yaml b/helm/syfon/templates/deployment.yaml index 165d4a9bd..fda551d86 100644 --- a/helm/syfon/templates/deployment.yaml +++ b/helm/syfon/templates/deployment.yaml @@ -14,6 +14,8 @@ spec: {{- include "syfon.selectorLabels" . | nindent 6 }} template: metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/config-secret.yaml") . | sha256sum }} labels: {{- include "syfon.selectorLabels" . | nindent 8 }} spec: diff --git a/helm/syfon/templates/postgres-schema-configmap.yaml b/helm/syfon/templates/postgres-schema-configmap.yaml index 731e921ae..a401390f5 100644 --- a/helm/syfon/templates/postgres-schema-configmap.yaml +++ b/helm/syfon/templates/postgres-schema-configmap.yaml @@ -13,6 +13,7 @@ data: created_time TIMESTAMPTZ, updated_time TIMESTAMPTZ, name TEXT, + file_name TEXT, version TEXT, description TEXT ); @@ -33,12 +34,31 @@ data: FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE ); + CREATE TABLE IF NOT EXISTS drs_object_controlled_access ( + object_id TEXT NOT NULL, + resource TEXT NOT NULL, + FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE + ); + CREATE TABLE IF NOT EXISTS drs_object_alias ( alias_id TEXT PRIMARY KEY, object_id TEXT NOT NULL, FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE ); + ALTER TABLE drs_object + ADD COLUMN IF NOT EXISTS file_name TEXT; + + CREATE TABLE IF NOT EXISTS drs_object_browse_index ( + object_id TEXT NOT NULL, + resource TEXT NOT NULL, + normalized_path TEXT NOT NULL, + parent_path TEXT NOT NULL, + entry_name TEXT NOT NULL, + PRIMARY KEY (resource, object_id), + FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE + ); + ALTER TABLE drs_object_access_method ADD COLUMN IF NOT EXISTS org TEXT NOT NULL DEFAULT ''; @@ -46,7 +66,8 @@ data: ADD COLUMN IF NOT EXISTS project TEXT NOT NULL DEFAULT ''; CREATE TABLE IF NOT EXISTS s3_credential ( - bucket TEXT PRIMARY KEY, + credential_id TEXT PRIMARY KEY, + bucket TEXT NOT NULL, provider TEXT NOT NULL DEFAULT 's3', region TEXT, access_key TEXT, @@ -54,18 +75,60 @@ data: endpoint TEXT ); + ALTER TABLE s3_credential + ADD COLUMN IF NOT EXISTS credential_id TEXT; + ALTER TABLE s3_credential ADD COLUMN IF NOT EXISTS provider TEXT NOT NULL DEFAULT 's3'; + UPDATE s3_credential + SET credential_id = bucket + WHERE COALESCE(BTRIM(credential_id), '') = ''; + + ALTER TABLE s3_credential + ALTER COLUMN credential_id SET NOT NULL; + + ALTER TABLE s3_credential + DROP CONSTRAINT IF EXISTS s3_credential_pkey; + + ALTER TABLE s3_credential + ADD PRIMARY KEY (credential_id); + + CREATE INDEX IF NOT EXISTS idx_s3_credential_bucket + ON s3_credential(bucket); CREATE TABLE IF NOT EXISTS bucket_scope ( organization TEXT NOT NULL, project_id TEXT NOT NULL, + credential_id TEXT NOT NULL, bucket TEXT NOT NULL, path_prefix TEXT NULL, PRIMARY KEY (organization, project_id) ); + ALTER TABLE bucket_scope + ADD COLUMN IF NOT EXISTS credential_id TEXT; + + UPDATE bucket_scope + SET credential_id = bucket + WHERE COALESCE(BTRIM(credential_id), '') = ''; + + ALTER TABLE bucket_scope + ALTER COLUMN credential_id SET NOT NULL; + + ALTER TABLE bucket_scope + ADD COLUMN IF NOT EXISTS bucket TEXT; + + UPDATE bucket_scope + SET bucket = credential_id + WHERE COALESCE(BTRIM(bucket), '') = ''; + + ALTER TABLE bucket_scope + ALTER COLUMN bucket SET NOT NULL; + + CREATE INDEX IF NOT EXISTS idx_bucket_scope_credential_id + ON bucket_scope(credential_id); + CREATE TABLE IF NOT EXISTS lfs_pending_metadata ( oid TEXT PRIMARY KEY, candidate_json JSONB NOT NULL, @@ -202,12 +265,33 @@ data: CREATE INDEX IF NOT EXISTS drs_object_checksum_checksum_idx ON drs_object_checksum(checksum); + CREATE INDEX IF NOT EXISTS drs_object_controlled_access_object_id_idx + ON drs_object_controlled_access(object_id); + + CREATE INDEX IF NOT EXISTS drs_object_controlled_access_resource_idx + ON drs_object_controlled_access(resource); + + CREATE INDEX IF NOT EXISTS drs_object_controlled_access_resource_object_id_idx + ON drs_object_controlled_access(resource, object_id); + + CREATE INDEX IF NOT EXISTS drs_object_controlled_access_object_id_resource_idx + ON drs_object_controlled_access(object_id, resource); + CREATE INDEX IF NOT EXISTS drs_object_access_method_scope_idx ON drs_object_access_method(org, project); CREATE INDEX IF NOT EXISTS drs_object_alias_object_id_idx ON drs_object_alias(object_id); + CREATE INDEX IF NOT EXISTS drs_object_browse_index_resource_parent_object_id_idx + ON drs_object_browse_index(resource, parent_path, object_id); + + CREATE INDEX IF NOT EXISTS drs_object_browse_index_resource_parent_entry_name_idx + ON drs_object_browse_index(resource, parent_path, entry_name); + + CREATE INDEX IF NOT EXISTS drs_object_browse_index_resource_normalized_path_idx + ON drs_object_browse_index(resource, normalized_path); + CREATE INDEX IF NOT EXISTS idx_bucket_scope_bucket ON bucket_scope(bucket); @@ -259,6 +343,41 @@ data: CREATE INDEX IF NOT EXISTS idx_provider_transfer_scope_time ON provider_transfer_event(organization, project, direction, event_time); + INSERT INTO drs_object_browse_index (object_id, resource, normalized_path, parent_path, entry_name) + SELECT + scoped.object_id, + scoped.resource, + scoped.normalized_path, + CASE + WHEN position('/' in scoped.normalized_path) = 0 THEN '' + ELSE regexp_replace(scoped.normalized_path, '/[^/]+$', '') + END AS parent_path, + regexp_replace(scoped.normalized_path, '^.*/', '') AS entry_name + FROM ( + SELECT + ca.object_id, + ca.resource, + regexp_replace( + regexp_replace( + btrim(replace(COALESCE(o.file_name, o.name, ''), E'\\', '/'), '/'), + '/+', + '/', + 'g' + ), + '^/+|/+$', + '', + 'g' + ) AS normalized_path + FROM drs_object_controlled_access ca + INNER JOIN drs_object o ON o.id = ca.object_id + ) scoped + WHERE scoped.normalized_path <> '' + AND scoped.normalized_path !~ '(^|/)\.\.?(/|$)' + ON CONFLICT (resource, object_id) DO UPDATE SET + normalized_path = EXCLUDED.normalized_path, + parent_path = EXCLUDED.parent_path, + entry_name = EXCLUDED.entry_name; + CREATE INDEX IF NOT EXISTS idx_provider_transfer_actor_time ON provider_transfer_event(actor_email, actor_subject, event_time); diff --git a/helm/syfon/templates/service-creds-secret.yaml b/helm/syfon/templates/service-creds-secret.yaml new file mode 100644 index 000000000..a4889dcaa --- /dev/null +++ b/helm/syfon/templates/service-creds-secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceCreds.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.serviceCreds.secretName }} + labels: + {{- include "syfon.labels" . | nindent 4 }} +type: Opaque +data: + fence: {{ include "syfon.getOrGenSecret" (list .Values.serviceCreds.users.fence .Values.serviceCreds.secretName "fence" 20 .Release.Namespace) }} + sheepdog: {{ include "syfon.getOrGenSecret" (list .Values.serviceCreds.users.sheepdog .Values.serviceCreds.secretName "sheepdog" 20 .Release.Namespace) }} +{{- end }} diff --git a/helm/syfon/values.yaml b/helm/syfon/values.yaml index 09457ef14..fe0283219 100644 --- a/helm/syfon/values.yaml +++ b/helm/syfon/values.yaml @@ -27,14 +27,25 @@ config: signing: default_expiry_seconds: 900 credential_encryption: {} - # Preferred Syfon config shape. Each bucket may carry nested org/project routing - # resources, which Syfon flattens into runtime bucket scope records on startup. + # Preferred Syfon config shape. Buckets are credentials; Syfon derives the + # internal credential key from provider/endpoint/bucket/region/access_key. buckets: [] - # Legacy inputs still accepted by Syfon, but the chart no longer synthesizes - # bucket_scopes from credential resources. + # Legacy inputs are still accepted by Syfon. bucket_scopes use physical bucket + # plus organization/project_id/path_prefix. s3_credentials: [] bucket_scopes: [] +# Backward-compatible shorthand accepted by the chart and rendered into +# config.credential_encryption. Prefer config.credential_encryption for new +# values, but existing local values often set this at syfon.credential_encryption. +credential_encryption: {} + +credentialEncryption: + # Refuse to deploy without a stable credential encryption key. Without this, + # Syfon falls back to /app/.syfon-credential-kek, which is pod-local unless the + # operator explicitly persists it. + requireStableKey: true + postgres: app: existingSecret: "" @@ -58,6 +69,13 @@ postgres: image: postgres:16 waitTimeoutSeconds: 180 +serviceCreds: + enabled: true + secretName: indexd-service-creds + users: + fence: "" + sheepdog: "" + resources: {} extraEnv: []