Feature/drs server

helm/arborist/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.11
+version: 0.1.12
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/arborist/README.md

@@ -60,11 +60,14 @@ A Helm chart for gen3 arborist
 | global.publicDataSets | bool | `true` | Whether public datasets are enabled. |
 | global.revproxyArn | string | `"arn:aws:acm:us-east-1:123456:certificate"` | ARN of the reverse proxy certificate. |
 | global.tierAccessLevel | string | `"libre"` | Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` |
-| image | map | `{"pullPolicy":"IfNotPresent","repository":"quay.io/cdis/arborist","tag":""}` | Docker image information. |
+| image | map | `{"pullPolicy":"IfNotPresent","repository":"quay.io/ohsu-comp-bio/arborist","tag":""}` | Docker image information. |
 | image.pullPolicy | string | `"IfNotPresent"` | Docker pull policy. |
-| image.repository | string | `"quay.io/cdis/arborist"` | Docker repository. |
+| image.repository | string | `"quay.io/ohsu-comp-bio/arborist"` | Docker repository. |
 | image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. |
 | imagePullSecrets | list | `[]` | Docker image pull secrets. |
+| migrations | map | `{"command":"/app/migrations/latest","enabled":true}` | Database migration settings. Arborist migrations are packaged in the application image; deploying a new image with new migration files is enough for this chart to apply them before starting Arborist. |
+| migrations.command | string | `"/app/migrations/latest"` | Migration command inside the Arborist image. |
+| migrations.enabled | bool | `true` | Whether to run Arborist database migrations at container startup. |
 | nameOverride | string | `""` | Override the name of the chart. |
 | nodeSelector | map | `{}` | Node selector to apply to the pod |
 | partOf | string | `"Authentication"` | Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. |
@@ -104,4 +107,3 @@ A Helm chart for gen3 arborist
 | tolerations | list | `[]` | Tolerations to apply to the pod |
 | volumeMounts | list | `[]` | Volume mounts to attach to the container |
 | volumes | list | `[]` | Volumes to attach to the pod |
-

helm/arborist/templates/deployment.yaml

@@ -37,6 +37,27 @@ spec:
       serviceAccountName: {{ include "arborist.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- if .Values.authzSnapshotCache.waitForRedis.enabled }}
+      initContainers:
+        - name: wait-for-authz-cache
+          image: "{{ .Values.authzSnapshotCache.waitForRedis.image }}"
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.authzSnapshotCache.waitForRedis.secretName }}
+                  key: {{ .Values.authzSnapshotCache.waitForRedis.passwordKey }}
+                  optional: false
+          command:
+            - /bin/sh
+            - -c
+            - |
+              until redis-cli -h {{ .Values.authzSnapshotCache.waitForRedis.host }} -p {{ .Values.authzSnapshotCache.waitForRedis.port }} -a "$REDIS_PASSWORD" --no-auth-warning ping | grep -q PONG; do
+                echo "waiting for authz cache redis at {{ .Values.authzSnapshotCache.waitForRedis.host }}:{{ .Values.authzSnapshotCache.waitForRedis.port }}"
+                sleep 2
+              done
+      {{- end }}
       containers:
         - name: {{ .Chart.Name }}
           securityContext:
@@ -68,11 +89,13 @@ spec:
               # set env vars
               export PGSSLMODE="disable"
 
+              {{- if .Values.migrations.enabled }}
               # bring the database schema up to the latest version
-              /go/src/github.com/uc-cdis/arborist/migrations/latest
+              {{ .Values.migrations.command }}
+              {{- end }}
 
               # run arborist
-              /go/src/github.com/uc-cdis/arborist/bin/arborist
+              /usr/local/bin/arborist
           env:
             {{- if .Values.global.ddEnabled }}
             {{- include "common.datadogEnvVar" . | nindent 12 }}
@@ -132,4 +155,4 @@ spec:
       {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
-      {{- end }}
+      {{- end }}

helm/arborist/values.yaml

@@ -109,12 +109,21 @@ replicaCount: 1
 # -- (map) Docker image information.
 image:
   # -- (string) Docker repository.
-  repository: quay.io/cdis/arborist
+  repository: quay.io/ohsu-comp-bio/arborist
   # -- (string) Docker pull policy.
-  pullPolicy: IfNotPresent
+  pullPolicy: Always
   # -- (string) Overrides the image tag whose default is the chart appVersion.
   tag: ""
 
+# -- (map) Database migration settings. Arborist migrations are packaged in the
+# application image; deploying a new image with new migration files is enough for
+# this chart to apply them before starting Arborist.
+migrations:
+  # -- (bool) Whether to run Arborist database migrations at container startup.
+  enabled: true
+  # -- (string) Migration command inside the Arborist image.
+  command: /app/migrations/latest
+
 # -- (list) Docker image pull secrets.
 imagePullSecrets: []
 
@@ -214,7 +223,23 @@ env:
   # -- (string) The URL of the JSON Web Key Set (JWKS) endpoint for authentication
   - name: JWKS_ENDPOINT
     value: "http://fence-service/.well-known/jwks"
-
+  - name: AUTHZ_SNAPSHOT_CACHE_REDIS_URL
+    value: "redis://authz-cache-service:6379/0"
+  - name: AUTHZ_SNAPSHOT_CACHE_REDIS_PASSWORD
+    valueFrom:
+      secretKeyRef:
+        name: authz-cache-credentials
+        key: redis-password
+        optional: false
+
+authzSnapshotCache:
+  waitForRedis:
+    enabled: true
+    image: "redis:7.2-alpine"
+    host: "authz-cache-service"
+    port: 6379
+    secretName: "authz-cache-credentials"
+    passwordKey: "redis-password"
 
 # Values to determine the labels that are used for the deployment, pod, etc.
 # -- (string) Valid options are "production" or "dev". If invalid option is set- the value will default to "dev".

helm/fence/fence-secret/config_helper.py

@@ -59,7 +59,6 @@ def inject_creds_into_fence_config(creds_file_path, config_file_path):
     db_password = _get_nested_value(creds, "db_password")
     db_database = _get_nested_value(creds, "db_database")
     hostname = _get_nested_value(creds, "hostname")
-    indexd_password = _get_nested_value(creds, "indexd_password")
     google_client_secret = _get_nested_value(creds, "google_client_secret")
     google_client_id = _get_nested_value(creds, "google_client_id")
     hmac_key = _get_nested_value(creds, "hmac_key")
@@ -75,10 +74,6 @@ def inject_creds_into_fence_config(creds_file_path, config_file_path):
     print("  BASE_URL injected with value(s) from creds.json")
     config_file = _replace(config_file, "BASE_URL", "https://{}/user".format(hostname))
 
-    print("  INDEXD_PASSWORD injected with value(s) from creds.json")
-    config_file = _replace(config_file, "INDEXD_PASSWORD", indexd_password)
-    config_file = _replace(config_file, "INDEXD_USERNAME", "fence")
-
     print("  ENCRYPTION_KEY injected with value(s) from creds.json")
     config_file = _replace(config_file, "ENCRYPTION_KEY", hmac_key)
 
@@ -121,9 +116,6 @@ def set_prod_defaults(config_file_path):
         "/var/www/fence/fence_google_storage_creds_secret.json",
     )
 
-    print("  INDEXD set as http://indexd-service/")
-    config_file = _replace(config_file, "INDEXD", "http://indexd-service/")
-
     print("  ARBORIST set as http://arborist-service/")
     config_file = _replace(config_file, "ARBORIST", "http://arborist-service/")
 

helm/fence/fence-secret/fence_settings.py

@@ -165,6 +165,4 @@ def get_from_dict(dictionary, key, default=""):
 
 DEFAULT_LOGIN_URL_REDIRECT_PARAM = "redirect"
 
-INDEXD = "http://indexd-service/"
-
 ARBORIST = "http://arborist-service/"

helm/fence/templates/fence-creds.yaml

@@ -11,9 +11,7 @@ stringData:
         "db_password": "{{include "gen3.service-postgres" (dict "key" "password" "service" $.Chart.Name "context" $) }}",
         "db_database": "{{ include "gen3.service-postgres" (dict "key" "database" "service" $.Chart.Name "context" $)}}",
         "hostname": "{{ .Values.global.hostname }}",
-        "indexd_password": "",
         "google_client_secret": "YOUR.GOOGLE.SECRET",
         "google_client_id": "YOUR.GOOGLE.CLIENT",
         "hmac_key": ""
       }
-

helm/fence/templates/fence-deployment.yaml

@@ -74,6 +74,10 @@ spec:
                 cp /var/run/fence-secrets/fence-config-secret.yaml /var/www/fence/fence-config.yaml
               fi
 
+              if [[ -n "${AUTHZ_SNAPSHOT_CACHE_REDIS_PASSWORD:-""}" ]]; then
+                python -c $'import os, yaml\npath = "/var/www/fence/fence-config.yaml"\nwith open(path) as f:\n    data = yaml.safe_load(f) or {}\nurl = data.get("AUTHZ_SNAPSHOT_CACHE_REDIS_URL")\npassword = os.environ.get("AUTHZ_SNAPSHOT_CACHE_REDIS_PASSWORD", "")\nif url and password and url.startswith("redis://") and "@" not in url:\n    data["AUTHZ_SNAPSHOT_CACHE_REDIS_URL"] = f"redis://:{password}@{url[8:]}"\n    with open(path, "w") as f:\n        yaml.safe_dump(data, f, sort_keys=False)'
+              fi
+
               if [[ -f /var/run/fence-secrets/jwt_private_key.pem ]]; then
                 mkdir -p /fence/keys/key
                 cp /var/run/fence-secrets/jwt_private_key.pem /fence/keys/key/jwt_private_key.pem
@@ -90,6 +94,26 @@ spec:
           volumeMounts:
             {{- toYaml .Values.volumeMounts | nindent 12 }}
       initContainers:
+        {{- if .Values.authzSnapshotCache.waitForRedis.enabled }}
+        - name: wait-for-authz-cache
+          image: "{{ .Values.authzSnapshotCache.waitForRedis.image }}"
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.authzSnapshotCache.waitForRedis.secretName }}
+                  key: {{ .Values.authzSnapshotCache.waitForRedis.passwordKey }}
+                  optional: false
+          command:
+            - /bin/sh
+            - -c
+            - |
+              until redis-cli -h {{ .Values.authzSnapshotCache.waitForRedis.host }} -p {{ .Values.authzSnapshotCache.waitForRedis.port }} -a "$REDIS_PASSWORD" --no-auth-warning ping | grep -q PONG; do
+                echo "waiting for authz cache redis at {{ .Values.authzSnapshotCache.waitForRedis.host }}:{{ .Values.authzSnapshotCache.waitForRedis.port }}"
+                sleep 2
+              done
+        {{- end }}
         - name: fence-init
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}

helm/fence/templates/usersync-cron.yaml

@@ -124,7 +124,7 @@ spec:
           containers:
           - name: usersync
             image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
-            imagePullPolicy: Always
+            imagePullPolicy: {{ .Values.image.pullPolicy }}
             env:
             - name: SYNC_FROM_DBGAP
               value: {{ .Values.usersync.syncFromDbgap | quote }}
@@ -211,4 +211,4 @@ spec:
                 echo "Exit code: $exitcode"
                 exit "$exitcode"
           restartPolicy: "Never"
-{{- end }}
+{{- end }}

helm/fence/templates/useryaml-job.yaml

@@ -27,7 +27,7 @@ spec:
       containers:
       - name: fence
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
-        imagePullPolicy: Always
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
         env:
           {{- toYaml .Values.env | nindent 10 }}
         volumeMounts:
@@ -44,5 +44,7 @@ spec:
           # Script always succeeds if it runs (echo exits with 0)
           - |
             # can be removed once this is merged: https://github.com/uc-cdis/fence/pull/1096
-            fence-create sync --arborist http://arborist-service --yaml /var/www/fence/user.yaml
+            cd /fence
+            export PYTHONPATH=/fence${PYTHONPATH:+:$PYTHONPATH}
+            /fence/.venv/bin/python -m bin.fence_create sync --arborist http://arborist-service --yaml /var/www/fence/user.yaml
       restartPolicy: OnFailure

helm/fence/values.yaml

@@ -297,6 +297,12 @@ env:
         name: manifest-fence
         key: fence-config-public.yaml
         optional: true
+  - name: AUTHZ_SNAPSHOT_CACHE_REDIS_PASSWORD
+    valueFrom:
+      secretKeyRef:
+        name: authz-cache-credentials
+        key: redis-password
+        optional: false
   - name: PGHOST
     valueFrom:
       secretKeyRef:
@@ -329,17 +335,21 @@ env:
         optional: false
   - name: DB
     value: postgresql://$(PGUSER):$(PGPASSWORD)@$(PGHOST):5432/$(PGDB)
-  - name: INDEXD_PASSWORD
-    valueFrom:
-      secretKeyRef:
-        name: indexd-service-creds
-        key: fence
   - name: gen3Env
     valueFrom:
       configMapKeyRef:
         name: manifest-global
         key: hostname
 
+authzSnapshotCache:
+  waitForRedis:
+    enabled: true
+    image: "redis:7.2-alpine"
+    host: "authz-cache-service"
+    port: 6379
+    secretName: "authz-cache-credentials"
+    passwordKey: "redis-password"
+
 # -- (list) Volumes to attach to the container.
 volumes:
   - name: old-config-volume
@@ -1677,6 +1687,7 @@ FENCE_CONFIG:
     - "openid"
     - "user"
     - "data"
+    - "github_credentials"
     - "google_credentials"
     - "google_service_account"
     - "google_link"
@@ -1690,6 +1701,7 @@ FENCE_CONFIG:
     - "user"
     - "data"
     - "admin"
+    - "github_credentials"
     - "google_credentials"
     - "google_service_account"
     - "google_link"
@@ -1708,6 +1720,19 @@ FENCE_CONFIG:
     - "google_link"
     - "ga4gh_passport_v1"
 
+  # -- (map) GitHub App configuration for Fence-managed installation token brokering
+  GITHUB_APP:
+    # -- (string) GitHub App ID
+    app_id: ''
+    # -- (string) PEM-encoded GitHub App private key
+    private_key: ''
+    # -- (string) Optional path to a PEM file mounted in the container. Mutually exclusive with private_key.
+    private_key_file: ''
+    # -- (string) GitHub API base URL. Leave default for github.com, override for GitHub Enterprise.
+    api_base_url: 'https://api.github.com'
+    # -- (int) Timeout in seconds for outbound GitHub API requests
+    timeout_seconds: 30
+
   # //////////////////////////////////////////////////////////////////////////////////////
   # LOGIN
   #   - Modify based on which OIDC provider(s) you configured above
@@ -2018,20 +2043,6 @@ FENCE_CONFIG:
     host: null
     port: 3128
 
-  # //////////////////////////////////////////////////////////////////////////////////////
-  # MICROSERVICE PATHS
-  #   - Support `/data` endpoints & authz functionality
-  # //////////////////////////////////////////////////////////////////////////////////////
-  # url where indexd microservice is running (for signed urls primarily)
-  # NOTE: Leaving as null will force fence to default to {{BASE_URL}}/index
-  # example value: 'https://example.com/index'
-  INDEXD: http://indexd-service
-
-  # this is the username which fence uses to make authenticated requests to indexd
-  INDEXD_USERNAME: 'fence'
-  # this is the password which fence uses to make authenticated requests to indexd
-  INDEXD_PASSWORD: ''
-
   # //////////////////////////////////////////////////////////////////////////////////////
   # AZURE STORAGE BLOB CONFIGURATION
   #   - Support Azure Blob Data Access Methods
@@ -2048,6 +2059,9 @@ FENCE_CONFIG:
 
   # url where authz microservice is running
   ARBORIST: http://arborist-service
+  AUTHZ_SNAPSHOT_CACHE_ENABLED: true
+  AUTHZ_SNAPSHOT_CACHE_REDIS_URL: redis://authz-cache-service:6379/0
+  AUTHZ_SNAPSHOT_CACHE_TTL_SECONDS: 3600
 
   # url where the audit-service is running
   AUDIT_SERVICE: 'http://audit-service'

helm/funnel/Chart.yaml

@@ -17,7 +17,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.76
+version: 0.1.77
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
@@ -26,12 +26,6 @@ version: 0.1.76
 appVersion: "2025-12-22"
 
 dependencies:
-  - name: postgresql
-    version: 18.1.15
-    repository: https://charts.bitnami.com/bitnami
-    condition: postgresql.enabled
-
-  - name: mongodb
-    version: 13.9.4
-    repository: https://charts.bitnami.com/bitnami
-    condition: mongodb.enabled
+  - name: common
+    version: 0.1.10
+    repository: file://../common