diff --git a/.github/workflows/lint_test.yaml b/.github/workflows/lint_test.yaml index 2bf831608..516f6aa6b 100644 --- a/.github/workflows/lint_test.yaml +++ b/.github/workflows/lint_test.yaml @@ -22,8 +22,12 @@ jobs: helm repo add elastic https://helm.elastic.co helm repo add bitnami https://charts.bitnami.com/bitnami helm repo add qdrant https://qdrant.github.io/qdrant-helm + helm repo add clickstack https://clickhouse.github.io/ClickStack-helm-charts helm repo update + - name: Build Loom chart dependencies + run: helm dependency build ./helm/loom + - uses: actions/setup-python@v4 with: python-version: '3.9' diff --git a/README.md b/README.md index 6c6da9fca..f8c430cc9 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,18 @@ In a production deployment: ## Configuration +### Loom FHIR dataframe service + +The optional `helm/loom` chart deploys the Loom FHIR dataframe GraphQL server +alongside a single chart-owned ClickHouse instance. See +[`helm/loom/README.md`](./helm/loom/README.md) for the local-cluster smoke test, +managed-backend configuration, and persistence settings. + +The local ClickHouse service is `clickhouse:9000`. ArangoDB is not +silently deployed by the Gen3 parent chart: production installs must provide a +managed/operator-owned ArangoDB service, while local smoke tests may enable +the disposable backend in `helm/loom/values-local.yaml`. + For a full set of configuration options see the [CONFIGURATION.md](./docs/CONFIGURATION.md) for a more in depth instructions on how to configure each service. There's also an auto-generated table of basic configuration options here: diff --git a/examples/gke_values.yaml b/examples/gke_values.yaml index 6e83543c2..8b18ae7ed 100644 --- a/examples/gke_values.yaml +++ b/examples/gke_values.yaml @@ -13,6 +13,10 @@ global: username: "postgres" password: "postgres-password" +elasticsearch: + # Use the managed Elasticsearch/OpenSearch endpoint instead of deploying ES in-cluster. + enabled: false + # configuration for fence helm chart. You can add it for all our services. fence: # Override image @@ -81,4 +85,4 @@ revproxy: - qureshi.planx-pla.net -# Add configuration overrides for fence and other services below as needed \ No newline at end of file +# Add configuration overrides for fence and other services below as needed diff --git a/helm/audit/Chart.yaml b/helm/audit/Chart.yaml deleted file mode 100644 index 2295c6023..000000000 --- a/helm/audit/Chart.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: v2 -name: audit -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.12 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -appVersion: "master" - -dependencies: -- name: common - version: 0.1.10 - repository: file://../common -- name: postgresql - version: 11.9.13 - repository: "https://charts.bitnami.com/bitnami" - condition: postgres.separate diff --git a/helm/audit/README.md b/helm/audit/README.md deleted file mode 100644 index 8d4ffa2ca..000000000 --- a/helm/audit/README.md +++ /dev/null @@ -1,125 +0,0 @@ -# audit - -![Version: 0.1.12](https://img.shields.io/badge/Version-0.1.12-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) - -A Helm chart for Kubernetes - -## Requirements - -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.10 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | - -## Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| affinity | map | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["audit"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]}}` | Affinity to use for the deployment. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution | map | `[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["audit"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]` | Option for scheduling to be required or preferred. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0] | int | `{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["audit"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}` | Weight value for preferred scheduling. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0] | list | `{"key":"app","operator":"In","values":["audit"]}` | Label key for match expression. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].operator | string | `"In"` | Operation type for the match expression. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].values | list | `["audit"]` | Value for the match expression key. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey | string | `"kubernetes.io/hostname"` | Value for topology key label. | -| api.QUERY_PAGE_SIZE | int | `1000` | The maximum number of entires the query endpoint will return. | -| api.QUERY_TIMEBOX_MAX_DAYS | int | `nil` | Amount to time-box queries. | -| api.QUERY_USERNAMES | bool | `true` | Whether to return usernames in query responses and allow querying by username. | -| autoscaling | map | `{"enabled":false,"maxReplicas":4,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Configuration for autoscaling the number of replicas | -| autoscaling.enabled | bool | `false` | Whether autoscaling is enabled or not | -| autoscaling.maxReplicas | int | `4` | The maximum number of replicas to scale up to | -| autoscaling.minReplicas | int | `1` | The minimum number of replicas to scale down to | -| autoscaling.targetCPUUtilizationPercentage | int | `80` | The target CPU utilization percentage for autoscaling | -| commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | -| criticalService | string | `"false"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | -| datadogLogsInjection | bool | `true` | If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. | -| datadogProfilingEnabled | bool | `true` | If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. | -| datadogTraceSampleRate | int | `1` | A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. | -| env | list | `[{"name":"DEBUG","value":"false"},{"name":"ARBORIST_URL","valueFrom":{"configMapKeyRef":{"key":"arborist_url","name":"manifest-global","optional":true}}}]` | Environment variables to pass to the container | -| externalSecrets | map | `{"auditG3auto":null,"createK8sAuditSecret":false,"dbcreds":null}` | External Secrets settings. | -| externalSecrets.auditG3auto | string | `nil` | Will override the name of the aws secrets manager secret. Default is "audit-g3auto" | -| externalSecrets.createK8sAuditSecret | string | `false` | Will create the Helm "audit-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. | -| externalSecrets.dbcreds | string | `nil` | Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" | -| fullnameOverride | string | `""` | Override the full name of the chart, which is used as the name of resources created by the chart | -| global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false}` | AWS configuration | -| global.aws.awsAccessKeyId | string | `nil` | Credentials for AWS stuff. | -| global.aws.awsSecretAccessKey | string | `nil` | Credentials for AWS stuff. | -| global.aws.enabled | bool | `false` | Set to true if deploying to AWS. Controls ingress annotations. | -| global.ddEnabled | bool | `false` | Whether Datadog is enabled. | -| global.dev | bool | `true` | Whether the deployment is for development purposes. | -| global.dictionaryUrl | string | `"https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json"` | URL of the data dictionary. | -| global.dispatcherJobNum | int | `"10"` | Number of dispatcher jobs. | -| global.environment | string | `"default"` | Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. | -| global.externalSecrets | map | `{"deploy":false,"separateSecretStore":false}` | External Secrets settings. | -| global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any audit secrets you have deployed. | -| global.externalSecrets.separateSecretStore | string | `false` | Will deploy a separate External Secret Store for this service. | -| global.hostname | string | `"localhost"` | Hostname for the deployment. | -| global.kubeBucket | string | `"kube-gen3"` | S3 bucket name for Kubernetes manifest files. | -| global.logsBucket | string | `"logs-gen3"` | S3 bucket name for log files. | -| global.minAvialable | int | `1` | The minimum amount of pods that are available at all times if the PDB is deployed. | -| global.netPolicy | bool | `true` | Whether network policies are enabled. | -| global.pdb | bool | `false` | If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. | -| global.portalApp | string | `"gitops"` | Portal application name. | -| global.postgres.dbCreate | bool | `true` | Whether the database should be created. | -| global.postgres.externalSecret | string | `""` | Name of external secret. Disabled if empty | -| global.postgres.master | map | `{"host":null,"password":null,"port":"5432","username":"postgres"}` | Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres | -| global.postgres.master.host | string | `nil` | hostname of postgres server | -| global.postgres.master.password | string | `nil` | password for superuser in postgres. This is used to create or restore databases | -| global.postgres.master.port | string | `"5432"` | Port for Postgres. | -| global.postgres.master.username | string | `"postgres"` | username of superuser in postgres. This is used to create or restore databases | -| global.publicDataSets | bool | `true` | Whether public datasets are enabled. | -| global.revproxyArn | string | `"arn:aws:acm:us-east-1:123456:certificate"` | ARN of the reverse proxy certificate. | -| global.tierAccessLevel | string | `"libre"` | Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` | -| image | map | `{"pullPolicy":"Always","repository":"quay.io/cdis/audit-service","tag":"master"}` | Docker image information. | -| image.pullPolicy | string | `"Always"` | When to pull the image. This value should be "Always" to ensure the latest image is used. | -| image.repository | string | `"quay.io/cdis/audit-service"` | The Docker image repository for the audit service | -| image.tag | string | `"master"` | Overrides the image tag whose default is the chart appVersion. | -| imagePullSecrets | list | `[]` | Docker image pull secrets. | -| initEnv | list | `{}` | Volumes to attach to the init container. | -| initVolumeMounts | list | `[]` | Volumes to mount to the init container. | -| nameOverride | string | `""` | Override the name of the chart. This can be used to provide a unique name for a chart | -| nodeSelector | map | `{}` | Node Selector for the pods | -| partOf | string | `"Logging"` | Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. | -| podAnnotations | map | `{}` | Annotations to add to the pod | -| podSecurityContext | map | `{}` | Security context for the pod | -| postgres | map | `{"database":null,"dbCreate":null,"host":null,"password":null,"port":"5432","separate":false,"username":null}` | Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you | -| postgres.database | string | `nil` | Database name for postgres. This is a service override, defaults to - | -| postgres.dbCreate | bool | `nil` | Whether the database should be created. Default to global.postgres.dbCreate | -| postgres.host | string | `nil` | Hostname for postgres server. This is a service override, defaults to global.postgres.host | -| postgres.password | string | `nil` | Password for Postgres. Will be autogenerated if left empty. | -| postgres.port | string | `"5432"` | Port for Postgres. | -| postgres.separate | string | `false` | Will create a Database for the individual service to help with developing it. | -| postgres.username | string | `nil` | Username for postgres. This is a service override, defaults to - | -| postgresql | map | `{"primary":{"persistence":{"enabled":false}}}` | Postgresql subchart settings if deployed separately option is set to "true". Disable persistence by default so we can spin up and down ephemeral environments | -| postgresql.primary.persistence.enabled | bool | `false` | Option to persist the dbs data. | -| release | string | `"production"` | Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". | -| replicaCount | int | `1` | Number of desired replicas | -| resources | map | `{"limits":{"cpu":1,"memory":"512Mi"},"requests":{"cpu":0.2,"memory":"120Mi"}}` | Resource requests and limits for the containers in the pod | -| resources.limits | map | `{"cpu":1,"memory":"512Mi"}` | The maximum amount of resources that the container is allowed to use | -| resources.limits.cpu | string | `1` | The maximum amount of CPU the container can use | -| resources.limits.memory | string | `"512Mi"` | The maximum amount of memory the container can use | -| resources.requests | map | `{"cpu":0.2,"memory":"120Mi"}` | The amount of resources that the container requests | -| resources.requests.cpu | string | `0.2` | The amount of CPU requested | -| resources.requests.memory | string | `"120Mi"` | The amount of memory requested | -| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information for External Secrets. | -| secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. | -| secrets.awsSecretAccessKey | str | `nil` | AWS secret access key ID. Overrides global key. | -| securityContext | map | `{}` | Security context for the containers in the pod | -| selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | -| server.AWS_CREDENTIALS | map | `{}` | AWS credentials to access SQS queue. | -| server.debug | bool | `false` | Whether to enable or disable debug mode. | -| server.pull_from_queue | bool | `false` | Whether to pull logs from sqs queue. | -| server.sqs | map | `{"region":"us-east-1","url":"http://sqs.com"}` | AWS SQS queue information. | -| server.sqs.region | string | `"us-east-1"` | SQS queue AWS region. | -| server.sqs.url | string | `"http://sqs.com"` | The URL for the SQS queue. | -| service | map | `{"port":80,"type":"ClusterIP"}` | Configuration for the service | -| service.port | int | `80` | Port on which the service is exposed | -| service.type | string | `"ClusterIP"` | Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". | -| serviceAccount | map | `{"annotations":{"eks.amazonaws.com/role-arn":null},"create":true,"name":"audit-service-sa"}` | Service account to use or create. | -| serviceAccount.annotations."eks.amazonaws.com/role-arn" | string | `nil` | The Amazon Resource Name (ARN) of the role to associate with the service account | -| serviceAccount.create | bool | `true` | Whether to create a service account | -| serviceAccount.name | string | `"audit-service-sa"` | The name of the service account | -| tolerations | list | `[]` | Tolerations for the pods | -| volumeMounts | list | `[]` | Volumes to mount to the container. | -| volumes | list | `[]` | Volumes to attach to the container. | - diff --git a/helm/audit/templates/NOTES.txt b/helm/audit/templates/NOTES.txt deleted file mode 100644 index c1e7e1aef..000000000 --- a/helm/audit/templates/NOTES.txt +++ /dev/null @@ -1 +0,0 @@ -{{ .Chart.Name }} has been deployed successfully. diff --git a/helm/audit/templates/_helpers.tpl b/helm/audit/templates/_helpers.tpl deleted file mode 100644 index 6f70cc66b..000000000 --- a/helm/audit/templates/_helpers.tpl +++ /dev/null @@ -1,89 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "audit.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "audit.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "audit.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "audit.labels" -}} -{{- if .Values.commonLabels }} - {{- with .Values.commonLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.commonLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "audit.selectorLabels" -}} -{{- if .Values.selectorLabels }} - {{- with .Values.selectorLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.selectorLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "audit.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "audit.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} - -{{/* - Postgres Password lookup -*/}} -{{- define "audit.postgres.password" -}} -{{- $localpass := (lookup "v1" "Secret" "postgres" "postgres-postgresql" ) -}} -{{- if $localpass }} -{{- default (index $localpass.data "postgres-password" | b64dec) }} -{{- else }} -{{- default .Values.postgres.password }} -{{- end }} -{{- end }} - - -{{/* - Audit g3 Auto Secrets Manager Name -*/}} -{{- define "audit-g3auto" -}} -{{- default "audit-g3auto" .Values.externalSecrets.auditG3auto }} -{{- end }} - diff --git a/helm/audit/templates/aws-config.yaml b/helm/audit/templates/aws-config.yaml deleted file mode 100644 index 398770d36..000000000 --- a/helm/audit/templates/aws-config.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if or (.Values.secrets.awsSecretAccessKey) (.Values.global.aws.awsSecretAccessKey ) }} -{{ include "common.awsconfig" . }} -{{- end -}} \ No newline at end of file diff --git a/helm/audit/templates/db-init.yaml b/helm/audit/templates/db-init.yaml deleted file mode 100644 index 5ef14e87e..000000000 --- a/helm/audit/templates/db-init.yaml +++ /dev/null @@ -1,6 +0,0 @@ -{{ include "common.db-secret" . }} ---- -{{ include "common.db_setup_job" . }} ---- -{{ include "common.db_setup_sa" . }} ---- \ No newline at end of file diff --git a/helm/audit/templates/deployment.yaml b/helm/audit/templates/deployment.yaml deleted file mode 100644 index 6d4db6f6b..000000000 --- a/helm/audit/templates/deployment.yaml +++ /dev/null @@ -1,162 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: audit-deployment - labels: - {{- include "audit.labels" . | nindent 4 }} - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 4 }} - {{- end }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "audit.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "audit.selectorLabels" . | nindent 8 }} - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 8 }} - {{- end }} - spec: - serviceAccountName: {{ include "audit.serviceAccountName" . }} - volumes: - - name: config-volume - secret: - secretName: "audit-g3auto" - {{- with .Values.volumes }} - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: audit - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - containerPort: 80 - name: http - protocol: TCP - livenessProbe: - httpGet: - path: /_status - port: 80 - initialDelaySeconds: 30 - periodSeconds: 60 - timeoutSeconds: 30 - readinessProbe: - httpGet: - path: /_status - port: 80 - resources: - {{- toYaml .Values.resources | nindent 12 }} - env: - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogEnvVar" . | nindent 12 }} - {{- end }} - - name: DB_HOST - valueFrom: - secretKeyRef: - name: audit-dbcreds - key: host - optional: false - - name: DB_USER - valueFrom: - secretKeyRef: - name: audit-dbcreds - key: username - optional: false - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: audit-dbcreds - key: password - optional: false - - name: DB_DATABASE - valueFrom: - secretKeyRef: - name: audit-dbcreds - key: database - optional: false - - name: DBREADY - valueFrom: - secretKeyRef: - name: audit-dbcreds - key: dbcreated - optional: false - {{- toYaml .Values.env | nindent 12 }} - volumeMounts: - - name: "config-volume" - readOnly: true - mountPath: "/src/audit-service-config.yaml" - subPath: "audit-service-config.yaml" - {{- with .Values.volumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - initContainers: - - name: audit-init - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: DB_HOST - valueFrom: - secretKeyRef: - name: audit-dbcreds - key: host - optional: false - - name: DB_USER - valueFrom: - secretKeyRef: - name: audit-dbcreds - key: username - optional: false - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: audit-dbcreds - key: password - optional: false - - name: DB_DATABASE - valueFrom: - secretKeyRef: - name: audit-dbcreds - key: database - optional: false - - name: DBREADY - valueFrom: - secretKeyRef: - name: audit-dbcreds - key: dbcreated - optional: false - resources: - {{- toYaml .Values.resources | nindent 12 }} - command: ["/bin/bash"] - args: - - "-c" - - | - /env/bin/alembic upgrade head - volumeMounts: - - name: "config-volume" - readOnly: true - mountPath: "/src/audit-service-config.yaml" - subPath: "audit-service-config.yaml" - {{- with .Values.volumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/audit/templates/external-secret.yaml b/helm/audit/templates/external-secret.yaml deleted file mode 100644 index d925feaac..000000000 --- a/helm/audit/templates/external-secret.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{ if .Values.global.externalSecrets.deploy }} -apiVersion: external-secrets.io/v1beta1 -kind: ExternalSecret -metadata: - name: audit-g3auto -spec: - refreshInterval: 5m - secretStoreRef: - name: {{include "common.SecretStore" .}} - kind: SecretStore - target: - name: audit-g3auto - creationPolicy: Owner - data: - - secretKey: audit-service-config.yaml - remoteRef: - #name of secret in secrets manager - key: {{include "audit-g3auto" .}} -{{- end }} ---- -{{ include "common.externalSecret.db" . }} \ No newline at end of file diff --git a/helm/audit/templates/pdb.yaml b/helm/audit/templates/pdb.yaml deleted file mode 100644 index 2ef2de13d..000000000 --- a/helm/audit/templates/pdb.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if and .Values.global.pdb (gt (int .Values.replicaCount) 1) }} -{{ include "common.pod_disruption_budget" . }} -{{- end }} \ No newline at end of file diff --git a/helm/audit/templates/secret-store.yaml b/helm/audit/templates/secret-store.yaml deleted file mode 100644 index 771c7760d..000000000 --- a/helm/audit/templates/secret-store.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{ if .Values.global.externalSecrets.separateSecretStore }} -{{ include "common.secretstore" . }} -{{- end }} \ No newline at end of file diff --git a/helm/audit/templates/secrets.yaml b/helm/audit/templates/secrets.yaml deleted file mode 100644 index b10733818..000000000 --- a/helm/audit/templates/secrets.yaml +++ /dev/null @@ -1,54 +0,0 @@ -{{- if or (not .Values.global.externalSecrets.deploy) (and .Values.global.externalSecrets.deploy .Values.externalSecrets.createK8sAuditSecret) }} -apiVersion: v1 -kind: Secret -metadata: - name: audit-g3auto - labels: - {{- include "audit.labels" . | nindent 4 }} -stringData: - audit-service-config.yaml: |- - #################### - # SERVER # - #################### - # whether to enable debug logs - DEBUG: {{ .Values.server.debug }} - - # If `PULL_FROM_QUEUE` is true, `QUEUE_CONFIG` is required. Otherwise, - # logs can only be created by hitting the API's log creation endpoint. - PULL_FROM_QUEUE: {{ .Values.server.pull_from_queue }} - - # `QUEUE_CONFIG.type` is one of: [api, aws_sqs]. - # - if type == aws_sqs: logs are pulled from an SQS and `aws_sqs_config` - # fields `sqs_url` and `region` are required. Field `aws_cred` is optional and - # it should be a key in section `AWS_CREDENTIALS`. - QUEUE_CONFIG: - type: aws_sqs - aws_sqs_config: - sqs_url: {{ .Values.server.sqs.url }} - region: {{ .Values.server.sqs.region }} - aws_cred: {{ .Values.server.sqs.awsCred }} - # defaults to "http://arborist-service/" - ARBORIST_URL: {{ .Values.server.arboristUrl }} - # how often to check the queue for new audit logs after - # seeing an empty queue - PULL_FREQUENCY_SECONDS: 300 # default: 5 min - - # NOTE: Remove the {} and supply creds if needed. Example in comments below - AWS_CREDENTIALS: {} - # CRED1: - # aws_access_key_id: '' - # aws_secret_access_key: '' - - #################### - # API # - #################### - - # if left empty, queries are not time-boxed - QUERY_TIMEBOX_MAX_DAYS: - - QUERY_PAGE_SIZE: 1000 - - # whether to return usernames in query responses, - # and to allow querying by username - QUERY_USERNAMES: true -{{- end }} \ No newline at end of file diff --git a/helm/audit/templates/service.yaml b/helm/audit/templates/service.yaml deleted file mode 100644 index aeb8e725c..000000000 --- a/helm/audit/templates/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: audit-service - labels: - {{- include "audit.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "audit.selectorLabels" . | nindent 4 }} diff --git a/helm/audit/templates/serviceaccount.yaml b/helm/audit/templates/serviceaccount.yaml deleted file mode 100644 index 5fb0e2cdd..000000000 --- a/helm/audit/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "audit.serviceAccountName" . }} - labels: - {{- include "audit.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/helm/audit/templates/tests/test-connection.yaml b/helm/audit/templates/tests/test-connection.yaml deleted file mode 100644 index d0a13d3af..000000000 --- a/helm/audit/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "audit.fullname" . }}-test-connection" - labels: - {{- include "audit.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['audit-service:{{ .Values.service.port }}/_status'] - restartPolicy: Never diff --git a/helm/audit/values.yaml b/helm/audit/values.yaml deleted file mode 100644 index 0112fb8a6..000000000 --- a/helm/audit/values.yaml +++ /dev/null @@ -1,288 +0,0 @@ -# Default values for audit. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -# Global configuration -global: - # -- (map) AWS configuration - aws: - # -- (bool) Set to true if deploying to AWS. Controls ingress annotations. - enabled: false - # -- (string) Credentials for AWS stuff. - awsAccessKeyId: - # -- (string) Credentials for AWS stuff. - awsSecretAccessKey: - # -- (bool) Whether the deployment is for development purposes. - dev: true - - postgres: - # -- (bool) Whether the database should be created. - dbCreate: true - # -- (string) Name of external secret. Disabled if empty - externalSecret: "" - # -- (map) Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres - master: - # -- (string) hostname of postgres server - host: - # -- (string) username of superuser in postgres. This is used to create or restore databases - username: postgres - # -- (string) password for superuser in postgres. This is used to create or restore databases - password: - # -- (string) Port for Postgres. - port: "5432" - # -- (string) Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. - environment: default - # -- (string) Hostname for the deployment. - hostname: localhost - # -- (string) ARN of the reverse proxy certificate. - revproxyArn: arn:aws:acm:us-east-1:123456:certificate - # -- (string) URL of the data dictionary. - dictionaryUrl: https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json - # -- (string) Portal application name. - portalApp: gitops - # -- (string) S3 bucket name for Kubernetes manifest files. - kubeBucket: kube-gen3 - # -- (string) S3 bucket name for log files. - logsBucket: logs-gen3 - # -- (bool) Whether public datasets are enabled. - publicDataSets: true - # -- (string) Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` - tierAccessLevel: libre - # -- (bool) Whether network policies are enabled. - netPolicy: true - # -- (int) Number of dispatcher jobs. - dispatcherJobNum: "10" - # -- (bool) Whether Datadog is enabled. - ddEnabled: false - # -- (bool) If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. - pdb: false - # -- (int) The minimum amount of pods that are available at all times if the PDB is deployed. - minAvialable: 1 - # -- (map) External Secrets settings. - externalSecrets: - # -- (bool) Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any audit secrets you have deployed. - deploy: false - # -- (string) Will deploy a separate External Secret Store for this service. - separateSecretStore: false - -# -- (map) External Secrets settings. -externalSecrets: - # -- (string) Will create the Helm "audit-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. - createK8sAuditSecret: false - # -- (string) Will override the name of the aws secrets manager secret. Default is "audit-g3auto" - auditG3auto: - # -- (string) Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" - dbcreds: -# -- (map) Secret information for External Secrets. -secrets: - # -- (str) AWS access key ID. Overrides global key. - awsAccessKeyId: - # -- (str) AWS secret access key ID. Overrides global key. - awsSecretAccessKey: - -# -- (map) Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you -postgres: - # -- (bool) Whether the database should be created. Default to global.postgres.dbCreate - dbCreate: - # -- (string) Hostname for postgres server. This is a service override, defaults to global.postgres.host - host: - # -- (string) Database name for postgres. This is a service override, defaults to - - database: - # -- (string) Username for postgres. This is a service override, defaults to - - username: - # -- (string) Port for Postgres. - port: "5432" - # -- (string) Password for Postgres. Will be autogenerated if left empty. - password: - # -- (string) Will create a Database for the individual service to help with developing it. - separate: false - -# -- (map) Postgresql subchart settings if deployed separately option is set to "true". -# Disable persistence by default so we can spin up and down ephemeral environments -postgresql: - primary: - persistence: - # -- (bool) Option to persist the dbs data. - enabled: false - -# -- (int) Number of desired replicas -replicaCount: 1 - -# -- (map) Docker image information. -image: - # -- (string) The Docker image repository for the audit service - repository: quay.io/cdis/audit-service - # -- (string) When to pull the image. This value should be "Always" to ensure the latest image is used. - pullPolicy: Always - # -- (string) Overrides the image tag whose default is the chart appVersion. - tag: "master" - -# -- (list) Docker image pull secrets. -imagePullSecrets: [] - -# -- (string) Override the name of the chart. This can be used to provide a unique name for a chart -nameOverride: "" - -# -- (string) Override the full name of the chart, which is used as the name of resources created by the chart -fullnameOverride: "" - -# -- (map) Service account to use or create. -serviceAccount: - # Specifies whether a service account should be created - # -- (bool) Whether to create a service account - create: true - # Annotations to add to the service account - annotations: - # -- (string) The Amazon Resource Name (ARN) of the role to associate with the service account - eks.amazonaws.com/role-arn: - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - # -- (string) The name of the service account - name: "audit-service-sa" - -# -- (map) Annotations to add to the pod -podAnnotations: {} - -# -- (map) Security context for the pod -podSecurityContext: {} - -# -- (map) Security context for the containers in the pod -securityContext: {} - - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -# -- (map) Configuration for the service -service: - # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". - type: ClusterIP - # -- (int) Port on which the service is exposed - port: 80 - -# -- (map) Resource requests and limits for the containers in the pod -resources: - # -- (map) The amount of resources that the container requests - requests: - # -- (string) The amount of CPU requested - cpu: 0.2 - # -- (string) The amount of memory requested - memory: 120Mi - # -- (map) The maximum amount of resources that the container is allowed to use - limits: - # -- (string) The maximum amount of CPU the container can use - cpu: 1.0 - # -- (string) The maximum amount of memory the container can use - memory: 512Mi - -# -- (map) Configuration for autoscaling the number of replicas -autoscaling: - # -- (bool) Whether autoscaling is enabled or not - enabled: false - # -- (int) The minimum number of replicas to scale down to - minReplicas: 1 - # -- (int) The maximum number of replicas to scale up to - maxReplicas: 4 - # -- (int) The target CPU utilization percentage for autoscaling - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - - -# -- (map) Node Selector for the pods -nodeSelector: {} - -# -- (list) Tolerations for the pods -tolerations: [] - -# -- (map) Affinity to use for the deployment. -affinity: - podAntiAffinity: - # -- (map) Option for scheduling to be required or preferred. - preferredDuringSchedulingIgnoredDuringExecution: - # -- (int) Weight value for preferred scheduling. - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - # -- (list) Label key for match expression. - - key: app - # -- (string) Operation type for the match expression. - operator: In - # -- (list) Value for the match expression key. - values: - - audit - # -- (string) Value for topology key label. - topologyKey: "kubernetes.io/hostname" - -# -- (list) Environment variables to pass to the container -env: - - name: DEBUG - value: "false" - - name: ARBORIST_URL - valueFrom: - configMapKeyRef: - name: manifest-global - key: arborist_url - optional: true - -# -- (list) Volumes to attach to the container. -volumes: [] - -# -- (list) Volumes to mount to the container. -volumeMounts: [] - -# -- (list) Volumes to mount to the init container. -initVolumeMounts: [] - -# -- (list) Volumes to attach to the init container. -initEnv: {} - -server: - # -- (bool) Whether to pull logs from sqs queue. - pull_from_queue: false - # -- (bool) Whether to enable or disable debug mode. - debug: false - # -- (map) AWS SQS queue information. - sqs: - # -- (string) SQS queue AWS region. - region: "us-east-1" - # -- (string) The URL for the SQS queue. - url: "http://sqs.com" - # -- (map) AWS credentials to access SQS queue. - AWS_CREDENTIALS: {} - # cred1: - # aws_access_key_id: - # aws_secret_access_key: - -api: - # -- (int) Amount to time-box queries. - QUERY_TIMEBOX_MAX_DAYS: - - # -- (int) The maximum number of entires the query endpoint will return. - QUERY_PAGE_SIZE: 1000 - - # -- (bool) Whether to return usernames in query responses and allow querying by username. - QUERY_USERNAMES: true - -# Values to determine the labels that are used for the deployment, pod, etc. -# -- (string) Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". -release: "production" -# -- (string) Valid options are "true" or "false". If invalid option is set- the value will default to "false". -criticalService: "false" -# -- (string) Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. -partOf: "Logging" -# -- (map) Will completely override the selectorLabels defined in the common chart's _label_setup.tpl -selectorLabels: -# -- (map) Will completely override the commonLabels defined in the common chart's _label_setup.tpl -commonLabels: - -# Values to configure datadog if ddEnabled is set to "true". -# -- (bool) If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. -datadogLogsInjection: true -# -- (bool) If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. -datadogProfilingEnabled: true -# -- (int) A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. -datadogTraceSampleRate: 1 diff --git a/helm/dicom-server/.helmignore b/helm/dicom-server/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/helm/dicom-server/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm/dicom-server/Chart.yaml b/helm/dicom-server/Chart.yaml deleted file mode 100644 index 4741141b0..000000000 --- a/helm/dicom-server/Chart.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: v2 -name: dicom-server -description: A Helm chart for gen3 Dicom Server - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.8 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "master" - -dependencies: -- name: common - version: 0.1.10 - repository: file://../common diff --git a/helm/dicom-server/README.md b/helm/dicom-server/README.md deleted file mode 100644 index f95924f0d..000000000 --- a/helm/dicom-server/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# dicom-server - -![Version: 0.1.8](https://img.shields.io/badge/Version-0.1.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) - -A Helm chart for gen3 Dicom Server - -## Requirements - -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.10 | - -## Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| autoscaling | map | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Configuration for autoscaling the number of replicas | -| autoscaling.enabled | bool | `false` | Whether autoscaling is enabled | -| autoscaling.maxReplicas | int | `100` | The maximum number of replicas to scale up to | -| autoscaling.minReplicas | int | `1` | The minimum number of replicas to scale down to | -| autoscaling.targetCPUUtilizationPercentage | int | `80` | The target CPU utilization percentage for autoscaling | -| commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | -| criticalService | string | `"false"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | -| datadogLogsInjection | bool | `true` | If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. | -| datadogProfilingEnabled | bool | `true` | If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. | -| datadogTraceSampleRate | int | `1` | A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. | -| global.ddEnabled | bool | `false` | Whether Datadog is enabled. | -| global.environment | string | `"default"` | Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. | -| global.minAvialable | int | `1` | The minimum amount of pods that are available at all times if the PDB is deployed. | -| global.pdb | bool | `false` | If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. | -| image | map | `{"pullPolicy":"Always","repository":"quay.io/cdis/gen3-orthanc","tag":"master"}` | Docker image information. | -| image.pullPolicy | string | `"Always"` | Docker pull policy. | -| image.repository | string | `"quay.io/cdis/gen3-orthanc"` | Docker repository. | -| image.tag | string | `"master"` | Overrides the image tag whose default is the chart appVersion. | -| partOf | string | `"Imaging"` | Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. | -| release | string | `"production"` | Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". | -| replicaCount | int | `1` | Number of replicas for the deployment. | -| secrets | map | `{"authenticationEnabled":false,"dataBase":"postgres","enableIndex":true,"enableStorage":true,"host":"postgres-postgresql.postgres.svc.cluster.local","indexConnectionsCount":5,"lock":false,"password":"postgres","port":"5432","userName":"postgres"}` | Secret information | -| secrets.authenticationEnabled | bool | `false` | Whether or not the password protection is enabled. | -| secrets.dataBase | string | `"postgres"` | Database name for postgres. | -| secrets.enableIndex | bool | `true` | Whether to enable index. If set to "false", Orthanc will continue to use its default SQLite back-end. | -| secrets.enableStorage | bool | `true` | Whether to enable storage. If set to "false", Orthanc will continue to use its default filesystem storage area. | -| secrets.host | string | `"postgres-postgresql.postgres.svc.cluster.local"` | Hostname for postgres server. | -| secrets.indexConnectionsCount | int | `5` | The number of connections from the index plugin to the PostgreSQL database. | -| secrets.lock | bool | `false` | Whether to lock the database. | -| secrets.password | string | `"postgres"` | Password for Postgres. | -| secrets.port | string | `"5432"` | Port for Postgres. | -| secrets.userName | string | `"postgres"` | Username for postgres. | -| selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | -| service | map | `{"port":80,"targetport":8042}` | Kubernetes service information. | -| service.port | int | `80` | The port number that the service exposes. | -| service.targetport | int | `8042` | The port on the host machine that traffic is directed to. | -| volumeMounts | list | `[{"mountPath":"/etc/orthanc/orthanc_config_overwrites.json","name":"config-volume-g3auto","readOnly":true,"subPath":"orthanc_config_overwrites.json"}]` | Volumes to mount to the pod. | -| volumes | list | `[{"name":"config-volume-g3auto","secret":{"secretName":"orthanc-g3auto"}}]` | Volumes to attach to the pod. | - diff --git a/helm/dicom-server/templates/NOTES.txt b/helm/dicom-server/templates/NOTES.txt deleted file mode 100644 index c1e7e1aef..000000000 --- a/helm/dicom-server/templates/NOTES.txt +++ /dev/null @@ -1 +0,0 @@ -{{ .Chart.Name }} has been deployed successfully. diff --git a/helm/dicom-server/templates/_helpers.tpl b/helm/dicom-server/templates/_helpers.tpl deleted file mode 100644 index 0ad87443d..000000000 --- a/helm/dicom-server/templates/_helpers.tpl +++ /dev/null @@ -1,80 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "dicom-server.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "dicom-server.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "dicom-server.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "dicom-server.labels" -}} -{{- if .Values.commonLabels }} - {{- with .Values.commonLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.commonLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "dicom-server.selectorLabels" -}} -{{- if .Values.selectorLabels }} - {{- with .Values.selectorLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.selectorLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "dicom-server.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "dicom-server.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} - -{{/* - Postgres Password lookup -*/}} -{{- define "dicom.postgres.password" -}} -{{- $localpass := (lookup "v1" "Secret" "postgres" "postgres-postgresql" ) -}} -{{- if $localpass }} -{{- default (index $localpass.data "postgres-password" | b64dec) }} -{{- else }} -{{- default .Values.secrets.password }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/helm/dicom-server/templates/config-volume-g3auto.yaml b/helm/dicom-server/templates/config-volume-g3auto.yaml deleted file mode 100644 index dd238b4ef..000000000 --- a/helm/dicom-server/templates/config-volume-g3auto.yaml +++ /dev/null @@ -1,45 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: orthanc-g3auto -type: Opaque -stringData: - dbcreds.json: | - { - "db_host": {{ .Values.secrets.host | quote }}, - "db_username": {{ .Values.secrets.userName | quote }}, - "db_password": {{ include "dicom.postgres.password" . | quote }}, - "db_database": {{ .Values.secrets.dataBase | quote }} - } - orthanc_config_overwrites.json: | - { - "AuthenticationEnabled": {{ .Values.secrets.authenticationEnabled }}, - "PostgreSQL": { - "EnableIndex": {{ .Values.secrets.enableIndex }}, - "EnableStorage": {{ .Values.secrets.enableStorage }}, - "Port": {{ .Values.secrets.port }}, - "Host": {{ .Values.secrets.host | quote }}, - "Database": {{ .Values.secrets.dataBase | quote }}, - "Username": {{ .Values.secrets.userName | quote }}, - "Password": {{ include "dicom.postgres.password" . | quote }}, - "IndexConnectionsCount": {{ .Values.secrets.indexConnectionsCount }}, - "Lock": {{ .Values.secrets.lock }} - }, - "PythonScript": "/etc/orthanc/authz_filter.py" - } - orthanc_config_overwrites.json_backup: | - { - "AuthenticationEnabled": {{ .Values.secrets.authenticationEnabled }}, - "PostgreSQL": { - "EnableIndex": {{ .Values.secrets.enableIndex }}, - "EnableStorage": {{ .Values.secrets.enableStorage }}, - "Port": {{ .Values.secrets.port }}, - "Host": {{ .Values.secrets.host | quote }}, - "Database": {{ .Values.secrets.dataBase | quote }}, - "Username": {{ .Values.secrets.userName | quote }}, - "Password": {{ .Values.secrets.password | quote }}, - "IndexConnectionsCount": {{ .Values.secrets.indexConnectionsCount }}, - "Lock": {{ .Values.secrets.lock }} - }, - "PythonScript": "/etc/orthanc/authz_filter.py" - } \ No newline at end of file diff --git a/helm/dicom-server/templates/deployment.yaml b/helm/dicom-server/templates/deployment.yaml deleted file mode 100644 index d7090ab90..000000000 --- a/helm/dicom-server/templates/deployment.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dicom-server-deployment - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} - labels: - {{- include "dicom-server.labels" . | nindent 4 }} - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 4 }} - {{- end }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "dicom-server.selectorLabels" . | nindent 6 }} - release: "production" - public: "yes" - template: - metadata: - labels: - {{- include "dicom-server.selectorLabels" . | nindent 8 }} - public: "yes" - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: "dicom-server" - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - readinessProbe: - httpGet: - path: /system - port: 8042 - initialDelaySeconds: 5 - periodSeconds: 20 - timeoutSeconds: 30 - livenessProbe: - httpGet: - path: /system - port: 8042 - initialDelaySeconds: 5 - periodSeconds: 60 - timeoutSeconds: 30 - ports: - - containerPort: 8042 - env: - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogEnvVar" . | nindent 12 }} - {{- end }} - {{- with .Values.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 10 }} - {{- end }} \ No newline at end of file diff --git a/helm/dicom-server/templates/pdb.yaml b/helm/dicom-server/templates/pdb.yaml deleted file mode 100644 index 2ef2de13d..000000000 --- a/helm/dicom-server/templates/pdb.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if and .Values.global.pdb (gt (int .Values.replicaCount) 1) }} -{{ include "common.pod_disruption_budget" . }} -{{- end }} \ No newline at end of file diff --git a/helm/dicom-server/templates/service.yaml b/helm/dicom-server/templates/service.yaml deleted file mode 100644 index e8364cc17..000000000 --- a/helm/dicom-server/templates/service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: "dicom-server-service" - labels: - {{- include "dicom-server.labels" . | nindent 4 }} -spec: - ports: - - port: {{ .Values.service.port }} - targetPort: {{ .Values.service.targetport }} - protocol: TCP - name: https - selector: - {{- include "dicom-server.selectorLabels" . | nindent 4 }} diff --git a/helm/dicom-server/values.yaml b/helm/dicom-server/values.yaml deleted file mode 100644 index 3cf06900b..000000000 --- a/helm/dicom-server/values.yaml +++ /dev/null @@ -1,104 +0,0 @@ -# Default values for dicom-server. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - - -# Global configuration -global: - # -- (string) Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. - environment: default - # -- (bool) Whether Datadog is enabled. - ddEnabled: false - # -- (bool) If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. - pdb: false - # -- (int) The minimum amount of pods that are available at all times if the PDB is deployed. - minAvialable: 1 - -# Deployment - -# -- (map) Configuration for autoscaling the number of replicas -autoscaling: - # -- (bool) Whether autoscaling is enabled - enabled: false - # -- (int) The minimum number of replicas to scale down to - minReplicas: 1 - # -- (int) The maximum number of replicas to scale up to - maxReplicas: 100 - # -- (int) The target CPU utilization percentage for autoscaling - targetCPUUtilizationPercentage: 80 - -# -- (int) Number of replicas for the deployment. -replicaCount: 1 - -# -- (list) Volumes to attach to the pod. -volumes: - - name: config-volume-g3auto - secret: - secretName: orthanc-g3auto - -# -- (map) Docker image information. -image: - # -- (string) Docker repository. - repository: quay.io/cdis/gen3-orthanc - # -- (string) Docker pull policy. - pullPolicy: Always - # -- (string) Overrides the image tag whose default is the chart appVersion. - tag: "master" - -# -- (list) Volumes to mount to the pod. -volumeMounts: - - name: config-volume-g3auto - readOnly: true - mountPath: /etc/orthanc/orthanc_config_overwrites.json - subPath: orthanc_config_overwrites.json - -# Service and Pod -# -- (map) Kubernetes service information. -service: - # -- (int) The port number that the service exposes. - port: 80 - # -- (int) The port on the host machine that traffic is directed to. - targetport: 8042 - -# -- (map) Secret information -secrets: - # -- (bool) Whether or not the password protection is enabled. - authenticationEnabled: false - # -- (bool) Whether to enable index. If set to "false", Orthanc will continue to use its default SQLite back-end. - enableIndex: true - # -- (bool) Whether to enable storage. If set to "false", Orthanc will continue to use its default filesystem storage area. - enableStorage: true - # -- (string) Port for Postgres. - port: "5432" - # -- (string) Hostname for postgres server. - host: postgres-postgresql.postgres.svc.cluster.local - # -- (string) Database name for postgres. - dataBase: postgres - # -- (string) Username for postgres. - userName: postgres - # -- (string) Password for Postgres. - password: postgres - # -- (int) The number of connections from the index plugin to the PostgreSQL database. - indexConnectionsCount: 5 - # -- (bool) Whether to lock the database. - lock: false - -# Values to determine the labels that are used for the deployment, pod, etc. -# -- (string) Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". -release: "production" -# -- (string) Valid options are "true" or "false". If invalid option is set- the value will default to "false". -criticalService: "false" -# -- (string) Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. -partOf: "Imaging" -# -- (map) Will completely override the selectorLabels defined in the common chart's _label_setup.tpl -selectorLabels: -# -- (map) Will completely override the commonLabels defined in the common chart's _label_setup.tpl -commonLabels: - -# Values to configure datadog if ddEnabled is set to "true". -# -- (bool) If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. -datadogLogsInjection: true -# -- (bool) If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. -datadogProfilingEnabled: true -# -- (int) A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. -datadogTraceSampleRate: 1 diff --git a/helm/dicom-viewer/.helmignore b/helm/dicom-viewer/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/helm/dicom-viewer/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm/dicom-viewer/Chart.yaml b/helm/dicom-viewer/Chart.yaml deleted file mode 100644 index 4835cfea6..000000000 --- a/helm/dicom-viewer/Chart.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: v2 -name: dicom-viewer -description: A Helm chart for gen3 Dicom Viewer - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.8 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "master" - -dependencies: -- name: common - version: 0.1.10 - repository: file://../common diff --git a/helm/dicom-viewer/README.md b/helm/dicom-viewer/README.md deleted file mode 100644 index 28eec5171..000000000 --- a/helm/dicom-viewer/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# dicom-viewer - -![Version: 0.1.8](https://img.shields.io/badge/Version-0.1.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) - -A Helm chart for gen3 Dicom Viewer - -## Requirements - -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.10 | - -## Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| autoscaling | map | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Configuration for autoscaling the number of replicas | -| autoscaling.enabled | bool | `false` | Whether autoscaling is enabled | -| autoscaling.maxReplicas | int | `100` | The maximum number of replicas to scale up to | -| autoscaling.minReplicas | int | `1` | The minimum number of replicas to scale down to | -| autoscaling.targetCPUUtilizationPercentage | int | `80` | The target CPU utilization percentage for autoscaling | -| commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | -| criticalService | string | `"false"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | -| datadogLogsInjection | bool | `true` | If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. | -| datadogProfilingEnabled | bool | `true` | If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. | -| datadogTraceSampleRate | int | `1` | A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. | -| global.ddEnabled | bool | `false` | Whether Datadog is enabled. | -| global.environment | string | `"default"` | Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. | -| global.minAvialable | int | `1` | The minimum amount of pods that are available at all times if the PDB is deployed. | -| global.pdb | bool | `false` | If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. | -| image | map | `{"pullPolicy":"Always","repository":"quay.io/cdis/ohif-viewer","tag":"master"}` | Docker image information. | -| image.pullPolicy | string | `"Always"` | Docker pull policy. | -| image.repository | string | `"quay.io/cdis/ohif-viewer"` | Docker repository. | -| image.tag | string | `"master"` | Overrides the image tag whose default is the chart appVersion. | -| partOf | string | `"Imaging"` | Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. | -| release | string | `"production"` | Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". | -| replicaCount | int | `1` | Number of replicas for the deployment. | -| selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | -| service | map | `{"port":80,"type":"ClusterIP"}` | Kubernetes service information. | -| service.port | int | `80` | The port number that the service exposes. | -| service.type | string | `"ClusterIP"` | Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". | - diff --git a/helm/dicom-viewer/templates/NOTES.txt b/helm/dicom-viewer/templates/NOTES.txt deleted file mode 100644 index c1e7e1aef..000000000 --- a/helm/dicom-viewer/templates/NOTES.txt +++ /dev/null @@ -1 +0,0 @@ -{{ .Chart.Name }} has been deployed successfully. diff --git a/helm/dicom-viewer/templates/_helpers.tpl b/helm/dicom-viewer/templates/_helpers.tpl deleted file mode 100644 index 7bbc396ec..000000000 --- a/helm/dicom-viewer/templates/_helpers.tpl +++ /dev/null @@ -1,68 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "dicom-viewer.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "dicom-viewer.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "dicom-viewer.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "dicom-viewer.labels" -}} -{{- if .Values.commonLabels }} - {{- with .Values.commonLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.commonLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "dicom-viewer.selectorLabels" -}} -{{- if .Values.selectorLabels }} - {{- with .Values.selectorLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.selectorLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "dicom-viewer.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "dicom-viewer.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/helm/dicom-viewer/templates/deployment.yaml b/helm/dicom-viewer/templates/deployment.yaml deleted file mode 100644 index a34e40891..000000000 --- a/helm/dicom-viewer/templates/deployment.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dicom-viewer-deployment - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} - labels: - {{- include "dicom-viewer.labels" . | nindent 4 }} - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 4 }} - {{- end }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "dicom-viewer.selectorLabels" . | nindent 6 }} - public: "yes" - template: - metadata: - labels: - {{- include "dicom-viewer.selectorLabels" . | nindent 8 }} - public: "yes" - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 8 }} - {{- end }} - spec: - containers: - - name: "dicom-viewer" - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - readinessProbe: - httpGet: - path: / - port: 80 - initialDelaySeconds: 5 - periodSeconds: 20 - timeoutSeconds: 30 - livenessProbe: - httpGet: - path: / - port: 80 - initialDelaySeconds: 5 - periodSeconds: 60 - timeoutSeconds: 30 - ports: - - containerPort: 80 - env: - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogEnvVar" . | nindent 12 }} - {{- end }} - {{- with .Values.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 10 }} - {{- end }} \ No newline at end of file diff --git a/helm/dicom-viewer/templates/hpa.yaml b/helm/dicom-viewer/templates/hpa.yaml deleted file mode 100644 index 93d39c969..000000000 --- a/helm/dicom-viewer/templates/hpa.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "dicom-viewer.fullname" . }} - labels: - {{- include "dicom-viewer.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "dicom-viewer.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/helm/dicom-viewer/templates/pdb.yaml b/helm/dicom-viewer/templates/pdb.yaml deleted file mode 100644 index 2ef2de13d..000000000 --- a/helm/dicom-viewer/templates/pdb.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if and .Values.global.pdb (gt (int .Values.replicaCount) 1) }} -{{ include "common.pod_disruption_budget" . }} -{{- end }} \ No newline at end of file diff --git a/helm/dicom-viewer/templates/service.yaml b/helm/dicom-viewer/templates/service.yaml deleted file mode 100644 index 1075df23d..000000000 --- a/helm/dicom-viewer/templates/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: "dicom-viewer-service" - labels: - {{- include "dicom-viewer.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: {{ .Values.service.port }} - protocol: TCP - name: https - selector: - {{- include "dicom-viewer.selectorLabels" . | nindent 4 }} diff --git a/helm/dicom-viewer/templates/tests/test-connection.yaml b/helm/dicom-viewer/templates/tests/test-connection.yaml deleted file mode 100644 index 85af6bd6e..000000000 --- a/helm/dicom-viewer/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "dicom-viewer-test-connection" - labels: - {{- include "dicom-viewer.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "dicom-viewer.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/helm/dicom-viewer/values.yaml b/helm/dicom-viewer/values.yaml deleted file mode 100644 index 01c3995b4..000000000 --- a/helm/dicom-viewer/values.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# Default values for dicom-viewer. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - - -# Global configuration -global: - # -- (string) Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. - environment: default - # -- (bool) Whether Datadog is enabled. - ddEnabled: false - # -- (bool) If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. - pdb: false - # -- (int) The minimum amount of pods that are available at all times if the PDB is deployed. - minAvialable: 1 - -# Deployment - -# -- (map) Configuration for autoscaling the number of replicas -autoscaling: - # -- (bool) Whether autoscaling is enabled - enabled: false - # -- (int) The minimum number of replicas to scale down to - minReplicas: 1 - # -- (int) The maximum number of replicas to scale up to - maxReplicas: 100 - # -- (int) The target CPU utilization percentage for autoscaling - targetCPUUtilizationPercentage: 80 - -# -- (int) Number of replicas for the deployment. -replicaCount: 1 - -# -- (map) Docker image information. -image: - # -- (string) Docker repository. - repository: quay.io/cdis/ohif-viewer - # -- (string) Docker pull policy. - pullPolicy: Always - # -- (string) Overrides the image tag whose default is the chart appVersion. - tag: "master" - -# Service and Pod -# -- (map) Kubernetes service information. -service: - # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". - type: ClusterIP - # -- (int) The port number that the service exposes. - port: 80 - -# Values to determine the labels that are used for the deployment, pod, etc. -# -- (string) Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". -release: "production" -# -- (string) Valid options are "true" or "false". If invalid option is set- the value will default to "false". -criticalService: "false" -# -- (string) Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. -partOf: "Imaging" -# -- (map) Will completely override the selectorLabels defined in the common chart's _label_setup.tpl -selectorLabels: -# -- (map) Will completely override the commonLabels defined in the common chart's _label_setup.tpl -commonLabels: - -# Values to configure datadog if ddEnabled is set to "true". -# -- (bool) If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. -datadogLogsInjection: true -# -- (bool) If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. -datadogProfilingEnabled: true -# -- (int) A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. -datadogTraceSampleRate: 1 diff --git a/helm/etl-pod/.helmignore b/helm/etl-pod/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/helm/etl-pod/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm/etl-pod/README.md b/helm/etl-pod/README.md deleted file mode 100644 index e80384dfb..000000000 --- a/helm/etl-pod/README.md +++ /dev/null @@ -1,23 +0,0 @@ - - -## Specifying S3 Data to be copied to Etl-pod for Elastic Indexing -Specify new data to be copied by changing helm/etl-pod/values.yaml:L8-9 then uninstall the etl deployment with: - -``` -helm uninstall etl-pod -``` - -Then deploy just the etl-pod with: -``` -helm install etl-pod ./helm/etl-pod -f ./helm/etl-pod/values.yaml -``` - -The S3_BUCKET and S3_PATH variables are string templating the following command: - -``` -aws s3 cp s3://$S3_BUCKET/$S3_PATH -``` - -where S3_BUCKET is the name of the bucket that you want -and S3_PATH is the remainder of the path to be coped from the bucket -into the ETL pod diff --git a/helm/etl-pod/templates/deployment.yaml b/helm/etl-pod/templates/deployment.yaml deleted file mode 100644 index 434380648..000000000 --- a/helm/etl-pod/templates/deployment.yaml +++ /dev/null @@ -1,133 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "etl-pod.fullname" . }} - labels: - {{- include "etl-pod.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "etl-pod.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "etl-pod.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - - name: gen3-credentials - configMap: - name: gen3-credentials - - - name: aws-creds - configMap: - name: aws-creds - - - name: sheepdog-dbcreds - secret: - secretName: sheepdog-dbcreds - - - name: useryaml - configMap: - name: useryaml - - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "etl-pod.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - env: - {{- range $key, $value := .Values.env }} - - name: {{ $key }} - value: {{ $value | quote }} - {{- end }} - - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - #ports: - #- name: http - #containerPort: {{ .Values.service.port }} - #protocol: TCP - #livenessProbe: - #httpGet: - #path: / - #port: http - #readinessProbe: - #httpGet: - #path: / - #port: http - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - # Gen3 credentials file - - name: gen3-credentials - mountPath: /creds/credentials.json - subPath: credentials.json - - - name: aws-creds - mountPath: /root/.aws/ - - # Sheepdog creds, secrets defined in helm/sheepdog/templates/deployment.yaml - - name: sheepdog-dbcreds - mountPath: /creds/sheepdog-creds - - # User yaml, configmap defined in helm/fence/templates/useryaml-job.yaml - - name: useryaml - mountPath: /creds/user.yaml - subPath: useryaml - - command: ["/bin/bash" ] - args: - - "-c" - - | - cat << EOF >> ~/.bashrc - export PGDB=`cat /creds/sheepdog-creds/database` - export PGPASSWORD=`cat /creds/sheepdog-creds/password` - export PGUSER=`cat /creds/sheepdog-creds/username` - export PGHOST=`cat /creds/sheepdog-creds/host` - export DBREADY=`cat /creds/sheepdog-creds/dbcreated` - export PGPORT=`cat /creds/sheepdog-creds/port` - EOF - - #this varifies credentials work - apt-get update - apt-get install less - aws sts get-caller-identity - - echo "aws s3 cp s3://$S3_BUCKET/$S3_PATH" - aws s3 cp "s3://$S3_BUCKET/$S3_PATH" . - - echo "Done setting up data_model" - while true; do sleep 30; done; - #while true; do echo "$S3_BUCKET/$S3_PATH"; sleep 10; done; - - {{ if eq .Values.global.deployment "development" }} - nodeSelector: - eks.amazonaws.com/nodegroup: aced-commons-development-etl-node-group-large - {{ else if eq .Values.global.deployment "staging" }} - nodeSelector: - eks.amazonaws.com/nodegroup: aced-commons-staging-etl-node-group - {{ else if eq .Values.global.deployment "production" }} - nodeSelector: - eks.amazonaws.com/nodegroup: aced-commons-production-etl-node-group - {{- end }} diff --git a/helm/etl-pod/templates/hpa.yaml b/helm/etl-pod/templates/hpa.yaml deleted file mode 100644 index 30c6d85f2..000000000 --- a/helm/etl-pod/templates/hpa.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "etl-pod.fullname" . }} - labels: - {{- include "etl-pod.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "etl-pod.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/helm/etl-pod/templates/ingress.yaml b/helm/etl-pod/templates/ingress.yaml deleted file mode 100644 index e13e3d948..000000000 --- a/helm/etl-pod/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "etl-pod.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "etl-pod.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/etl-pod/templates/tests/test-connection.yaml b/helm/etl-pod/templates/tests/test-connection.yaml deleted file mode 100644 index 5ce36503b..000000000 --- a/helm/etl-pod/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "etl-pod.fullname" . }}-test-connection" - labels: - {{- include "etl-pod.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "etl-pod.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/helm/etl-pod/values.yaml b/helm/etl-pod/values.yaml deleted file mode 100644 index 831495b9b..000000000 --- a/helm/etl-pod/values.yaml +++ /dev/null @@ -1,109 +0,0 @@ -# Default values for etl-pod. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -# expected form: "aws s3 cp s3://$S3_BUCKET/$S3_PATH" -# where S3_BUCKET is the bucket name and S3_PATH is the rest of the path -env: - S3_BUCKET: aced-development - S3_PATH: meta_graph_load.py - -global: - environment: default - ddEnabled: false - - # this chooses what node group to use - deployment: development - # as example variables just going to get a small python file - - -replicaCount: 1 - -image: - repository: quay.io/ohsu-comp-bio/aced-etl - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "x86" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: - limits: - cpu: 1.0 - memory: 1Gi - requests: - cpu: 0.2 - memory: 256Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -tolerations: [] - -affinity: {} - -release: "production" -# -- (string) Valid options are "true" or "false". If invalid option is set- the value will default to "false". -criticalService: "true" -# -- (string) Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. -partOf: "Core-Service" -# -- (map) Will completely override the selectorLabels defined in the common chart's _label_setup.tpl -selectorLabels: -# -- (map) Will completely override the commonLabels defined in the common chart's _label_setup.tpl -commonLabels: - -# Values to configure datadog if ddEnabled is set to "true". -# -- (bool) If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. -datadogLogsInjection: true -# -- (bool) If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. -datadogProfilingEnabled: true -# -- (int) A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. -datadogTraceSampleRate: 1 diff --git a/helm/etl/Chart.yaml b/helm/etl/Chart.yaml deleted file mode 100644 index c5b08a7b0..000000000 --- a/helm/etl/Chart.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: v2 -name: etl -description: A Helm chart for gen3 etl - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.1 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -appVersion: "master" - -dependencies: [] diff --git a/helm/etl/README.md b/helm/etl/README.md deleted file mode 100644 index faa4d2f2e..000000000 --- a/helm/etl/README.md +++ /dev/null @@ -1,105 +0,0 @@ -# etl - -![Version: 0.1.1](https://img.shields.io/badge/Version-0.1.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) - -A Helm chart for gen3 etl - -## Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| esEndpoint | string | `"gen3-elasticsearch-master"` | | -| etlMapping.mappings[0].aggregated_props[0].fn | string | `"count"` | | -| etlMapping.mappings[0].aggregated_props[0].name | string | `"_samples_count"` | | -| etlMapping.mappings[0].aggregated_props[0].path | string | `"samples"` | | -| etlMapping.mappings[0].aggregated_props[1].fn | string | `"count"` | | -| etlMapping.mappings[0].aggregated_props[1].name | string | `"_aliquots_count"` | | -| etlMapping.mappings[0].aggregated_props[1].path | string | `"samples.aliquots"` | | -| etlMapping.mappings[0].aggregated_props[2].fn | string | `"count"` | | -| etlMapping.mappings[0].aggregated_props[2].name | string | `"_submitted_methylations_count"` | | -| etlMapping.mappings[0].aggregated_props[2].path | string | `"samples.aliquots.submitted_methylation_files"` | | -| etlMapping.mappings[0].aggregated_props[3].fn | string | `"count"` | | -| etlMapping.mappings[0].aggregated_props[3].name | string | `"_submitted_copy_number_files_on_aliquots_count"` | | -| etlMapping.mappings[0].aggregated_props[3].path | string | `"samples.aliquots.submitted_copy_number_files"` | | -| etlMapping.mappings[0].aggregated_props[4].fn | string | `"count"` | | -| etlMapping.mappings[0].aggregated_props[4].name | string | `"_read_groups_count"` | | -| etlMapping.mappings[0].aggregated_props[4].path | string | `"samples.aliquots.read_groups"` | | -| etlMapping.mappings[0].aggregated_props[5].fn | string | `"count"` | | -| etlMapping.mappings[0].aggregated_props[5].name | string | `"_submitted_aligned_reads_count"` | | -| etlMapping.mappings[0].aggregated_props[5].path | string | `"samples.aliquots.read_groups.submitted_aligned_reads_files"` | | -| etlMapping.mappings[0].aggregated_props[6].fn | string | `"count"` | | -| etlMapping.mappings[0].aggregated_props[6].name | string | `"_submitted_unaligned_reads_count"` | | -| etlMapping.mappings[0].aggregated_props[6].path | string | `"samples.aliquots.read_groups.submitted_unaligned_reads_files"` | | -| etlMapping.mappings[0].aggregated_props[7].fn | string | `"count"` | | -| etlMapping.mappings[0].aggregated_props[7].name | string | `"_submitted_copy_number_files_on_read_groups_count"` | | -| etlMapping.mappings[0].aggregated_props[7].path | string | `"samples.aliquots.read_groups.submitted_copy_number_files"` | | -| etlMapping.mappings[0].aggregated_props[8].fn | string | `"count"` | | -| etlMapping.mappings[0].aggregated_props[8].name | string | `"_submitted_somatic_mutations_count"` | | -| etlMapping.mappings[0].aggregated_props[8].path | string | `"samples.aliquots.read_groups.submitted_somatic_mutations"` | | -| etlMapping.mappings[0].doc_type | string | `"case"` | | -| etlMapping.mappings[0].flatten_props[0].path | string | `"demographics"` | | -| etlMapping.mappings[0].flatten_props[0].props[0].name | string | `"gender"` | | -| etlMapping.mappings[0].flatten_props[0].props[0].value_mappings[0].female | string | `"F"` | | -| etlMapping.mappings[0].flatten_props[0].props[0].value_mappings[1].male | string | `"M"` | | -| etlMapping.mappings[0].flatten_props[0].props[1].name | string | `"race"` | | -| etlMapping.mappings[0].flatten_props[0].props[1].value_mappings[0]."american indian or alaskan native" | string | `"Indian"` | | -| etlMapping.mappings[0].flatten_props[0].props[2].name | string | `"ethnicity"` | | -| etlMapping.mappings[0].flatten_props[0].props[3].name | string | `"year_of_birth"` | | -| etlMapping.mappings[0].joining_props[0].index | string | `"file"` | | -| etlMapping.mappings[0].joining_props[0].join_on | string | `"_case_id"` | | -| etlMapping.mappings[0].joining_props[0].props[0].fn | string | `"set"` | | -| etlMapping.mappings[0].joining_props[0].props[0].name | string | `"data_format"` | | -| etlMapping.mappings[0].joining_props[0].props[0].src | string | `"data_format"` | | -| etlMapping.mappings[0].joining_props[0].props[1].fn | string | `"set"` | | -| etlMapping.mappings[0].joining_props[0].props[1].name | string | `"data_type"` | | -| etlMapping.mappings[0].joining_props[0].props[1].src | string | `"data_type"` | | -| etlMapping.mappings[0].joining_props[0].props[2].fn | string | `"set"` | | -| etlMapping.mappings[0].joining_props[0].props[2].name | string | `"_file_id"` | | -| etlMapping.mappings[0].joining_props[0].props[2].src | string | `"_file_id"` | | -| etlMapping.mappings[0].name | string | `"dev_case"` | | -| etlMapping.mappings[0].props[0].name | string | `"submitter_id"` | | -| etlMapping.mappings[0].props[1].name | string | `"project_id"` | | -| etlMapping.mappings[0].props[2].name | string | `"disease_type"` | | -| etlMapping.mappings[0].props[3].name | string | `"primary_site"` | | -| etlMapping.mappings[0].root | string | `"case"` | | -| etlMapping.mappings[0].type | string | `"aggregator"` | | -| etlMapping.mappings[1].category | string | `"data_file"` | | -| etlMapping.mappings[1].doc_type | string | `"file"` | | -| etlMapping.mappings[1].injecting_props.case.props[0].fn | string | `"set"` | | -| etlMapping.mappings[1].injecting_props.case.props[0].name | string | `"_case_id"` | | -| etlMapping.mappings[1].injecting_props.case.props[0].src | string | `"id"` | | -| etlMapping.mappings[1].injecting_props.case.props[1].name | string | `"project_id"` | | -| etlMapping.mappings[1].name | string | `"dev_file"` | | -| etlMapping.mappings[1].props[0].name | string | `"object_id"` | | -| etlMapping.mappings[1].props[1].name | string | `"md5sum"` | | -| etlMapping.mappings[1].props[2].name | string | `"file_name"` | | -| etlMapping.mappings[1].props[3].name | string | `"file_size"` | | -| etlMapping.mappings[1].props[4].name | string | `"data_format"` | | -| etlMapping.mappings[1].props[5].name | string | `"data_type"` | | -| etlMapping.mappings[1].props[6].name | string | `"state"` | | -| etlMapping.mappings[1].root | string | `"None"` | | -| etlMapping.mappings[1].target_nodes[0].name | string | `"slide_image"` | | -| etlMapping.mappings[1].target_nodes[0].path | string | `"slides.samples.cases"` | | -| etlMapping.mappings[1].type | string | `"collector"` | | -| image.spark.pullPolicy | string | `"Always"` | When to pull the image. This value should be "Always" to ensure the latest image is used. | -| image.spark.repository | string | `"quay.io/cdis/gen3-spark"` | The Docker image repository for the spark service | -| image.spark.tag | string | `"master"` | Overrides the image tag whose default is the chart appVersion. | -| image.tube.pullPolicy | string | `"Always"` | When to pull the image. This value should be "Always" to ensure the latest image is used. | -| image.tube.repository | string | `"quay.io/cdis/tube"` | The Docker image repository for the fence service | -| image.tube.tag | string | `"master"` | Overrides the image tag whose default is the chart appVersion. | -| imagePullSecrets | list | `[]` | Docker image pull secrets. | -| podAnnotations | map | `{}` | Annotations to add to the pod | -| resources | map | `{"spark":{"limits":{"cpu":1,"memory":"2Gi"},"requests":{"cpu":0.3,"memory":"128Mi"}},"tube":{"limits":{"cpu":1,"memory":"2Gi"},"requests":{"cpu":0.3,"memory":"128Mi"}}}` | Resource requests and limits for the containers in the pod | -| resources.spark.limits | map | `{"cpu":1,"memory":"2Gi"}` | The maximum amount of resources that the container is allowed to use | -| resources.spark.limits.cpu | string | `1` | The maximum amount of CPU the container can use | -| resources.spark.limits.memory | string | `"2Gi"` | The maximum amount of memory the container can use | -| resources.spark.requests | map | `{"cpu":0.3,"memory":"128Mi"}` | The amount of resources that the container requests | -| resources.spark.requests.cpu | string | `0.3` | The amount of CPU requested | -| resources.spark.requests.memory | string | `"128Mi"` | The amount of memory requested | -| resources.tube.limits | map | `{"cpu":1,"memory":"2Gi"}` | The maximum amount of resources that the container is allowed to use | -| resources.tube.limits.cpu | string | `1` | The maximum amount of CPU the container can use | -| resources.tube.limits.memory | string | `"2Gi"` | The maximum amount of memory the container can use | -| resources.tube.requests | map | `{"cpu":0.3,"memory":"128Mi"}` | The amount of resources that the container requests | -| resources.tube.requests.cpu | string | `0.3` | The amount of CPU requested | -| resources.tube.requests.memory | string | `"128Mi"` | The amount of memory requested | - diff --git a/helm/etl/templates/etl-job.yaml b/helm/etl/templates/etl-job.yaml deleted file mode 100644 index 0b306d078..000000000 --- a/helm/etl/templates/etl-job.yaml +++ /dev/null @@ -1,211 +0,0 @@ -apiVersion: batch/v1 -kind: CronJob -metadata: - name: etl-cronjob -spec: - schedule: "0 0 1 1 */5" - jobTemplate: - spec: - backoffLimit: 0 - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 12 }} - {{- end }} - labels: - app: gen3job - spec: - shareProcessNamespace: true - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - preference: - matchExpressions: - - key: karpenter.sh/capacity-type - operator: In - values: - - on-demand - - weight: 99 - preference: - matchExpressions: - - key: eks.amazonaws.com/capacityType - operator: In - values: - - ONDEMAND - volumes: - - name: signal-volume - emptyDir: {} - - name: creds-volume - secret: - secretName: "peregrine-dbcreds" - - name: etl-mapping - configMap: - name: etl-mapping - - name: fence-yaml - configMap: - name: useryaml - containers: - - name: gen3-spark - image: {{ .Values.image.spark.repository }}:{{ .Values.image.spark.tag }} - ports: - - containerPort: 22 - - containerPort: 9000 - - containerPort: 8030 - - containerPort: 8031 - - containerPort: 8032 - - containerPort: 7077 - readinessProbe: - tcpSocket: - port: 9000 - initialDelaySeconds: 10 - periodSeconds: 30 - env: - - name: DICTIONARY_URL - valueFrom: - configMapKeyRef: - name: manifest-global - key: dictionary_url - - name: HADOOP_URL - value: hdfs://0.0.0.0:9000 - - name: HADOOP_HOST - value: 0.0.0.0 - volumeMounts: - - mountPath: /usr/share/pod - name: signal-volume - readOnly: true - imagePullPolicy: {{ .Values.image.spark.pullPolicy }} - resources: - requests: - cpu: {{ .Values.resources.spark.requests.cpu }} - memory: {{ .Values.resources.spark.requests.memory }} - # limits: - # cpu: {{ .Values.resources.spark.limits.cpu }} - # memory: {{ .Values.resources.spark.limits.memory }} - command: ["/bin/bash" ] - args: - - "-c" - - | - trap 'exit 0' SIGINT SIGQUIT SIGTERM - # get /usr/local/share/ca-certificates/cdis-ca.crt into system bundle - ssh server sudo /etc/init.d/ssh start - # update-ca-certificates - python run_config.py - hdfs namenode -format - hdfs --daemon start namenode - hdfs --daemon start datanode - yarn --daemon start resourcemanager - yarn --daemon start nodemanager - hdfs dfsadmin -safemode leave - hdfs dfs -mkdir /result - hdfs dfs -mkdir /jars - hdfs dfs -mkdir /archive - /spark/sbin/start-all.sh - while true; do sleep 5; done - - name: tube - imagePullPolicy: IfNotPresent - # image: quay.io/cdis/tube:feat_helm_test - image: {{ .Values.image.tube.repository }}:{{ .Values.image.tube.tag }} - ports: - - containerPort: 80 - env: - - name: DB_HOST - valueFrom: - secretKeyRef: - name: peregrine-dbcreds - key: host - - name: DB_DATABASE - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: database - - name: DB_USERNAME - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: username - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: password - - name: DB_PORT - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: port - - name: DICTIONARY_URL - valueFrom: - configMapKeyRef: - name: manifest-global - key: dictionary_url - - name: HADOOP_URL - value: hdfs://localhost:9000 - - name: ES_URL - value: {{ .Values.esEndpoint }} - - name: HADOOP_HOST - value: localhost - - name: HADOOP_CLIENT_OPTS - value: -Xmx1g - - name: SPARK_EXECUTOR_MEMORY - value: 4g - - name: SPARK_DRIVER_MEMORY - value: 6g - - name: ETL_FORCED - value: "TRUE" - - name: gen3Env - valueFrom: - configMapKeyRef: - name: manifest-global - key: hostname - - name: slackWebHook - valueFrom: - configMapKeyRef: - name: global - key: slack_webhook - optional: true - volumeMounts: - # - name: "creds-volume" - # readOnly: true - # mountPath: "/gen3/tube/creds.json" - # subPath: creds.json - # Volume to signal when to kill spark - - mountPath: /usr/share/pod - name: signal-volume - - name: "etl-mapping" - readOnly: true - mountPath: "/gen3/tube/etlMapping.yaml" - subPath: "etlMapping.yaml" - - name: "fence-yaml" - readOnly: true - mountPath: "/gen3/tube/user.yaml" - subPath: useryaml - resources: - requests: - cpu: {{ .Values.resources.tube.requests.cpu }} - memory: {{ .Values.resources.tube.requests.memory }} - # limits: - # cpu: {{ .Values.resources.tube.limits.cpu }} - # memory: {{ .Values.resources.tube.limits.memory }} - command: ["/bin/bash"] - args: - - "-c" - - | - while ! bash -c "echo >/dev/tcp/localhost/9000"; do - echo "Spark is not ready on port 9000... waiting for 10 seconds." - sleep 10 - done - - # Port 9000 is open, continue with the rest of the script - echo "Port 9000 is now open. Continuing with the script..." - - echo "python run_config.py && python run_etl.py" - python run_config.py && python run_etl.py - exitcode=$? - - # Kill sidecar and all processes - echo "Exit code: $exitcode" - pkill -u root && exit $exitcode - exit "$exitcode" & - restartPolicy: Never \ No newline at end of file diff --git a/helm/etl/templates/etl-mapping.yaml b/helm/etl/templates/etl-mapping.yaml deleted file mode 100644 index 184a3e256..000000000 --- a/helm/etl/templates/etl-mapping.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: etl-mapping -data: - etlMapping.yaml: | - {{- with .Values.etlMapping }} - {{- toYaml . | nindent 4 }} - {{ end }} ---- \ No newline at end of file diff --git a/helm/etl/values.yaml b/helm/etl/values.yaml deleted file mode 100644 index 1db9765e9..000000000 --- a/helm/etl/values.yaml +++ /dev/null @@ -1,145 +0,0 @@ -# populate with normal values from a regular chart created by helm create - -image: - tube: - # -- (string) The Docker image repository for the fence service - repository: quay.io/cdis/tube - # -- (string) When to pull the image. This value should be "Always" to ensure the latest image is used. - pullPolicy: Always - # -- (string) Overrides the image tag whose default is the chart appVersion. - tag: "master" - spark: - # -- (string) The Docker image repository for the spark service - repository: quay.io/cdis/gen3-spark - # -- (string) When to pull the image. This value should be "Always" to ensure the latest image is used. - pullPolicy: Always - # -- (string) Overrides the image tag whose default is the chart appVersion. - tag: "master" - - -# -- (list) Docker image pull secrets. -imagePullSecrets: [] - -# -- (map) Annotations to add to the pod -podAnnotations: {} - - -# -- (map) Resource requests and limits for the containers in the pod -resources: - tube: - # -- (map) The amount of resources that the container requests - requests: - # -- (string) The amount of CPU requested - cpu: 0.3 - # -- (string) The amount of memory requested - memory: 128Mi - # -- (map) The maximum amount of resources that the container is allowed to use - limits: - # -- (string) The maximum amount of CPU the container can use - cpu: 1.0 - # -- (string) The maximum amount of memory the container can use - memory: 2Gi - spark: - # -- (map) The amount of resources that the container requests - requests: - # -- (string) The amount of CPU requested - cpu: 0.3 - # -- (string) The amount of memory requested - memory: 128Mi - # -- (map) The maximum amount of resources that the container is allowed to use - limits: - # -- (string) The maximum amount of CPU the container can use - cpu: 1.0 - # -- (string) The maximum amount of memory the container can use - memory: 2Gi - - -esEndpoint: gen3-elasticsearch-master - -etlMapping: - mappings: - - name: dev_case - doc_type: case - type: aggregator - root: case - props: - - name: submitter_id - - name: project_id - - name: disease_type - - name: primary_site - flatten_props: - - path: demographics - props: - - name: gender - value_mappings: - - female: F - - male: M - - name: race - value_mappings: - - american indian or alaskan native: Indian - - name: ethnicity - - name: year_of_birth - aggregated_props: - - name: _samples_count - path: samples - fn: count - - name: _aliquots_count - path: samples.aliquots - fn: count - - name: _submitted_methylations_count - path: samples.aliquots.submitted_methylation_files - fn: count - - name: _submitted_copy_number_files_on_aliquots_count - path: samples.aliquots.submitted_copy_number_files - fn: count - - name: _read_groups_count - path: samples.aliquots.read_groups - fn: count - - name: _submitted_aligned_reads_count - path: samples.aliquots.read_groups.submitted_aligned_reads_files - fn: count - - name: _submitted_unaligned_reads_count - path: samples.aliquots.read_groups.submitted_unaligned_reads_files - fn: count - - name: _submitted_copy_number_files_on_read_groups_count - path: samples.aliquots.read_groups.submitted_copy_number_files - fn: count - - name: _submitted_somatic_mutations_count - path: samples.aliquots.read_groups.submitted_somatic_mutations - fn: count - joining_props: - - index: file - join_on: _case_id - props: - - name: data_format - src: data_format - fn: set - - name: data_type - src: data_type - fn: set - - name: _file_id - src: _file_id - fn: set - - name: dev_file - doc_type: file - type: collector - root: None - category: data_file - props: - - name: object_id - - name: md5sum - - name: file_name - - name: file_size - - name: data_format - - name: data_type - - name: state - injecting_props: - case: - props: - - name: _case_id - src: id - fn: set - - name: project_id - target_nodes: - - name: slide_image - path: slides.samples.cases diff --git a/helm/fence/templates/fence-deployment.yaml b/helm/fence/templates/fence-deployment.yaml index 211871564..afa697942 100644 --- a/helm/fence/templates/fence-deployment.yaml +++ b/helm/fence/templates/fence-deployment.yaml @@ -49,12 +49,23 @@ spec: path: /_status port: http initialDelaySeconds: 60 - periodSeconds: 60 - timeoutSeconds: 30 + periodSeconds: 15 + timeoutSeconds: 3 + failureThreshold: 3 readinessProbe: httpGet: path: /_status port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 2 + startupProbe: + httpGet: + path: /_status + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 24 resources: {{- toYaml .Values.resources | nindent 12 }} command: ["/bin/bash"] @@ -109,10 +120,18 @@ spec: - /bin/sh - -c - | - until redis-cli -h {{ .Values.authzSnapshotCache.waitForRedis.host }} -p {{ .Values.authzSnapshotCache.waitForRedis.port }} -a "$REDIS_PASSWORD" --no-auth-warning ping | grep -q PONG; do - echo "waiting for authz cache redis at {{ .Values.authzSnapshotCache.waitForRedis.host }}:{{ .Values.authzSnapshotCache.waitForRedis.port }}" + attempt=1 + while [ "$attempt" -le "{{ .Values.authzSnapshotCache.waitForRedis.maxAttempts }}" ]; do + if redis-cli -h {{ .Values.authzSnapshotCache.waitForRedis.host }} -p {{ .Values.authzSnapshotCache.waitForRedis.port }} -a "$REDIS_PASSWORD" --no-auth-warning ping | grep -q PONG; then + echo "authz cache redis is ready" + exit 0 + fi + echo "waiting for authz cache redis at {{ .Values.authzSnapshotCache.waitForRedis.host }}:{{ .Values.authzSnapshotCache.waitForRedis.port }} (attempt ${attempt}/{{ .Values.authzSnapshotCache.waitForRedis.maxAttempts }})" + attempt=$((attempt + 1)) sleep 2 done + echo "authz cache redis did not become ready before the startup deadline" >&2 + exit 1 {{- end }} - name: fence-init image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" diff --git a/helm/fence/templates/usersync-cron.yaml b/helm/fence/templates/usersync-cron.yaml index dfb18c7bc..f24613796 100644 --- a/helm/fence/templates/usersync-cron.yaml +++ b/helm/fence/templates/usersync-cron.yaml @@ -170,7 +170,7 @@ spec: if [[ "$SYNC_FROM_DBGAP" != "true" && "$ADD_DBGAP" != "true" ]]; then if [[ -f /mnt/shared/user.yaml ]]; then echo "running fence-create" - time fence-create sync --arborist http://arborist-service --yaml /mnt/shared/user.yaml + time fence-create sync{{- if .Values.usersync.preserveExistingArboristState }} --preserve-existing-arborist-state{{- end }} --arborist http://arborist-service --yaml /mnt/shared/user.yaml else echo "/mnt/shared/user.yaml did not appear within timeout :-(" false # non-zero exit code @@ -180,7 +180,7 @@ spec: output=$(mktemp "/tmp/fence-create-output_XXXXXX") if [[ -f /mnt/shared/user.yaml && "$ONLY_DBGAP" != "true" ]]; then echo "Running fence-create dbgap-sync with user.yaml - see $output" - time fence-create sync --arborist http://arborist-service --sync_from_dbgap "True" --projects /var/www/fence/projects.yaml --yaml /mnt/shared/user.yaml 2>&1 | tee "$output" + time fence-create sync{{- if .Values.usersync.preserveExistingArboristState }} --preserve-existing-arborist-state{{- end }} --arborist http://arborist-service --sync_from_dbgap "True" --projects /var/www/fence/projects.yaml --yaml /mnt/shared/user.yaml 2>&1 | tee "$output" else echo "Running fence-create dbgap-sync without user.yaml - see $output" time fence-create sync --arborist http://arborist-service --sync_from_dbgap "True" --projects /var/www/fence/projects.yaml 2>&1 | tee "$output" diff --git a/helm/fence/templates/useryaml-job.yaml b/helm/fence/templates/useryaml-job.yaml index 7b1de16d4..c64ad2e50 100644 --- a/helm/fence/templates/useryaml-job.yaml +++ b/helm/fence/templates/useryaml-job.yaml @@ -46,5 +46,5 @@ spec: # can be removed once this is merged: https://github.com/uc-cdis/fence/pull/1096 cd /fence export PYTHONPATH=/fence${PYTHONPATH:+:$PYTHONPATH} - /fence/.venv/bin/python -m bin.fence_create sync --arborist http://arborist-service --yaml /var/www/fence/user.yaml + /fence/.venv/bin/python -m bin.fence_create sync{{- if .Values.usersync.preserveExistingArboristState }} --preserve-existing-arborist-state{{- end }} --arborist http://arborist-service --yaml /var/www/fence/user.yaml restartPolicy: OnFailure diff --git a/helm/fence/values.yaml b/helm/fence/values.yaml index 335806b55..16a92e82a 100644 --- a/helm/fence/values.yaml +++ b/helm/fence/values.yaml @@ -103,6 +103,8 @@ usersync: usersync: true # -- (string) The cron schedule expression to use in the usersync cronjob. Runs every 30 minutes by default. schedule: "*/30 * * * *" + # -- (bool) Preserve existing Arborist groups and policy bindings when applying user.yaml. Requires Fence with --preserve-existing-arborist-state support. + preserveExistingArboristState: false # -- (string) To set a custom image for pulling the user.yaml file from S3. Default is the Gen3 Awshelper image. custom_image: # -- (bool) Whether to sync data from dbGaP. @@ -347,6 +349,7 @@ authzSnapshotCache: image: "redis:7.2-alpine" host: "authz-cache-service" port: 6379 + maxAttempts: 60 secretName: "authz-cache-credentials" passwordKey: "redis-password" @@ -2062,6 +2065,10 @@ FENCE_CONFIG: AUTHZ_SNAPSHOT_CACHE_ENABLED: true AUTHZ_SNAPSHOT_CACHE_REDIS_URL: redis://authz-cache-service:6379/0 AUTHZ_SNAPSHOT_CACHE_TTL_SECONDS: 3600 + AUTHZ_SNAPSHOT_CACHE_CONNECT_TIMEOUT_SECONDS: 0.5 + AUTHZ_SNAPSHOT_CACHE_READ_TIMEOUT_SECONDS: 1.0 + AUTHZ_SNAPSHOT_CACHE_FAILURE_COOLDOWN_SECONDS: 30 + AUTHZ_SNAPSHOT_CACHE_HEALTH_CHECK_INTERVAL_SECONDS: 30 # url where the audit-service is running AUDIT_SERVICE: 'http://audit-service' diff --git a/helm/gecko/templates/_helpers.tpl b/helm/gecko/templates/_helpers.tpl index bcb318a0c..e444a4877 100644 --- a/helm/gecko/templates/_helpers.tpl +++ b/helm/gecko/templates/_helpers.tpl @@ -19,8 +19,13 @@ If release name contains chart name it will be used as a full name. {{- .Release.Name | trunc 63 | trimSuffix "-" }} {{- else }} {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} + {{- end }} {{- end }} + +{{- define "gecko.dbInitName" -}} +gecko {{- end }} {{/* diff --git a/helm/gecko/templates/db-init-job.yaml b/helm/gecko/templates/db-init-job.yaml index 3a15cb46e..8f62d6299 100644 --- a/helm/gecko/templates/db-init-job.yaml +++ b/helm/gecko/templates/db-init-job.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: {{ printf "%s-db-init-%s" (include "gecko.fullname" . | trunc 45 | trimSuffix "-") (include (print $.Template.BasePath "/job-db-init.yaml") . | sha256sum | trunc 8) }} + name: {{ printf "%s-db-init-%s" (include "gecko.dbInitName" . | trunc 45 | trimSuffix "-") (include (print $.Template.BasePath "/job-db-init.yaml") . | sha256sum | trunc 8) }} spec: backoffLimit: 10 template: @@ -146,7 +146,7 @@ spec: volumes: - name: db-init-data-volume configMap: - name: {{ include "gecko.fullname" . }}-db-init-data + name: {{ include "gecko.dbInitName" . }}-db-init-data optional: false {{- with .Values.nodeSelector }} diff --git a/helm/gecko/templates/deployment.yaml b/helm/gecko/templates/deployment.yaml index b6c753f37..956a0a5a3 100644 --- a/helm/gecko/templates/deployment.yaml +++ b/helm/gecko/templates/deployment.yaml @@ -113,18 +113,6 @@ spec: - name: STORAGE_CHAIN_AUDIT_CACHE_ENABLED value: "false" {{- end }} - - name: GRIP_GRAPH - valueFrom: - configMapKeyRef: - name: grip-values - key: graphName - - name: GRIP_HOST - valueFrom: - configMapKeyRef: - name: grip-values - key: serviceName - - name: GRIP_PORT - value: "8202" {{- if .Values.qdrant.enabled }} - name: QDRANT_HOST value: {{ printf "%s-qdrant" .Release.Name | quote }} diff --git a/helm/gecko/templates/job-db-init.yaml b/helm/gecko/templates/job-db-init.yaml index 7a8d199c1..f18a6e09b 100644 --- a/helm/gecko/templates/job-db-init.yaml +++ b/helm/gecko/templates/job-db-init.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "gecko.fullname" . }}-db-init-data + name: {{ include "gecko.dbInitName" . }}-db-init-data data: # Reads content from the chart filesystem at files/init-data/nav.json nav.json: | diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index 2b3b4c5a9..ea51c119b 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -16,10 +16,6 @@ dependencies: version: 0.1.7 repository: "file://../argo-wrapper" condition: argo-wrapper.enabled -- name: audit - version: 0.1.12 - repository: "file://../audit" - condition: audit.enabled - name: aws-es-proxy version: 0.1.9 repository: "file://../aws-es-proxy" @@ -27,10 +23,6 @@ dependencies: - name: common version: 0.1.10 repository: file://../common -- name: etl - version: 0.1.1 - repository: file://../etl - condition: etl.enabled - name: frontend-framework version: 0.1.2 repository: "file://../frontend-framework" @@ -63,18 +55,10 @@ dependencies: version: 0.1.9 repository: "file://../hatchery" condition: hatchery.enabled -- name: indexd - version: 0.1.14 - repository: "file://../indexd" - condition: indexd.enabled - name: kafka version: 32.1.3 repository: "file://../kafka" condition: kafka.enabled -- name: metadata - version: 0.1.12 - repository: "file://../metadata" - condition: metadata.enabled - name: portal version: 0.1.13 repository: "file://../portal" @@ -91,22 +75,14 @@ dependencies: version: 0.1.14 repository: "file://../revproxy" condition: revproxy.enabled -- name: ssjdispatcher - version: 0.1.9 - repository: "file://../ssjdispatcher" - condition: ssjdispatcher.enabled - name: sower version: "0.1.6" condition: sower.enabled repository: "file://../sower" -- name: wts - version: 0.1.13 - repository: "file://../wts" - condition: wts.enabled - name: elasticsearch version: 7.17.3 repository: "https://helm.elastic.co" - condition: global.dev + condition: elasticsearch.enabled - name: postgresql version: 11.9.13 repository: "https://charts.bitnami.com/bitnami" @@ -123,6 +99,10 @@ dependencies: version: 0.1.0 repository: "file://../syfon" condition: syfon.enabled +- name: loom + version: 0.1.0 + repository: "file://../loom" + condition: loom.enabled #- name: qdrant #version: 1.15.4 #repository: "https://qdrant.github.io/qdrant-helm" diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 7351eb327..135ce964b 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -57,6 +57,7 @@ Helm chart to deploy Gen3 Data Commons | aws-es-proxy.secrets | map | `{"awsAccessKeyId":"","awsSecretAccessKey":""}` | Secret information | | aws-es-proxy.secrets.awsAccessKeyId | str | `""` | AWS access key ID for aws-es-proxy | | aws-es-proxy.secrets.awsSecretAccessKey | str | `""` | AWS secret access key for aws-es-proxy | +| elasticsearch.enabled | bool | `true` | Whether to deploy the in-cluster Elasticsearch dependency. | | elasticsearch.clusterHealthCheckParams | string | `"wait_for_status=yellow&timeout=1s"` | | | elasticsearch.clusterName | string | `"gen3-elasticsearch"` | | | elasticsearch.maxUnavailable | int | `0` | | @@ -162,4 +163,3 @@ Helm chart to deploy Gen3 Data Commons | sheepdog.enabled | bool | `true` | Whether to deploy the sheepdog subchart. | | ssjdispatcher.enabled | bool | `false` | Whether to deploy the ssjdispatcher subchart. | | wts.enabled | bool | `true` | Whether to deploy the wts subchart. | - diff --git a/helm/gen3/ci/revproxy-vm-values.yaml b/helm/gen3/ci/revproxy-vm-values.yaml new file mode 100644 index 000000000..ede827e47 --- /dev/null +++ b/helm/gen3/ci/revproxy-vm-values.yaml @@ -0,0 +1,24 @@ +# Example overrides for a VM-hosted external nginx that forwards into the +# in-cluster revproxy over stable NodePorts. +# +# Apply with: +# helm upgrade --install ./helm/gen3 -f helm/gen3/ci/revproxy-vm-values.yaml +# +# Keep local Docker Desktop/kind/minikube setups on their own values file instead +# of adding a generic "vm" switch to the chart. + +revproxy: + replicaCount: 2 + service: + type: NodePort + port: + - protocol: TCP + port: 80 + targetPort: 80 + nodePort: 31551 + name: http + - protocol: TCP + port: 443 + targetPort: 443 + nodePort: 30912 + name: https diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml index 6aa840aa2..bea5bc28c 100644 --- a/helm/gen3/values.yaml +++ b/helm/gen3/values.yaml @@ -73,10 +73,12 @@ global: # Dependancy Charts -ambassador: - # -- (bool) Whether to deploy the ambassador subchart. +elasticsearch: + # -- (bool) Whether to deploy the in-cluster Elasticsearch dependency. + # Set this to false when using an external Elasticsearch/OpenSearch service. enabled: true + arborist: # -- (bool) Whether to deploy the arborist subchart. enabled: true @@ -85,9 +87,6 @@ argo-wrapper: # -- (bool) Whether to deploy the argo-wrapper subchart. enabled: false -audit: - # -- (bool) Whether to deploy the audit subchart. - enabled: true aws-es-proxy: @@ -102,16 +101,6 @@ aws-es-proxy: # -- (str) AWS secret access key for aws-es-proxy awsSecretAccessKey: "" -etl-pod: - # -- (bool) Whether to deploy the etl-pod subchart. - enabled: true - # -- (map) Docker image information. - image: - # -- (string) The Docker image repository for the etl-po service. - repository: - # -- (string) Overrides the image tag whose default is the chart appVersion. - tag: - fence: # -- (bool) Whether to deploy the fence subchart. enabled: true @@ -121,6 +110,8 @@ fence: usersync: false # -- (string) The cron schedule expression to use in the usersync cronjob. Runs every 30 minutes by default. schedule: "*/30 * * * *" + # -- (bool) Preserve existing Arborist groups and policy bindings when applying user.yaml. Requires Fence with --preserve-existing-arborist-state support. + preserveExistingArboristState: false # -- (bool) Whether to sync data from dbGaP. syncFromDbgap: false # -- (bool) Force attempting a dbgap sync if "true", falls back on user.yaml @@ -157,7 +148,7 @@ guppy: hatchery: # -- (bool) Whether to deploy the hatchery subchart. - enabled: true + enabled: false hatchery: sidecarContainer: @@ -221,35 +212,10 @@ hatchery: user-volume-location: "/home/jovyan/pd" gen3-volume-location: "/home/jovyan/.gen3" - -indexd: - # -- (bool) Whether to deploy the indexd subchart. - enabled: true - # -- (string) the default prefix for indexd records - defaultPrefix: "PREFIX/" - - -manifestservice: - # -- (bool) Whether to deploy the manifest service subchart. - enabled: true - -metadata: - # -- (bool) Whether to deploy the metadata subchart. - enabled: true - -peregrine: - # -- (bool) Whether to deploy the peregrine subchart. - enabled: true - -pidgin: - # -- (bool) Whether to deploy the pidgin subchart. - enabled: true - portal: # -- (bool) Whether to deploy the portal subchart. enabled: true - requestor: # -- (bool) Whether to deploy the requestor subchart. enabled: true @@ -263,6 +229,7 @@ requestor: redis: # -- (bool) Whether to deploy the shared authz cache subchart. enabled: true + fullnameOverride: redis revproxy: # -- (bool) Whether to deploy the revproxy subchart. @@ -281,25 +248,23 @@ revproxy: # -- (list) To secure an Ingress by specifying a secret that contains a TLS private key and certificate. tls: [] -sheepdog: - # -- (bool) Whether to deploy the sheepdog subchart. - enabled: true - -ssjdispatcher: - # -- (bool) Whether to deploy the ssjdispatcher subchart. - enabled: false - -wts: - # -- (bool) Whether to deploy the wts subchart. - enabled: true syfon: # -- (bool) Whether to deploy the syfon subchart. - enabled: false + enabled: true + +loom: + enabled: true + server: + clickhouse: + # -- (bool) Whether Loom should connect to ClickHouse. + enabled: true + # -- (string) ClickHouse native URL used by Loom when enabled. + url: clickhouse://clickhouse:9000 gecko: + enabled: true qdrant: - # -- (bool) Whether Gecko should connect to Qdrant. enabled: false # Disable persistence by default so we can spin up and down ephemeral environments @@ -327,7 +292,7 @@ qdrant: cpu: 200m memory: 256Mi persistence: - enabled: true + enabled: false storageClass: "qdrant-manual-storage" size: 26Gi accessModes: diff --git a/helm/audit/.helmignore b/helm/gen3workflow/.helmignore similarity index 100% rename from helm/audit/.helmignore rename to helm/gen3workflow/.helmignore diff --git a/helm/etl-pod/Chart.yaml b/helm/gen3workflow/Chart.yaml similarity index 95% rename from helm/etl-pod/Chart.yaml rename to helm/gen3workflow/Chart.yaml index 2eca1a916..3c143bbcc 100644 --- a/helm/etl-pod/Chart.yaml +++ b/helm/gen3workflow/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: etl-pod +name: gen3workflow description: A Helm chart for Kubernetes # A chart can be either an 'application' or a 'library' chart. @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.1.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.16.0" +appVersion: "master" diff --git a/helm/gen3workflow/README.md b/helm/gen3workflow/README.md new file mode 100644 index 000000000..463763a60 --- /dev/null +++ b/helm/gen3workflow/README.md @@ -0,0 +1,54 @@ +# gen3workflow + +![Version: 0.1.1](https://img.shields.io/badge/Version-0.1.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) + +A Helm chart for Kubernetes + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| autoscaling.enabled | bool | `false` | | +| autoscaling.maxReplicas | int | `100` | | +| autoscaling.minReplicas | int | `1` | | +| autoscaling.targetCPUUtilizationPercentage | int | `80` | | +| fullnameOverride | string | `""` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"quay.io/cdis/gen3-workflow"` | | +| image.tag | string | `""` | | +| imagePullSecrets | list | `[]` | | +| ingress.annotations | object | `{}` | | +| ingress.className | string | `""` | | +| ingress.enabled | bool | `false` | | +| ingress.hosts[0].host | string | `"chart-example.local"` | | +| ingress.hosts[0].paths[0].path | string | `"/"` | | +| ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` | | +| ingress.tls | list | `[]` | | +| livenessProbe.httpGet.path | string | `"/_version"` | | +| livenessProbe.httpGet.port | int | `8000` | | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| podSecurityContext | object | `{}` | | +| readinessProbe.httpGet.path | string | `"/_version"` | | +| readinessProbe.httpGet.port | int | `8000` | | +| replicaCount | int | `1` | | +| resources | object | `{}` | | +| securityContext | object | `{}` | | +| service.port | int | `80` | | +| service.type | string | `"ClusterIP"` | | +| serviceAccount.annotations | object | `{}` | | +| serviceAccount.automount | bool | `true` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `""` | | +| tolerations | list | `[]` | | +| volumeMounts[0].mountPath | string | `"/home/gen3/.gen3/gen3-workflow"` | | +| volumeMounts[0].name | string | `"config-volume"` | | +| volumeMounts[0].readOnly | bool | `true` | | +| volumes[0].configMap.name | string | `"gen3workflow-config"` | | +| volumes[0].name | string | `"config-volume"` | | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) diff --git a/helm/etl-pod/templates/NOTES.txt b/helm/gen3workflow/templates/NOTES.txt similarity index 73% rename from helm/etl-pod/templates/NOTES.txt rename to helm/gen3workflow/templates/NOTES.txt index b57cfd518..5e5d59ebe 100644 --- a/helm/etl-pod/templates/NOTES.txt +++ b/helm/gen3workflow/templates/NOTES.txt @@ -6,16 +6,16 @@ {{- end }} {{- end }} {{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "etl-pod.fullname" . }}) + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "gen3workflow.fullname" . }}) export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") echo http://$NODE_IP:$NODE_PORT {{- else if contains "LoadBalancer" .Values.service.type }} NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "etl-pod.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "etl-pod.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + You can watch its status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "gen3workflow.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "gen3workflow.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") echo http://$SERVICE_IP:{{ .Values.service.port }} {{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "etl-pod.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "gen3workflow.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") echo "Visit http://127.0.0.1:8080 to use your application" kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT diff --git a/helm/etl-pod/templates/_helpers.tpl b/helm/gen3workflow/templates/_helpers.tpl similarity index 73% rename from helm/etl-pod/templates/_helpers.tpl rename to helm/gen3workflow/templates/_helpers.tpl index 0f6d4f227..88040ef4c 100644 --- a/helm/etl-pod/templates/_helpers.tpl +++ b/helm/gen3workflow/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{/* Expand the name of the chart. */}} -{{- define "etl-pod.name" -}} +{{- define "gen3workflow.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} @@ -10,7 +10,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "etl-pod.fullname" -}} +{{- define "gen3workflow.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} @@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "etl-pod.chart" -}} +{{- define "gen3workflow.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} -{{- define "etl-pod.labels" -}} -helm.sh/chart: {{ include "etl-pod.chart" . }} -{{ include "etl-pod.selectorLabels" . }} +{{- define "gen3workflow.labels" -}} +helm.sh/chart: {{ include "gen3workflow.chart" . }} +{{ include "gen3workflow.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -45,17 +45,17 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{/* Selector labels */}} -{{- define "etl-pod.selectorLabels" -}} -app.kubernetes.io/name: {{ include "etl-pod.name" . }} +{{- define "gen3workflow.selectorLabels" -}} +app.kubernetes.io/name: {{ include "gen3workflow.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* Create the name of the service account to use */}} -{{- define "etl-pod.serviceAccountName" -}} +{{- define "gen3workflow.serviceAccountName" -}} {{- if .Values.serviceAccount.create }} -{{- default (include "etl-pod.fullname" .) .Values.serviceAccount.name }} +{{- default (include "gen3workflow.fullname" .) .Values.serviceAccount.name }} {{- else }} {{- default "default" .Values.serviceAccount.name }} {{- end }} diff --git a/helm/gen3workflow/templates/deployment.yaml b/helm/gen3workflow/templates/deployment.yaml new file mode 100644 index 000000000..7bebf45cc --- /dev/null +++ b/helm/gen3workflow/templates/deployment.yaml @@ -0,0 +1,78 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "gen3workflow.fullname" . }} + labels: + {{- include "gen3workflow.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "gen3workflow.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "gen3workflow.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "gen3workflow.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/dicom-server/templates/hpa.yaml b/helm/gen3workflow/templates/hpa.yaml similarity index 58% rename from helm/dicom-server/templates/hpa.yaml rename to helm/gen3workflow/templates/hpa.yaml index bd8ab9c0e..7d84e501d 100644 --- a/helm/dicom-server/templates/hpa.yaml +++ b/helm/gen3workflow/templates/hpa.yaml @@ -2,14 +2,14 @@ apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: - name: {{ include "dicom-server.fullname" . }} + name: {{ include "gen3workflow.fullname" . }} labels: - {{- include "dicom-server.labels" . | nindent 4 }} + {{- include "gen3workflow.labels" . | nindent 4 }} spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: {{ include "dicom-server.fullname" . }} + name: {{ include "gen3workflow.fullname" . }} minReplicas: {{ .Values.autoscaling.minReplicas }} maxReplicas: {{ .Values.autoscaling.maxReplicas }} metrics: @@ -17,12 +17,16 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} diff --git a/helm/gen3workflow/templates/ingress.yaml b/helm/gen3workflow/templates/ingress.yaml new file mode 100644 index 000000000..4363bda46 --- /dev/null +++ b/helm/gen3workflow/templates/ingress.yaml @@ -0,0 +1,43 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "gen3workflow.fullname" . }} + labels: + {{- include "gen3workflow.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- with .pathType }} + pathType: {{ . }} + {{- end }} + backend: + service: + name: {{ include "gen3workflow.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/ssjdispatcher/templates/service.yaml b/helm/gen3workflow/templates/service.yaml similarity index 55% rename from helm/ssjdispatcher/templates/service.yaml rename to helm/gen3workflow/templates/service.yaml index 30ee24a0f..4a2aa5e5d 100644 --- a/helm/ssjdispatcher/templates/service.yaml +++ b/helm/gen3workflow/templates/service.yaml @@ -1,9 +1,9 @@ apiVersion: v1 kind: Service metadata: - name: ssjdispatcher-service + name: {{ include "gen3workflow.fullname" . }} labels: - {{- include "ssjdispatcher.labels" . | nindent 4 }} + {{- include "gen3workflow.labels" . | nindent 4 }} spec: type: {{ .Values.service.type }} ports: @@ -12,4 +12,4 @@ spec: protocol: TCP name: http selector: - {{- include "ssjdispatcher.selectorLabels" . | nindent 4 }} + {{- include "gen3workflow.selectorLabels" . | nindent 4 }} diff --git a/helm/etl-pod/templates/serviceaccount.yaml b/helm/gen3workflow/templates/serviceaccount.yaml similarity index 54% rename from helm/etl-pod/templates/serviceaccount.yaml rename to helm/gen3workflow/templates/serviceaccount.yaml index f95d262cb..9213e1d49 100644 --- a/helm/etl-pod/templates/serviceaccount.yaml +++ b/helm/gen3workflow/templates/serviceaccount.yaml @@ -2,11 +2,12 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "etl-pod.serviceAccountName" . }} + name: {{ include "gen3workflow.serviceAccountName" . }} labels: - {{- include "etl-pod.labels" . | nindent 4 }} + {{- include "gen3workflow.labels" . | nindent 4 }} {{- with .Values.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} {{- end }} diff --git a/helm/dicom-server/templates/tests/test-connection.yaml b/helm/gen3workflow/templates/tests/test-connection.yaml similarity index 53% rename from helm/dicom-server/templates/tests/test-connection.yaml rename to helm/gen3workflow/templates/tests/test-connection.yaml index 46fe07351..e973565f1 100644 --- a/helm/dicom-server/templates/tests/test-connection.yaml +++ b/helm/gen3workflow/templates/tests/test-connection.yaml @@ -1,9 +1,9 @@ apiVersion: v1 kind: Pod metadata: - name: "dicom-server-test-connection" + name: "{{ include "gen3workflow.fullname" . }}-test-connection" labels: - {{- include "dicom-server.labels" . | nindent 4 }} + {{- include "gen3workflow.labels" . | nindent 4 }} annotations: "helm.sh/hook": test spec: @@ -11,5 +11,5 @@ spec: - name: wget image: busybox command: ['wget'] - args: ['{{ include "dicom-server.fullname" . }}:{{ .Values.service.port }}'] + args: ['{{ include "gen3workflow.fullname" . }}:{{ .Values.service.port }}'] restartPolicy: Never diff --git a/helm/gen3workflow/values.yaml b/helm/gen3workflow/values.yaml new file mode 100644 index 000000000..5121fe8b7 --- /dev/null +++ b/helm/gen3workflow/values.yaml @@ -0,0 +1,129 @@ +# Default values for gen3workflow. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ +replicaCount: 1 + +# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ +image: + repository: quay.io/cdis/gen3-workflow + # This sets the pull policy for images. + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] +# This is to override the chart name. +nameOverride: "" +fullnameOverride: "" + +# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/ +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# This is for setting Kubernetes Annotations to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} +# This is for setting Kubernetes Labels to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/ +service: + # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: ClusterIP + # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports + port: 80 + +# This block is for setting up the ingress for more information can be found here: https://kubernetes.io/docs/concepts/services-networking/ingress/ +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +livenessProbe: + httpGet: + path: /_version + port: 8000 +readinessProbe: + httpGet: + path: /_version + port: 8000 + +# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# Additional volumes on the output Deployment definition. +volumes: +- name: config-volume + configMap: + name: gen3workflow-config +# - name: foo +# secret: +# secretName: mysecret +# optional: false + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: +- name: config-volume + mountPath: /home/gen3/.gen3/gen3-workflow + readOnly: true +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/helm/indexd/.helmignore b/helm/indexd/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/helm/indexd/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm/indexd/Chart.yaml b/helm/indexd/Chart.yaml deleted file mode 100644 index 19e781264..000000000 --- a/helm/indexd/Chart.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: v2 -name: indexd -description: A Helm chart for gen3 indexd - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.14 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "master" - - -dependencies: -- name: common - version: 0.1.10 - repository: file://../common -- name: postgresql - version: 11.9.13 - repository: "https://charts.bitnami.com/bitnami" - condition: postgres.separate diff --git a/helm/indexd/README.md b/helm/indexd/README.md deleted file mode 100644 index 8d7057cb4..000000000 --- a/helm/indexd/README.md +++ /dev/null @@ -1,109 +0,0 @@ -# indexd - -![Version: 0.1.14](https://img.shields.io/badge/Version-0.1.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) - -A Helm chart for gen3 indexd - -## Requirements - -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.10 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | - -## Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| affinity | map | `{}` | Affinity to use for the deployment. | -| autoscaling | map | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Autoscaling options. | -| autoscaling.maxReplicas | int | `100` | Maximum number of replicas | -| autoscaling.minReplicas | int | `1` | Minimum number of replicas | -| autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage | -| commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | -| criticalService | string | `"true"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | -| datadogLogsInjection | bool | `true` | If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. | -| datadogProfilingEnabled | bool | `true` | If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. | -| datadogTraceSampleRate | int | `1` | A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. | -| defaultPrefix | string | `"PREFIX/"` | default prefix for indexd | -| env | list | `[{"name":"ARBORIST","value":"true"},{"name":"GEN3_DEBUG","value":"False"}]` | Environment variables to pass to the container | -| externalSecrets | map | `{"createK8sServiceCredsSecret":false,"dbcreds":null,"serviceCreds":"indexd-service-creds"}` | External Secrets settings. | -| externalSecrets.createK8sServiceCredsSecret | string | `false` | Will create the Helm "indexd-service-creds" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. | -| externalSecrets.dbcreds | string | `nil` | Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" | -| fullnameOverride | string | `""` | Override the full name of the deployment. | -| global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false}` | AWS configuration | -| global.aws.awsAccessKeyId | string | `nil` | Credentials for AWS stuff. | -| global.aws.awsSecretAccessKey | string | `nil` | Credentials for AWS stuff. | -| global.aws.enabled | bool | `false` | Set to true if deploying to AWS. Controls ingress annotations. | -| global.ddEnabled | bool | `false` | Whether Datadog is enabled. | -| global.dev | bool | `true` | Whether the deployment is for development purposes. | -| global.dictionaryUrl | string | `"https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json"` | URL of the data dictionary. | -| global.dispatcherJobNum | int | `"10"` | Number of dispatcher jobs. | -| global.environment | string | `"default"` | Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. | -| global.externalSecrets | map | `{"deploy":false,"separateSecretStore":false}` | External Secrets settings. | -| global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any indexd secrets you have deployed. | -| global.externalSecrets.separateSecretStore | string | `false` | Will deploy a separate External Secret Store for this service. | -| global.hostname | string | `"localhost"` | Hostname for the deployment. | -| global.kubeBucket | string | `"kube-gen3"` | S3 bucket name for Kubernetes manifest files. | -| global.logsBucket | string | `"logs-gen3"` | S3 bucket name for log files. | -| global.minAvialable | int | `1` | The minimum amount of pods that are available at all times if the PDB is deployed. | -| global.netPolicy | bool | `true` | Whether network policies are enabled. | -| global.pdb | bool | `false` | If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. | -| global.portalApp | string | `"gitops"` | Portal application name. | -| global.postgres.dbCreate | bool | `true` | Whether the database should be created. | -| global.postgres.externalSecret | string | `""` | Name of external secret. Disabled if empty | -| global.postgres.master | map | `{"host":null,"password":null,"port":"5432","username":"postgres"}` | Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres | -| global.postgres.master.host | string | `nil` | hostname of postgres server | -| global.postgres.master.password | string | `nil` | password for superuser in postgres. This is used to create or restore databases | -| global.postgres.master.port | string | `"5432"` | Port for Postgres. | -| global.postgres.master.username | string | `"postgres"` | username of superuser in postgres. This is used to create or restore databases | -| global.publicDataSets | bool | `true` | Whether public datasets are enabled. | -| global.revproxyArn | string | `"arn:aws:acm:us-east-1:123456:certificate"` | ARN of the reverse proxy certificate. | -| global.tierAccessLevel | string | `"libre"` | Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` | -| global.tierAccessLimit | int | `"1000"` | Only relevant if tireAccessLevel is set to "regular". Summary charts below this limit will not appear for aggregated data. | -| image | map | `{"pullPolicy":"IfNotPresent","repository":"quay.io/cdis/indexd","tag":""}` | Docker image information. | -| image.pullPolicy | string | `"IfNotPresent"` | When to pull the image. | -| image.repository | string | `"quay.io/cdis/indexd"` | The Docker image repository for the indexd service | -| image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | -| imagePullSecrets | list | `[]` | Docker image pull secrets. | -| nameOverride | string | `""` | Override the name of the chart. | -| nodeSelector | map | `{}` | Node Selector for the pods | -| partOf | string | `"S3-GS"` | Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. | -| podAnnotations | map | `{}` | Annotations to add to the pod | -| podSecurityContext | map | `{}` | Security context for the pod | -| postgres | map | `{"database":null,"dbCreate":null,"dbRestore":false,"host":null,"password":null,"port":"5432","separate":false,"username":null}` | Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you | -| postgres.database | string | `nil` | Database name for postgres. This is a service override, defaults to - | -| postgres.dbCreate | bool | `nil` | Whether the database should be created. Default to global.postgres.dbCreate | -| postgres.host | string | `nil` | Hostname for postgres server. This is a service override, defaults to global.postgres.host | -| postgres.password | string | `nil` | Password for Postgres. Will be autogenerated if left empty. | -| postgres.port | string | `"5432"` | Port for Postgres. | -| postgres.separate | string | `false` | Will create a Database for the individual service to help with developing it. | -| postgres.username | string | `nil` | Username for postgres. This is a service override, defaults to - | -| postgresql | map | `{"primary":{"persistence":{"enabled":false}}}` | Postgresql subchart settings if deployed separately option is set to "true". Disable persistence by default so we can spin up and down ephemeral environments | -| postgresql.primary.persistence.enabled | bool | `false` | Option to persist the dbs data. | -| release | string | `"production"` | Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". | -| replicaCount | int | `1` | Number of desired replicas | -| resources | map | `{"limits":{"cpu":1,"memory":"512Mi"},"requests":{"cpu":0.1,"memory":"12Mi"}}` | Resource requests and limits for the containers in the pod | -| resources.limits | map | `{"cpu":1,"memory":"512Mi"}` | The maximum amount of resources that the container is allowed to use | -| resources.limits.cpu | string | `1` | The maximum amount of CPU the container can use | -| resources.limits.memory | string | `"512Mi"` | The maximum amount of memory the container can use | -| resources.requests | map | `{"cpu":0.1,"memory":"12Mi"}` | The amount of resources that the container requests | -| resources.requests.cpu | string | `0.1` | The amount of CPU requested | -| resources.requests.memory | string | `"12Mi"` | The amount of memory requested | -| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"userdb":{"fence":null,"sheepdog":null}}` | Values for indexd secret. | -| secrets.awsAccessKeyId | str | `nil` | AWS access key ID to access the db restore job S3 bucket. Overrides global key. | -| secrets.awsSecretAccessKey | str | `nil` | AWS secret access key ID to access the db restore job S3 bucket. Overrides global key. | -| securityContext | map | `{}` | Security context for the containers in the pod | -| selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | -| service | map | `{"port":80,"type":"ClusterIP"}` | Kubernetes service information. | -| service.port | int | `80` | The port number that the service exposes. | -| service.type | string | `"ClusterIP"` | Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". | -| serviceAccount | map | `{"annotations":{},"create":false,"name":""}` | Service account to use or create. | -| serviceAccount.annotations | map | `{}` | Annotations to add to the service account. | -| serviceAccount.create | bool | `false` | Specifies whether a service account should be created. | -| serviceAccount.name | string | `""` | The name of the service account | -| tolerations | list | `[]` | Tolerations for the pods | -| uwsgi | map | `{"listen":1024}` | Values for overriding uwsgi settings | -| volumeMounts | list | `[{"mountPath":"/var/www/indexd/local_settings.py","name":"config-volume","readOnly":true,"subPath":"local_settings.py"}]` | Volumes to mount to the container. | -| volumes | list | `[{"configMap":{"name":"indexd-uwsgi"},"name":"uwsgi-config"},{"name":"config-volume","secret":{"secretName":"indexd-settings"}}]` | Volumes to attach to the pod | - diff --git a/helm/indexd/indexd-settings/local_settings.py b/helm/indexd/indexd-settings/local_settings.py deleted file mode 100644 index 30fc5f906..000000000 --- a/helm/indexd/indexd-settings/local_settings.py +++ /dev/null @@ -1,69 +0,0 @@ -from indexd.index.drivers.alchemy import SQLAlchemyIndexDriver -from indexd.alias.drivers.alchemy import SQLAlchemyAliasDriver -from indexd.auth.drivers.alchemy import SQLAlchemyAuthDriver - - -from os import environ -import json - -APP_NAME = "indexd" - - -usr = environ.get("PGUSER", "indexd") -db = environ.get("PGDB", "indexd") -psw = environ.get("PGPASSWORD") -pghost = environ.get("PGHOST") -pgport = environ.get("PGPORT", 5432) - -# TODO: FIX THIS TO READ FROM ENV VARS -index_config = { - "DEFAULT_PREFIX": environ.get("DEFAULT_PREFIX", "testprefix/"), - "PREPEND_PREFIX": environ.get("PREPEND_PREFIX", True), -} - -CONFIG = {} - -CONFIG["JSONIFY_PRETTYPRINT_REGULAR"] = False - -dist = environ.get("DIST", None) -if dist: - CONFIG["DIST"] = json.loads(dist) - -arborist = environ.get("ARBORIST", "false").lower() == "true" - -CONFIG["INDEX"] = { - "driver": SQLAlchemyIndexDriver( - "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format( - usr=usr, psw=psw, pghost=pghost, pgport=pgport, db=db - ), - index_config=index_config, - ) -} - -CONFIG["ALIAS"] = { - "driver": SQLAlchemyAliasDriver( - "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format( - usr=usr, psw=psw, pghost=pghost, pgport=pgport, db=db - ) - ) -} - -if arborist: - AUTH = SQLAlchemyAuthDriver( - "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format( - usr=usr, psw=psw, pghost=pghost, pgport=pgport, db=db - ), - arborist="http://arborist-service/", - ) -else: - AUTH = SQLAlchemyAuthDriver( - "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format( - usr=usr, psw=psw, pghost=pghost, pgport=pgport, db=db - ) - ) - -rbac = environ.get("RBAC", "false").lower() == "true" -if rbac: - CONFIG["RBAC"] = True - -settings = {"config": CONFIG, "auth": AUTH} diff --git a/helm/indexd/templates/NOTES.txt b/helm/indexd/templates/NOTES.txt deleted file mode 100644 index c1e7e1aef..000000000 --- a/helm/indexd/templates/NOTES.txt +++ /dev/null @@ -1 +0,0 @@ -{{ .Chart.Name }} has been deployed successfully. diff --git a/helm/indexd/templates/_helpers.tpl b/helm/indexd/templates/_helpers.tpl deleted file mode 100644 index 5778d5100..000000000 --- a/helm/indexd/templates/_helpers.tpl +++ /dev/null @@ -1,104 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "indexd.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "indexd.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "indexd.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "indexd.labels" -}} -{{- if .Values.commonLabels }} - {{- with .Values.commonLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.commonLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "indexd.selectorLabels" -}} -{{- if .Values.selectorLabels }} - {{- with .Values.selectorLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.selectorLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "indexd.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "indexd.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} - - -{{/* - Postgres Password lookup -*/}} -{{- define "indexd.postgres.password" -}} -{{- $localpass := (lookup "v1" "Secret" "postgres" "postgres-postgresql" ) -}} -{{- if $localpass }} -{{- default (index $localpass.data "postgres-password" | b64dec) }} -{{- else }} -{{- default .Values.postgres.password }} -{{- end }} -{{- end }} - - - -{{/* - Indexd Fence Creds -*/}} -{{- define "indexd-fence-creds" -}} -{{- default (randAlphaNum 32) .Values.secrets.userdb.fence }} -{{- end }} - -{{/* - Indexd sheepdog Creds -*/}} -{{- define "indexd-sheepdog-creds" -}} -{{- default (randAlphaNum 32) .Values.secrets.userdb.sheepdog }} -{{- end }} - -{{/* - Indexd Gateway Creds -*/}} -{{- define "indexd-gateway-creds" -}} -{{- default (randAlphaNum 32) .Values.secrets.userdb.gateway }} -{{- end }} diff --git a/helm/indexd/templates/aws-config.yaml b/helm/indexd/templates/aws-config.yaml deleted file mode 100644 index 4723e6b37..000000000 --- a/helm/indexd/templates/aws-config.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if or (.Values.secrets.awsSecretAccessKey) (.Values.global.aws.awsSecretAccessKey ) }} -{{ include "common.awsconfig" . }} -{{- end }} \ No newline at end of file diff --git a/helm/indexd/templates/db-init.yaml b/helm/indexd/templates/db-init.yaml deleted file mode 100644 index 0393aa732..000000000 --- a/helm/indexd/templates/db-init.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{ include "common.db-secret" . }} ---- -{{ include "common.db_setup_sa" . }} ---- -{{- if .Values.dbRestore }} -{{ include "common.s3_pg_restore" . }} -{{- else }} -{{ include "common.db_setup_job" . }} -{{- end -}} \ No newline at end of file diff --git a/helm/indexd/templates/deployment.yaml b/helm/indexd/templates/deployment.yaml deleted file mode 100644 index ac1698545..000000000 --- a/helm/indexd/templates/deployment.yaml +++ /dev/null @@ -1,118 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: indexd-deployment - labels: - {{- include "indexd.labels" . | nindent 4 }} - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 4 }} - {{- end }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "indexd.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "indexd.selectorLabels" . | nindent 8 }} - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "indexd.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: indexd - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogEnvVar" . | nindent 12 }} - {{- end }} - - name: PGHOST - valueFrom: - secretKeyRef: - name: indexd-dbcreds - key: host - optional: false - - name: PGUSER - valueFrom: - secretKeyRef: - name: indexd-dbcreds - key: username - optional: false - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: indexd-dbcreds - key: password - optional: false - - name: PGDB - valueFrom: - secretKeyRef: - name: indexd-dbcreds - key: database - optional: false - - name: DBREADY - valueFrom: - secretKeyRef: - name: indexd-dbcreds - key: dbcreated - optional: false - - name: DEFAULT_PREFIX - value: {{ .Values.defaultPrefix }} - {{- toYaml .Values.env | nindent 12 }} - volumeMounts: - - name: "uwsgi-config" - mountPath: "/etc/uwsgi/uwsgi.ini" - subPath: uwsgi.ini - - name: "config-volume" - readOnly: true - mountPath: "/var/www/indexd/local_settings.py" - subPath: "local_settings.py" - ports: - - name: http - containerPort: 80 - protocol: TCP - livenessProbe: - httpGet: - path: /_status - port: http - initialDelaySeconds: 30 - periodSeconds: 10 - readinessProbe: - httpGet: - path: /_status - port: http - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/indexd/templates/external-secrets.yaml b/helm/indexd/templates/external-secrets.yaml deleted file mode 100644 index b8cb2a385..000000000 --- a/helm/indexd/templates/external-secrets.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{ include "common.externalSecret.db" . }} ---- -{{ if .Values.global.externalSecrets.deploy }} -apiVersion: external-secrets.io/v1beta1 -kind: ExternalSecret -metadata: - name: indexd-service-creds -spec: - refreshInterval: 5m - secretStoreRef: - name: {{include "common.SecretStore" .}} - kind: SecretStore - target: - name: indexd-service-creds - creationPolicy: Owner - dataFrom: - - extract: - key: {{ .Values.externalSecrets.serviceCreds }} - conversionStrategy: Default - decodingStrategy: None -{{- end }} \ No newline at end of file diff --git a/helm/indexd/templates/hpa.yaml b/helm/indexd/templates/hpa.yaml deleted file mode 100644 index 81742fe8c..000000000 --- a/helm/indexd/templates/hpa.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "indexd.fullname" . }} - labels: - {{- include "indexd.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "indexd.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/helm/indexd/templates/indexd-secret.yaml b/helm/indexd/templates/indexd-secret.yaml deleted file mode 100644 index 5b296ccfd..000000000 --- a/helm/indexd/templates/indexd-secret.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: indexd-settings -type: Opaque -data: -{{ (.Files.Glob "indexd-settings/*").AsSecrets | indent 2 }} ---- -{{- if or (not .Values.global.externalSecrets.deploy) (and .Values.global.externalSecrets.deploy .Values.externalSecrets.createK8sServiceCredsSecret) }} -apiVersion: v1 -kind: Secret -metadata: - name: indexd-service-creds -type: Opaque -data: - fence: {{ include "common.getOrGenSecret" (list .Values.secrets.userdb.fence "indexd-service-creds" "fence" 20 .Release.Namespace) }} - sheepdog: {{ include "common.getOrGenSecret" (list .Values.secrets.userdb.sheepdog "indexd-service-creds" "sheepdog" 20 .Release.Namespace) }} -{{- end }} \ No newline at end of file diff --git a/helm/indexd/templates/pdb.yaml b/helm/indexd/templates/pdb.yaml deleted file mode 100644 index 2ef2de13d..000000000 --- a/helm/indexd/templates/pdb.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if and .Values.global.pdb (gt (int .Values.replicaCount) 1) }} -{{ include "common.pod_disruption_budget" . }} -{{- end }} \ No newline at end of file diff --git a/helm/indexd/templates/secret-store.yaml b/helm/indexd/templates/secret-store.yaml deleted file mode 100644 index 771c7760d..000000000 --- a/helm/indexd/templates/secret-store.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{ if .Values.global.externalSecrets.separateSecretStore }} -{{ include "common.secretstore" . }} -{{- end }} \ No newline at end of file diff --git a/helm/indexd/templates/service.yaml b/helm/indexd/templates/service.yaml deleted file mode 100644 index 6fb671d1d..000000000 --- a/helm/indexd/templates/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: indexd-service - labels: - {{- include "indexd.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "indexd.selectorLabels" . | nindent 4 }} diff --git a/helm/indexd/templates/serviceaccount.yaml b/helm/indexd/templates/serviceaccount.yaml deleted file mode 100644 index 106deba9d..000000000 --- a/helm/indexd/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "indexd.serviceAccountName" . }} - labels: - {{- include "indexd.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/helm/indexd/templates/tests/test-connection.yaml b/helm/indexd/templates/tests/test-connection.yaml deleted file mode 100644 index fc5d39350..000000000 --- a/helm/indexd/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "indexd-test-connection" - labels: - {{- include "indexd.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['indexd-service:{{ .Values.service.port }}/_status'] - restartPolicy: Never diff --git a/helm/indexd/templates/uwsgi.yaml b/helm/indexd/templates/uwsgi.yaml deleted file mode 100644 index a6eef58c1..000000000 --- a/helm/indexd/templates/uwsgi.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: indexd-uwsgi -data: - uwsgi.ini: | - [uwsgi] - protocol = uwsgi - socket = /var/run/gen3/uwsgi.sock - buffer-size = 32768 - uid = nginx - gid = nginx - chown-socket = nginx:nginx - chmod-socket = 666 - master = true - harakiri-verbose = true - # No global HARAKIRI, using only user HARAKIRI, because export overwrites it - # Cannot overwrite global HARAKIRI with user's: https://git.io/fjYuD - # harakiri = 45 - ; If VIRTUAL_ENV is set then use its value to specify the virtualenv directory - if-env = VIRTUAL_ENV - virtualenv = %(_) - endif = - http-timeout = 45 - socket-timeout = 45 - worker-reload-mercy = 45 - reload-mercy = 45 - mule-reload-mercy = 45 - disable-logging = true - wsgi-file=/indexd/wsgi.py - plugins = python3 - vacuum = true - pythonpath = /indexd/ - stats = 127.0.0.1:9191 - stats-http = true - env = prometheus_multiproc_dir=/var/tmp/uwsgi_flask_metrics - exec-asap = /indexd/clear_prometheus_multiproc /var/tmp/uwsgi_flask_metrics - # Initialize application in worker processes, not master. This prevents the - # workers from all trying to open the same database connections at startup. - lazy = true - lazy-apps = true - listen = {{ .Values.uwsgi.listen }} diff --git a/helm/indexd/values.yaml b/helm/indexd/values.yaml deleted file mode 100644 index 60e51717f..000000000 --- a/helm/indexd/values.yaml +++ /dev/null @@ -1,254 +0,0 @@ -# Default values for indexd. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - - -# Global configuration -global: - # -- (map) AWS configuration - aws: - # -- (bool) Set to true if deploying to AWS. Controls ingress annotations. - enabled: false - # -- (string) Credentials for AWS stuff. - awsAccessKeyId: - # -- (string) Credentials for AWS stuff. - awsSecretAccessKey: - # -- (bool) Whether the deployment is for development purposes. - dev: true - - postgres: - # -- (bool) Whether the database should be created. - dbCreate: true - # -- (string) Name of external secret. Disabled if empty - externalSecret: "" - # -- (map) Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres - master: - # -- (string) hostname of postgres server - host: - # -- (string) username of superuser in postgres. This is used to create or restore databases - username: postgres - # -- (string) password for superuser in postgres. This is used to create or restore databases - password: - # -- (string) Port for Postgres. - port: "5432" - # -- (string) Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. - environment: default - # -- (string) Hostname for the deployment. - hostname: localhost - # -- (string) ARN of the reverse proxy certificate. - revproxyArn: arn:aws:acm:us-east-1:123456:certificate - # -- (string) URL of the data dictionary. - dictionaryUrl: https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json - # -- (string) Portal application name. - portalApp: gitops - # -- (string) S3 bucket name for Kubernetes manifest files. - kubeBucket: kube-gen3 - # -- (string) S3 bucket name for log files. - logsBucket: logs-gen3 - # -- (bool) Whether public datasets are enabled. - publicDataSets: true - # -- (string) Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` - tierAccessLevel: libre - # -- (int) Only relevant if tireAccessLevel is set to "regular". Summary charts below this limit will not appear for aggregated data. - tierAccessLimit: "1000" - # -- (bool) Whether network policies are enabled. - netPolicy: true - # -- (int) Number of dispatcher jobs. - dispatcherJobNum: "10" - # -- (bool) Whether Datadog is enabled. - ddEnabled: false - # -- (bool) If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. - pdb: false - # -- (int) The minimum amount of pods that are available at all times if the PDB is deployed. - minAvialable: 1 - # -- (map) External Secrets settings. - externalSecrets: - # -- (bool) Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any indexd secrets you have deployed. - deploy: false - # -- (string) Will deploy a separate External Secret Store for this service. - separateSecretStore: false - -# -- (map) External Secrets settings. -externalSecrets: - # -- (string) Will create the Helm "indexd-service-creds" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. - createK8sServiceCredsSecret: false - # -- (string) Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" - dbcreds: - serviceCreds: "indexd-service-creds" - -# -- (map) Values for indexd secret. -secrets: - userdb: - fence: - sheepdog: - # gateway: - # -- (str) AWS access key ID to access the db restore job S3 bucket. Overrides global key. - awsAccessKeyId: - # -- (str) AWS secret access key ID to access the db restore job S3 bucket. Overrides global key. - awsSecretAccessKey: - -# -- (map) Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you -postgres: - # (bool) Whether the database should be restored from s3. Default to global.postgres.dbRestore - dbRestore: false - # -- (bool) Whether the database should be created. Default to global.postgres.dbCreate - dbCreate: - # -- (string) Hostname for postgres server. This is a service override, defaults to global.postgres.host - host: - # -- (string) Database name for postgres. This is a service override, defaults to - - database: - # -- (string) Username for postgres. This is a service override, defaults to - - username: - # -- (string) Port for Postgres. - port: "5432" - # -- (string) Password for Postgres. Will be autogenerated if left empty. - password: - # -- (string) Will create a Database for the individual service to help with developing it. - separate: false - -# -- (map) Postgresql subchart settings if deployed separately option is set to "true". -# Disable persistence by default so we can spin up and down ephemeral environments -postgresql: - primary: - persistence: - # -- (bool) Option to persist the dbs data. - enabled: false - -# -- (int) Number of desired replicas -replicaCount: 1 - -# -- (map) Docker image information. -image: - # -- (string) The Docker image repository for the indexd service - repository: quay.io/cdis/indexd - # -- (string) When to pull the image. - pullPolicy: IfNotPresent - # -- (string) Overrides the image tag whose default is the chart appVersion. - tag: "jawadqur-patch-1" - -# -- (list) Docker image pull secrets. -imagePullSecrets: [] - -# -- (string) Override the name of the chart. -nameOverride: "" - -# -- (string) Override the full name of the deployment. -fullnameOverride: "" - -# -- (map) Service account to use or create. -serviceAccount: - # -- (bool) Specifies whether a service account should be created. - create: false - # -- (map) Annotations to add to the service account. - annotations: {} - # If not set and create is true, a name is generated using the fullname template - # -- (string) The name of the service account - name: "" - -# -- (map) Annotations to add to the pod -podAnnotations: {} - -# -- (map) Security context for the pod -podSecurityContext: {} - # fsGroup: 2000 - -# -- (map) Security context for the containers in the pod -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -# -- (map) Kubernetes service information. -service: - # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". - type: ClusterIP - # -- (int) The port number that the service exposes. - port: 80 - -# -- (map) Resource requests and limits for the containers in the pod -resources: - # -- (map) The amount of resources that the container requests - requests: - # -- (string) The amount of CPU requested - cpu: 0.1 - # -- (string) The amount of memory requested - memory: 12Mi - # -- (map) The maximum amount of resources that the container is allowed to use - limits: - # -- (string) The maximum amount of CPU the container can use - cpu: 1.0 - # -- (string) The maximum amount of memory the container can use - memory: 512Mi - -# -- (map) Autoscaling options. -autoscaling: - enabled: false - # -- (int) Minimum number of replicas - minReplicas: 1 - # -- (int) Maximum number of replicas - maxReplicas: 100 - # -- (int) Target CPU utilization percentage - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -# -- (map) Node Selector for the pods -nodeSelector: {} - -# -- (list) Tolerations for the pods -tolerations: [] - -# -- (map) Affinity to use for the deployment. -affinity: {} - -# -- (list) Volumes to attach to the pod -volumes: -- name: uwsgi-config - configMap: - name: indexd-uwsgi -- name: config-volume - secret: - secretName: "indexd-settings" - -# -- (list) Volumes to mount to the container. -volumeMounts: -- name: "config-volume" - readOnly: true - mountPath: "/var/www/indexd/local_settings.py" - subPath: "local_settings.py" - -# -- (list) Environment variables to pass to the container -env: - - name: "ARBORIST" - value: "true" - - name: "GEN3_DEBUG" - value: "False" - -# -- (map) Values for overriding uwsgi settings -uwsgi: - listen: 1024 - -# -- (string) default prefix for indexd -defaultPrefix: "PREFIX/" - -# Values to determine the labels that are used for the deployment, pod, etc. -# -- (string) Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". -release: "production" -# -- (string) Valid options are "true" or "false". If invalid option is set- the value will default to "false". -criticalService: "true" -# -- (string) Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. -partOf: "S3-GS" -# -- (map) Will completely override the selectorLabels defined in the common chart's _label_setup.tpl -selectorLabels: -# -- (map) Will completely override the commonLabels defined in the common chart's _label_setup.tpl -commonLabels: - -# Values to configure datadog if ddEnabled is set to "true". -# -- (bool) If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. -datadogLogsInjection: true -# -- (bool) If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. -datadogProfilingEnabled: true -# -- (int) A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. -datadogTraceSampleRate: 1 diff --git a/helm/loom/Chart.yaml b/helm/loom/Chart.yaml new file mode 100644 index 000000000..84428b976 --- /dev/null +++ b/helm/loom/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +name: loom +description: Loom FHIR dataframe GraphQL server +type: application +version: 0.1.0 +appVersion: "0.1.0" +keywords: + - fhir + - arangodb + - graphql + - dataframe diff --git a/helm/loom/README.md b/helm/loom/README.md new file mode 100644 index 000000000..efe71f3a6 --- /dev/null +++ b/helm/loom/README.md @@ -0,0 +1,48 @@ +# Loom Helm chart + +The chart deploys the Loom FHIR dataframe GraphQL service, one ClickHouse +StatefulSet, and the ArangoDB service it requires. No ClickHouse operator or +Keeper cluster is involved. The in-cluster endpoints are fixed at +`clickhouse:9000` and `arangodb:8529`. + +## Local kind smoke test + +```bash +cd /path/to/gen3-helm +export LOOM_REPO=/path/to/loom +docker build -t loom:dev "$LOOM_REPO" +kind create cluster --name loom +kind load docker-image loom:dev --name loom +helm upgrade --install loom ./helm/loom \ + --namespace loom --create-namespace \ + -f ./helm/loom/values-local.yaml +kubectl -n loom rollout status deployment/loom-loom --timeout=5m +kubectl -n loom port-forward svc/loom-loom 8080:8080 +curl http://127.0.0.1:8080/healthz +``` + +The local values use `--no-auth`, an ephemeral ArangoDB, and an ephemeral +single-node ClickHouse. Set `clickstack.persistence.enabled: true` when the +ClickHouse data must survive pod replacement. +Load a resource file through +the existing import API after port-forwarding, for example: + +```bash +curl -F project=ARANGODB_PROTO \ + -F resource_type=Patient \ + -F use_generic=true \ + -F file=@"$LOOM_REPO/META/Patient.ndjson" \ + http://127.0.0.1:8080/api/v1/imports +``` + +For a real cluster, use a managed Loom image registry. If the Loom server is +configured to use ClickHouse, set `server.clickhouse.url` (or +`server.clickhouse.host` and `port`) and set `server.waitForBackends` false +unless the endpoint is reachable from a BusyBox init container. The chart's +liveness/readiness probes use `/healthz`; the endpoint is process-level health +and does not hide backend connection failures during startup. + +To run Loom without ClickHouse, set `server.clickhouse.enabled: false`. The +chart then emits an empty ClickHouse URL and omits the ClickHouse wait +init-container. The Loom image must include the matching disabled-backend +support. diff --git a/helm/loom/templates/NOTES.txt b/helm/loom/templates/NOTES.txt new file mode 100644 index 000000000..3eda95f35 --- /dev/null +++ b/helm/loom/templates/NOTES.txt @@ -0,0 +1,11 @@ +Loom is installed as {{ include "loom.fullname" . }}. + +Port-forward the GraphQL server: + kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "loom.fullname" . }} 8080:{{ .Values.service.port }} + +Then open: + http://127.0.0.1:8080/healthz + http://127.0.0.1:8080/graphql + +This release includes an ArangoDB service managed by the Loom chart. +All data is lost when the release is removed. diff --git a/helm/loom/templates/_helpers.tpl b/helm/loom/templates/_helpers.tpl new file mode 100644 index 000000000..d64098902 --- /dev/null +++ b/helm/loom/templates/_helpers.tpl @@ -0,0 +1,60 @@ +{{- define "loom.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "loom.fullname" -}} +{{- default (include "loom.name" .) .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "loom.deploymentName" -}} +loom-deployment +{{- end }} + +{{- define "loom.labels" -}} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +app.kubernetes.io/name: {{ include "loom.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{- define "loom.selectorLabels" -}} +app.kubernetes.io/name: {{ include "loom.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{- define "loom.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "loom.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "loom.image" -}} +{{- printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- end }} + +{{- define "loom.arangoURL" -}} +{{- if .Values.server.arango.url -}} +{{- .Values.server.arango.url -}} +{{- else -}} +{{- "http://arangodb:8529" -}} +{{- end -}} +{{- end -}} + +{{- define "loom.clickhouseHost" -}} +{{- if .Values.server.clickhouse.host -}} +{{- .Values.server.clickhouse.host -}} +{{- else -}} +{{- "clickhouse" -}} +{{- end -}} +{{- end -}} + +{{- define "loom.clickhouseURL" -}} +{{- if .Values.server.clickhouse.url -}} +{{- .Values.server.clickhouse.url -}} +{{- else -}} +{{- "clickhouse://clickhouse:9000" -}} +{{- end -}} +{{- end -}} diff --git a/helm/loom/templates/clickhouse.yaml b/helm/loom/templates/clickhouse.yaml new file mode 100644 index 000000000..24343aac8 --- /dev/null +++ b/helm/loom/templates/clickhouse.yaml @@ -0,0 +1,115 @@ +{{- if .Values.clickstack.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: clickhouse-users + labels: + {{- include "loom.labels" . | nindent 4 }} + app.kubernetes.io/component: clickhouse +type: Opaque +stringData: + loom.xml: | + + + + {{ .Values.clickstack.auth.password }} + + 0.0.0.0/0 + ::/0 + + + + +--- +apiVersion: v1 +kind: Service +metadata: + name: clickhouse + labels: + {{- include "loom.labels" . | nindent 4 }} + app.kubernetes.io/component: clickhouse +spec: + selector: + app.kubernetes.io/name: clickhouse + app.kubernetes.io/instance: {{ .Release.Name }} + ports: + - name: http + port: 8123 + targetPort: http + - name: native + port: 9000 + targetPort: native +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: clickhouse + labels: + {{- include "loom.labels" . | nindent 4 }} + app.kubernetes.io/name: clickhouse + app.kubernetes.io/component: clickhouse +spec: + serviceName: clickhouse + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: clickhouse + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: clickhouse + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: clickhouse + spec: + containers: + - name: clickhouse + image: "{{ .Values.clickstack.image.repository }}:{{ .Values.clickstack.image.tag }}" + imagePullPolicy: {{ .Values.clickstack.image.pullPolicy }} + ports: + - name: http + containerPort: 8123 + - name: native + containerPort: 9000 + readinessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + resources: + {{- toYaml .Values.clickstack.resources | nindent 12 }} + volumeMounts: + - name: clickhouse-data + mountPath: /var/lib/clickhouse + - name: clickhouse-users + mountPath: /etc/clickhouse-server/users.d/loom.xml + subPath: loom.xml + volumes: + - name: clickhouse-users + secret: + secretName: clickhouse-users + {{- if not .Values.clickstack.persistence.enabled }} + - name: clickhouse-data + emptyDir: {} + {{- end }} + {{- if .Values.clickstack.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: clickhouse-data + spec: + accessModes: ["ReadWriteOnce"] + {{- with .Values.clickstack.persistence.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.clickstack.persistence.size }} + {{- end }} +{{- end }} diff --git a/helm/loom/templates/config-secret.yaml b/helm/loom/templates/config-secret.yaml new file mode 100644 index 000000000..d76259107 --- /dev/null +++ b/helm/loom/templates/config-secret.yaml @@ -0,0 +1,33 @@ +{{- $cfg := deepCopy (.Values.config | default dict) -}} +{{- $server := deepCopy (get $cfg "server" | default dict) -}} +{{- if not (hasKey $server "listen") }}{{- $_ := set $server "listen" (printf ":%v" .Values.service.port) -}}{{- end }} +{{- if not (hasKey $server "schema") }}{{- $_ := set $server "schema" "/app/schemas/graph-fhir.json" -}}{{- end }} +{{- if not (hasKey $server "database") }}{{- $_ := set $server "database" .Values.server.arango.database -}}{{- end }} +{{- $arango := deepCopy (get $server "arango" | default dict) -}} +{{- if not (get $arango "url" | default "") }}{{- $_ := set $arango "url" (include "loom.arangoURL" .) -}}{{- end }} +{{- $_ := set $server "arango" $arango -}} +{{- $clickhouse := deepCopy (get $server "clickhouse" | default dict) -}} +{{- $clickhouseEnabled := and .Values.server.clickhouse.enabled (or .Values.clickstack.enabled (ne .Values.server.clickhouse.url "clickhouse://clickhouse:9000")) -}} +{{- $_ := set $clickhouse "enabled" $clickhouseEnabled -}} +{{- if $clickhouseEnabled }} +{{- $_ := set $clickhouse "username" .Values.clickstack.auth.username -}} +{{- $_ := set $clickhouse "password" .Values.clickstack.auth.password -}} +{{- end }} +{{- if $clickhouseEnabled }} +{{- if not (get $clickhouse "url" | default "") }}{{- $_ := set $clickhouse "url" (include "loom.clickhouseURL" .) -}}{{- end }} +{{- else }} +{{- $_ := set $clickhouse "url" "" -}} +{{- end }} +{{- if not (get $clickhouse "database" | default "") }}{{- $_ := set $clickhouse "database" .Values.server.clickhouse.database -}}{{- end }} +{{- $_ := set $server "clickhouse" $clickhouse -}} +{{- $_ := set $cfg "server" $server -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "loom.fullname" . }}-config + labels: + {{- include "loom.labels" . | nindent 4 }} +type: Opaque +stringData: + config.yaml: | +{{ toYaml $cfg | nindent 4 }} diff --git a/helm/loom/templates/deployment.yaml b/helm/loom/templates/deployment.yaml new file mode 100644 index 000000000..8b68685c3 --- /dev/null +++ b/helm/loom/templates/deployment.yaml @@ -0,0 +1,140 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "loom.deploymentName" . }} + labels: + {{- include "loom.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "loom.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/config-secret.yaml") . | sha256sum }} + labels: + {{- include "loom.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "loom.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: 30 + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + - name: wait-for-arango + image: busybox:1.36 + imagePullPolicy: IfNotPresent + command: ["sh", "-c"] + args: + - >- + until wget -q -O - {{ include "loom.arangoURL" . | trimSuffix "/" }}/_api/version; + do echo waiting for ArangoDB; sleep 2; done + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 65532 + {{- if and .Values.server.clickhouse.enabled .Values.clickstack.enabled .Values.server.waitForBackends }} + - name: wait-for-clickhouse + image: busybox:1.36 + imagePullPolicy: IfNotPresent + command: ["sh", "-c"] + args: + - >- + until wget -q -O - http://{{ include "loom.clickhouseHost" . }}:8123/ping; + do echo waiting for ClickHouse; sleep 2; done + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 65532 + {{- end }} + containers: + - name: loom + image: {{ include "loom.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + {{- with .Values.auth.basic.existingSecret }} + env: + - name: LOOM_AUTH_BASIC_USERNAME + valueFrom: + secretKeyRef: + name: {{ . | quote }} + key: {{ $.Values.auth.basic.usernameKey | quote }} + - name: LOOM_AUTH_BASIC_PASSWORD + valueFrom: + secretKeyRef: + name: {{ . | quote }} + key: {{ $.Values.auth.basic.passwordKey | quote }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + args: + - --config + - /etc/loom/config.yaml + {{- if .Values.server.noAuth }} + # Deprecated local-development override. Production config must + # select auth.mode explicitly. + - --no-auth + {{- end }} + {{- with .Values.server.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: {{ .Values.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.probes.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.probes.liveness.timeoutSeconds }} + failureThreshold: {{ .Values.probes.liveness.failureThreshold }} + readinessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: {{ .Values.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.probes.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }} + failureThreshold: {{ .Values.probes.readiness.failureThreshold }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: tmp + mountPath: /tmp + - name: config + mountPath: /etc/loom + readOnly: true + volumes: + - name: tmp + emptyDir: {} + - name: config + secret: + secretName: {{ include "loom.fullname" . }}-config + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/loom/templates/dev-backends.yaml b/helm/loom/templates/dev-backends.yaml new file mode 100644 index 000000000..e75d6c976 --- /dev/null +++ b/helm/loom/templates/dev-backends.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: arangodb + labels: + app.kubernetes.io/name: arangodb + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: arangodb + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: arangodb + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + containers: + - name: arangodb + image: {{ .Values.arango.image | quote }} + env: + - name: ARANGO_NO_AUTH + value: {{ ternary "1" "0" .Values.arango.noAuth | quote }} + ports: + - name: http + containerPort: 8529 + volumeMounts: + - name: data + mountPath: /var/lib/arangodb3 + volumes: + - name: data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: arangodb + labels: + app.kubernetes.io/name: arangodb + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + selector: + app.kubernetes.io/name: arangodb + app.kubernetes.io/instance: {{ .Release.Name }} + ports: + - name: http + port: 8529 + targetPort: http diff --git a/helm/loom/templates/service.yaml b/helm/loom/templates/service.yaml new file mode 100644 index 000000000..1fa554cf3 --- /dev/null +++ b/helm/loom/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "loom.fullname" . }} + labels: + {{- include "loom.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - name: http + port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + selector: + {{- include "loom.selectorLabels" . | nindent 4 }} diff --git a/helm/loom/templates/serviceaccount.yaml b/helm/loom/templates/serviceaccount.yaml new file mode 100644 index 000000000..fbabaa5af --- /dev/null +++ b/helm/loom/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "loom.serviceAccountName" . }} + labels: + {{- include "loom.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/helm/loom/templates/tests/test-connection.yaml b/helm/loom/templates/tests/test-connection.yaml new file mode 100644 index 000000000..85ec2cbc7 --- /dev/null +++ b/helm/loom/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "loom.fullname" . }}-test-connection" + labels: + {{- include "loom.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + restartPolicy: Never + containers: + - name: wget + image: busybox:1.36 + command: ["wget"] + args: ["-q", "-O", "-", "http://{{ include "loom.fullname" . }}:{{ .Values.service.port }}/healthz"] diff --git a/helm/loom/values-local.yaml b/helm/loom/values-local.yaml new file mode 100644 index 000000000..8c20faee0 --- /dev/null +++ b/helm/loom/values-local.yaml @@ -0,0 +1,10 @@ +image: + repository: loom + tag: dev + pullPolicy: IfNotPresent + +server: + noAuth: true + clickhouse: + database: default + waitForBackends: true diff --git a/helm/loom/values.yaml b/helm/loom/values.yaml new file mode 100644 index 000000000..7ddf1bb0f --- /dev/null +++ b/helm/loom/values.yaml @@ -0,0 +1,116 @@ +replicaCount: 1 + +image: + repository: quay.io/ohsu-comp-bio/loom + tag: main + pullPolicy: IfNotPresent +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + automount: true + annotations: {} + name: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + +service: + type: ClusterIP + port: 8080 + +server: + listen: ":8080" + noAuth: false + datasetGenerations: false + schema: /app/schemas/graph-fhir.json + arango: + # Override only when the Loom-provided ArangoDB service is replaced. + url: "" + database: fhir_proto + clickhouse: + enabled: true + url: clickhouse://clickhouse:9000 + host: clickhouse + port: 9000 + database: default + waitForBackends: false + extraArgs: [] + +# Rendered into /etc/loom/config.yaml. The Gen3 umbrella uses Calypr/Fence +# authorization by default; standalone deployments may set mode: basic and +# provide credentials through an existing Secret-backed environment. +config: + auth: + mode: calypr + calypr: + request_timeout: 5s + cache_ttl: 30s + +auth: + basic: + # Optional Secret for standalone mode. The server reads these via + # LOOM_AUTH_BASIC_USERNAME/PASSWORD; credentials are never rendered into + # config.yaml. + existingSecret: "" + usernameKey: username + passwordKey: password + +probes: + liveness: + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 6 + readiness: + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 6 + +resources: {} +nodeSelector: {} +tolerations: [] +affinity: {} + +arango: + image: arangodb:3.12 + noAuth: true + +clickstack: + enabled: true + image: + repository: clickhouse/clickhouse-server + tag: 25.7-alpine + pullPolicy: IfNotPresent + persistence: + enabled: false + size: 2Gi + storageClass: "" + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 1 + memory: 1Gi + auth: + username: default + password: loom-clickhouse diff --git a/helm/manifestservice/.helmignore b/helm/manifestservice/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/helm/manifestservice/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm/manifestservice/Chart.yaml b/helm/manifestservice/Chart.yaml deleted file mode 100644 index 382e01657..000000000 --- a/helm/manifestservice/Chart.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: v2 -name: manifestservice -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.14 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "master" - -dependencies: -- name: common - version: 0.1.10 - repository: file://../common diff --git a/helm/manifestservice/README.md b/helm/manifestservice/README.md deleted file mode 100644 index 11fc1f396..000000000 --- a/helm/manifestservice/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# manifestservice - -![Version: 0.1.14](https://img.shields.io/badge/Version-0.1.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) - -A Helm chart for Kubernetes - -## Requirements - -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.10 | - -## Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| affinity | map | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["manifestservice"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]}}` | Affinity to use for the deployment. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution | map | `[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["manifestservice"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]` | Option for scheduling to be required or preferred. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0] | int | `{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["manifestservice"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}` | Weight value for preferred scheduling. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0] | list | `{"key":"app","operator":"In","values":["manifestservice"]}` | Label key for match expression. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].operator | string | `"In"` | Operation type for the match expression. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].values | list | `["manifestservice"]` | Value for the match expression key. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey | string | `"kubernetes.io/hostname"` | Value for topology key label. | -| automountServiceAccountToken | bool | `false` | Automount the default service account token | -| autoscaling | map | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Configuration for autoscaling the number of replicas | -| autoscaling.enabled | bool | `false` | Whether autoscaling is enabled | -| autoscaling.maxReplicas | int | `100` | The maximum number of replicas to scale up to | -| autoscaling.minReplicas | int | `1` | The minimum number of replicas to scale down to | -| autoscaling.targetCPUUtilizationPercentage | int | `80` | The target CPU utilization percentage for autoscaling | -| commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | -| criticalService | string | `"true"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | -| datadogLogsInjection | bool | `true` | If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. | -| datadogProfilingEnabled | bool | `true` | If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. | -| datadogTraceSampleRate | int | `1` | A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. | -| env | list | `[{"name":"REQUESTS_CA_BUNDLE","value":"/etc/ssl/certs/ca-certificates.crt"},{"name":"MANIFEST_SERVICE_CONFIG_PATH","value":"/var/gen3/config/config.json"},{"name":"GEN3_DEBUG","value":"False"}]` | Environment variables to pass to the container | -| externalSecrets | map | `{"createK8sManifestServiceSecret":false,"manifestserviceG3auto":null}` | External Secrets settings. | -| externalSecrets.createK8sManifestServiceSecret | string | `false` | Will create the Helm "manifestservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. | -| externalSecrets.manifestserviceG3auto | string | `nil` | Will override the name of the aws secrets manager secret. Default is "manifestservice-g3auto" | -| global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false}` | AWS configuration | -| global.aws.awsAccessKeyId | string | `nil` | Credentials for AWS stuff. | -| global.aws.awsSecretAccessKey | string | `nil` | Credentials for AWS stuff. | -| global.aws.enabled | bool | `false` | Set to true if deploying to AWS. Controls ingress annotations. | -| global.ddEnabled | bool | `false` | Whether Datadog is enabled. | -| global.environment | string | `"default"` | Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. | -| global.externalSecrets | map | `{"deploy":false,"separateSecretStore":false}` | External Secrets settings. | -| global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any manifestservice secrets you have deployed. | -| global.externalSecrets.separateSecretStore | string | `false` | Will deploy a separate External Secret Store for this service. | -| global.minAvialable | int | `1` | The minimum amount of pods that are available at all times if the PDB is deployed. | -| global.pdb | bool | `false` | If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. | -| image | map | `{"pullPolicy":"Always","repository":"quay.io/cdis/manifestservice","tag":""}` | Docker image information. | -| image.pullPolicy | string | `"Always"` | Docker pull policy. | -| image.repository | string | `"quay.io/cdis/manifestservice"` | Docker repository. | -| image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | -| manifestserviceG3auto | map | `{"awsaccesskey":"","awssecretkey":"","bucketName":"testbucket","hostname":"testinstall","prefix":"test"}` | Values for manifestservice secret. | -| manifestserviceG3auto.awsaccesskey | string | `""` | AWS access key. | -| manifestserviceG3auto.awssecretkey | string | `""` | AWS secret access key. | -| manifestserviceG3auto.bucketName | string | `"testbucket"` | Bucket for the manifestservice to read and write to. | -| manifestserviceG3auto.prefix | string | `"test"` | Directory name to use within the s3 bucket. | -| partOf | string | `"Workspace-tab"` | Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. | -| release | string | `"production"` | Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". | -| replicaCount | int | `1` | Number of replicas for the deployment. | -| resources | map | `{"limits":{"cpu":1,"memory":"512Mi"},"requests":{"cpu":0.1,"memory":"12Mi"}}` | Resource requests and limits for the containers in the pod | -| resources.limits | map | `{"cpu":1,"memory":"512Mi"}` | The maximum amount of resources that the container is allowed to use | -| resources.limits.cpu | string | `1` | The maximum amount of CPU the container can use | -| resources.limits.memory | string | `"512Mi"` | The maximum amount of memory the container can use | -| resources.requests | map | `{"cpu":0.1,"memory":"12Mi"}` | The amount of resources that the container requests | -| resources.requests.cpu | string | `0.1` | The amount of CPU requested | -| resources.requests.memory | string | `"12Mi"` | The amount of memory requested | -| revisionHistoryLimit | int | `2` | Number of old revisions to retain | -| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information for External Secrets. | -| secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. | -| secrets.awsSecretAccessKey | str | `nil` | AWS secret access key ID. Overrides global key. | -| selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | -| service | map | `{"port":80,"type":"ClusterIP"}` | Kubernetes service information. | -| service.port | int | `80` | The port number that the service exposes. | -| service.type | string | `"ClusterIP"` | Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". | -| serviceAccount | map | `{"annotations":{},"create":true,"name":""}` | Service account to use or create. | -| serviceAccount.annotations | map | `{}` | Annotations to add to the service account. | -| serviceAccount.create | bool | `true` | Specifies whether a service account should be created. | -| serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | -| strategy | map | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | Rolling update deployment strategy | -| strategy.rollingUpdate.maxSurge | int | `1` | Number of additional replicas to add during rollout. | -| strategy.rollingUpdate.maxUnavailable | int | `0` | Maximum amount of pods that can be unavailable during the update. | -| terminationGracePeriodSeconds | int | `50` | Grace period that applies to the total time it takes for both the PreStop hook to execute and for the Container to stop normally. | -| volumeMounts | list | `[{"mountPath":"/var/gen3/config/","name":"config-volume","readOnly":true}]` | Volumes to mount to the container. | -| volumes | list | `[{"name":"config-volume","secret":{"secretName":"manifestservice-g3auto"}}]` | Volumes to attach to the container. | - diff --git a/helm/manifestservice/templates/NOTES.txt b/helm/manifestservice/templates/NOTES.txt deleted file mode 100644 index 70b82c54c..000000000 --- a/helm/manifestservice/templates/NOTES.txt +++ /dev/null @@ -1 +0,0 @@ -{{ .Chart.Name }} has been deployed \ No newline at end of file diff --git a/helm/manifestservice/templates/_helpers.tpl b/helm/manifestservice/templates/_helpers.tpl deleted file mode 100644 index fb9b68c7d..000000000 --- a/helm/manifestservice/templates/_helpers.tpl +++ /dev/null @@ -1,74 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "manifestservice.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "manifestservice.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "manifestservice.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -Common labels -*/}} -{{- define "manifestservice.labels" -}} -{{- if .Values.commonLabels }} - {{- with .Values.commonLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.commonLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "manifestservice.selectorLabels" -}} -{{- if .Values.selectorLabels }} - {{- with .Values.selectorLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.selectorLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "manifestservice.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "manifestservice.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} - -{{/* - Audit g3 Auto Secrets Manager Name -*/}} -{{- define "manifestservice-g3auto" -}} -{{- default "manifestservice-g3auto" .Values.externalSecrets.manifestserviceG3auto }} -{{- end }} diff --git a/helm/manifestservice/templates/aws-config.yaml b/helm/manifestservice/templates/aws-config.yaml deleted file mode 100644 index 398770d36..000000000 --- a/helm/manifestservice/templates/aws-config.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if or (.Values.secrets.awsSecretAccessKey) (.Values.global.aws.awsSecretAccessKey ) }} -{{ include "common.awsconfig" . }} -{{- end -}} \ No newline at end of file diff --git a/helm/manifestservice/templates/deployment.yaml b/helm/manifestservice/templates/deployment.yaml deleted file mode 100644 index 6923a5c0f..000000000 --- a/helm/manifestservice/templates/deployment.yaml +++ /dev/null @@ -1,64 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: manifestservice-deployment - labels: - {{- include "manifestservice.labels" . | nindent 4 }} - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 4 }} - {{- end }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "manifestservice.selectorLabels" . | nindent 6 }} - revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} - strategy: - {{- toYaml .Values.strategy | nindent 8 }} - template: - metadata: - labels: - {{- include "manifestservice.selectorLabels" . | nindent 8 }} - s3: "yes" - public: "yes" - userhelper: "yes" - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - automountServiceAccountToken: {{ .Values.automountServiceAccountToken}} - volumes: - {{- toYaml .Values.volumes | nindent 8 }} - terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds}} - containers: - - name: manifestservice - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogEnvVar" . | nindent 12 }} - {{- end }} - {{- toYaml .Values.env | nindent 12 }} - volumeMounts: - {{- toYaml .Values.volumeMounts | nindent 12 }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - ports: - - containerPort: 80 - livenessProbe: - httpGet: - path: /_status - port: 80 - initialDelaySeconds: 10 - periodSeconds: 60 - timeoutSeconds: 30 - readinessProbe: - httpGet: - path: /_status - port: 80 diff --git a/helm/manifestservice/templates/external-secret.yaml b/helm/manifestservice/templates/external-secret.yaml deleted file mode 100644 index 7d94f5c93..000000000 --- a/helm/manifestservice/templates/external-secret.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{ if .Values.global.externalSecrets.deploy }} -apiVersion: external-secrets.io/v1beta1 -kind: ExternalSecret -metadata: - name: manifestservice-g3auto -spec: - refreshInterval: 5m - secretStoreRef: - name: {{include "common.SecretStore" .}} - kind: SecretStore - target: - name: manifestservice-g3auto - creationPolicy: Owner - data: - - secretKey: config.json - remoteRef: - #name of secret in secrets manager - key: {{include "manifestservice-g3auto" .}} -{{- end }} \ No newline at end of file diff --git a/helm/manifestservice/templates/hpa.yaml b/helm/manifestservice/templates/hpa.yaml deleted file mode 100644 index a180530b1..000000000 --- a/helm/manifestservice/templates/hpa.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "manifestservice.fullname" . }} - labels: - {{- include "manifestservice.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "manifestservice.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/helm/manifestservice/templates/manifestservice-creds.yaml b/helm/manifestservice/templates/manifestservice-creds.yaml deleted file mode 100644 index 54c5f29ae..000000000 --- a/helm/manifestservice/templates/manifestservice-creds.yaml +++ /dev/null @@ -1,16 +0,0 @@ -{{- if or (not .Values.global.externalSecrets.deploy) (and .Values.global.externalSecrets.deploy .Values.externalSecrets.createK8sManifestServiceSecret) }} -apiVersion: v1 -kind: Secret -metadata: - name: manifestservice-g3auto -type: Opaque -stringData: - config.json: |- - { - "manifest_bucket_name": "{{ .Values.manifestserviceG3auto.bucketName }}", - "hostname": "{{ .Values.global.hostname }}", - "aws_access_key_id": "{{ .Values.manifestserviceG3auto.awsaccesskey }}", - "aws_secret_access_key": "{{ .Values.manifestserviceG3auto.awssecretkey }}", - "prefix": "{{ .Values.manifestserviceG3auto.prefix }}" - } -{{- end }} \ No newline at end of file diff --git a/helm/manifestservice/templates/pdb.yaml b/helm/manifestservice/templates/pdb.yaml deleted file mode 100644 index 2ef2de13d..000000000 --- a/helm/manifestservice/templates/pdb.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if and .Values.global.pdb (gt (int .Values.replicaCount) 1) }} -{{ include "common.pod_disruption_budget" . }} -{{- end }} \ No newline at end of file diff --git a/helm/manifestservice/templates/secret-store.yaml b/helm/manifestservice/templates/secret-store.yaml deleted file mode 100644 index 771c7760d..000000000 --- a/helm/manifestservice/templates/secret-store.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{ if .Values.global.externalSecrets.separateSecretStore }} -{{ include "common.secretstore" . }} -{{- end }} \ No newline at end of file diff --git a/helm/manifestservice/templates/service.yaml b/helm/manifestservice/templates/service.yaml deleted file mode 100644 index 173ba48c2..000000000 --- a/helm/manifestservice/templates/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: manifestservice-service - labels: - {{- include "manifestservice.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: 80 - protocol: TCP - name: http - selector: - {{- include "manifestservice.selectorLabels" . | nindent 4 }} diff --git a/helm/manifestservice/templates/serviceaccount.yaml b/helm/manifestservice/templates/serviceaccount.yaml deleted file mode 100644 index 25c0070bf..000000000 --- a/helm/manifestservice/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "manifestservice.serviceAccountName" . }} - labels: - {{- include "manifestservice.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/helm/manifestservice/templates/tests/test-connection.yaml b/helm/manifestservice/templates/tests/test-connection.yaml deleted file mode 100644 index 3d4b1d87c..000000000 --- a/helm/manifestservice/templates/tests/test-connection.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "manifestservice-test-connection" - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['manifestservice-service:{{ .Values.service.port }}/_status'] - restartPolicy: Never diff --git a/helm/manifestservice/values.yaml b/helm/manifestservice/values.yaml deleted file mode 100644 index 09cd04f36..000000000 --- a/helm/manifestservice/values.yaml +++ /dev/null @@ -1,188 +0,0 @@ -# Default values for manifestservice. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - - -# Global configuration -global: - # -- (map) AWS configuration - aws: - # -- (bool) Set to true if deploying to AWS. Controls ingress annotations. - enabled: false - # -- (string) Credentials for AWS stuff. - awsAccessKeyId: - # -- (string) Credentials for AWS stuff. - awsSecretAccessKey: - # -- (string) Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. - environment: default - # -- (bool) Whether Datadog is enabled. - ddEnabled: false - # -- (bool) If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. - pdb: false - # -- (int) The minimum amount of pods that are available at all times if the PDB is deployed. - minAvialable: 1 - # -- (map) External Secrets settings. - externalSecrets: - # -- (bool) Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any manifestservice secrets you have deployed. - deploy: false - # -- (string) Will deploy a separate External Secret Store for this service. - separateSecretStore: false - -# -- (map) External Secrets settings. -externalSecrets: - # -- (string) Will create the Helm "manifestservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. - createK8sManifestServiceSecret: false - # -- (string) Will override the name of the aws secrets manager secret. Default is "manifestservice-g3auto" - manifestserviceG3auto: -# -- (map) Secret information for External Secrets. -secrets: - # -- (str) AWS access key ID. Overrides global key. - awsAccessKeyId: - # -- (str) AWS secret access key ID. Overrides global key. - awsSecretAccessKey: - -# -- (int) Number of old revisions to retain -revisionHistoryLimit: 2 - -# -- (int) Number of replicas for the deployment. -replicaCount: 1 - -# -- (map) Docker image information. -image: - # -- (string) Docker repository. - repository: quay.io/cdis/manifestservice - # -- (string) Docker pull policy. - pullPolicy: Always - # -- (string) Overrides the image tag whose default is the chart appVersion. - tag: "" - -# -- (map) Kubernetes service information. -service: - # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". - type: ClusterIP - # -- (int) The port number that the service exposes. - port: 80 - -# -- (map) Service account to use or create. -serviceAccount: - # -- (bool) Specifies whether a service account should be created. - create: true - # -- (map) Annotations to add to the service account. - annotations: {} - # -- (string) The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -# -- (map) Configuration for autoscaling the number of replicas -autoscaling: - # -- (bool) Whether autoscaling is enabled - enabled: false - # -- (int) The minimum number of replicas to scale down to - minReplicas: 1 - # -- (int) The maximum number of replicas to scale up to - maxReplicas: 100 - # -- (int) The target CPU utilization percentage for autoscaling - targetCPUUtilizationPercentage: 80 - -# -- (map) Rolling update deployment strategy -strategy: - type: RollingUpdate - rollingUpdate: - # -- (int) Number of additional replicas to add during rollout. - maxSurge: 1 - # -- (int) Maximum amount of pods that can be unavailable during the update. - maxUnavailable: 0 - -# -- (map) Affinity to use for the deployment. -affinity: - podAntiAffinity: - # -- (map) Option for scheduling to be required or preferred. - preferredDuringSchedulingIgnoredDuringExecution: - # -- (int) Weight value for preferred scheduling. - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - # -- (list) Label key for match expression. - - key: app - # -- (string) Operation type for the match expression. - operator: In - # -- (list) Value for the match expression key. - values: - - manifestservice - # -- (string) Value for topology key label. - topologyKey: "kubernetes.io/hostname" - -# -- (bool) Automount the default service account token -automountServiceAccountToken: false - -# -- (list) Volumes to attach to the container. -volumes: - - name: config-volume - secret: - secretName: "manifestservice-g3auto" - -# -- (int) Grace period that applies to the total time it takes for both the PreStop hook to execute and for the Container to stop normally. -terminationGracePeriodSeconds: 50 - -# -- (list) Environment variables to pass to the container -env: - - name: REQUESTS_CA_BUNDLE - value: /etc/ssl/certs/ca-certificates.crt - - name: MANIFEST_SERVICE_CONFIG_PATH - value: /var/gen3/config/config.json - - name: GEN3_DEBUG - value: "False" - -# -- (list) Volumes to mount to the container. -volumeMounts: - - name: "config-volume" - readOnly: true - mountPath: "/var/gen3/config/" - -# -- (map) Resource requests and limits for the containers in the pod -resources: - # -- (map) The amount of resources that the container requests - requests: - # -- (string) The amount of CPU requested - cpu: 0.1 - # -- (string) The amount of memory requested - memory: 12Mi - # -- (map) The maximum amount of resources that the container is allowed to use - limits: - # -- (string) The maximum amount of CPU the container can use - cpu: 1.0 - # -- (string) The maximum amount of memory the container can use - memory: 512Mi - -# -- (map) Values for manifestservice secret. -manifestserviceG3auto: - hostname: testinstall - # -- (string) Bucket for the manifestservice to read and write to. - bucketName: testbucket - # -- (string) Directory name to use within the s3 bucket. - prefix: test - # -- (string) AWS access key. - awsaccesskey: "" - # -- (string) AWS secret access key. - awssecretkey: "" - -# Values to determine the labels that are used for the deployment, pod, etc. -# -- (string) Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". -release: "production" -# -- (string) Valid options are "true" or "false". If invalid option is set- the value will default to "false". -criticalService: "true" -# -- (string) Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. -partOf: "Workspace-tab" -# -- (map) Will completely override the selectorLabels defined in the common chart's _label_setup.tpl -selectorLabels: -# -- (map) Will completely override the commonLabels defined in the common chart's _label_setup.tpl -commonLabels: - -# Values to configure datadog if ddEnabled is set to "true". -# -- (bool) If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. -datadogLogsInjection: true -# -- (bool) If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. -datadogProfilingEnabled: true -# -- (int) A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. -datadogTraceSampleRate: 1 diff --git a/helm/metadata/Chart.yaml b/helm/metadata/Chart.yaml deleted file mode 100644 index d38d06e54..000000000 --- a/helm/metadata/Chart.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: v2 -name: metadata -description: A Helm chart for gen3 Metadata Service - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.12 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "master" - -dependencies: -- name: common - version: 0.1.10 - repository: file://../common -- name: postgresql - version: 11.9.13 - repository: "https://charts.bitnami.com/bitnami" - condition: postgres.separate -- name: elasticsearch - version: "7.17.1" - repository: "https://helm.elastic.co" - condition: elasticsearch.separate diff --git a/helm/metadata/README.md b/helm/metadata/README.md deleted file mode 100644 index c9553ba94..000000000 --- a/helm/metadata/README.md +++ /dev/null @@ -1,126 +0,0 @@ -# metadata - -![Version: 0.1.12](https://img.shields.io/badge/Version-0.1.12-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) - -A Helm chart for gen3 Metadata Service - -## Requirements - -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.10 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | -| https://helm.elastic.co | elasticsearch | 7.17.1 | - -## Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| affinity | map | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["metadata"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]}}` | Affinity to use for the deployment. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution | map | `[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["metadata"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]` | Option for scheduling to be required or preferred. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0] | int | `{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["metadata"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}` | Weight value for preferred scheduling. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0] | list | `{"key":"app","operator":"In","values":["metadata"]}` | Label key for match expression. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].operator | string | `"In"` | Operation type for the match expression. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].values | list | `["metadata"]` | Value for the match expression key. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey | string | `"kubernetes.io/hostname"` | Value for topology key label. | -| aggMdsConfig | string | `"{\n \"configuration\": {\n \"schema\": {\n \"_subjects_count\": {\n \"type\": \"integer\"\n },\n \"__manifest\": {\n \"description\": \"an array of filename (usually DRS ids and its size\",\n \"type\": \"array\",\n \"properties\": {\n \"file_name\": {\n \"type\": \"string\"\n },\n \"file_size\": {\n \"type\": \"integer\"\n }\n }\n },\n \"tags\": {\n \"type\": \"array\"\n },\n \"_unique_id\": {},\n \"study_description\": {},\n \"study_id\": {},\n \"study_url\": {},\n \"project_id\": {},\n \"short_name\": {\n \"default\": \"not_set\"\n },\n \"year\": {\n \"default\": \"not_set\"\n },\n \"full_name\": {},\n \"commons_url\": {},\n \"commons\": {}\n },\n \"settings\": {\n \"cache_drs\": true\n }\n },\n \"adapter_commons\": {\n \"Gen3\": {\n \"mds_url\": \"https://gen3.datacommons.io/\",\n \"commons_url\": \"gen3.datacommons.io/\",\n \"adapter\": \"gen3\",\n \"config\": {\n \"guid_type\": \"discovery_metadata\",\n \"study_field\": \"gen3_discovery\"\n },\n \"keep_original_fields\": false,\n \"field_mappings\": {\n \"tags\": \"path:tags\",\n \"_unique_id\": \"path:_unique_id\",\n \"study_description\": \"path:summary\",\n \"full_name\": \"path:study_title\",\n \"short_name\": \"path:short_name\",\n \"year\": \"path:year\",\n \"accession_number\": \"path:accession_number\",\n \"commons\": \"Gen3 Data Commons\",\n \"study_url\": {\n \"path\": \"link\",\n \"default\": \"unknown\"\n }\n }\n }\n }\n}\n"` | | -| aggMdsNamespace | string | `"default"` | Namespae to use if AggMds is enabled. | -| args | list | `["-c","/env/bin/alembic upgrade head\n"]` | Arguments to pass to the init container. | -| automountServiceAccountToken | bool | `false` | Automount the default service account token | -| autoscaling | map | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Configuration for autoscaling the number of replicas | -| autoscaling.enabled | bool | `false` | Whether autoscaling is enabled | -| autoscaling.maxReplicas | int | `100` | The maximum number of replicas to scale up to | -| autoscaling.minReplicas | int | `1` | The minimum number of replicas to scale down to | -| autoscaling.targetCPUUtilizationPercentage | int | `80` | The target CPU utilization percentage for autoscaling | -| command | list | `["/bin/sh"]` | Command to run for the init container. | -| commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | -| criticalService | string | `"true"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | -| datadogLogsInjection | bool | `true` | If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. | -| datadogProfilingEnabled | bool | `true` | If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. | -| datadogTraceSampleRate | int | `1` | A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. | -| debug | bool | `false` | | -| elasticsearch.clusterName | string | `"gen3-elasticsearch"` | | -| elasticsearch.esConfig."elasticsearch.yml" | string | `"# Here we can add elasticsearch config\n"` | | -| elasticsearch.maxUnavailable | int | `0` | | -| elasticsearch.replicas | int | `1` | | -| elasticsearch.separate | bool | `false` | | -| elasticsearch.singleNode | bool | `true` | | -| esEndpoint | string | `"http://gen3-elasticsearch-master:9200"` | Elasticsearch endpoint. | -| externalSecrets | map | `{"createK8sMetadataSecret":false,"dbcreds":null,"metadataG3auto":null}` | External Secrets settings. | -| externalSecrets.createK8sMetadataSecret | string | `false` | Will create the Helm "metadata-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. | -| externalSecrets.dbcreds | string | `nil` | Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" | -| externalSecrets.metadataG3auto | string | `nil` | Will override the name of the aws secrets manager secret. Default is "metadata-g3auto" | -| global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false}` | AWS configuration | -| global.aws.awsAccessKeyId | string | `nil` | Credentials for AWS stuff. | -| global.aws.awsSecretAccessKey | string | `nil` | Credentials for AWS stuff. | -| global.aws.enabled | bool | `false` | Set to true if deploying to AWS. Controls ingress annotations. | -| global.ddEnabled | bool | `false` | Whether Datadog is enabled. | -| global.dev | bool | `true` | Whether the deployment is for development purposes. | -| global.dictionaryUrl | string | `"https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json"` | URL of the data dictionary. | -| global.dispatcherJobNum | int | `"10"` | Number of dispatcher jobs. | -| global.environment | string | `"default"` | Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. | -| global.externalSecrets | map | `{"deploy":false,"separateSecretStore":false}` | External Secrets settings. | -| global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any metadata secrets you have deployed. | -| global.externalSecrets.separateSecretStore | string | `false` | Will deploy a separate External Secret Store for this service. | -| global.hostname | string | `"localhost"` | Hostname for the deployment. | -| global.kubeBucket | string | `"kube-gen3"` | S3 bucket name for Kubernetes manifest files. | -| global.logsBucket | string | `"logs-gen3"` | S3 bucket name for log files. | -| global.minAvialable | int | `1` | The minimum amount of pods that are available at all times if the PDB is deployed. | -| global.netPolicy | bool | `true` | Whether network policies are enabled. | -| global.pdb | bool | `false` | If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. | -| global.portalApp | string | `"gitops"` | Portal application name. | -| global.postgres.dbCreate | bool | `true` | Whether the database should be created. | -| global.postgres.externalSecret | string | `""` | Name of external secret. Disabled if empty | -| global.postgres.master | map | `{"host":null,"password":null,"port":"5432","username":"postgres"}` | Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres | -| global.postgres.master.host | string | `nil` | hostname of postgres server | -| global.postgres.master.password | string | `nil` | password for superuser in postgres. This is used to create or restore databases | -| global.postgres.master.port | string | `"5432"` | Port for Postgres. | -| global.postgres.master.username | string | `"postgres"` | username of superuser in postgres. This is used to create or restore databases | -| global.publicDataSets | bool | `true` | Whether public datasets are enabled. | -| global.revproxyArn | string | `"arn:aws:acm:us-east-1:123456:certificate"` | ARN of the reverse proxy certificate. | -| global.tierAccessLevel | string | `"libre"` | Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` | -| image | map | `{"pullPolicy":"Always","repository":"quay.io/cdis/metadata-service","tag":"feat_es-7"}` | Docker image information. | -| image.pullPolicy | string | `"Always"` | Docker pull policy. | -| image.repository | string | `"quay.io/cdis/metadata-service"` | Docker repository. | -| image.tag | string | `"feat_es-7"` | Overrides the image tag whose default is the chart appVersion. | -| initContainerName | string | `"metadata-db-migrate"` | Name of the init container. | -| initResources | map | `{"limits":{"cpu":0.8,"memory":"512Mi"}}` | Resource limits for the init container. | -| initResources.limits | map | `{"cpu":0.8,"memory":"512Mi"}` | The maximum amount of resources that the container is allowed to use | -| initResources.limits.cpu | string | `0.8` | The maximum amount of CPU the container can use | -| initResources.limits.memory | string | `"512Mi"` | The maximum amount of memory the container can use | -| initVolumeMounts | list | `[{"mountPath":"/src/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"}]` | Volumes to mount to the init container. | -| partOf | string | `"Discovery-Tab"` | Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. | -| postgres | map | `{"database":null,"dbCreate":null,"dbRestore":false,"host":null,"password":null,"port":"5432","separate":false,"username":null}` | Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you | -| postgres.database | string | `nil` | Database name for postgres. This is a service override, defaults to - | -| postgres.dbCreate | bool | `nil` | Whether the database should be created. Default to global.postgres.dbCreate | -| postgres.host | string | `nil` | Hostname for postgres server. This is a service override, defaults to global.postgres.host | -| postgres.password | string | `nil` | Password for Postgres. Will be autogenerated if left empty. | -| postgres.port | string | `"5432"` | Port for Postgres. | -| postgres.separate | string | `false` | Will create a Database for the individual service to help with developing it. | -| postgres.username | string | `nil` | Username for postgres. This is a service override, defaults to - | -| postgresql | map | `{"primary":{"persistence":{"enabled":false}}}` | Postgresql subchart settings if deployed separately option is set to "true". Disable persistence by default so we can spin up and down ephemeral environments | -| postgresql.primary.persistence.enabled | bool | `false` | Option to persist the dbs data. | -| release | string | `"production"` | Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". | -| replicaCount | int | `1` | Number of replicas for the deployment. | -| resources | map | `{"limits":{"cpu":1,"memory":"512Mi"},"requests":{"cpu":0.1,"memory":"12Mi"}}` | Resource requests and limits for the containers in the pod | -| resources.limits | map | `{"cpu":1,"memory":"512Mi"}` | The maximum amount of resources that the container is allowed to use | -| resources.limits.cpu | string | `1` | The maximum amount of CPU the container can use | -| resources.limits.memory | string | `"512Mi"` | The maximum amount of memory the container can use | -| resources.requests | map | `{"cpu":0.1,"memory":"12Mi"}` | The amount of resources that the container requests | -| resources.requests.cpu | string | `0.1` | The amount of CPU requested | -| resources.requests.memory | string | `"12Mi"` | The amount of memory requested | -| revisionHistoryLimit | int | `2` | Number of old revisions to retain | -| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information to access the db restore job S3 bucket. | -| secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. | -| secrets.awsSecretAccessKey | str | `nil` | AWS secret access key ID. Overrides global key. | -| selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | -| service | map | `{"port":[{"name":"http","port":80,"protocol":"TCP","targetPort":80}],"type":"ClusterIP"}` | Kubernetes service information. | -| service.port | int | `[{"name":"http","port":80,"protocol":"TCP","targetPort":80}]` | The port number that the service exposes. | -| service.type | string | `"ClusterIP"` | Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". | -| serviceAnnotations."getambassador.io/config" | string | `"---\napiVersion: ambassador/v1\nambassador_id: \"gen3\"\nkind: Mapping\nname: metadata_mapping\nprefix: /index/\nservice: http://metadata-service:80\n"` | | -| strategy | map | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | Rolling update deployment strategy | -| strategy.rollingUpdate.maxSurge | int | `1` | Number of additional replicas to add during rollout. | -| strategy.rollingUpdate.maxUnavailable | int | `0` | Maximum amount of pods that can be unavailable during the update. | -| useAggMds | bool | `"True"` | Set to true to aggregate metadata from multiple other Metadata Service instances. | -| volumeMounts | list | `[{"mountPath":"/src/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"},{"mountPath":"/aggregate_config.json","name":"config-volume","readOnly":true,"subPath":"aggregate_config.json"},{"mountPath":"/metadata.json","name":"config-manifest","readOnly":true,"subPath":"json"}]` | Volumes to mount to the container. | - diff --git a/helm/metadata/templates/NOTES.txt b/helm/metadata/templates/NOTES.txt deleted file mode 100644 index c1e7e1aef..000000000 --- a/helm/metadata/templates/NOTES.txt +++ /dev/null @@ -1 +0,0 @@ -{{ .Chart.Name }} has been deployed successfully. diff --git a/helm/metadata/templates/_helpers.tpl b/helm/metadata/templates/_helpers.tpl deleted file mode 100644 index 91c790fd3..000000000 --- a/helm/metadata/templates/_helpers.tpl +++ /dev/null @@ -1,86 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "metadata.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "metadata.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "metadata.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -Common labels -*/}} -{{- define "metadata.labels" -}} -{{- if .Values.commonLabels }} - {{- with .Values.commonLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.commonLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "metadata.selectorLabels" -}} -{{- if .Values.selectorLabels }} - {{- with .Values.selectorLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.selectorLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "metadata.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "metadata.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} - -{{/* - Postgres Password lookup -*/}} -{{- define "metadata.postgres.password" -}} -{{- $localpass := (lookup "v1" "Secret" "postgres" "postgres-postgresql" ) -}} -{{- if $localpass }} -{{- default (index $localpass.data "postgres-password" | b64dec) }} -{{- else }} -{{- default .Values.postgres.password }} -{{- end }} -{{- end }} - -{{/* - Metadata g3 Auto Secrets Manager Name -*/}} -{{- define "metadata-g3auto" -}} -{{- default "metadata-g3auto" .Values.externalSecrets.metadataG3auto }} -{{- end }} \ No newline at end of file diff --git a/helm/metadata/templates/agg-mds-sync.yaml b/helm/metadata/templates/agg-mds-sync.yaml deleted file mode 100644 index 08754b401..000000000 --- a/helm/metadata/templates/agg-mds-sync.yaml +++ /dev/null @@ -1,160 +0,0 @@ -{{- if .Values.useAggMds }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: agg-mds-config -data: - aggregate_config.json: | - {{ .Values.aggMdsConfig | default "{}" | nindent 4 }} ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: metadata-aggregate-sync -spec: - schedule: "0 0 1 1 */5" - jobTemplate: - spec: - template: - metadata: - labels: - app: gen3job - spec: - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - preference: - matchExpressions: - - key: karpenter.sh/capacity-type - operator: In - values: - - on-demand - - weight: 99 - preference: - matchExpressions: - - key: eks.amazonaws.com/capacityType - operator: In - values: - - ONDEMAND - volumes: - - name: config-volume - configMap: - name: agg-mds-config - - name: shared-data - emptyDir: {} - initContainers: - - name: wait-for-es - image: alpine/curl - env: - - name: GEN3_ES_ENDPOINT - value: {{ .Values.esEndpoint | default "http://gen3-elasticsearch-master:9200" }} - imagePullPolicy: IfNotPresent - command: ["/bin/sh"] - args: - - "-c" - - | - echo "Waiting for Elasticsearch to be ready..." - until curl -s -XGET $GEN3_ES_ENDPOINT; do - echo "Elasticsearch is not ready yet..." - sleep 5 - done - echo "Elasticsearch is ready!" - - name: wait-for-metadata - image: alpine/curl - env: - - name: GEN3_ES_ENDPOINT - value: {{ .Values.esEndpoint | default "http://gen3-elasticsearch-master:9200" }} - imagePullPolicy: IfNotPresent - command: ["/bin/sh"] - args: - - "-c" - - | - echo "Waiting for metadata service to be ready" - until curl -s -XGET http://metadata-service; do - echo "Metadata service is not ready yet..." - sleep 5 - done - containers: - - name: metadata-sync - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - volumeMounts: - # - name: config-volume-g3auto - # readOnly: true - # mountPath: /src/.env - # subPath: metadata.env - - name: config-volume - readOnly: true - mountPath: /aggregate_config.json - subPath: aggregate_config.json - # - name: config-manifest - # readOnly: true - # mountPath: /metadata.json - # subPath: json - - name: shared-data - mountPath: /mnt/shared - env: - - name: GEN3_DEBUG - value: "False" - - name: GEN3_ES_ENDPOINT - value: {{ .Values.esEndpoint | default "http://gen3-elasticsearch-master:9200" }} - - name: USE_AGG_MDS - value: {{ (.Values.useAggMds | quote | default "True") }} - - name: AGG_MDS_NAMESPACE - value: {{ .Values.aggMdsNamespace | default .Release.Name }} - imagePullPolicy: Always - command: ["/bin/sh"] - args: - - "-c" - - | - cat /aggregate_config.json - /env/bin/python /src/src/mds/populate.py --config /aggregate_config.json - if [ $? -ne 0 ]; then - echo "WARNING: non zero exit code: $?" - echo "WARNING: non zero exit code: $?" > /mnt/shared/status - else - echo "Success" > /mnt/shared/status - fi - - name: slack-alert - env: - - name: slackWebHook - valueFrom: - configMapKeyRef: - name: global - key: slack_webhook - optional: true - - name: gen3Env - valueFrom: - configMapKeyRef: - name: manifest-global - key: hostname - optional: true - image: quay.io/cdis/awshelper:master - volumeMounts: - - name: shared-data - mountPath: /mnt/shared - command: ["/bin/bash"] - args: - - "-c" - - | - if [[ ! "$slackWebHook" =~ ^http ]]; then - echo "Slack webhook not set" - exit 0 - fi - while [ ! -f /mnt/shared/status ]; do - echo "Waiting for status file..." - sleep 5 - done - if ! [[ $(cat /mnt/shared/status) =~ "Success" ]]; then - success="SUCCESS" - color="2EB67D" - else - success="FAILED" - color="FF0000" - fi - echo "Sending ${success} message to slack..." - payload="{\"attachments\": [{\"fallback\": \"JOB ${success}: metadata-aggregate-sync cronjob on ${gen3Env}\",\"color\": \"#${color}\",\"title\": \"JOB ${success}: metadata-aggregate-sync cronjob on ${gen3Env}\",\"text\": \"Pod name: ${HOSTNAME}\",\"ts\": \"$(date +%s)\"}]}" - echo "Payload=${payload}" - curl -X POST --data-urlencode "payload=${payload}" "${slackWebHook}" - restartPolicy: Never -{{- end}} \ No newline at end of file diff --git a/helm/metadata/templates/aws-config.yaml b/helm/metadata/templates/aws-config.yaml deleted file mode 100644 index 4723e6b37..000000000 --- a/helm/metadata/templates/aws-config.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if or (.Values.secrets.awsSecretAccessKey) (.Values.global.aws.awsSecretAccessKey ) }} -{{ include "common.awsconfig" . }} -{{- end }} \ No newline at end of file diff --git a/helm/metadata/templates/db-init.yaml b/helm/metadata/templates/db-init.yaml deleted file mode 100644 index 0393aa732..000000000 --- a/helm/metadata/templates/db-init.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{ include "common.db-secret" . }} ---- -{{ include "common.db_setup_sa" . }} ---- -{{- if .Values.dbRestore }} -{{ include "common.s3_pg_restore" . }} -{{- else }} -{{ include "common.db_setup_job" . }} -{{- end -}} \ No newline at end of file diff --git a/helm/metadata/templates/deployment.yaml b/helm/metadata/templates/deployment.yaml deleted file mode 100644 index d0723397a..000000000 --- a/helm/metadata/templates/deployment.yaml +++ /dev/null @@ -1,169 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: metadata-deployment - labels: - {{- include "metadata.labels" . | nindent 4 }} - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 4 }} - {{- end }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "metadata.selectorLabels" . | nindent 6 }} - revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} - {{- with .Values.strategy }} - strategy: - {{- toYaml . | nindent 4 }} - {{- end }} - template: - metadata: - labels: - {{- include "metadata.selectorLabels" . | nindent 8 }} - # gen3 networkpolicy labels - netnolimit: 'yes' - public: 'yes' - userhelper: 'yes' - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} - volumes: - - name: config-volume-g3auto - secret: - secretName: metadata-g3auto - - name: config-volume - secret: - secretName: metadata-config - optional: true - - name: config-manifest - configMap: - name: manifest-metadata - optional: true - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - env: - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogEnvVar" . | nindent 12 }} - {{- end }} - - name: GEN3_DEBUG - value: "False" - - name: GEN3_ES_ENDPOINT - value: {{ .Values.esEndpoint }} - - name: DB_HOST - valueFrom: - secretKeyRef: - name: metadata-dbcreds - key: host - optional: false - - name: DB_USER - valueFrom: - secretKeyRef: - name: metadata-dbcreds - key: username - optional: false - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: metadata-dbcreds - key: password - optional: false - - name: DB_DATABASE - valueFrom: - secretKeyRef: - name: metadata-dbcreds - key: database - optional: false - - name: DBREADY - valueFrom: - secretKeyRef: - name: metadata-dbcreds - key: dbcreated - optional: false - {{- with .Values.useAggMds }} - - name: USE_AGG_MDS - value: {{ . | quote }} - {{- end }} - {{- with .Values.aggMdsNamespace}} - - name: AGG_MDS_NAMESPACE - value: {{ . }} - {{- end }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - livenessProbe: - httpGet: - path: /_status - port: 80 - initialDelaySeconds: 30 - periodSeconds: 60 - timeoutSeconds: 30 - readinessProbe: - httpGet: - path: /_status - port: 80 - ports: - - containerPort: 80 - {{- with .Values.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 10 }} - {{- end }} - {{- with .Values.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - initContainers: - - name: {{ .Values.initContainerName }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- with .Values.initVolumeMounts }} - env: - - name: DB_HOST - valueFrom: - secretKeyRef: - name: metadata-dbcreds - key: host - optional: false - - name: DB_USER - valueFrom: - secretKeyRef: - name: metadata-dbcreds - key: username - optional: false - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: metadata-dbcreds - key: password - optional: false - - name: DB_DATABASE - valueFrom: - secretKeyRef: - name: metadata-dbcreds - key: database - optional: false - - name: DBREADY - valueFrom: - secretKeyRef: - name: metadata-dbcreds - key: dbcreated - optional: false - volumeMounts: - {{- toYaml . | nindent 10 }} - {{- end }} - {{- with .Values.initResources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - command: {{ .Values.command }} - {{- with .Values.args }} - args: - {{- toYaml . | nindent 12}} - {{- end }} \ No newline at end of file diff --git a/helm/metadata/templates/external-secret.yaml b/helm/metadata/templates/external-secret.yaml deleted file mode 100644 index c3bb34653..000000000 --- a/helm/metadata/templates/external-secret.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{ if .Values.global.externalSecrets.deploy }} -apiVersion: external-secrets.io/v1beta1 -kind: ExternalSecret -metadata: - name: metadata-g3auto -spec: - refreshInterval: 5m - secretStoreRef: - name: {{include "common.SecretStore" .}} - kind: SecretStore - target: - name: metadata-g3auto - creationPolicy: Owner - data: - - secretKey: base64Authz.txt - remoteRef: - #name of secret in secrets manager - key: {{include "metadata-g3auto" .}} - property: base64Authz.txt - - secretKey: dbcreds.json - remoteRef: - #name of secret in secrets manager - key: {{include "metadata-g3auto" .}} - property: dbcreds.json - - secretKey: metadata.env - remoteRef: - #name of secret in secrets manager - key: {{include "metadata-g3auto" .}} - property: metadata.env -{{- end }} ---- -{{ include "common.externalSecret.db" . }} \ No newline at end of file diff --git a/helm/metadata/templates/hpa.yaml b/helm/metadata/templates/hpa.yaml deleted file mode 100644 index e66ae9042..000000000 --- a/helm/metadata/templates/hpa.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "metadata.fullname" . }} - labels: - {{- include "metadata.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "metadata.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/helm/metadata/templates/metadata.yaml b/helm/metadata/templates/metadata.yaml deleted file mode 100644 index 58700a6ef..000000000 --- a/helm/metadata/templates/metadata.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: manifest-metadata -data: - metadata.json: | - { - "USE_AGG_MDS": "{{ .Values.USE_AGG_MDS }}", - "AGG_MDS_NAMESPACE": "{{ .Values.AGG_MDS_NAMESPACE }}" - } \ No newline at end of file diff --git a/helm/metadata/templates/pdb.yaml b/helm/metadata/templates/pdb.yaml deleted file mode 100644 index 2ef2de13d..000000000 --- a/helm/metadata/templates/pdb.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if and .Values.global.pdb (gt (int .Values.replicaCount) 1) }} -{{ include "common.pod_disruption_budget" . }} -{{- end }} \ No newline at end of file diff --git a/helm/metadata/templates/secret-store.yaml b/helm/metadata/templates/secret-store.yaml deleted file mode 100644 index 771c7760d..000000000 --- a/helm/metadata/templates/secret-store.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{ if .Values.global.externalSecrets.separateSecretStore }} -{{ include "common.secretstore" . }} -{{- end }} \ No newline at end of file diff --git a/helm/metadata/templates/secrets.yaml b/helm/metadata/templates/secrets.yaml deleted file mode 100644 index 0bd639d73..000000000 --- a/helm/metadata/templates/secrets.yaml +++ /dev/null @@ -1,16 +0,0 @@ -{{- if or (not .Values.global.externalSecrets.deploy) (and .Values.global.externalSecrets.deploy .Values.externalSecrets.createK8sMetadataSecret) }} -apiVersion: v1 -kind: Secret -metadata: - name: metadata-g3auto -stringData: - {{- $randomPass := printf "%s%s" "gateway:" (randAlphaNum 32) }} - base64Authz.txt: {{ $randomPass | quote | b64enc }} - metadata.env: | - DEBUG={{ .Values.debug}} - DB_HOST={{ .Values.postgres.host }} - DB_USER={{ .Values.postgres.user }} - DB_PASSWORD={{ include "metadata.postgres.password" . }} - DB_DATABASE={{ .Values.postgres.dbname }} - ADMIN_LOGINS={{ $randomPass }} -{{- end }} \ No newline at end of file diff --git a/helm/metadata/templates/service.yaml b/helm/metadata/templates/service.yaml deleted file mode 100644 index 0cc0cb061..000000000 --- a/helm/metadata/templates/service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: "metadata-service" - {{- with .Values.serviceAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} - labels: - {{- include "metadata.labels" . | nindent 4 }} -spec: - selector: - {{- include "metadata.selectorLabels" . | nindent 4 }} - {{- with .Values.service.port }} - ports: - {{- toYaml . | nindent 8 }} - {{- end }} - type: {{ .Values.service.type }} \ No newline at end of file diff --git a/helm/metadata/templates/tests/test-connection.yaml b/helm/metadata/templates/tests/test-connection.yaml deleted file mode 100644 index 4bafd3c82..000000000 --- a/helm/metadata/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "metadata-test-connection" - labels: - {{- include "metadata.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['metadata-service:80/_status'] - restartPolicy: Never diff --git a/helm/metadata/values.yaml b/helm/metadata/values.yaml deleted file mode 100644 index d39538089..000000000 --- a/helm/metadata/values.yaml +++ /dev/null @@ -1,354 +0,0 @@ -# Default values for metadata. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - - -# Global configuration -global: - # -- (map) AWS configuration - aws: - # -- (bool) Set to true if deploying to AWS. Controls ingress annotations. - enabled: false - # -- (string) Credentials for AWS stuff. - awsAccessKeyId: - # -- (string) Credentials for AWS stuff. - awsSecretAccessKey: - # -- (bool) Whether the deployment is for development purposes. - dev: true - - postgres: - # -- (bool) Whether the database should be created. - dbCreate: true - # -- (string) Name of external secret. Disabled if empty - externalSecret: "" - # -- (map) Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres - master: - # -- (string) hostname of postgres server - host: - # -- (string) username of superuser in postgres. This is used to create or restore databases - username: postgres - # -- (string) password for superuser in postgres. This is used to create or restore databases - password: - # -- (string) Port for Postgres. - port: "5432" - # -- (string) Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. - environment: default - # -- (string) Hostname for the deployment. - hostname: localhost - # -- (string) ARN of the reverse proxy certificate. - revproxyArn: arn:aws:acm:us-east-1:123456:certificate - # -- (string) URL of the data dictionary. - dictionaryUrl: https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json - # -- (string) Portal application name. - portalApp: gitops - # -- (string) S3 bucket name for Kubernetes manifest files. - kubeBucket: kube-gen3 - # -- (string) S3 bucket name for log files. - logsBucket: logs-gen3 - # -- (bool) Whether public datasets are enabled. - publicDataSets: true - # -- (string) Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` - tierAccessLevel: libre - # -- (bool) Whether network policies are enabled. - netPolicy: true - # -- (int) Number of dispatcher jobs. - dispatcherJobNum: "10" - # -- (bool) Whether Datadog is enabled. - ddEnabled: false - # -- (bool) If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. - pdb: false - # -- (int) The minimum amount of pods that are available at all times if the PDB is deployed. - minAvialable: 1 - # -- (map) External Secrets settings. - externalSecrets: - # -- (bool) Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any metadata secrets you have deployed. - deploy: false - # -- (string) Will deploy a separate External Secret Store for this service. - separateSecretStore: false - -# -- (map) External Secrets settings. -externalSecrets: - # -- (string) Will create the Helm "metadata-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. - createK8sMetadataSecret: false - # -- (string) Will override the name of the aws secrets manager secret. Default is "metadata-g3auto" - metadataG3auto: - # -- (string) Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" - dbcreds: -# -- (map) Secret information to access the db restore job S3 bucket. -secrets: - # -- (str) AWS access key ID. Overrides global key. - awsAccessKeyId: - # -- (str) AWS secret access key ID. Overrides global key. - awsSecretAccessKey: - -# -- (map) Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you -postgres: - # (bool) Whether the database should be restored from s3. Default to global.postgres.dbRestore - dbRestore: false - # -- (bool) Whether the database should be created. Default to global.postgres.dbCreate - dbCreate: - # -- (string) Hostname for postgres server. This is a service override, defaults to global.postgres.host - host: - # -- (string) Database name for postgres. This is a service override, defaults to - - database: - # -- (string) Username for postgres. This is a service override, defaults to - - username: - # -- (string) Port for Postgres. - port: "5432" - # -- (string) Password for Postgres. Will be autogenerated if left empty. - password: - # -- (string) Will create a Database for the individual service to help with developing it. - separate: false - -# -- (map) Postgresql subchart settings if deployed separately option is set to "true". -# Disable persistence by default so we can spin up and down ephemeral environments -postgresql: - primary: - persistence: - # -- (bool) Option to persist the dbs data. - enabled: false - -# -- (map) Configuration for autoscaling the number of replicas -autoscaling: - # -- (bool) Whether autoscaling is enabled - enabled: false - # -- (int) The minimum number of replicas to scale down to - minReplicas: 1 - # -- (int) The maximum number of replicas to scale up to - maxReplicas: 100 - # -- (int) The target CPU utilization percentage for autoscaling - targetCPUUtilizationPercentage: 80 - -# -- (int) Number of replicas for the deployment. -replicaCount: 1 - -# -- (int) Number of old revisions to retain -revisionHistoryLimit: 2 - -# -- (map) Rolling update deployment strategy -strategy: - type: RollingUpdate - rollingUpdate: - # -- (int) Number of additional replicas to add during rollout. - maxSurge: 1 - # -- (int) Maximum amount of pods that can be unavailable during the update. - maxUnavailable: 0 - -# -- (map) Affinity to use for the deployment. -affinity: - podAntiAffinity: - # -- (map) Option for scheduling to be required or preferred. - preferredDuringSchedulingIgnoredDuringExecution: - # -- (int) Weight value for preferred scheduling. - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - # -- (list) Label key for match expression. - - key: app - # -- (string) Operation type for the match expression. - operator: In - # -- (list) Value for the match expression key. - values: - - metadata - # -- (string) Value for topology key label. - topologyKey: "kubernetes.io/hostname" - -# -- (bool) Automount the default service account token -automountServiceAccountToken: false - -# -- (map) Docker image information. -image: - # -- (string) Docker repository. - repository: quay.io/cdis/metadata-service - # -- (string) Docker pull policy. - pullPolicy: Always - # -- (string) Overrides the image tag whose default is the chart appVersion. - tag: "feat_es-7" - -debug: false - -# Environment Variables -# -- (string) Elasticsearch endpoint. -esEndpoint: http://gen3-elasticsearch-master:9200 -# -- (bool) Set to true to aggregate metadata from multiple other Metadata Service instances. -useAggMds: "True" -# -- (string) Namespae to use if AggMds is enabled. -aggMdsNamespace: default - - -aggMdsConfig: | - { - "configuration": { - "schema": { - "_subjects_count": { - "type": "integer" - }, - "__manifest": { - "description": "an array of filename (usually DRS ids and its size", - "type": "array", - "properties": { - "file_name": { - "type": "string" - }, - "file_size": { - "type": "integer" - } - } - }, - "tags": { - "type": "array" - }, - "_unique_id": {}, - "study_description": {}, - "study_id": {}, - "study_url": {}, - "project_id": {}, - "short_name": { - "default": "not_set" - }, - "year": { - "default": "not_set" - }, - "full_name": {}, - "commons_url": {}, - "commons": {} - }, - "settings": { - "cache_drs": true - } - }, - "adapter_commons": { - "Gen3": { - "mds_url": "https://gen3.datacommons.io/", - "commons_url": "gen3.datacommons.io/", - "adapter": "gen3", - "config": { - "guid_type": "discovery_metadata", - "study_field": "gen3_discovery" - }, - "keep_original_fields": false, - "field_mappings": { - "tags": "path:tags", - "_unique_id": "path:_unique_id", - "study_description": "path:summary", - "full_name": "path:study_title", - "short_name": "path:short_name", - "year": "path:year", - "accession_number": "path:accession_number", - "commons": "Gen3 Data Commons", - "study_url": { - "path": "link", - "default": "unknown" - } - } - } - } - } - -# -- (list) Volumes to mount to the container. -volumeMounts: - - name: config-volume-g3auto - readOnly: true - mountPath: /src/.env - subPath: metadata.env - - name: config-volume - readOnly: true - mountPath: /aggregate_config.json - subPath: aggregate_config.json - - name: config-manifest - readOnly: true - mountPath: /metadata.json - subPath: json - -# -- (map) Resource requests and limits for the containers in the pod -resources: - # -- (map) The amount of resources that the container requests - requests: - # -- (string) The amount of CPU requested - cpu: 0.1 - # -- (string) The amount of memory requested - memory: 12Mi - # -- (map) The maximum amount of resources that the container is allowed to use - limits: - # -- (string) The maximum amount of CPU the container can use - cpu: 1.0 - # -- (string) The maximum amount of memory the container can use - memory: 512Mi - -# Init Container -# -- (string) Name of the init container. -initContainerName: metadata-db-migrate -# -- (list) Volumes to mount to the init container. -initVolumeMounts: - - name: config-volume-g3auto - readOnly: true - mountPath: /src/.env - subPath: metadata.env -# -- (map) Resource limits for the init container. -initResources: - # -- (map) The maximum amount of resources that the container is allowed to use - limits: - # -- (string) The maximum amount of CPU the container can use - cpu: 0.8 - # -- (string) The maximum amount of memory the container can use - memory: 512Mi -# -- (list) Command to run for the init container. -command: ["/bin/sh"] -# -- (list) Arguments to pass to the init container. -args: - - "-c" - - | - /env/bin/alembic upgrade head - -# Service and Pod -serviceAnnotations: - getambassador.io/config: | - --- - apiVersion: ambassador/v1 - ambassador_id: "gen3" - kind: Mapping - name: metadata_mapping - prefix: /index/ - service: http://metadata-service:80 - -# -- (map) Kubernetes service information. -service: - # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". - type: ClusterIP - # -- (int) The port number that the service exposes. - port: - - protocol: TCP - port: 80 - targetPort: 80 - name: http - -# Values to determine the labels that are used for the deployment, pod, etc. -# -- (string) Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". -release: "production" -# -- (string) Valid options are "true" or "false". If invalid option is set- the value will default to "false". -criticalService: "true" -# -- (string) Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. -partOf: "Discovery-Tab" -# -- (map) Will completely override the selectorLabels defined in the common chart's _label_setup.tpl -selectorLabels: -# -- (map) Will completely override the commonLabels defined in the common chart's _label_setup.tpl -commonLabels: - -# Values to configure datadog if ddEnabled is set to "true". -# -- (bool) If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. -datadogLogsInjection: true -# -- (bool) If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. -datadogProfilingEnabled: true -# -- (int) A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. -datadogTraceSampleRate: 1 - -elasticsearch: - separate: false - clusterName: gen3-elasticsearch - maxUnavailable: 0 - singleNode: true - replicas: 1 - esConfig: - elasticsearch.yml: | - # Here we can add elasticsearch config diff --git a/helm/peregrine/.helmignore b/helm/peregrine/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/helm/peregrine/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm/peregrine/Chart.yaml b/helm/peregrine/Chart.yaml deleted file mode 100644 index 10f79b5d2..000000000 --- a/helm/peregrine/Chart.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: v2 -name: peregrine -description: A Helm chart for gen3 Peregrine service - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.13 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "master" - - -dependencies: -- name: common - version: 0.1.10 - repository: file://../common -- name: postgresql - version: 11.9.13 - repository: "https://charts.bitnami.com/bitnami" - condition: postgres.separate diff --git a/helm/peregrine/README.md b/helm/peregrine/README.md deleted file mode 100644 index 8d9884c55..000000000 --- a/helm/peregrine/README.md +++ /dev/null @@ -1,104 +0,0 @@ -# peregrine - -![Version: 0.1.13](https://img.shields.io/badge/Version-0.1.13-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) - -A Helm chart for gen3 Peregrine service - -## Requirements - -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.10 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | - -## Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| affinity | map | `{}` | Affinity to use for the deployment. | -| arboristUrl | string | `nil` | URL for the arborist service | -| autoscaling | map | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Configuration for autoscaling the number of replicas | -| autoscaling.enabled | bool | `false` | Whether autoscaling is enabled | -| autoscaling.maxReplicas | int | `100` | The maximum number of replicas to scale up to | -| autoscaling.minReplicas | int | `1` | The minimum number of replicas to scale down to | -| autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage | -| commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | -| criticalService | string | `"true"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | -| datadogLogsInjection | bool | `true` | If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. | -| datadogProfilingEnabled | bool | `true` | If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. | -| datadogTraceSampleRate | int | `1` | A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. | -| env | list | `nil` | Environment variables to pass to the container | -| externalSecrets | map | `{"dbcreds":null}` | External Secrets settings. | -| externalSecrets.dbcreds | string | `nil` | Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" | -| fullnameOverride | string | `""` | Override the full name of the deployment. | -| global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false}` | AWS configuration | -| global.aws.awsAccessKeyId | string | `nil` | Credentials for AWS stuff. | -| global.aws.awsSecretAccessKey | string | `nil` | Credentials for AWS stuff. | -| global.aws.enabled | bool | `false` | Set to true if deploying to AWS. Controls ingress annotations. | -| global.ddEnabled | bool | `false` | Whether Datadog is enabled. | -| global.dev | bool | `true` | Whether the deployment is for development purposes. | -| global.dictionaryUrl | string | `"https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json"` | URL of the data dictionary. | -| global.dispatcherJobNum | int | `"10"` | Number of dispatcher jobs. | -| global.environment | string | `"default"` | Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. | -| global.externalSecrets | map | `{"deploy":false,"separateSecretStore":false}` | External Secrets settings. | -| global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any peregrine secrets you have deployed. | -| global.externalSecrets.separateSecretStore | string | `false` | Will deploy a separate External Secret Store for this service. | -| global.hostname | string | `"localhost"` | Hostname for the deployment. | -| global.kubeBucket | string | `"kube-gen3"` | S3 bucket name for Kubernetes manifest files. | -| global.logsBucket | string | `"logs-gen3"` | S3 bucket name for log files. | -| global.minAvialable | int | `1` | The minimum amount of pods that are available at all times if the PDB is deployed. | -| global.netPolicy | bool | `true` | Whether network policies are enabled. | -| global.pdb | bool | `false` | If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. | -| global.portalApp | string | `"gitops"` | Portal application name. | -| global.postgres.dbCreate | bool | `true` | Whether the database should be created. | -| global.postgres.externalSecret | string | `""` | Name of external secret. Disabled if empty | -| global.postgres.master | map | `{"host":null,"password":null,"port":"5432","username":"postgres"}` | Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres | -| global.postgres.master.host | string | `nil` | hostname of postgres server | -| global.postgres.master.password | string | `nil` | password for superuser in postgres. This is used to create or restore databases | -| global.postgres.master.port | string | `"5432"` | Port for Postgres. | -| global.postgres.master.username | string | `"postgres"` | username of superuser in postgres. This is used to create or restore databases | -| global.revproxyArn | string | `"arn:aws:acm:us-east-1:123456:certificate"` | ARN of the reverse proxy certificate. | -| global.tierAccessLevel | string | `"libre"` | Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` | -| image.pullPolicy | string | `"IfNotPresent"` | When to pull the image. | -| image.repository | string | `"quay.io/cdis/peregrine"` | The Docker image repository for the fence service | -| image.tag | string | `"feat_jq-audience"` | Overrides the image tag whose default is the chart appVersion. | -| imagePullSecrets | list | `[]` | Docker image pull secrets. | -| nameOverride | string | `""` | Override the name of the chart. | -| nodeSelector | map | `{}` | Node Selector for the pods | -| partOf | string | `"Core-Service"` | Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. | -| podAnnotations | map | `{}` | Annotations to add to the pod | -| podSecurityContext | map | `{}` | Security context for the pod | -| postgres | map | `{"database":null,"dbCreate":null,"dbRestore":false,"host":null,"password":null,"port":"5432","separate":false,"username":null}` | Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you | -| postgres.database | string | `nil` | Database name for postgres. This is a service override, defaults to - | -| postgres.dbCreate | bool | `nil` | Whether the database should be created. Default to global.postgres.dbCreate | -| postgres.host | string | `nil` | Hostname for postgres server. This is a service override, defaults to global.postgres.host | -| postgres.password | string | `nil` | Password for Postgres. Will be autogenerated if left empty. | -| postgres.port | string | `"5432"` | Port for Postgres. | -| postgres.separate | string | `false` | Will create a Database for the individual service to help with developing it. | -| postgres.username | string | `nil` | Username for postgres. This is a service override, defaults to - | -| postgresql | map | `{"primary":{"persistence":{"enabled":false}}}` | Postgresql subchart settings if deployed separately option is set to "true". Disable persistence by default so we can spin up and down ephemeral environments | -| release | string | `"production"` | Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". | -| replicaCount | int | `1` | Number of desired replicas | -| resources | map | `{"limits":{"cpu":1,"memory":"512Mi"},"requests":{"cpu":0.1,"memory":"12Mi"}}` | Resource requests and limits for the containers in the pod | -| resources.limits | map | `{"cpu":1,"memory":"512Mi"}` | The maximum amount of resources that the container is allowed to use | -| resources.limits.cpu | string | `1` | The maximum amount of CPU the container can use | -| resources.limits.memory | string | `"512Mi"` | The maximum amount of memory the container can use | -| resources.requests | map | `{"cpu":0.1,"memory":"12Mi"}` | The amount of resources that the container requests | -| resources.requests.cpu | string | `0.1` | The amount of CPU requested | -| resources.requests.memory | string | `"12Mi"` | The amount of memory requested | -| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information for External Secrets. | -| secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. | -| secrets.awsSecretAccessKey | str | `nil` | AWS secret access key ID. Overrides global key. | -| securityContext | map | `{}` | Security context for the containers in the pod | -| selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | -| service | map | `{"port":80,"type":"ClusterIP"}` | Kubernetes service information. | -| service.port | int | `80` | The port number that the service exposes. | -| service.type | string | `"ClusterIP"` | Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". | -| serviceAccount | map | `{"annotations":{},"create":true,"name":""}` | Service account to use or create. | -| serviceAccount.annotations | map | `{}` | Annotations to add to the service account. | -| serviceAccount.create | bool | `true` | Specifies whether a service account should be created. | -| serviceAccount.name | string | `""` | The name of the service account | -| tolerations | list | `[]` | Tolerations for the pods | -| volumeMounts | list | `[{"mountPath":"/var/www/peregrine/settings.py","name":"config-volume","readOnly":true,"subPath":"settings.py"}]` | Volumes to mount to the container. | -| volumes | list | `[{"emptyDir":{},"name":"shared-data"},{"name":"config-volume","secret":{"secretName":"peregrine-secret"}}]` | Volumes to attach to the container. | - diff --git a/helm/peregrine/peregrine-secret/config_helper.py b/helm/peregrine/peregrine-secret/config_helper.py deleted file mode 100644 index 6b303beac..000000000 --- a/helm/peregrine/peregrine-secret/config_helper.py +++ /dev/null @@ -1,376 +0,0 @@ -import json -import os -import copy -import argparse -import re -import types - -# -# make it easy to change this for testing -XDG_DATA_HOME = os.getenv("XDG_DATA_HOME", "/usr/share/") - - -def default_search_folders(app_name): - """ - Return the list of folders to search for configuration files - """ - return [ - "%s/cdis/%s" % (XDG_DATA_HOME, app_name), - "/usr/share/cdis/%s" % app_name, - "%s/gen3/%s" % (XDG_DATA_HOME, app_name), - "/usr/share/gen3/%s" % app_name, - "/var/www/%s" % app_name, - "/etc/gen3/%s" % app_name, - ] - - -def find_paths(file_name, app_name, search_folders=None): - """ - Search the given folders for file_name - search_folders defaults to default_search_folders if not specified - return the first path to file_name found - """ - search_folders = search_folders or default_search_folders(app_name) - possible_files = [os.path.join(folder, file_name) for folder in search_folders] - return [path for path in possible_files if os.path.exists(path)] - - -def load_json(file_name, app_name, search_folders=None): - """ - json.load(file_name) after finding file_name in search_folders - - return the loaded json data or None if file not found - """ - actual_files = find_paths(file_name, app_name, search_folders) - if not actual_files: - return None - with open(actual_files[0], "r") as reader: - return json.load(reader) - - -def inject_creds_into_fence_config(creds_file_path, config_file_path): - creds_file = open(creds_file_path, "r") - creds = json.load(creds_file) - creds_file.close() - - # get secret values from creds.json file - db_host = _get_nested_value(creds, "db_host") - db_username = _get_nested_value(creds, "db_username") - db_password = _get_nested_value(creds, "db_password") - db_database = _get_nested_value(creds, "db_database") - hostname = _get_nested_value(creds, "hostname") - indexd_password = _get_nested_value(creds, "indexd_password") - google_client_secret = _get_nested_value(creds, "google_client_secret") - google_client_id = _get_nested_value(creds, "google_client_id") - hmac_key = _get_nested_value(creds, "hmac_key") - db_path = "postgresql://{}:{}@{}:5432/{}".format( - db_username, db_password, db_host, db_database - ) - - config_file = open(config_file_path, "r").read() - - print(" DB injected with value(s) from creds.json") - config_file = _replace(config_file, "DB", db_path) - - print(" BASE_URL injected with value(s) from creds.json") - config_file = _replace(config_file, "BASE_URL", "https://{}/user".format(hostname)) - - print(" INDEXD_PASSWORD injected with value(s) from creds.json") - config_file = _replace(config_file, "INDEXD_PASSWORD", indexd_password) - config_file = _replace(config_file, "INDEXD_USERNAME", "fence") - - print(" ENCRYPTION_KEY injected with value(s) from creds.json") - config_file = _replace(config_file, "ENCRYPTION_KEY", hmac_key) - - print( - " OPENID_CONNECT/google/client_secret injected with value(s) " - "from creds.json" - ) - config_file = _replace( - config_file, "OPENID_CONNECT/google/client_secret", google_client_secret - ) - - print(" OPENID_CONNECT/google/client_id injected with value(s) from creds.json") - config_file = _replace( - config_file, "OPENID_CONNECT/google/client_id", google_client_id - ) - - open(config_file_path, "w+").write(config_file) - - -def set_prod_defaults(config_file_path): - config_file = open(config_file_path, "r").read() - - print( - " CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS set as " - "var/www/fence/fence_google_app_creds_secret.json" - ) - config_file = _replace( - config_file, - "CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS", - "/var/www/fence/fence_google_app_creds_secret.json", - ) - - print( - " CIRRUS_CFG/GOOGLE_STORAGE_CREDS set as " - "var/www/fence/fence_google_storage_creds_secret.json" - ) - config_file = _replace( - config_file, - "CIRRUS_CFG/GOOGLE_STORAGE_CREDS", - "/var/www/fence/fence_google_storage_creds_secret.json", - ) - - print(" INDEXD set as http://indexd-service/") - config_file = _replace(config_file, "INDEXD", "http://indexd-service/") - - print(" ARBORIST set as http://arborist-service/") - config_file = _replace(config_file, "ARBORIST", "http://arborist-service/") - - print(" HTTP_PROXY/host set as cloud-proxy.internal.io") - config_file = _replace(config_file, "HTTP_PROXY/host", "cloud-proxy.internal.io") - - print(" HTTP_PROXY/port set as 3128") - config_file = _replace(config_file, "HTTP_PROXY/port", 3128) - - print(" DEBUG set to false") - config_file = _replace(config_file, "DEBUG", False) - - print(" MOCK_AUTH set to false") - config_file = _replace(config_file, "MOCK_AUTH", False) - - print(" MOCK_GOOGLE_AUTH set to false") - config_file = _replace(config_file, "MOCK_GOOGLE_AUTH", False) - - print(" AUTHLIB_INSECURE_TRANSPORT set to true") - config_file = _replace(config_file, "AUTHLIB_INSECURE_TRANSPORT", True) - - print(" SESSION_COOKIE_SECURE set to true") - config_file = _replace(config_file, "SESSION_COOKIE_SECURE", True) - - print(" ENABLE_CSRF_PROTECTION set to true") - config_file = _replace(config_file, "ENABLE_CSRF_PROTECTION", True) - - open(config_file_path, "w+").write(config_file) - - -def inject_other_files_into_fence_config(other_files, config_file_path): - additional_cfgs = _get_all_additional_configs(other_files) - - config_file = open(config_file_path, "r").read() - - for key, value in additional_cfgs.iteritems(): - print(" {} set to {}".format(key, value)) - config_file = _nested_replace(config_file, key, value) - - open(config_file_path, "w+").write(config_file) - - -def _get_all_additional_configs(other_files): - """ - Attempt to parse given list of files and extract configuration variables and values - """ - additional_configs = dict() - for file_path in other_files: - try: - file_ext = file_path.strip().split(".")[-1] - if file_ext == "json": - json_file = open(file_path, "r") - configs = json.load(json_file) - json_file.close() - elif file_ext == "py": - configs = from_pyfile(file_path) - else: - print( - "Cannot load config vars from a file with extention: {}".format( - file_ext - ) - ) - except Exception as exc: - # if there's any issue reading the file, exit - print( - "Error reading {}. Cannot get configuration. Skipping this file. " - "Details: {}".format(other_files, str(exc)) - ) - continue - - if configs: - additional_configs.update(configs) - - return additional_configs - - -def _nested_replace(config_file, key, value, replacement_path=None): - replacement_path = replacement_path or key - try: - for inner_key, inner_value in value.iteritems(): - temp_path = replacement_path - temp_path = temp_path + "/" + inner_key - config_file = _nested_replace( - config_file, inner_key, inner_value, temp_path - ) - except AttributeError: - # not a dict so replace - if value is not None: - config_file = _replace(config_file, replacement_path, value) - - return config_file - - -def _replace(yaml_config, path_to_key, replacement_value, start=0, nested_level=0): - """ - Replace a nested value in a YAML file string with the given value without - losing comments. Uses a regex to do the replacement. - - Args: - yaml_config (str): a string representing a full configuration file - path_to_key (str): nested/path/to/key. The value of this key will be - replaced - replacement_value (str): Replacement value for the key from - path_to_key - """ - nested_path_to_replace = path_to_key.split("/") - - # our regex looks for a specific number of spaces to ensure correct - # level of nesting. It matches to the end of the line - search_string = ( - " " * nested_level + ".*" + nested_path_to_replace[0] + "(')?(\")?:.*\n" - ) - matches = re.search(search_string, yaml_config[start:]) - - # early return if we haven't found anything - if not matches: - return yaml_config - - # if we're on the last item in the path, we need to get the value and - # replace it in the original file - if len(nested_path_to_replace) == 1: - # replace the current key:value with the new replacement value - match_start = start + matches.start(0) + len(" " * nested_level) - match_end = start + matches.end(0) - yaml_config = ( - yaml_config[:match_start] - + "{}: {}\n".format( - nested_path_to_replace[0], - _get_yaml_replacement_value(replacement_value, nested_level), - ) - + yaml_config[match_end:] - ) - - return yaml_config - - # set new start point to past current match and move on to next match - start = matches.end(0) - nested_level += 1 - del nested_path_to_replace[0] - - return _replace( - yaml_config, - "/".join(nested_path_to_replace), - replacement_value, - start, - nested_level, - ) - - -def from_pyfile(filename, silent=False): - """ - Modeled after flask's ability to load in python files: - https://github.com/pallets/flask/blob/master/flask/config.py - - Some alterations were made but logic is essentially the same - """ - filename = os.path.abspath(filename) - d = types.ModuleType("config") - d.__file__ = filename - try: - with open(filename, mode="rb") as config_file: - exec(compile(config_file.read(), filename, "exec"), d.__dict__) - except IOError as e: - print("Unable to load configuration file ({})".format(e.strerror)) - if silent: - return False - raise - return _from_object(d) - - -def _from_object(obj): - configs = {} - for key in dir(obj): - if key.isupper(): - configs[key] = getattr(obj, key) - return configs - - -def _get_yaml_replacement_value(value, nested_level=0): - if isinstance(value, str): - return "'" + value + "'" - elif isinstance(value, bool): - return str(value).lower() - elif isinstance(value, list) or isinstance(value, set): - output = "" - for item in value: - # spaces for nested level then spaces and hyphen for each list item - output += ( - "\n" - + " " * nested_level - + " - " - + _get_yaml_replacement_value(item) - + "" - ) - return output - else: - return value - - -def _get_nested_value(dictionary, nested_path): - """ - Return a value from a dictionary given a path-like nesting of keys. - - Will default to an empty string if value cannot be found. - - Args: - dictionary (dict): a dictionary - nested_path (str): nested/path/to/key - - Returns: - ?: Value from dict - """ - replacement_value_path = nested_path.split("/") - replacement_value = copy.deepcopy(dictionary) - - for item in replacement_value_path: - replacement_value = replacement_value.get(item, {}) - - if replacement_value == {}: - replacement_value = "" - - return replacement_value - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "-i", - "--creds_file_to_inject", - default="creds.json", - help="creds file to inject into the configuration yaml", - ) - parser.add_argument( - "--other_files_to_inject", - nargs="+", - help="fence_credentials.json, local_settings.py, fence_settings.py file(s) to " - "inject into the configuration yaml", - ) - parser.add_argument( - "-c", "--config_file", default="config.yaml", help="configuration yaml" - ) - args = parser.parse_args() - - inject_creds_into_fence_config(args.creds_file_to_inject, args.config_file) - set_prod_defaults(args.config_file) - - if args.other_files_to_inject: - inject_other_files_into_fence_config( - args.other_files_to_inject, args.config_file - ) diff --git a/helm/peregrine/peregrine-secret/settings.py b/helm/peregrine/peregrine-secret/settings.py deleted file mode 100644 index 1a623a907..000000000 --- a/helm/peregrine/peregrine-secret/settings.py +++ /dev/null @@ -1,87 +0,0 @@ -##################################################### -# DO NOT CHANGE THIS FILE # -# config updates should be done in the service code # -##################################################### - -from peregrine.api import app, app_init -from os import environ -# import config_helper - -APP_NAME='peregrine' -# def load_json(file_name): -# return config_helper.load_json(file_name, APP_NAME) - -# conf_data = load_json('creds.json') -config = app.config - -# config["AUTH"] = 'https://auth.service.consul:5000/v3/' -# config["AUTH_ADMIN_CREDS"] = None -# config["INTERNAL_AUTH"] = None - -# ARBORIST deprecated, replaced by ARBORIST_URL -# ARBORIST_URL is initialized in app_init() directly -# config["ARBORIST"] = "http://arborist-service/" - -config['INDEX_CLIENT'] = { - 'host': environ.get('INDEX_CLIENT_HOST') or 'http://indexd-service', - 'version': 'v0', - 'auth': ('gdcapi', environ.get( "PGHOST") ), -} -# config["FAKE_AUTH"] = environ.get( "FAKE_AUTH", False) -config["PSQLGRAPH"] = { - 'host': environ.get( "PGHOST"), - 'user': environ.get( "PGUSER"), - 'password': environ.get( "PGPASSWORD"), - 'database': environ.get( "PGDB"), -} - -config['HMAC_ENCRYPTION_KEY'] = environ.get( "HMAC_ENCRYPTION_KEY") -config['FLASK_SECRET_KEY'] = environ.get( "FLASK_SECRET_KEY") - -fence_username = environ.get( "FENCE_DB_USER") -fence_password = environ.get( "FENCE_DB_PASS") -fence_host = environ.get( "FENCE_DB_HOST") -fence_database = environ.get( "FENCE_DB_DBNAME") -config['PSQL_USER_DB_CONNECTION'] = 'postgresql://%s:%s@%s:5432/%s' % (fence_username, fence_password, fence_host, fence_database) - -config['DICTIONARY_URL'] = environ.get('DICTIONARY_URL','https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json') - -# config['SUBMISSION'] = { -# 'bucket': conf_data.get( 'bagit_bucket', '' ) -# } - -# config['STORAGE'] = { -# "s3": -# { -# "access_key": conf_data.get( 's3_access', '' ), -# 'secret_key': conf_data.get( 's3_secret', '' ) -# } -# } - -hostname = environ.get("CONF_HOSTNAME") -config['OIDC_ISSUER'] = 'https://%s/user' % hostname - -config['OAUTH2'] = { - 'client_id': "conf_data.get('oauth2_client_id', '{{oauth2_client_id}}')", - 'client_secret': "conf_data.get('oauth2_client_secret', '{{oauth2_client_secret}}')", - 'api_base_url': 'https://%s/user/' % hostname, - 'authorize_url': 'https://%s/user/oauth2/authorize' % hostname, - 'access_token_url': 'https://%s/user/oauth2/token' % hostname, - 'refresh_token_url': 'https://%s/user/oauth2/token' % hostname, - 'client_kwargs': { - 'redirect_uri': 'https://%s/api/v0/oauth2/authorize' % hostname, - 'scope': 'openid data user', - }, - # deprecated key values, should be removed after all commons use new oidc - 'internal_oauth_provider': 'http://fence-service/oauth2/', - 'oauth_provider': 'https://%s/user/oauth2/' % hostname, - 'redirect_uri': 'https://%s/api/v0/oauth2/authorize' % hostname -} - -config['USER_API'] = environ.get('FENCE_URL') or 'http://fence-service/' -# use the USER_API URL instead of the public issuer URL to accquire JWT keys -config['FORCE_ISSUER'] = True -print(config) -app_init(app) -application = app -application.debug = (environ.get('GEN3_DEBUG') == "True") diff --git a/helm/peregrine/templates/NOTES.txt b/helm/peregrine/templates/NOTES.txt deleted file mode 100644 index c1e7e1aef..000000000 --- a/helm/peregrine/templates/NOTES.txt +++ /dev/null @@ -1 +0,0 @@ -{{ .Chart.Name }} has been deployed successfully. diff --git a/helm/peregrine/templates/_helpers.tpl b/helm/peregrine/templates/_helpers.tpl deleted file mode 100644 index 4d1439e08..000000000 --- a/helm/peregrine/templates/_helpers.tpl +++ /dev/null @@ -1,93 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "peregrine.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "peregrine.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "peregrine.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "peregrine.labels" -}} -{{- if .Values.commonLabels }} - {{- with .Values.commonLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.commonLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "peregrine.selectorLabels" -}} -{{- if .Values.selectorLabels }} - {{- with .Values.selectorLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.selectorLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "peregrine.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "peregrine.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} - - -{{/* - Postgres Password lookup -*/}} -{{- define "peregrine.postgres.password" -}} -{{- $masterpass := (lookup "v1" "Secret" "postgres" "postgres-postgresql" ) -}} -# {{- $localpass := (lookup "v1" "Secret" .Release.Namespace "{{ .Chart.Name }}-dbcreds" ) -}} -{{- if $localpass }} -{{- default (index $masterpass.data "postgres-password" | b64dec) }} -{{- else }} -{{- default .Values.postgres.password }} -{{- end }} -{{- end }} - -{{/* -Define dictionaryUrl -*/}} -{{- define "peregrine.dictionaryUrl" -}} -{{- if .Values.global }} -{{- .Values.global.dictionaryUrl }} -{{- else}} -{{- .Values.dictionaryUrl }} -{{- end }} -{{- end }} diff --git a/helm/peregrine/templates/aws-config.yaml b/helm/peregrine/templates/aws-config.yaml deleted file mode 100644 index 398770d36..000000000 --- a/helm/peregrine/templates/aws-config.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if or (.Values.secrets.awsSecretAccessKey) (.Values.global.aws.awsSecretAccessKey ) }} -{{ include "common.awsconfig" . }} -{{- end -}} \ No newline at end of file diff --git a/helm/peregrine/templates/db-init.yaml b/helm/peregrine/templates/db-init.yaml deleted file mode 100644 index 5ef14e87e..000000000 --- a/helm/peregrine/templates/db-init.yaml +++ /dev/null @@ -1,6 +0,0 @@ -{{ include "common.db-secret" . }} ---- -{{ include "common.db_setup_job" . }} ---- -{{ include "common.db_setup_sa" . }} ---- \ No newline at end of file diff --git a/helm/peregrine/templates/deployment.yaml b/helm/peregrine/templates/deployment.yaml deleted file mode 100644 index e554be55e..000000000 --- a/helm/peregrine/templates/deployment.yaml +++ /dev/null @@ -1,183 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: peregrine-deployment - labels: - {{- include "peregrine.labels" . | nindent 4 }} - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 4 }} - {{- end }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "peregrine.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "peregrine.selectorLabels" . | nindent 8 }} - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "peregrine.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogEnvVar" . | nindent 12 }} - {{- end }} - - name: FENCE_DB_USER - valueFrom: - secretKeyRef: - name: fence-dbcreds - key: username - optional: false - - name: FENCE_DB_PASS - valueFrom: - secretKeyRef: - name: fence-dbcreds - key: password - optional: false - - name: FENCE_DB_HOST - valueFrom: - secretKeyRef: - name: fence-dbcreds - key: host - optional: false - - name: FENCE_DB_DBNAME - valueFrom: - secretKeyRef: - name: fence-dbcreds - key: database - optional: false - - name: DBREADY_FENCE - valueFrom: - secretKeyRef: - name: fence-dbcreds - key: dbcreated - optional: false - - name: FLASK_SECRET_KEY - value: "TODO: FIX THIS!!!" - - name: PGHOST - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: host - optional: false - - name: PGUSER - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: username - optional: false - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: password - optional: false - - name: PGDB - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: database - optional: false - - name: DBREADY - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: dbcreated - optional: false - - name: GEN3_UWSGI_TIMEOUT - value: "600" - - name: DICTIONARY_URL - valueFrom: - configMapKeyRef: - name: manifest-global - key: dictionary_url - optional: true - - name: PUBLIC_DATASETS - valueFrom: - configMapKeyRef: - name: manifest-global - key: public_datasets - optional: true - - name: INDEX_CLIENT_HOST - valueFrom: - configMapKeyRef: - name: manifest-global - key: indexd_url - optional: true - - name: GRAPHQL_TIMEOUT - valueFrom: - configMapKeyRef: - name: manifest-peregrine - key: peregrine_timeout - optional: true - - name: FENCE_URL - valueFrom: - configMapKeyRef: - name: manifest-global - key: fence_url - optional: true - - name: ARBORIST_URL - value: {{ .Values.arborist_url }} - - name: GEN3_SIDECAR - value: "False" - - name: CONF_HOSTNAME - value: {{ .Values.global.hostname }} - {{- with .Values.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 10 }} - {{- end }} - ports: - - name: http - containerPort: 80 - protocol: TCP - livenessProbe: - httpGet: - path: /_status?timeout=20 - port: http - # peregrine can take forever to initialize - initialDelaySeconds: 60 - periodSeconds: 60 - timeoutSeconds: 30 - readinessProbe: - httpGet: - path: /_status?timeout=2 - port: http - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/peregrine/templates/external-secret.yaml b/helm/peregrine/templates/external-secret.yaml deleted file mode 100644 index 70c278fea..000000000 --- a/helm/peregrine/templates/external-secret.yaml +++ /dev/null @@ -1 +0,0 @@ -{{ include "common.externalSecret.db" . }} \ No newline at end of file diff --git a/helm/peregrine/templates/hpa.yaml b/helm/peregrine/templates/hpa.yaml deleted file mode 100644 index 09600fb71..000000000 --- a/helm/peregrine/templates/hpa.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "peregrine.fullname" . }} - labels: - {{- include "peregrine.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "peregrine.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/helm/peregrine/templates/pdb.yaml b/helm/peregrine/templates/pdb.yaml deleted file mode 100644 index 2ef2de13d..000000000 --- a/helm/peregrine/templates/pdb.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if and .Values.global.pdb (gt (int .Values.replicaCount) 1) }} -{{ include "common.pod_disruption_budget" . }} -{{- end }} \ No newline at end of file diff --git a/helm/peregrine/templates/peregrine-secret.yaml b/helm/peregrine/templates/peregrine-secret.yaml deleted file mode 100644 index e75f43a23..000000000 --- a/helm/peregrine/templates/peregrine-secret.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: peregrine-secret -type: Opaque -data: -{{ (.Files.Glob "peregrine-secret/*").AsSecrets | indent 2 }} diff --git a/helm/peregrine/templates/secret-store.yaml b/helm/peregrine/templates/secret-store.yaml deleted file mode 100644 index 771c7760d..000000000 --- a/helm/peregrine/templates/secret-store.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{ if .Values.global.externalSecrets.separateSecretStore }} -{{ include "common.secretstore" . }} -{{- end }} \ No newline at end of file diff --git a/helm/peregrine/templates/service.yaml b/helm/peregrine/templates/service.yaml deleted file mode 100644 index afff1c22c..000000000 --- a/helm/peregrine/templates/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: peregrine-service - labels: - {{- include "peregrine.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "peregrine.selectorLabels" . | nindent 4 }} diff --git a/helm/peregrine/templates/serviceaccount.yaml b/helm/peregrine/templates/serviceaccount.yaml deleted file mode 100644 index 3c27db5bf..000000000 --- a/helm/peregrine/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "peregrine.serviceAccountName" . }} - labels: - {{- include "peregrine.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/helm/peregrine/templates/tests/test-connection.yaml b/helm/peregrine/templates/tests/test-connection.yaml deleted file mode 100644 index 56bcf4b57..000000000 --- a/helm/peregrine/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: peregrine-test-connection - labels: - {{- include "peregrine.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['peregrine-service:{{ .Values.service.port }}/_status'] - restartPolicy: Never diff --git a/helm/peregrine/values.yaml b/helm/peregrine/values.yaml deleted file mode 100644 index 460866585..000000000 --- a/helm/peregrine/values.yaml +++ /dev/null @@ -1,232 +0,0 @@ -# Default values for peregrine. -# This is a YAML-formatted file. - - -# Global configuration -global: - # -- (map) AWS configuration - aws: - # -- (bool) Set to true if deploying to AWS. Controls ingress annotations. - enabled: false - # -- (string) Credentials for AWS stuff. - awsAccessKeyId: - # -- (string) Credentials for AWS stuff. - awsSecretAccessKey: - # -- (bool) Whether the deployment is for development purposes. - dev: true - - postgres: - # -- (bool) Whether the database should be created. - dbCreate: true - # -- (string) Name of external secret. Disabled if empty - externalSecret: "" - # -- (map) Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres - master: - # -- (string) hostname of postgres server - host: - # -- (string) username of superuser in postgres. This is used to create or restore databases - username: postgres - # -- (string) password for superuser in postgres. This is used to create or restore databases - password: - # -- (string) Port for Postgres. - port: "5432" - # -- (string) Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. - environment: default - # -- (string) Hostname for the deployment. - hostname: localhost - # -- (string) ARN of the reverse proxy certificate. - revproxyArn: arn:aws:acm:us-east-1:123456:certificate - # -- (string) URL of the data dictionary. - dictionaryUrl: https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json - # -- (string) Portal application name. - portalApp: gitops - # -- (string) S3 bucket name for Kubernetes manifest files. - kubeBucket: kube-gen3 - # -- (string) S3 bucket name for log files. - logsBucket: logs-gen3 - # -- (string) Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` - tierAccessLevel: libre - # -- (bool) Whether network policies are enabled. - netPolicy: true - # -- (int) Number of dispatcher jobs. - dispatcherJobNum: "10" - # -- (bool) Whether Datadog is enabled. - ddEnabled: false - # -- (bool) If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. - pdb: false - # -- (int) The minimum amount of pods that are available at all times if the PDB is deployed. - minAvialable: 1 - # -- (map) External Secrets settings. - externalSecrets: - # -- (bool) Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any peregrine secrets you have deployed. - deploy: false - # -- (string) Will deploy a separate External Secret Store for this service. - separateSecretStore: false - -# -- (map) External Secrets settings. -externalSecrets: - # -- (string) Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" - dbcreds: -# -- (map) Secret information for External Secrets. -secrets: - # -- (str) AWS access key ID. Overrides global key. - awsAccessKeyId: - # -- (str) AWS secret access key ID. Overrides global key. - awsSecretAccessKey: - - -# -- (map) Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you -postgres: - # (bool) Whether the database should be restored from s3. Default to global.postgres.dbRestore - dbRestore: false - # -- (bool) Whether the database should be created. Default to global.postgres.dbCreate - dbCreate: - # -- (string) Hostname for postgres server. This is a service override, defaults to global.postgres.host - host: - # -- (string) Database name for postgres. This is a service override, defaults to - - database: - # -- (string) Username for postgres. This is a service override, defaults to - - username: - # -- (string) Port for Postgres. - port: "5432" - # -- (string) Password for Postgres. Will be autogenerated if left empty. - password: - # -- (string) Will create a Database for the individual service to help with developing it. - separate: false - -# -- (map) Postgresql subchart settings if deployed separately option is set to "true". -# Disable persistence by default so we can spin up and down ephemeral environments -postgresql: - primary: - persistence: - enabled: false - -# -- (string) URL for the arborist service -arboristUrl: - -# -- (int) Number of desired replicas -replicaCount: 1 - -image: - # -- (string) The Docker image repository for the fence service - repository: quay.io/cdis/peregrine - # -- (string) When to pull the image. - pullPolicy: IfNotPresent - # -- (string) Overrides the image tag whose default is the chart appVersion. - tag: "feat_jq-audience" - -# -- (list) Docker image pull secrets. -imagePullSecrets: [] - -# -- (string) Override the name of the chart. -nameOverride: "" - -# -- (string) Override the full name of the deployment. -fullnameOverride: "" - -# -- (map) Service account to use or create. -serviceAccount: - # -- (bool) Specifies whether a service account should be created. - create: true - # -- (map) Annotations to add to the service account. - annotations: {} - # If not set and create is true, a name is generated using the fullname template - # -- (string) The name of the service account - name: "" - -# -- (map) Annotations to add to the pod -podAnnotations: {} - -# -- (map) Security context for the pod -podSecurityContext: {} - # fsGroup: 2000 - -# -- (map) Security context for the containers in the pod -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -# -- (map) Kubernetes service information. -service: - # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". - type: ClusterIP - # -- (int) The port number that the service exposes. - port: 80 - -# -- (map) Resource requests and limits for the containers in the pod -resources: - # -- (map) The amount of resources that the container requests - requests: - # -- (string) The amount of CPU requested - cpu: 0.1 - # -- (string) The amount of memory requested - memory: 12Mi - # -- (map) The maximum amount of resources that the container is allowed to use - limits: - # -- (string) The maximum amount of CPU the container can use - cpu: 1.0 - # -- (string) The maximum amount of memory the container can use - memory: 512Mi - -# -- (map) Configuration for autoscaling the number of replicas -autoscaling: - # -- (bool) Whether autoscaling is enabled - enabled: false - # -- (int) The minimum number of replicas to scale down to - minReplicas: 1 - # -- (int) The maximum number of replicas to scale up to - maxReplicas: 100 - # -- (int) Target CPU utilization percentage - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -# -- (map) Node Selector for the pods -nodeSelector: {} - -# -- (list) Tolerations for the pods -tolerations: [] - -# -- (map) Affinity to use for the deployment. -affinity: {} - -# -- (list) Environment variables to pass to the container -env: - -# -- (list) Volumes to attach to the container. -volumes: -- name: shared-data - emptyDir: {} -- name: config-volume - secret: - secretName: "peregrine-secret" - -# -- (list) Volumes to mount to the container. -volumeMounts: - - name: "config-volume" - readOnly: true - mountPath: "/var/www/peregrine/settings.py" - subPath: "settings.py" - -# Values to determine the labels that are used for the deployment, pod, etc. -# -- (string) Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". -release: "production" -# -- (string) Valid options are "true" or "false". If invalid option is set- the value will default to "false". -criticalService: "true" -# -- (string) Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. -partOf: "Core-Service" -# -- (map) Will completely override the selectorLabels defined in the common chart's _label_setup.tpl -selectorLabels: -# -- (map) Will completely override the commonLabels defined in the common chart's _label_setup.tpl -commonLabels: - -# Values to configure datadog if ddEnabled is set to "true". -# -- (bool) If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. -datadogLogsInjection: true -# -- (bool) If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. -datadogProfilingEnabled: true -# -- (int) A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. -datadogTraceSampleRate: 1 diff --git a/helm/pidgin/.helmignore b/helm/pidgin/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/helm/pidgin/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm/pidgin/Chart.yaml b/helm/pidgin/Chart.yaml deleted file mode 100644 index be9c20863..000000000 --- a/helm/pidgin/Chart.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: v2 -name: pidgin -description: A Helm chart for gen3 Pidgin Service - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.10 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "master" - -dependencies: -- name: common - version: 0.1.10 - repository: file://../common diff --git a/helm/pidgin/README.md b/helm/pidgin/README.md deleted file mode 100644 index 21914338e..000000000 --- a/helm/pidgin/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# pidgin - -![Version: 0.1.10](https://img.shields.io/badge/Version-0.1.10-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) - -A Helm chart for gen3 Pidgin Service - -## Requirements - -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.10 | - -## Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| affinity | map | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["pidgin"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]}}` | Affinity to use for the deployment. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution | map | `[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["pidgin"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]` | Option for scheduling to be required or preferred. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0] | int | `{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["pidgin"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}` | Weight value for preferred scheduling. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0] | list | `{"key":"app","operator":"In","values":["pidgin"]}` | Label key for match expression. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].operator | string | `"In"` | Operation type for the match expression. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].values | list | `["pidgin"]` | Value for the match expression key. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey | string | `"kubernetes.io/hostname"` | Value for topology key label. | -| automountServiceAccountToken | bool | `false` | Automount the default service account token | -| autoscaling | map | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Configuration for autoscaling the number of replicas | -| autoscaling.enabled | bool | `false` | Whether autoscaling is enabled | -| autoscaling.maxReplicas | int | `100` | The maximum number of replicas to scale up to | -| autoscaling.minReplicas | int | `1` | The minimum number of replicas to scale down to | -| autoscaling.targetCPUUtilizationPercentage | int | `80` | The target CPU utilization percentage for autoscaling | -| commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | -| criticalService | string | `"false"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | -| dataDog | bool | `{"enabled":false,"env":"dev"}` | Whether Datadog is enabled. | -| datadogLogsInjection | bool | `true` | If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. | -| datadogProfilingEnabled | bool | `true` | If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. | -| datadogTraceSampleRate | int | `1` | A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. | -| global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false}` | AWS configuration | -| global.aws.awsAccessKeyId | string | `nil` | Credentials for AWS stuff. | -| global.aws.awsSecretAccessKey | string | `nil` | Credentials for AWS stuff. | -| global.aws.enabled | bool | `false` | Set to true if deploying to AWS. Controls ingress annotations. | -| global.ddEnabled | bool | `false` | Whether Datadog is enabled. | -| global.dev | bool | `true` | Whether the deployment is for development purposes. | -| global.dictionaryUrl | string | `"https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json"` | URL of the data dictionary. | -| global.dispatcherJobNum | int | `"10"` | Number of dispatcher jobs. | -| global.environment | string | `"default"` | Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. | -| global.hostname | string | `"localhost"` | Hostname for the deployment. | -| global.kubeBucket | string | `"kube-gen3"` | S3 bucket name for Kubernetes manifest files. | -| global.logsBucket | string | `"logs-gen3"` | S3 bucket name for log files. | -| global.minAvialable | int | `1` | The minimum amount of pods that are available at all times if the PDB is deployed. | -| global.netPolicy | bool | `true` | Whether network policies are enabled. | -| global.pdb | bool | `false` | If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. | -| global.portalApp | string | `"gitops"` | Portal application name. | -| global.postgres.dbCreate | bool | `true` | Whether the database should be created. | -| global.postgres.externalSecret | string | `""` | Name of external secret. Disabled if empty | -| global.postgres.master | map | `{"host":null,"password":null,"port":"5432","username":"postgres"}` | Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres | -| global.postgres.master.host | string | `nil` | hostname of postgres server | -| global.postgres.master.password | string | `nil` | password for superuser in postgres. This is used to create or restore databases | -| global.postgres.master.port | string | `"5432"` | Port for Postgres. | -| global.postgres.master.username | string | `"postgres"` | username of superuser in postgres. This is used to create or restore databases | -| global.publicDataSets | bool | `true` | Whether public datasets are enabled. | -| global.revproxyArn | string | `"arn:aws:acm:us-east-1:123456:certificate"` | ARN of the reverse proxy certificate. | -| global.tierAccessLevel | string | `"libre"` | Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` | -| image.pullPolicy | string | `"Always"` | When to pull the image. | -| image.repository | string | `"quay.io/cdis/pidgin"` | The Docker image repository for the fence service | -| image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | -| partOf | string | `"Peregrine"` | Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. | -| postgres | map | `{"database":null,"dbCreate":null,"dbRestore":false,"host":null,"password":null,"port":"5432","username":null}` | Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you | -| postgres.database | string | `nil` | Database name for postgres. This is a service override, defaults to - | -| postgres.dbCreate | bool | `nil` | Whether the database should be created. Default to global.postgres.dbCreate | -| postgres.host | string | `nil` | Hostname for postgres server. This is a service override, defaults to global.postgres.host | -| postgres.password | string | `nil` | Password for Postgres. Will be autogenerated if left empty. | -| postgres.port | string | `"5432"` | Port for Postgres. | -| postgres.username | string | `nil` | Username for postgres. This is a service override, defaults to - | -| release | string | `"production"` | Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". | -| replicaCount | int | `1` | Number of desired replicas | -| resources | map | `nil` | Resource requests and limits for the containers in the pod | -| revisionHistoryLimit | int | `2` | Number of old revisions to retain | -| selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | -| service | map | `{"port":[{"name":"http","port":80,"protocol":"TCP","targetPort":80},{"name":"https","port":443,"protocol":"TCP","targetPort":443}],"type":"ClusterIP"}` | Kubernetes service information. | -| service.port | list | `[{"name":"http","port":80,"protocol":"TCP","targetPort":80},{"name":"https","port":443,"protocol":"TCP","targetPort":443}]` | The port numbers that the service exposes. | -| service.type | string | `"ClusterIP"` | Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". | -| strategy | map | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | Rolling update deployment strategy | -| strategy.rollingUpdate.maxSurge | int | `1` | Number of additional replicas to add during rollout. | -| strategy.rollingUpdate.maxUnavailable | int | `0` | Maximum amount of pods that can be unavailable during the update. | - diff --git a/helm/pidgin/templates/NOTES.txt b/helm/pidgin/templates/NOTES.txt deleted file mode 100644 index c1e7e1aef..000000000 --- a/helm/pidgin/templates/NOTES.txt +++ /dev/null @@ -1 +0,0 @@ -{{ .Chart.Name }} has been deployed successfully. diff --git a/helm/pidgin/templates/_helpers.tpl b/helm/pidgin/templates/_helpers.tpl deleted file mode 100644 index 9a3571d96..000000000 --- a/helm/pidgin/templates/_helpers.tpl +++ /dev/null @@ -1,68 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "pidgin.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "pidgin.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "pidgin.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "pidgin.labels" -}} -{{- if .Values.commonLabels }} - {{- with .Values.commonLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.commonLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "pidgin.selectorLabels" -}} -{{- if .Values.selectorLabels }} - {{- with .Values.selectorLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.selectorLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "pidgin.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "pidgin.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/helm/pidgin/templates/deployment.yaml b/helm/pidgin/templates/deployment.yaml deleted file mode 100644 index 90d52d48c..000000000 --- a/helm/pidgin/templates/deployment.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: pidgin-deployment - labels: - {{- include "pidgin.labels" . | nindent 4 }} - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 4 }} - {{- end }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "pidgin.selectorLabels" . | nindent 6 }} - revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} - {{- with .Values.strategy }} - strategy: - {{- toYaml . | nindent 4 }} - {{- end }} - template: - metadata: - labels: - {{- include "pidgin.selectorLabels" . | nindent 8 }} - # gen3 networkpolicy labels - netnolimit: 'yes' - public: 'yes' - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - env: - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogEnvVar" . | nindent 12 }} - {{- end }} - - name: GEN3_DEBUG - value: "False" - livenessProbe: - httpGet: - path: /_status - port: 80 - initialDelaySeconds: 30 - periodSeconds: 60 - timeoutSeconds: 30 - readinessProbe: - httpGet: - path: /_status - port: 80 - ports: - - containerPort: 80 - - containerPort: 443 - imagePullPolicy: {{ .Values.image.pullPolicy }} - resources: - {{- toYaml .Values.resources | nindent 12 }} \ No newline at end of file diff --git a/helm/pidgin/templates/hpa.yaml b/helm/pidgin/templates/hpa.yaml deleted file mode 100644 index d16ecf056..000000000 --- a/helm/pidgin/templates/hpa.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "pidgin.fullname" . }} - labels: - {{- include "pidgin.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "pidgin.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/helm/pidgin/templates/pdb.yaml b/helm/pidgin/templates/pdb.yaml deleted file mode 100644 index 2ef2de13d..000000000 --- a/helm/pidgin/templates/pdb.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if and .Values.global.pdb (gt (int .Values.replicaCount) 1) }} -{{ include "common.pod_disruption_budget" . }} -{{- end }} \ No newline at end of file diff --git a/helm/pidgin/templates/service.yaml b/helm/pidgin/templates/service.yaml deleted file mode 100644 index 42f6936b9..000000000 --- a/helm/pidgin/templates/service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: "pidgin-service" - labels: - {{- include "pidgin.labels" . | nindent 4 }} -spec: - selector: - {{- include "pidgin.selectorLabels" . | nindent 4 }} - {{- with .Values.service.port }} - ports: - {{- toYaml . | nindent 8 }} - {{- end }} - type: {{ .Values.service.type }} \ No newline at end of file diff --git a/helm/pidgin/templates/tests/test-connection.yaml b/helm/pidgin/templates/tests/test-connection.yaml deleted file mode 100644 index 0fc4b8f42..000000000 --- a/helm/pidgin/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "pidgin-test-connection" - labels: - {{- include "pidgin.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['pidgin-service:80/_status'] - restartPolicy: Never diff --git a/helm/pidgin/values.yaml b/helm/pidgin/values.yaml deleted file mode 100644 index 414d642be..000000000 --- a/helm/pidgin/values.yaml +++ /dev/null @@ -1,183 +0,0 @@ -# Default values for pidgin. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - - -# Global configuration -global: - # -- (map) AWS configuration - aws: - # -- (bool) Set to true if deploying to AWS. Controls ingress annotations. - enabled: false - # -- (string) Credentials for AWS stuff. - awsAccessKeyId: - # -- (string) Credentials for AWS stuff. - awsSecretAccessKey: - # -- (bool) Whether the deployment is for development purposes. - dev: true - - postgres: - # -- (bool) Whether the database should be created. - dbCreate: true - # -- (string) Name of external secret. Disabled if empty - externalSecret: "" - # -- (map) Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres - master: - # -- (string) hostname of postgres server - host: - # -- (string) username of superuser in postgres. This is used to create or restore databases - username: postgres - # -- (string) password for superuser in postgres. This is used to create or restore databases - password: - # -- (string) Port for Postgres. - port: "5432" - # -- (string) Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. - environment: default - # -- (string) Hostname for the deployment. - hostname: localhost - # -- (string) ARN of the reverse proxy certificate. - revproxyArn: arn:aws:acm:us-east-1:123456:certificate - # -- (string) URL of the data dictionary. - dictionaryUrl: https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json - # -- (string) Portal application name. - portalApp: gitops - # -- (string) S3 bucket name for Kubernetes manifest files. - kubeBucket: kube-gen3 - # -- (string) S3 bucket name for log files. - logsBucket: logs-gen3 - # -- (bool) Whether public datasets are enabled. - publicDataSets: true - # -- (string) Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` - tierAccessLevel: libre - # -- (bool) Whether network policies are enabled. - netPolicy: true - # -- (int) Number of dispatcher jobs. - dispatcherJobNum: "10" - # -- (bool) Whether Datadog is enabled. - ddEnabled: false - # -- (bool) If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. - pdb: false - # -- (int) The minimum amount of pods that are available at all times if the PDB is deployed. - minAvialable: 1 - -# -- (map) Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you -postgres: - # (bool) Whether the database should be restored from s3. Default to global.postgres.dbRestore - dbRestore: false - # -- (bool) Whether the database should be created. Default to global.postgres.dbCreate - dbCreate: - # -- (string) Hostname for postgres server. This is a service override, defaults to global.postgres.host - host: - # -- (string) Database name for postgres. This is a service override, defaults to - - database: - # -- (string) Username for postgres. This is a service override, defaults to - - username: - # -- (string) Port for Postgres. - port: "5432" - # -- (string) Password for Postgres. Will be autogenerated if left empty. - password: - -# Deployment -# -- (map) Configuration for autoscaling the number of replicas -autoscaling: - # -- (bool) Whether autoscaling is enabled - enabled: false - # -- (int) The minimum number of replicas to scale down to - minReplicas: 1 - # -- (int) The maximum number of replicas to scale up to - maxReplicas: 100 - # -- (int) The target CPU utilization percentage for autoscaling - targetCPUUtilizationPercentage: 80 - -# -- (int) Number of desired replicas -replicaCount: 1 - -# -- (int) Number of old revisions to retain -revisionHistoryLimit: 2 - -# -- (map) Rolling update deployment strategy -strategy: - type: RollingUpdate - rollingUpdate: - # -- (int) Number of additional replicas to add during rollout. - maxSurge: 1 - # -- (int) Maximum amount of pods that can be unavailable during the update. - maxUnavailable: 0 - -# -- (bool) Whether Datadog is enabled. -dataDog: - enabled: false - env: dev - -# -- (map) Affinity to use for the deployment. -affinity: - podAntiAffinity: - # -- (map) Option for scheduling to be required or preferred. - preferredDuringSchedulingIgnoredDuringExecution: - # -- (int) Weight value for preferred scheduling. - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - # -- (list) Label key for match expression. - - key: app - # -- (string) Operation type for the match expression. - operator: In - # -- (list) Value for the match expression key. - values: - - pidgin - # -- (string) Value for topology key label. - topologyKey: "kubernetes.io/hostname" - -# -- (bool) Automount the default service account token -automountServiceAccountToken: false - -image: - # -- (string) The Docker image repository for the fence service - repository: quay.io/cdis/pidgin - # -- (string) When to pull the image. - pullPolicy: Always - # -- (string) Overrides the image tag whose default is the chart appVersion. - tag: "" - -# -- (map) Resource requests and limits for the containers in the pod -resources: - # limits: - # cpu: 1 - # memory: 512Mi - -# Service and Pod -# -- (map) Kubernetes service information. -service: - # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". - type: ClusterIP - # -- (list) The port numbers that the service exposes. - port: - - protocol: TCP - port: 80 - targetPort: 80 - name: http - - protocol: TCP - port: 443 - targetPort: 443 - name: https - -# Values to determine the labels that are used for the deployment, pod, etc. -# -- (string) Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". -release: "production" -# -- (string) Valid options are "true" or "false". If invalid option is set- the value will default to "false". -criticalService: "false" -# -- (string) Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. -partOf: "Peregrine" -# -- (map) Will completely override the selectorLabels defined in the common chart's _label_setup.tpl -selectorLabels: -# -- (map) Will completely override the commonLabels defined in the common chart's _label_setup.tpl -commonLabels: - -# Values to configure datadog if ddEnabled is set to "true". -# -- (bool) If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. -datadogLogsInjection: true -# -- (bool) If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. -datadogProfilingEnabled: true -# -- (int) A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. -datadogTraceSampleRate: 1 diff --git a/helm/redis/templates/deployment.yaml b/helm/redis/templates/deployment.yaml index 4cc54e91b..e2efed9a9 100644 --- a/helm/redis/templates/deployment.yaml +++ b/helm/redis/templates/deployment.yaml @@ -41,6 +41,48 @@ spec: exec redis-server --save "" --appendonly no --requirepass "$REDIS_PASSWORD" fi exec redis-server --save "" --appendonly no + startupProbe: + exec: + command: + - /bin/sh + - -c + - >- + {{- if .Values.auth.enabled }} + redis-cli -h 127.0.0.1 -p 6379 -a "$REDIS_PASSWORD" --no-auth-warning ping | grep -q PONG + {{- else }} + redis-cli -h 127.0.0.1 -p 6379 ping | grep -q PONG + {{- end }} + periodSeconds: {{ .Values.probes.startup.periodSeconds }} + timeoutSeconds: {{ .Values.probes.startup.timeoutSeconds }} + failureThreshold: {{ .Values.probes.startup.failureThreshold }} + readinessProbe: + exec: + command: + - /bin/sh + - -c + - >- + {{- if .Values.auth.enabled }} + redis-cli -h 127.0.0.1 -p 6379 -a "$REDIS_PASSWORD" --no-auth-warning ping | grep -q PONG + {{- else }} + redis-cli -h 127.0.0.1 -p 6379 ping | grep -q PONG + {{- end }} + periodSeconds: {{ .Values.probes.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }} + failureThreshold: {{ .Values.probes.readiness.failureThreshold }} + livenessProbe: + exec: + command: + - /bin/sh + - -c + - >- + {{- if .Values.auth.enabled }} + redis-cli -h 127.0.0.1 -p 6379 -a "$REDIS_PASSWORD" --no-auth-warning ping | grep -q PONG + {{- else }} + redis-cli -h 127.0.0.1 -p 6379 ping | grep -q PONG + {{- end }} + periodSeconds: {{ .Values.probes.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.probes.liveness.timeoutSeconds }} + failureThreshold: {{ .Values.probes.liveness.failureThreshold }} resources: {{- toYaml .Values.resources | nindent 12 }} {{- with .Values.nodeSelector }} diff --git a/helm/redis/values.yaml b/helm/redis/values.yaml index be43e37ab..51c7f4816 100644 --- a/helm/redis/values.yaml +++ b/helm/redis/values.yaml @@ -44,6 +44,20 @@ auth: secretName: authz-cache-credentials password: "" +probes: + startup: + periodSeconds: 2 + timeoutSeconds: 1 + failureThreshold: 30 + readiness: + periodSeconds: 5 + timeoutSeconds: 1 + failureThreshold: 3 + liveness: + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + nodeSelector: {} tolerations: [] affinity: {} diff --git a/helm/revproxy/gen3.nginx.conf/fence-service.conf b/helm/revproxy/gen3.nginx.conf/fence-service.conf index 37400d81b..732fc7e58 100644 --- a/helm/revproxy/gen3.nginx.conf/fence-service.conf +++ b/helm/revproxy/gen3.nginx.conf/fence-service.conf @@ -22,6 +22,9 @@ location /user/register { set $proxy_service "fence"; set $upstream http://fence-service$des_domain; rewrite ^/user/(.*) /$1 break; + proxy_connect_timeout 3s; + proxy_send_timeout 20s; + proxy_read_timeout 20s; proxy_pass $upstream; } @@ -38,6 +41,9 @@ location ^~ /user/ { set $proxy_service "fence"; set $upstream http://fence-service$des_domain; rewrite ^/user/(.*) /$1 break; + proxy_connect_timeout 3s; + proxy_send_timeout 20s; + proxy_read_timeout 20s; proxy_pass $upstream; } diff --git a/helm/revproxy/gen3.nginx.conf/loom-service.conf b/helm/revproxy/gen3.nginx.conf/loom-service.conf new file mode 100644 index 000000000..00786bda7 --- /dev/null +++ b/helm/revproxy/gen3.nginx.conf/loom-service.conf @@ -0,0 +1,29 @@ +# Loom's chart exposes the server as loom-loom on a standalone release. +# Keep the public /loom prefix out of Loom's own route matching. +location = /loom { + return 308 /loom/; +} + +location ^~ /loom/ { + set $proxy_service "loom"; + set $upstream http://loom-loom.$namespace.svc.cluster.local:8080; + + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + include /etc/nginx/snippets/gen3-authenticated-proxy-headers.conf; + + rewrite ^/loom/(.*)$ /$1 break; + proxy_pass $upstream; + + # Generation uploads are complete META bundles and may be large. Stream + # both uploads and NDJSON exports without proxy buffering. + client_max_body_size 0; + proxy_request_buffering off; + proxy_buffering off; + proxy_connect_timeout 600s; + proxy_send_timeout 3600s; + proxy_read_timeout 3600s; +} diff --git a/helm/revproxy/nginx/nginx.conf b/helm/revproxy/nginx/nginx.conf index 1c0a03fe4..ef59c7631 100644 --- a/helm/revproxy/nginx/nginx.conf +++ b/helm/revproxy/nginx/nginx.conf @@ -100,6 +100,11 @@ http { '"http_referer": "$http_referer", ' '"http_useragent": "$http_user_agent", ' '"http_upstream": "$upstream", ' + '"upstream_addr": "$upstream_addr", ' + '"upstream_status": "$upstream_status", ' + '"upstream_connect_secs": "$upstream_connect_time", ' + '"upstream_header_secs": "$upstream_header_time", ' + '"upstream_response_secs": "$upstream_response_time", ' '"proxy_service": "$proxy_service", ' '"message": "$request" }'; diff --git a/helm/sheepdog/.helmignore b/helm/sheepdog/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/helm/sheepdog/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm/sheepdog/Chart.yaml b/helm/sheepdog/Chart.yaml deleted file mode 100644 index ea4bcc323..000000000 --- a/helm/sheepdog/Chart.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: v2 -name: sheepdog -description: A Helm chart for gen3 Sheepdog Service - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.14 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "master" - -dependencies: -- name: common - version: 0.1.10 - repository: file://../common -- name: postgresql - version: 11.9.13 - repository: "https://charts.bitnami.com/bitnami" - condition: postgres.separate diff --git a/helm/sheepdog/README.md b/helm/sheepdog/README.md deleted file mode 100644 index afbdd1898..000000000 --- a/helm/sheepdog/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# sheepdog - -![Version: 0.1.14](https://img.shields.io/badge/Version-0.1.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) - -A Helm chart for gen3 Sheepdog Service - -## Requirements - -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.10 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | - -## Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| affinity | map | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["sheepdog"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]}}` | Affinity to use for the deployment. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution | map | `[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["sheepdog"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]` | Option for scheduling to be required or preferred. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0] | int | `{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["sheepdog"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}` | Weight value for preferred scheduling. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0] | list | `{"key":"app","operator":"In","values":["sheepdog"]}` | Label key for match expression. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].operator | string | `"In"` | Operation type for the match expression. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].values | list | `["sheepdog"]` | Value for the match expression key. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey | string | `"kubernetes.io/hostname"` | Value for topology key label. | -| arboristUrl | string | `"http://arborist-service"` | URL for the arborist service | -| authNamespace | string | `"default"` | | -| automountServiceAccountToken | bool | `false` | Automount the default service account token | -| autoscaling | map | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Configuration for autoscaling the number of replicas | -| autoscaling.enabled | bool | `false` | Whether autoscaling is enabled | -| autoscaling.maxReplicas | int | `100` | The maximum number of replicas to scale up to | -| autoscaling.minReplicas | int | `1` | The minimum number of replicas to scale down to | -| autoscaling.targetCPUUtilizationPercentage | int | `80` | The target CPU utilization percentage for autoscaling | -| commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | -| criticalService | string | `"true"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | -| dataDog | bool | `{"enabled":false,"env":"dev"}` | Whether Datadog is enabled. | -| datadogLogsInjection | bool | `true` | If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. | -| datadogProfilingEnabled | bool | `true` | If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. | -| datadogTraceSampleRate | int | `1` | A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. | -| dictionaryUrl | string | `"https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json"` | URL of the data dictionary. | -| externalSecrets | map | `{"dbcreds":null}` | External Secrets settings. | -| externalSecrets.dbcreds | string | `nil` | Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" | -| fenceUrl | string | `"http://fence-service"` | URL for the fence service | -| global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false}` | AWS configuration | -| global.aws.awsAccessKeyId | string | `nil` | Credentials for AWS stuff. | -| global.aws.awsSecretAccessKey | string | `nil` | Credentials for AWS stuff. | -| global.aws.enabled | bool | `false` | Set to true if deploying to AWS. Controls ingress annotations. | -| global.ddEnabled | bool | `false` | Whether Datadog is enabled. | -| global.dev | bool | `true` | Whether the deployment is for development purposes. | -| global.dictionaryUrl | string | `"https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json"` | URL of the data dictionary. | -| global.dispatcherJobNum | int | `"10"` | Number of dispatcher jobs. | -| global.environment | string | `"default"` | Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. | -| global.externalSecrets | map | `{"deploy":false,"separateSecretStore":false}` | External Secrets settings. | -| global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any sheepdog secrets you have deployed. | -| global.externalSecrets.separateSecretStore | string | `false` | Will deploy a separate External Secret Store for this service. | -| global.hostname | string | `"localhost"` | Hostname for the deployment. | -| global.kubeBucket | string | `"kube-gen3"` | S3 bucket name for Kubernetes manifest files. | -| global.logsBucket | string | `"logs-gen3"` | S3 bucket name for log files. | -| global.minAvialable | int | `1` | The minimum amount of pods that are available at all times if the PDB is deployed. | -| global.netPolicy | bool | `true` | Whether network policies are enabled. | -| global.pdb | bool | `false` | If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. | -| global.portalApp | string | `"gitops"` | Portal application name. | -| global.postgres.dbCreate | bool | `true` | Whether the database should be created. | -| global.postgres.externalSecret | string | `""` | Name of external secret. Disabled if empty | -| global.postgres.master | map | `{"host":null,"password":null,"port":"5432","username":"postgres"}` | Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres | -| global.postgres.master.host | string | `nil` | hostname of postgres server | -| global.postgres.master.password | string | `nil` | password for superuser in postgres. This is used to create or restore databases | -| global.postgres.master.port | string | `"5432"` | Port for Postgres. | -| global.postgres.master.username | string | `"postgres"` | username of superuser in postgres. This is used to create or restore databases | -| global.publicDataSets | bool | `true` | Whether public datasets are enabled. | -| global.revproxyArn | string | `"arn:aws:acm:us-east-1:123456:certificate"` | ARN of the reverse proxy certificate. | -| global.tierAccessLevel | string | `"libre"` | Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` | -| image | map | `{"pullPolicy":"Always","repository":"quay.io/cdis/sheepdog","tag":"bug_auth-audience"}` | Docker image information. | -| image.pullPolicy | string | `"Always"` | Docker pull policy. | -| image.repository | string | `"quay.io/cdis/sheepdog"` | Docker repository. | -| image.tag | string | `"bug_auth-audience"` | Overrides the image tag whose default is the chart appVersion. | -| indexdUrl | string | `"http://indexd-service"` | URL for the indexd service | -| partOf | string | `"Core-Service"` | Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. | -| podAnnotations | map | `{"gen3.io/network-ingress":"sheepdog"}` | Annotations to add to the pod | -| postgres | map | `{"database":null,"dbCreate":null,"dbRestore":false,"host":null,"password":null,"port":"5432","separate":false,"username":null}` | Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you | -| postgres.database | string | `nil` | Database name for postgres. This is a service override, defaults to - | -| postgres.dbCreate | bool | `nil` | Whether the database should be created. Default to global.postgres.dbCreate | -| postgres.host | string | `nil` | Hostname for postgres server. This is a service override, defaults to global.postgres.host | -| postgres.password | string | `nil` | Password for Postgres. Will be autogenerated if left empty. | -| postgres.port | string | `"5432"` | Port for Postgres. | -| postgres.separate | string | `false` | Will create a Database for the individual service to help with developing it. | -| postgres.username | string | `nil` | Username for postgres. This is a service override, defaults to - | -| postgresql | map | `{"primary":{"persistence":{"enabled":false}}}` | Postgresql subchart settings if deployed separately option is set to "true". Disable persistence by default so we can spin up and down ephemeral environments | -| postgresql.primary.persistence.enabled | bool | `false` | Option to persist the dbs data. | -| release | string | `"production"` | Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". | -| releaseLabel | string | `"production"` | | -| replicaCount | int | `1` | Number of replicas for the deployment. | -| resources | map | `{"limits":{"cpu":1,"memory":"512Mi"},"requests":{"cpu":0.3,"memory":"12Mi"}}` | Resource requests and limits for the containers in the pod | -| resources.limits | map | `{"cpu":1,"memory":"512Mi"}` | The maximum amount of resources that the container is allowed to use | -| resources.limits.cpu | string | `1` | The maximum amount of CPU the container can use | -| resources.limits.memory | string | `"512Mi"` | The maximum amount of memory the container can use | -| resources.requests | map | `{"cpu":0.3,"memory":"12Mi"}` | The amount of resources that the container requests | -| resources.requests.cpu | string | `0.3` | The amount of CPU requested | -| resources.requests.memory | string | `"12Mi"` | The amount of memory requested | -| revisionHistoryLimit | int | `2` | Number of old revisions to retain | -| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Values for sheepdog secret. | -| secrets.awsAccessKeyId | str | `nil` | AWS access key ID to access the db restore job S3 bucket. Overrides global key. | -| secrets.awsSecretAccessKey | str | `nil` | AWS secret access key ID to access the db restore job S3 bucket. Overrides global key. | -| selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | -| service | map | `{"port":80,"type":"ClusterIP"}` | Kubernetes service information. | -| service.port | int | `80` | The port number that the service exposes. | -| service.type | string | `"ClusterIP"` | Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". | -| strategy | map | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | Rolling update deployment strategy | -| strategy.rollingUpdate.maxSurge | int | `1` | Number of additional replicas to add during rollout. | -| strategy.rollingUpdate.maxUnavailable | int | `0` | Maximum amount of pods that can be unavailable during the update. | -| terminationGracePeriodSeconds | int | `50` | sheepdog transactions take forever - try to let the complete before termination | -| volumeMounts | list | `[{"mountPath":"/var/www/sheepdog/settings.py","name":"config-volume","readOnly":true,"subPath":"settings.py"}]` | Volumes to mount to the container. | - diff --git a/helm/sheepdog/sheepdog-secret/config_helper.py b/helm/sheepdog/sheepdog-secret/config_helper.py deleted file mode 100644 index ab1805496..000000000 --- a/helm/sheepdog/sheepdog-secret/config_helper.py +++ /dev/null @@ -1,376 +0,0 @@ -import json -import os -import copy -import argparse -import re -import types - -# -# make it easy to change this for testing -XDG_DATA_HOME = os.getenv("XDG_DATA_HOME", "/usr/share/") - - -def default_search_folders(app_name): - """ - Return the list of folders to search for configuration files - """ - return [ - "%s/cdis/%s" % (XDG_DATA_HOME, app_name), - "/usr/share/cdis/%s" % app_name, - "%s/gen3/%s" % (XDG_DATA_HOME, app_name), - "/usr/share/gen3/%s" % app_name, - "/var/www/%s" % app_name, - "/etc/gen3/%s" % app_name, - ] - - -def find_paths(file_name, app_name, search_folders=None): - """ - Search the given folders for file_name - search_folders defaults to default_search_folders if not specified - return the first path to file_name found - """ - search_folders = search_folders or default_search_folders(app_name) - possible_files = [os.path.join(folder, file_name) for folder in search_folders] - return [path for path in possible_files if os.path.exists(path)] - - -def load_json(file_name, app_name, search_folders=None): - """ - json.load(file_name) after finding file_name in search_folders - - return the loaded json data or None if file not found - """ - actual_files = find_paths(file_name, app_name, search_folders) - if not actual_files: - return None - with open(actual_files[0], "r") as reader: - return json.load(reader) - - -def inject_creds_into_fence_config(creds_file_path, config_file_path): - creds_file = open(creds_file_path, "r") - creds = json.load(creds_file) - creds_file.close() - - # get secret values from creds.json file - db_host = _get_nested_value(creds, "db_host") - db_username = _get_nested_value(creds, "db_username") - db_password = _get_nested_value(creds, "db_password") - db_database = _get_nested_value(creds, "db_database") - hostname = _get_nested_value(creds, "hostname") - indexd_password = environ.get('INDEXD_PASS') - google_client_secret = _get_nested_value(creds, "google_client_secret") - google_client_id = _get_nested_value(creds, "google_client_id") - hmac_key = _get_nested_value(creds, "hmac_key") - db_path = "postgresql://{}:{}@{}:5432/{}".format( - db_username, db_password, db_host, db_database - ) - - config_file = open(config_file_path, "r").read() - - print(" DB injected with value(s) from creds.json") - config_file = _replace(config_file, "DB", db_path) - - print(" BASE_URL injected with value(s) from creds.json") - config_file = _replace(config_file, "BASE_URL", "https://{}/user".format(hostname)) - - print(" INDEXD_PASSWORD injected with value(s) from creds.json") - config_file = _replace(config_file, "INDEXD_PASSWORD", indexd_password) - config_file = _replace(config_file, "INDEXD_USERNAME", "fence") - - print(" ENCRYPTION_KEY injected with value(s) from creds.json") - config_file = _replace(config_file, "ENCRYPTION_KEY", hmac_key) - - print( - " OPENID_CONNECT/google/client_secret injected with value(s) " - "from creds.json" - ) - config_file = _replace( - config_file, "OPENID_CONNECT/google/client_secret", google_client_secret - ) - - print(" OPENID_CONNECT/google/client_id injected with value(s) from creds.json") - config_file = _replace( - config_file, "OPENID_CONNECT/google/client_id", google_client_id - ) - - open(config_file_path, "w+").write(config_file) - - -def set_prod_defaults(config_file_path): - config_file = open(config_file_path, "r").read() - - print( - " CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS set as " - "var/www/fence/fence_google_app_creds_secret.json" - ) - config_file = _replace( - config_file, - "CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS", - "/var/www/fence/fence_google_app_creds_secret.json", - ) - - print( - " CIRRUS_CFG/GOOGLE_STORAGE_CREDS set as " - "var/www/fence/fence_google_storage_creds_secret.json" - ) - config_file = _replace( - config_file, - "CIRRUS_CFG/GOOGLE_STORAGE_CREDS", - "/var/www/fence/fence_google_storage_creds_secret.json", - ) - - print(" INDEXD set as http://indexd-service/") - config_file = _replace(config_file, "INDEXD", "http://indexd-service/") - - print(" ARBORIST set as http://arborist-service/") - config_file = _replace(config_file, "ARBORIST", "http://arborist-service/") - - print(" HTTP_PROXY/host set as cloud-proxy.internal.io") - config_file = _replace(config_file, "HTTP_PROXY/host", "cloud-proxy.internal.io") - - print(" HTTP_PROXY/port set as 3128") - config_file = _replace(config_file, "HTTP_PROXY/port", 3128) - - print(" DEBUG set to false") - config_file = _replace(config_file, "DEBUG", False) - - print(" MOCK_AUTH set to false") - config_file = _replace(config_file, "MOCK_AUTH", False) - - print(" MOCK_GOOGLE_AUTH set to false") - config_file = _replace(config_file, "MOCK_GOOGLE_AUTH", False) - - print(" AUTHLIB_INSECURE_TRANSPORT set to true") - config_file = _replace(config_file, "AUTHLIB_INSECURE_TRANSPORT", True) - - print(" SESSION_COOKIE_SECURE set to true") - config_file = _replace(config_file, "SESSION_COOKIE_SECURE", True) - - print(" ENABLE_CSRF_PROTECTION set to true") - config_file = _replace(config_file, "ENABLE_CSRF_PROTECTION", True) - - open(config_file_path, "w+").write(config_file) - - -def inject_other_files_into_fence_config(other_files, config_file_path): - additional_cfgs = _get_all_additional_configs(other_files) - - config_file = open(config_file_path, "r").read() - - for key, value in additional_cfgs.iteritems(): - print(" {} set to {}".format(key, value)) - config_file = _nested_replace(config_file, key, value) - - open(config_file_path, "w+").write(config_file) - - -def _get_all_additional_configs(other_files): - """ - Attempt to parse given list of files and extract configuration variables and values - """ - additional_configs = dict() - for file_path in other_files: - try: - file_ext = file_path.strip().split(".")[-1] - if file_ext == "json": - json_file = open(file_path, "r") - configs = json.load(json_file) - json_file.close() - elif file_ext == "py": - configs = from_pyfile(file_path) - else: - print( - "Cannot load config vars from a file with extention: {}".format( - file_ext - ) - ) - except Exception as exc: - # if there's any issue reading the file, exit - print( - "Error reading {}. Cannot get configuration. Skipping this file. " - "Details: {}".format(other_files, str(exc)) - ) - continue - - if configs: - additional_configs.update(configs) - - return additional_configs - - -def _nested_replace(config_file, key, value, replacement_path=None): - replacement_path = replacement_path or key - try: - for inner_key, inner_value in value.iteritems(): - temp_path = replacement_path - temp_path = temp_path + "/" + inner_key - config_file = _nested_replace( - config_file, inner_key, inner_value, temp_path - ) - except AttributeError: - # not a dict so replace - if value is not None: - config_file = _replace(config_file, replacement_path, value) - - return config_file - - -def _replace(yaml_config, path_to_key, replacement_value, start=0, nested_level=0): - """ - Replace a nested value in a YAML file string with the given value without - losing comments. Uses a regex to do the replacement. - - Args: - yaml_config (str): a string representing a full configuration file - path_to_key (str): nested/path/to/key. The value of this key will be - replaced - replacement_value (str): Replacement value for the key from - path_to_key - """ - nested_path_to_replace = path_to_key.split("/") - - # our regex looks for a specific number of spaces to ensure correct - # level of nesting. It matches to the end of the line - search_string = ( - " " * nested_level + ".*" + nested_path_to_replace[0] + "(')?(\")?:.*\n" - ) - matches = re.search(search_string, yaml_config[start:]) - - # early return if we haven't found anything - if not matches: - return yaml_config - - # if we're on the last item in the path, we need to get the value and - # replace it in the original file - if len(nested_path_to_replace) == 1: - # replace the current key:value with the new replacement value - match_start = start + matches.start(0) + len(" " * nested_level) - match_end = start + matches.end(0) - yaml_config = ( - yaml_config[:match_start] - + "{}: {}\n".format( - nested_path_to_replace[0], - _get_yaml_replacement_value(replacement_value, nested_level), - ) - + yaml_config[match_end:] - ) - - return yaml_config - - # set new start point to past current match and move on to next match - start = matches.end(0) - nested_level += 1 - del nested_path_to_replace[0] - - return _replace( - yaml_config, - "/".join(nested_path_to_replace), - replacement_value, - start, - nested_level, - ) - - -def from_pyfile(filename, silent=False): - """ - Modeled after flask's ability to load in python files: - https://github.com/pallets/flask/blob/master/flask/config.py - - Some alterations were made but logic is essentially the same - """ - filename = os.path.abspath(filename) - d = types.ModuleType("config") - d.__file__ = filename - try: - with open(filename, mode="rb") as config_file: - exec(compile(config_file.read(), filename, "exec"), d.__dict__) - except IOError as e: - print("Unable to load configuration file ({})".format(e.strerror)) - if silent: - return False - raise - return _from_object(d) - - -def _from_object(obj): - configs = {} - for key in dir(obj): - if key.isupper(): - configs[key] = getattr(obj, key) - return configs - - -def _get_yaml_replacement_value(value, nested_level=0): - if isinstance(value, str): - return "'" + value + "'" - elif isinstance(value, bool): - return str(value).lower() - elif isinstance(value, list) or isinstance(value, set): - output = "" - for item in value: - # spaces for nested level then spaces and hyphen for each list item - output += ( - "\n" - + " " * nested_level - + " - " - + _get_yaml_replacement_value(item) - + "" - ) - return output - else: - return value - - -def _get_nested_value(dictionary, nested_path): - """ - Return a value from a dictionary given a path-like nesting of keys. - - Will default to an empty string if value cannot be found. - - Args: - dictionary (dict): a dictionary - nested_path (str): nested/path/to/key - - Returns: - ?: Value from dict - """ - replacement_value_path = nested_path.split("/") - replacement_value = copy.deepcopy(dictionary) - - for item in replacement_value_path: - replacement_value = replacement_value.get(item, {}) - - if replacement_value == {}: - replacement_value = "" - - return replacement_value - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "-i", - "--creds_file_to_inject", - default="creds.json", - help="creds file to inject into the configuration yaml", - ) - parser.add_argument( - "--other_files_to_inject", - nargs="+", - help="fence_credentials.json, local_settings.py, fence_settings.py file(s) to " - "inject into the configuration yaml", - ) - parser.add_argument( - "-c", "--config_file", default="config.yaml", help="configuration yaml" - ) - args = parser.parse_args() - - inject_creds_into_fence_config(args.creds_file_to_inject, args.config_file) - set_prod_defaults(args.config_file) - - if args.other_files_to_inject: - inject_other_files_into_fence_config( - args.other_files_to_inject, args.config_file - ) \ No newline at end of file diff --git a/helm/sheepdog/sheepdog-secret/settings.py b/helm/sheepdog/sheepdog-secret/settings.py deleted file mode 100644 index 2818d1699..000000000 --- a/helm/sheepdog/sheepdog-secret/settings.py +++ /dev/null @@ -1,82 +0,0 @@ -##################################################### -# DO NOT CHANGE THIS FILE # -# config updates should be done in the service code # -##################################################### - -from sheepdog.api import app, app_init -from os import environ -# import config_helper - -APP_NAME='sheepdog' -# def load_json(file_name): -# return config_helper.load_json(file_name, APP_NAME) - -# conf_data = load_json('creds.json') -config = app.config - - -config['INDEX_CLIENT'] = { - 'host': environ.get('INDEX_CLIENT_HOST') or 'http://indexd-service', - 'version': 'v0', - 'auth': (environ.get( "INDEXD_USER", 'sheepdog'), environ.get( "INDEXD_PASS") ), -} - -config["PSQLGRAPH"] = { - 'host': environ.get( "PGHOST"), - 'user': environ.get( "PGUSER"), - 'password': environ.get( "PGPASSWORD"), - 'database': environ.get( "PGDB"), -} - -config['HMAC_ENCRYPTION_KEY'] = environ.get( "HMAC_ENCRYPTION_KEY") -config['FLASK_SECRET_KEY'] = environ.get( "FLASK_SECRET_KEY") - -fence_username = environ.get( "FENCE_DB_USER") -fence_password = environ.get( "FENCE_DB_PASS") -fence_host = environ.get( "FENCE_DB_HOST") -fence_database = environ.get( "FENCE_DB_DBNAME") -config['PSQL_USER_DB_CONNECTION'] = 'postgresql://%s:%s@%s:5432/%s' % (fence_username, fence_password, fence_host, fence_database) - -config['DICTIONARY_URL'] = environ.get('DICTIONARY_URL','https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json') - - -# config['SUBMISSION'] = { -# 'bucket': conf_data.get( 'bagit_bucket', '{{bagit_bucket}}' ) -# } - -# config['STORAGE'] = { -# "s3": -# { -# "access_key": conf_data.get( 's3_access', '{{s3_access}}' ), -# 'secret_key': conf_data.get( 's3_secret', '{{s3_secret}}' ) -# } -# } - -hostname = environ.get("CONF_HOSTNAME", "localhost") - -config['OIDC_ISSUER'] = 'https://%s/user' % hostname - -config['OAUTH2'] = { - 'client_id': "conf_data.get('oauth2_client_id', '{{oauth2_client_id}}')", - 'client_secret': "conf_data.get('oauth2_client_secret', '{{oauth2_client_secret}}')", - 'api_base_url': 'https://%s/user/' % hostname, - 'authorize_url': 'https://%s/user/oauth2/authorize' % hostname, - 'access_token_url': 'https://%s/user/oauth2/token' % hostname, - 'refresh_token_url': 'https://%s/user/oauth2/token' % hostname, - 'client_kwargs': { - 'redirect_uri': 'https://%s/api/v0/oauth2/authorize' % hostname, - 'scope': 'openid data user', - }, - # deprecated key values, should be removed after all commons use new oidc - 'internal_oauth_provider': 'http://fence-service/oauth2/', - 'oauth_provider': 'https://%s/user/oauth2/' % hostname, - 'redirect_uri': 'https://%s/api/v0/oauth2/authorize' % hostname -} - -config['USER_API'] = environ.get('FENCE_URL') or 'http://fence-service/' -# use the USER_API URL instead of the public issuer URL to accquire JWT keys -config['FORCE_ISSUER'] = True -print(config) -app_init(app) -application = app -application.debug = (environ.get('GEN3_DEBUG') == "True") diff --git a/helm/sheepdog/templates/NOTES.txt b/helm/sheepdog/templates/NOTES.txt deleted file mode 100644 index c1e7e1aef..000000000 --- a/helm/sheepdog/templates/NOTES.txt +++ /dev/null @@ -1 +0,0 @@ -{{ .Chart.Name }} has been deployed successfully. diff --git a/helm/sheepdog/templates/_helpers.tpl b/helm/sheepdog/templates/_helpers.tpl deleted file mode 100644 index 1c935c37b..000000000 --- a/helm/sheepdog/templates/_helpers.tpl +++ /dev/null @@ -1,103 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "sheepdog.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "sheepdog.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "sheepdog.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "sheepdog.labels" -}} -{{- if .Values.commonLabels }} - {{- with .Values.commonLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.commonLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "sheepdog.selectorLabels" -}} -{{- if .Values.selectorLabels }} - {{- with .Values.selectorLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.selectorLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "sheepdog.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "sheepdog.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} - -{{/* - Postgres Password lookup Sheepdog -*/}} -{{- define "sheepdog.postgres.password" -}} -{{- $localpass := (lookup "v1" "Secret" "postgres" "postgres-postgresql" ) -}} -{{- if $localpass }} -{{- default (index $localpass.data "postgres-password" | b64dec) }} -{{- else }} -{{- default .Values.secrets.sheepdog.password }} -{{- end }} -{{- end }} - -{{/* - Postgres Password lookup Fence -*/}} -{{- define "fence.postgres.password" -}} -{{- $localpass := (lookup "v1" "Secret" "postgres" "postgres-postgresql" ) -}} -{{- if $localpass }} -{{- default (index $localpass.data "postgres-password" | b64dec) }} -{{- else }} -{{- default .Values.secrets.fence.password }} -{{- end }} -{{- end }} - -{{/* -Define dictionaryUrl -*/}} -{{- define "sheepdog.dictionaryUrl" -}} -{{- if .Values.global }} -{{- .Values.global.dictionaryUrl }} -{{- else}} -{{- .Values.dictionaryUrl }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/helm/sheepdog/templates/aws-config.yaml b/helm/sheepdog/templates/aws-config.yaml deleted file mode 100644 index 398770d36..000000000 --- a/helm/sheepdog/templates/aws-config.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if or (.Values.secrets.awsSecretAccessKey) (.Values.global.aws.awsSecretAccessKey ) }} -{{ include "common.awsconfig" . }} -{{- end -}} \ No newline at end of file diff --git a/helm/sheepdog/templates/db-init.yaml b/helm/sheepdog/templates/db-init.yaml deleted file mode 100644 index 0393aa732..000000000 --- a/helm/sheepdog/templates/db-init.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{ include "common.db-secret" . }} ---- -{{ include "common.db_setup_sa" . }} ---- -{{- if .Values.dbRestore }} -{{ include "common.s3_pg_restore" . }} -{{- else }} -{{ include "common.db_setup_job" . }} -{{- end -}} \ No newline at end of file diff --git a/helm/sheepdog/templates/deployment.yaml b/helm/sheepdog/templates/deployment.yaml deleted file mode 100644 index d56beb93f..000000000 --- a/helm/sheepdog/templates/deployment.yaml +++ /dev/null @@ -1,239 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: sheepdog-deployment - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} - labels: - {{- include "sheepdog.labels" . | nindent 4 }} - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 4 }} - {{- end }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "sheepdog.selectorLabels" . | nindent 6 }} - revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} - {{- with .Values.strategy }} - strategy: - {{- toYaml . | nindent 4 }} - {{- end }} - template: - metadata: - labels: - {{- include "sheepdog.selectorLabels" . | nindent 8 }} - # gen3 networkpolicy labels - netnolimit: 'yes' - public: 'yes' - s3: 'yes' - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} - volumes: - - name: config-volume - secret: - secretName: "sheepdog-secret" - - name: config-helper - configMap: - name: config-helper - initContainers: - - name: sheepdog-init - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - env: - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogEnvVar" . | nindent 12 }} - {{- end }} - - name: DICTIONARY_URL - value: {{ .Values.dictionaryUrl }} - - name: PGHOST - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: host - optional: false - - name: PGUSER - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: username - optional: false - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: password - optional: false - - name: PGDB - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: database - optional: false - - name: DBREADY - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: dbcreated - optional: false - - name: CONF_HOSTNAME - value: {{ .Values.global.hostname }} - command: ["/bin/bash" ] - args: - - "-c" - # Script always succeeds if it runs (echo exits with 0) - - | - # - # sheepdog sets up core data model now, - # but suffers from a race condition doing it ... - # - echo datamodel_postgres_admin create-all -U "${PGUSER}" -P XXXXXXX -H "${PGHOST}" -D "${PGDB}" - datamodel_postgres_admin create-all -U "${PGUSER}" -P "${PGPASSWORD}" -H "${PGHOST}" -D "${PGDB}" - - echo python /sheepdog/bin/setup_transactionlogs.py --user "${PGUSER}" --password XXXXX --host "${PGHOST}" --database "${PGDB}" - python /sheepdog/bin/setup_transactionlogs.py --user "${PGUSER}" --password "${PGPASSWORD}" --host "${PGHOST}" --database "${PGDB}" - containers: - - name: sheepdog - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - containerPort: 80 - - containerPort: 443 - livenessProbe: - httpGet: - path: /_status?timeout=20 - port: 80 - initialDelaySeconds: 30 - periodSeconds: 60 - timeoutSeconds: 30 - readinessProbe: - initialDelaySeconds: 30 - httpGet: - path: /_status?timeout=2 - port: 80 - # command: ["/bin/bash" ] - # args: - # - "-c" - # - "sleep infinity" - env: - - name: CONF_HOSTNAME - value: {{ .Values.global.hostname }} - - name: FENCE_DB_USER - valueFrom: - secretKeyRef: - name: fence-dbcreds - key: username - optional: false - - name: FENCE_DB_PASS - valueFrom: - secretKeyRef: - name: fence-dbcreds - key: password - optional: false - - name: FENCE_DB_HOST - valueFrom: - secretKeyRef: - name: fence-dbcreds - key: host - optional: false - - name: FENCE_DB_DBNAME - valueFrom: - secretKeyRef: - name: fence-dbcreds - key: database - optional: false - - name: DBREADY_FENCE - valueFrom: - secretKeyRef: - name: fence-dbcreds - key: dbcreated - optional: false - - name: FLASK_SECRET_KEY - value: "TODO: FIX THIS!!!" - - name: PGHOST - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: host - optional: false - - name: PGUSER - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: username - optional: false - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: password - optional: false - - name: PGDB - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: database - optional: false - - name: DBREADY - valueFrom: - secretKeyRef: - name: sheepdog-dbcreds - key: dbcreated - optional: false - - name: PUBLIC_DATASETS - valueFrom: - configMapKeyRef: - name: manifest-global - key: public_datasets - optional: true - - name: INDEXD_PASS - valueFrom: - secretKeyRef: - name: indexd-service-creds - key: sheepdog - optional: false - - name: GEN3_UWSGI_TIMEOUT - value: "600" - - name: DICTIONARY_URL - value: {{ include "sheepdog.dictionaryUrl" .}} - {{- with .Values.indexdUrl }} - - name: INDEX_CLIENT_HOST - value: {{ . }} - {{- end }} - {{- with .Values.fenceUrl }} - - name: FENCE_URL - value: {{ . }} - {{- end }} - {{- with .Values.arboristUrl }} - - name: ARBORIST_URL - value: {{ . }} - {{- end }} - {{- with .Values.authNamespace }} - - name: AUTH_NAMESPACE - value: {{ . }} - {{- end }} - # - name: REQUESTS_CA_BUNDLE - # # - # # override python 'requests' SSL certificate bundle - # # to use system trusted certs - # # which includes our private certificate authority - # # - # value: /etc/ssl/certs/ca-certificates.crt - - name: GEN3_DEBUG - value: "True" - {{- with .Values.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 10 }} - {{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} \ No newline at end of file diff --git a/helm/sheepdog/templates/external-secrets.yaml b/helm/sheepdog/templates/external-secrets.yaml deleted file mode 100644 index 70c278fea..000000000 --- a/helm/sheepdog/templates/external-secrets.yaml +++ /dev/null @@ -1 +0,0 @@ -{{ include "common.externalSecret.db" . }} \ No newline at end of file diff --git a/helm/sheepdog/templates/hpa.yaml b/helm/sheepdog/templates/hpa.yaml deleted file mode 100644 index 81d25de1d..000000000 --- a/helm/sheepdog/templates/hpa.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "sheepdog.fullname" . }} - labels: - {{- include "sheepdog.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "sheepdog.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/helm/sheepdog/templates/pdb.yaml b/helm/sheepdog/templates/pdb.yaml deleted file mode 100644 index 2ef2de13d..000000000 --- a/helm/sheepdog/templates/pdb.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if and .Values.global.pdb (gt (int .Values.replicaCount) 1) }} -{{ include "common.pod_disruption_budget" . }} -{{- end }} \ No newline at end of file diff --git a/helm/sheepdog/templates/secret-store.yaml b/helm/sheepdog/templates/secret-store.yaml deleted file mode 100644 index 771c7760d..000000000 --- a/helm/sheepdog/templates/secret-store.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{ if .Values.global.externalSecrets.separateSecretStore }} -{{ include "common.secretstore" . }} -{{- end }} \ No newline at end of file diff --git a/helm/sheepdog/templates/service.yaml b/helm/sheepdog/templates/service.yaml deleted file mode 100644 index eff84f425..000000000 --- a/helm/sheepdog/templates/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: "sheepdog-service" - labels: - {{- include "sheepdog.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: {{ .Values.service.port }} - protocol: TCP - name: http - selector: - {{- include "sheepdog.selectorLabels" . | nindent 4 }} diff --git a/helm/sheepdog/templates/sheepdog-secret.yaml b/helm/sheepdog/templates/sheepdog-secret.yaml deleted file mode 100644 index 936949084..000000000 --- a/helm/sheepdog/templates/sheepdog-secret.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: sheepdog-secret -type: Opaque -data: -{{ (.Files.Glob "sheepdog-secret/*").AsSecrets | indent 2 }} diff --git a/helm/sheepdog/templates/tests/test-connection.yaml b/helm/sheepdog/templates/tests/test-connection.yaml deleted file mode 100644 index dc94a1717..000000000 --- a/helm/sheepdog/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "sheepdog-test-connection" - labels: - {{- include "sheepdog.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['sheepdog-service:{{ .Values.service.port }}/_status?timeout=2'] - restartPolicy: Never diff --git a/helm/sheepdog/values.yaml b/helm/sheepdog/values.yaml deleted file mode 100644 index 59b0841ff..000000000 --- a/helm/sheepdog/values.yaml +++ /dev/null @@ -1,240 +0,0 @@ -# Default values for sheepdog. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - - -# Global configuration -global: - # -- (map) AWS configuration - aws: - # -- (bool) Set to true if deploying to AWS. Controls ingress annotations. - enabled: false - # -- (string) Credentials for AWS stuff. - awsAccessKeyId: - # -- (string) Credentials for AWS stuff. - awsSecretAccessKey: - # -- (bool) Whether the deployment is for development purposes. - dev: true - - postgres: - # -- (bool) Whether the database should be created. - dbCreate: true - # -- (string) Name of external secret. Disabled if empty - externalSecret: "" - # -- (map) Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres - master: - # -- (string) hostname of postgres server - host: - # -- (string) username of superuser in postgres. This is used to create or restore databases - username: postgres - # -- (string) password for superuser in postgres. This is used to create or restore databases - password: - # -- (string) Port for Postgres. - port: "5432" - # -- (string) Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. - environment: default - # -- (string) Hostname for the deployment. - hostname: localhost - # -- (string) ARN of the reverse proxy certificate. - revproxyArn: arn:aws:acm:us-east-1:123456:certificate - # -- (string) URL of the data dictionary. - dictionaryUrl: https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json - # -- (string) Portal application name. - portalApp: gitops - # -- (string) S3 bucket name for Kubernetes manifest files. - kubeBucket: kube-gen3 - # -- (string) S3 bucket name for log files. - logsBucket: logs-gen3 - # -- (bool) Whether public datasets are enabled. - publicDataSets: true - # -- (string) Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` - tierAccessLevel: libre - # -- (bool) Whether network policies are enabled. - netPolicy: true - # -- (int) Number of dispatcher jobs. - dispatcherJobNum: "10" - # -- (bool) Whether Datadog is enabled. - ddEnabled: false - # -- (bool) If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. - pdb: false - # -- (int) The minimum amount of pods that are available at all times if the PDB is deployed. - minAvialable: 1 - # -- (map) External Secrets settings. - externalSecrets: - # -- (bool) Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any sheepdog secrets you have deployed. - deploy: false - # -- (string) Will deploy a separate External Secret Store for this service. - separateSecretStore: false - -# -- (map) External Secrets settings. -externalSecrets: - # -- (string) Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" - dbcreds: - -# -- (map) Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you -postgres: - # (bool) Whether the database should be restored from s3. Default to global.postgres.dbRestore - dbRestore: false - # -- (bool) Whether the database should be created. Default to global.postgres.dbCreate - dbCreate: - # -- (string) Hostname for postgres server. This is a service override, defaults to global.postgres.host - host: - # -- (string) Database name for postgres. This is a service override, defaults to - - database: - # -- (string) Username for postgres. This is a service override, defaults to - - username: - # -- (string) Port for Postgres. - port: "5432" - # -- (string) Password for Postgres. Will be autogenerated if left empty. - password: - # -- (string) Will create a Database for the individual service to help with developing it. - separate: false - -# -- (map) Postgresql subchart settings if deployed separately option is set to "true". -# Disable persistence by default so we can spin up and down ephemeral environments -postgresql: - primary: - persistence: - # -- (bool) Option to persist the dbs data. - enabled: false - -# Deployment -releaseLabel: production - -# -- (map) Annotations to add to the pod -podAnnotations: {"gen3.io/network-ingress": "sheepdog"} - -# -- (map) Configuration for autoscaling the number of replicas -autoscaling: - # -- (bool) Whether autoscaling is enabled - enabled: false - # -- (int) The minimum number of replicas to scale down to - minReplicas: 1 - # -- (int) The maximum number of replicas to scale up to - maxReplicas: 100 - # -- (int) The target CPU utilization percentage for autoscaling - targetCPUUtilizationPercentage: 80 - -# -- (int) Number of replicas for the deployment. -replicaCount: 1 - -# -- (int) Number of old revisions to retain -revisionHistoryLimit: 2 - -# -- (map) Rolling update deployment strategy -strategy: - type: RollingUpdate - rollingUpdate: - # -- (int) Number of additional replicas to add during rollout. - maxSurge: 1 - # -- (int) Maximum amount of pods that can be unavailable during the update. - maxUnavailable: 0 - -# -- (bool) Whether Datadog is enabled. -dataDog: - enabled: false - env: dev - -# -- (map) Affinity to use for the deployment. -affinity: - podAntiAffinity: - # -- (map) Option for scheduling to be required or preferred. - preferredDuringSchedulingIgnoredDuringExecution: - # -- (int) Weight value for preferred scheduling. - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - # -- (list) Label key for match expression. - - key: app - # -- (string) Operation type for the match expression. - operator: In - # -- (list) Value for the match expression key. - values: - - sheepdog - # -- (string) Value for topology key label. - topologyKey: "kubernetes.io/hostname" - -# -- (bool) Automount the default service account token -automountServiceAccountToken: false - -# -- (int) sheepdog transactions take forever - try to let the complete before termination -terminationGracePeriodSeconds: 50 - -# -- (map) Docker image information. -image: - # -- (string) Docker repository. - repository: quay.io/cdis/sheepdog - # -- (string) Docker pull policy. - pullPolicy: Always - # -- (string) Overrides the image tag whose default is the chart appVersion. - tag: "bug_auth-audience" - -# Environment Variables -# -- (string) URL of the data dictionary. -dictionaryUrl: https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json -# -- (string) URL for the indexd service -indexdUrl: http://indexd-service -# -- (string) URL for the fence service -fenceUrl: http://fence-service -# -- (string) URL for the arborist service -arboristUrl: http://arborist-service -authNamespace: default - -# -- (list) Volumes to mount to the container. -volumeMounts: - - name: "config-volume" - readOnly: true - mountPath: "/var/www/sheepdog/settings.py" - subPath: "settings.py" - -# -- (map) Resource requests and limits for the containers in the pod -resources: - # -- (map) The amount of resources that the container requests - requests: - # -- (string) The amount of CPU requested - cpu: 0.3 - # -- (string) The amount of memory requested - memory: 12Mi - # -- (map) The maximum amount of resources that the container is allowed to use - limits: - # -- (string) The maximum amount of CPU the container can use - cpu: 1.0 - # -- (string) The maximum amount of memory the container can use - memory: 512Mi - -# Service and Pod -# -- (map) Kubernetes service information. -service: - # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". - type: ClusterIP - # -- (int) The port number that the service exposes. - port: 80 - -# Secrets -# -- (map) Values for sheepdog secret. -secrets: - # -- (str) AWS access key ID to access the db restore job S3 bucket. Overrides global key. - awsAccessKeyId: - # -- (str) AWS secret access key ID to access the db restore job S3 bucket. Overrides global key. - awsSecretAccessKey: - -# Values to determine the labels that are used for the deployment, pod, etc. -# -- (string) Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". -release: "production" -# -- (string) Valid options are "true" or "false". If invalid option is set- the value will default to "false". -criticalService: "true" -# -- (string) Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. -partOf: "Core-Service" -# -- (map) Will completely override the selectorLabels defined in the common chart's _label_setup.tpl -selectorLabels: -# -- (map) Will completely override the commonLabels defined in the common chart's _label_setup.tpl -commonLabels: - -# Values to configure datadog if ddEnabled is set to "true". -# -- (bool) If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. -datadogLogsInjection: true -# -- (bool) If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. -datadogProfilingEnabled: true -# -- (int) A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. -datadogTraceSampleRate: 1 diff --git a/helm/ssjdispatcher/.helmignore b/helm/ssjdispatcher/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/helm/ssjdispatcher/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm/ssjdispatcher/Chart.yaml b/helm/ssjdispatcher/Chart.yaml deleted file mode 100644 index ac6682874..000000000 --- a/helm/ssjdispatcher/Chart.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: v2 -name: ssjdispatcher -description: A Helm chart for gen3 ssjdispatcher - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.9 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "master" - -dependencies: -- name: common - version: 0.1.10 - repository: file://../common diff --git a/helm/ssjdispatcher/README.md b/helm/ssjdispatcher/README.md deleted file mode 100644 index 3bb1ab0a1..000000000 --- a/helm/ssjdispatcher/README.md +++ /dev/null @@ -1,114 +0,0 @@ -# ssjdispatcher - -![Version: 0.1.9](https://img.shields.io/badge/Version-0.1.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) - -A Helm chart for gen3 ssjdispatcher - -## Requirements - -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.10 | - -## Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| affinity | map | `{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["ssjdispatcher"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]}}` | Affinity to use for the deployment. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution | map | `[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["ssjdispatcher"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]` | Option for scheduling to be required or preferred. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0] | int | `{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["ssjdispatcher"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}` | Weight value for preferred scheduling. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0] | list | `{"key":"app","operator":"In","values":["ssjdispatcher"]}` | Label key for match expression. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].operator | string | `"In"` | Operation type for the match expression. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].values | list | `["ssjdispatcher"]` | Value for the match expression key. | -| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey | string | `"kubernetes.io/hostname"` | Value for topology key label. | -| automountServiceAccountToken | bool | `true` | Automount the default service account token | -| autoscaling | map | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Configuration for autoscaling the number of replicas | -| autoscaling.enabled | bool | `false` | Whether autoscaling is enabled | -| autoscaling.maxReplicas | int | `100` | The maximum number of replicas to scale up to | -| autoscaling.minReplicas | int | `1` | The minimum number of replicas to scale down to | -| autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage | -| awsRegion | string | `"us-east-1"` | AWS region to be used. | -| awsStsRegionalEndpoints | string | `"regional"` | AWS STS to issue temporary credentials to users and roles that make an AWS STS request. Values regional or global. | -| commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | -| criticalService | string | `"true"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | -| datadogLogsInjection | bool | `true` | If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. | -| datadogProfilingEnabled | bool | `true` | If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. | -| datadogTraceSampleRate | int | `1` | A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. | -| dispatcherJobNum | string | `"10"` | Ssjdispater job number. | -| fullnameOverride | string | `""` | Override the full name of the deployment. | -| gen3Namespace | string | `"default"` | Namespace to deploy the job. | -| global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false}` | AWS configuration | -| global.aws.awsAccessKeyId | string | `nil` | Credentials for AWS stuff. | -| global.aws.awsSecretAccessKey | string | `nil` | Credentials for AWS stuff. | -| global.aws.enabled | bool | `false` | Set to true if deploying to AWS. Controls ingress annotations. | -| global.ddEnabled | bool | `false` | Whether Datadog is enabled. | -| global.dev | bool | `true` | Whether the deployment is for development purposes. | -| global.dictionaryUrl | string | `"https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json"` | URL of the data dictionary. | -| global.dispatcherJobNum | int | `"10"` | Number of dispatcher jobs. | -| global.environment | string | `"default"` | Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. | -| global.hostname | string | `"localhost"` | Hostname for the deployment. | -| global.kubeBucket | string | `"kube-gen3"` | S3 bucket name for Kubernetes manifest files. | -| global.logsBucket | string | `"logs-gen3"` | S3 bucket name for log files. | -| global.minAvialable | int | `1` | The minimum amount of pods that are available at all times if the PDB is deployed. | -| global.netPolicy | bool | `true` | Whether network policies are enabled. | -| global.pdb | bool | `false` | If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. | -| global.portalApp | string | `"gitops"` | Portal application name. | -| global.postgres.dbCreate | bool | `true` | Whether the database should be created. | -| global.postgres.externalSecret | string | `""` | Name of external secret. Disabled if empty | -| global.postgres.master | map | `{"host":null,"password":null,"port":"5432","username":"postgres"}` | Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres | -| global.postgres.master.host | string | `nil` | hostname of postgres server | -| global.postgres.master.password | string | `nil` | password for superuser in postgres. This is used to create or restore databases | -| global.postgres.master.port | string | `"5432"` | Port for Postgres. | -| global.postgres.master.username | string | `"postgres"` | username of superuser in postgres. This is used to create or restore databases | -| global.publicDataSets | bool | `true` | Whether public datasets are enabled. | -| global.revproxyArn | string | `"arn:aws:acm:us-east-1:123456:certificate"` | ARN of the reverse proxy certificate. | -| global.tierAccessLevel | string | `"libre"` | Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` | -| image | map | `{"pullPolicy":"Always","repository":"quay.io/cdis/ssjdispatcher","tag":"2022.08"}` | Docker image information. | -| image.pullPolicy | string | `"Always"` | Docker pull policy. | -| image.repository | string | `"quay.io/cdis/ssjdispatcher"` | Docker repository. | -| image.tag | string | `"2022.08"` | Overrides the image tag whose default is the chart appVersion. | -| imagePullSecrets | list | `[]` | Docker image pull secrets. | -| indexing | string | `"707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/indexs3client:2022.08"` | Image to use for the "indexing" job. | -| nameOverride | string | `""` | Override the name of the chart. | -| nodeSelector | map | `{}` | Node Selector for the pods | -| partOf | string | `"Workspace-Tab"` | Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. | -| podSecurityContext | map | `{"fsGroup":1000,"runAsUser":1000}` | Security context to apply to the pod | -| podSecurityContext.fsGroup | int | `1000` | Group that Kubernetes will change the permissions of all files in volumes to when volumes are mounted by a pod. | -| podSecurityContext.runAsUser | int | `1000` | User that all the processes will run under in the container. | -| release | string | `"production"` | Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". | -| replicaCount | int | `1` | Number of replicas for the deployment. | -| resources | map | `{"limits":{"cpu":1,"memory":"2400Mi"},"requests":{"cpu":0.1,"memory":"128Mi"}}` | Resource requests and limits for the containers in the pod | -| resources.limits | map | `{"cpu":1,"memory":"2400Mi"}` | The maximum amount of resources that the container is allowed to use | -| resources.limits.cpu | string | `1` | The maximum amount of CPU the container can use | -| resources.limits.memory | string | `"2400Mi"` | The maximum amount of memory the container can use | -| resources.requests | map | `{"cpu":0.1,"memory":"128Mi"}` | The amount of resources that the container requests | -| resources.requests.cpu | string | `0.1` | The amount of CPU requested | -| resources.requests.memory | string | `"128Mi"` | The amount of memory requested | -| securityContext | map | `{}` | Security context for the containers in the pod | -| selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | -| service | map | `{"port":80,"type":"ClusterIP"}` | Kubernetes service information. | -| service.port | int | `80` | The port number that the service exposes. | -| service.type | string | `"ClusterIP"` | Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". | -| serviceAccount | map | `{"annotations":{},"create":true,"name":"ssjdispatcher-service-account"}` | Service account to use or create. | -| serviceAccount.annotations | map | `{}` | Annotations to add to the service account. | -| serviceAccount.create | bool | `true` | Specifies whether a service account should be created. | -| serviceAccount.name | string | `"ssjdispatcher-service-account"` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | -| ssjcreds | map | `{"jobName":"indexing","jobPassword":"replace_with_password","jobPattern":"s3://test-12345678901234-upload/*","jobRequestCpu":"500m","jobRequestMem":"0.5Gi","jobUrl":"http://indexd-service/index","jobUser":"ssj","metadataservicePassword":"replace_with_password","metadataserviceUrl":"http://revproxy-service/mds","metadataserviceUsername":"gateway","sqsUrl":"https://sqs.us-east-1.amazonaws.com/12345678901234/test-upload_data_upload"}` | Values for ssjdispatcher secret. | -| ssjcreds.jobName | string | `"indexing"` | Name of the ssj job. | -| ssjcreds.jobPassword | string | `"replace_with_password"` | Password for the job. | -| ssjcreds.jobPattern | string | `"s3://test-12345678901234-upload/*"` | URL upload pattern that will trigger an event in S3 to send a message to SQS. | -| ssjcreds.jobRequestCpu | string | `"500m"` | The amount of CPU the job requests. | -| ssjcreds.jobRequestMem | string | `"0.5Gi"` | The amount of memory the job requests. | -| ssjcreds.jobUrl | string | `"http://indexd-service/index"` | Indexd service URL. | -| ssjcreds.jobUser | string | `"ssj"` | Name of the user the job will run with. | -| ssjcreds.metadataservicePassword | string | `"replace_with_password"` | Password for the metadata service. | -| ssjcreds.metadataserviceUrl | string | `"http://revproxy-service/mds"` | URL to reach metadata service endpoint. | -| ssjcreds.metadataserviceUsername | string | `"gateway"` | Username for the metadata service. | -| ssjcreds.sqsUrl | string | `"https://sqs.us-east-1.amazonaws.com/12345678901234/test-upload_data_upload"` | Sqs queue to monitor. | -| strategy | map | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | Rolling update deployment strategy | -| strategy.rollingUpdate.maxSurge | int | `1` | Number of additional replicas to add during rollout. | -| strategy.rollingUpdate.maxUnavailable | int | `0` | Maximum amount of pods that can be unavailable during the update. | -| tolerations | list | `[]` | Tolerations for the pods | -| volumeMounts | list | `[{"mountPath":"/credentials.json","name":"ssjdispatcher-creds-volume","readOnly":true,"subPath":"credentials.json"}]` | Volumes to mount to the container. | -| volumes | list | `[{"name":"ssjdispatcher-creds-volume","secret":{"secretName":"ssjdispatcher-creds"}}]` | Volumes to attach to the container. | - diff --git a/helm/ssjdispatcher/templates/NOTES.txt b/helm/ssjdispatcher/templates/NOTES.txt deleted file mode 100644 index c1e7e1aef..000000000 --- a/helm/ssjdispatcher/templates/NOTES.txt +++ /dev/null @@ -1 +0,0 @@ -{{ .Chart.Name }} has been deployed successfully. diff --git a/helm/ssjdispatcher/templates/_helpers.tpl b/helm/ssjdispatcher/templates/_helpers.tpl deleted file mode 100644 index c7ffa9d8d..000000000 --- a/helm/ssjdispatcher/templates/_helpers.tpl +++ /dev/null @@ -1,68 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "ssjdispatcher.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "ssjdispatcher.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "ssjdispatcher.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "ssjdispatcher.labels" -}} -{{- if .Values.commonLabels }} - {{- with .Values.commonLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.commonLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "ssjdispatcher.selectorLabels" -}} -{{- if .Values.selectorLabels }} - {{- with .Values.selectorLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.selectorLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "ssjdispatcher.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "ssjdispatcher.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/helm/ssjdispatcher/templates/deployment.yaml b/helm/ssjdispatcher/templates/deployment.yaml deleted file mode 100644 index 2f0f86b95..000000000 --- a/helm/ssjdispatcher/templates/deployment.yaml +++ /dev/null @@ -1,88 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ssjdispatcher - labels: - {{- include "ssjdispatcher.labels" . | nindent 4 }} - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 4 }} - {{- end }} -spec: - selector: - matchLabels: - {{- include "ssjdispatcher.selectorLabels" . | nindent 8 }} - revisionHistoryLimit: 2 - strategy: - {{- toYaml .Values.strategy | nindent 8 }} - template: - metadata: - labels: - {{- include "ssjdispatcher.selectorLabels" . | nindent 8 }} - netnolimit: "yes" - public: "yes" - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 8 }} - {{- end }} - spec: - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - automountServiceAccountToken: {{ .Values.automountServiceAccountToken}} - serviceAccountName: {{ include "ssjdispatcher.serviceAccountName" . }} - volumes: - {{- toYaml .Values.volumes | nindent 8 }} - containers: - - name: ssjdispatcher - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - volumeMounts: - {{- toYaml .Values.volumeMounts | nindent 12 }} - env: - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogEnvVar" . | nindent 12 }} - {{- end }} - {{- with .Values.awsRegion }} - - name: AWS_REGION - value: {{ . }} - {{- end }} - {{- with .Values.awsStsRegionalEndpoints }} - - name: AWS_STS_REGIONAL_ENDPOINTS - value: {{ . }} - {{- end }} - {{- with .Values.gen3Namespace }} - - name: GEN3_NAMESPACE - value: {{ . }} - {{- end }} - - name: JOB_IMAGES - valueFrom: - configMapKeyRef: - name: manifest-ssjdispatcher - key: job_images - ports: - - name: http - containerPort: 8000 - protocol: TCP - livenessProbe: - httpGet: - path: /_status - port: 8000 - initialDelaySeconds: 5 - periodSeconds: 60 - timeoutSeconds: 30 - readinessProbe: - httpGet: - path: /_status - port: 8000 - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/ssjdispatcher/templates/hpa.yaml b/helm/ssjdispatcher/templates/hpa.yaml deleted file mode 100644 index b072bfd7a..000000000 --- a/helm/ssjdispatcher/templates/hpa.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "ssjdispatcher.fullname" . }} - labels: - {{- include "ssjdispatcher.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "ssjdispatcher.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/helm/ssjdispatcher/templates/manifest-ssjdispactcher.yaml b/helm/ssjdispatcher/templates/manifest-ssjdispactcher.yaml deleted file mode 100644 index 9a8efe1e3..000000000 --- a/helm/ssjdispatcher/templates/manifest-ssjdispactcher.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: manifest-ssjdispatcher -data: - job_images: |- - { - "indexing": {{ .Values.indexing | toJson }} - } - diff --git a/helm/ssjdispatcher/templates/pdb.yaml b/helm/ssjdispatcher/templates/pdb.yaml deleted file mode 100644 index 2ef2de13d..000000000 --- a/helm/ssjdispatcher/templates/pdb.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if and .Values.global.pdb (gt (int .Values.replicaCount) 1) }} -{{ include "common.pod_disruption_budget" . }} -{{- end }} \ No newline at end of file diff --git a/helm/ssjdispatcher/templates/serviceaccount.yaml b/helm/ssjdispatcher/templates/serviceaccount.yaml deleted file mode 100644 index ac52270fc..000000000 --- a/helm/ssjdispatcher/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "ssjdispatcher.serviceAccountName" . }} - labels: - {{- include "ssjdispatcher.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - {{- if .Values.global.aws.enabled }} - annotations: - eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.global.aws.account }}:role/{{ .Values.global.aws.hatchery_role }} - {{- end }} -{{- end }} diff --git a/helm/ssjdispatcher/templates/ssjdispatcher-secret.yaml b/helm/ssjdispatcher/templates/ssjdispatcher-secret.yaml deleted file mode 100644 index c05a72fa7..000000000 --- a/helm/ssjdispatcher/templates/ssjdispatcher-secret.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: ssjdispatcher-creds -type: Opaque -stringData: - credentials.json: |- - { - "SQS": { - "url": {{ .Values.ssjcreds.sqsUrl | quote }} - }, - "JOBS": [ - { - "name": {{ .Values.ssjcreds.jobName | quote }}, - "pattern": "{{ .Values.ssjcreds.jobPattern }}", - "imageConfig": { - "url": "{{ .Values.ssjcreds.jobUrl }}", - "username": "{{ .Values.ssjcreds.jobUser }}", - "password": "{{ .Values.ssjcreds.jobPassword }}", - "metadataService": { - "url": "{{ .Values.ssjcreds.metadataserviceUrl }}", - "username": "{{ .Values.ssjcreds.metadataserviceUsername }}", - "password": "{{ .Values.ssjcreds.metadataservicePassword }}" - } - }, - "RequestCPU": "{{ .Values.ssjcreds.jobRequestCpu }}", - "RequestMem": "{{ .Values.ssjcreds.jobRequestMem }}" - } - ] - } diff --git a/helm/ssjdispatcher/templates/tests/test-connection.yaml b/helm/ssjdispatcher/templates/tests/test-connection.yaml deleted file mode 100644 index 282184fe5..000000000 --- a/helm/ssjdispatcher/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "ssjdispatcher.fullname" . }}-test-connection" - labels: - {{- include "ssjdispatcher.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "ssjdispatcher.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/helm/ssjdispatcher/values.yaml b/helm/ssjdispatcher/values.yaml deleted file mode 100644 index 74ed990b0..000000000 --- a/helm/ssjdispatcher/values.yaml +++ /dev/null @@ -1,248 +0,0 @@ -# Default values for ssjdispatcher. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - - -# Global configuration -global: - # -- (map) AWS configuration - aws: - # -- (bool) Set to true if deploying to AWS. Controls ingress annotations. - enabled: false - # -- (string) Credentials for AWS stuff. - awsAccessKeyId: - # -- (string) Credentials for AWS stuff. - awsSecretAccessKey: - # -- (bool) Whether the deployment is for development purposes. - dev: true - - postgres: - # -- (bool) Whether the database should be created. - dbCreate: true - # -- (string) Name of external secret. Disabled if empty - externalSecret: "" - # -- (map) Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres - master: - # -- (string) hostname of postgres server - host: - # -- (string) username of superuser in postgres. This is used to create or restore databases - username: postgres - # -- (string) password for superuser in postgres. This is used to create or restore databases - password: - # -- (string) Port for Postgres. - port: "5432" - # -- (string) Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. - environment: default - # -- (string) Hostname for the deployment. - hostname: localhost - # -- (string) ARN of the reverse proxy certificate. - revproxyArn: arn:aws:acm:us-east-1:123456:certificate - # -- (string) URL of the data dictionary. - dictionaryUrl: https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json - # -- (string) Portal application name. - portalApp: gitops - # -- (string) S3 bucket name for Kubernetes manifest files. - kubeBucket: kube-gen3 - # -- (string) S3 bucket name for log files. - logsBucket: logs-gen3 - # -- (bool) Whether public datasets are enabled. - publicDataSets: true - # -- (string) Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` - tierAccessLevel: libre - # -- (bool) Whether network policies are enabled. - netPolicy: true - # -- (int) Number of dispatcher jobs. - dispatcherJobNum: "10" - # -- (bool) Whether Datadog is enabled. - ddEnabled: false - # -- (bool) If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. - pdb: false - # -- (int) The minimum amount of pods that are available at all times if the PDB is deployed. - minAvialable: 1 - -# -- (int) Number of replicas for the deployment. -replicaCount: 1 - -# -- (map) Docker image information. -image: - # -- (string) Docker repository. - repository: quay.io/cdis/ssjdispatcher - # -- (string) Docker pull policy. - pullPolicy: Always - # -- (string) Overrides the image tag whose default is the chart appVersion. - tag: "2022.08" - -# -- (list) Docker image pull secrets. -imagePullSecrets: [] - -# -- (string) Override the name of the chart. -nameOverride: "" - -# -- (string) Override the full name of the deployment. -fullnameOverride: "" - -# -- (map) Security context for the containers in the pod -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -# -- (map) Kubernetes service information. -service: - # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". - type: ClusterIP - # -- (int) The port number that the service exposes. - port: 80 - -# -- (map) Configuration for autoscaling the number of replicas -autoscaling: - # -- (bool) Whether autoscaling is enabled - enabled: false - # -- (int) The minimum number of replicas to scale down to - minReplicas: 1 - # -- (int) The maximum number of replicas to scale up to - maxReplicas: 100 - # -- (int) Target CPU utilization percentage - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -# -- (map) Node Selector for the pods -nodeSelector: {} - -# -- (list) Tolerations for the pods -tolerations: [] - -# -- (map) Security context to apply to the pod -podSecurityContext: - # -- (int) User that all the processes will run under in the container. - runAsUser: 1000 - # -- (int) Group that Kubernetes will change the permissions of all files in volumes to when volumes are mounted by a pod. - fsGroup: 1000 - -# -- (map) Affinity to use for the deployment. -affinity: - podAntiAffinity: - # -- (map) Option for scheduling to be required or preferred. - preferredDuringSchedulingIgnoredDuringExecution: - # -- (int) Weight value for preferred scheduling. - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - # -- (list) Label key for match expression. - - key: app - # -- (string) Operation type for the match expression. - operator: In - # -- (list) Value for the match expression key. - values: - - ssjdispatcher - # -- (string) Value for topology key label. - topologyKey: "kubernetes.io/hostname" - -# -- (list) Volumes to attach to the container. -volumes: - - name: ssjdispatcher-creds-volume - secret: - secretName: "ssjdispatcher-creds" -# -- (list) Volumes to mount to the container. -volumeMounts: - - name: "ssjdispatcher-creds-volume" - readOnly: true - mountPath: "/credentials.json" - subPath: credentials.json - -# -- (string) AWS region to be used. -awsRegion: us-east-1 -# -- (string) AWS STS to issue temporary credentials to users and roles that make an AWS STS request. Values regional or global. -awsStsRegionalEndpoints: regional -# -- (string) Namespace to deploy the job. -gen3Namespace: default -# -- (string) Ssjdispater job number. -dispatcherJobNum: "10" -# -- (string) Image to use for the "indexing" job. -indexing: 707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/indexs3client:2022.08 - -# -- (map) Resource requests and limits for the containers in the pod -resources: - # -- (map) The amount of resources that the container requests - requests: - # -- (string) The amount of CPU requested - cpu: 0.1 - # -- (string) The amount of memory requested - memory: 128Mi - # -- (map) The maximum amount of resources that the container is allowed to use - limits: - # -- (string) The maximum amount of CPU the container can use - cpu: 1.0 - # -- (string) The maximum amount of memory the container can use - memory: 2400Mi - -# -- (map) Rolling update deployment strategy -strategy: - type: RollingUpdate - rollingUpdate: - # -- (int) Number of additional replicas to add during rollout. - maxSurge: 1 - # -- (int) Maximum amount of pods that can be unavailable during the update. - maxUnavailable: 0 - -# -- (bool) Automount the default service account token -automountServiceAccountToken: true - -# -- (map) Values for ssjdispatcher secret. -ssjcreds: - # -- (string) Sqs queue to monitor. - sqsUrl: "https://sqs.us-east-1.amazonaws.com/12345678901234/test-upload_data_upload" - # -- (string) Name of the ssj job. - jobName: "indexing" - # -- (string) URL upload pattern that will trigger an event in S3 to send a message to SQS. - jobPattern: "s3://test-12345678901234-upload/*" - # -- (string) Indexd service URL. - jobUrl: "http://indexd-service/index" - # -- (string) Name of the user the job will run with. - jobUser: "ssj" - # -- (string) Password for the job. - jobPassword: "replace_with_password" - # -- (string) The amount of CPU the job requests. - jobRequestCpu: "500m" - # -- (string) The amount of memory the job requests. - jobRequestMem: "0.5Gi" - # -- (string) URL to reach metadata service endpoint. - metadataserviceUrl: "http://revproxy-service/mds" - # -- (string) Username for the metadata service. - metadataserviceUsername: "gateway" - # -- (string) Password for the metadata service. - metadataservicePassword: "replace_with_password" - -# -- (map) Service account to use or create. -serviceAccount: - # -- (bool) Specifies whether a service account should be created. - create: true - # -- (map) Annotations to add to the service account. - annotations: {} - # -- (string) The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "ssjdispatcher-service-account" - -# Values to determine the labels that are used for the deployment, pod, etc. -# -- (string) Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". -release: "production" -# -- (string) Valid options are "true" or "false". If invalid option is set- the value will default to "false". -criticalService: "true" -# -- (string) Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. -partOf: "Workspace-Tab" -# -- (map) Will completely override the selectorLabels defined in the common chart's _label_setup.tpl -selectorLabels: -# -- (map) Will completely override the commonLabels defined in the common chart's _label_setup.tpl -commonLabels: - -# Values to configure datadog if ddEnabled is set to "true". -# -- (bool) If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. -datadogLogsInjection: true -# -- (bool) If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. -datadogProfilingEnabled: true -# -- (int) A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. -datadogTraceSampleRate: 1 diff --git a/helm/syfon/templates/_helpers.tpl b/helm/syfon/templates/_helpers.tpl index 62cefbbde..df2a60c6a 100644 --- a/helm/syfon/templates/_helpers.tpl +++ b/helm/syfon/templates/_helpers.tpl @@ -11,8 +11,13 @@ {{- .Release.Name | trunc 63 | trimSuffix "-" -}} {{- else -}} {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{- end -}} {{- end -}} + +{{- define "syfon.deploymentName" -}} +syfon-deployment {{- end -}} {{- define "syfon.labels" -}} diff --git a/helm/syfon/templates/deployment.yaml b/helm/syfon/templates/deployment.yaml index fda551d86..5b91d98b4 100644 --- a/helm/syfon/templates/deployment.yaml +++ b/helm/syfon/templates/deployment.yaml @@ -4,7 +4,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "syfon.fullname" . }} + name: {{ include "syfon.deploymentName" . }} labels: {{- include "syfon.labels" . | nindent 4 }} spec: diff --git a/helm/wts/.helmignore b/helm/wts/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/helm/wts/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm/wts/Chart.yaml b/helm/wts/Chart.yaml deleted file mode 100644 index 6d9533e47..000000000 --- a/helm/wts/Chart.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: v2 -name: wts -description: A Helm chart for gen3 workspace token service - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.13 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "master" - -dependencies: -- name: common - version: 0.1.10 - repository: file://../common -- name: postgresql - version: 11.9.13 - repository: "https://charts.bitnami.com/bitnami" - condition: postgres.separate diff --git a/helm/wts/README.md b/helm/wts/README.md deleted file mode 100644 index f755b7994..000000000 --- a/helm/wts/README.md +++ /dev/null @@ -1,107 +0,0 @@ -# wts - -![Version: 0.1.13](https://img.shields.io/badge/Version-0.1.13-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) - -A Helm chart for gen3 workspace token service - -## Requirements - -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.10 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | - -## Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| affinity | map | `{}` | Affinity to use for the deployment. | -| autoscaling | map | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Configuration for autoscaling the number of replicas | -| autoscaling.enabled | bool | `false` | Whether autoscaling is enabled or not | -| autoscaling.maxReplicas | int | `100` | The maximum number of replicas to scale up to | -| autoscaling.minReplicas | int | `1` | The minimum number of replicas to scale down to | -| autoscaling.targetCPUUtilizationPercentage | int | `80` | The target CPU utilization percentage for autoscaling | -| commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | -| criticalService | string | `"true"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | -| datadogLogsInjection | bool | `true` | If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. | -| datadogProfilingEnabled | bool | `true` | If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. | -| datadogTraceSampleRate | int | `1` | A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. | -| externalSecrets | map | `{"dbcreds":null}` | External Secrets settings. | -| externalSecrets.dbcreds | string | `nil` | Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" | -| fullnameOverride | string | `""` | Override the full name of the deployment. | -| global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false}` | AWS configuration | -| global.aws.awsAccessKeyId | string | `nil` | Credentials for AWS stuff. | -| global.aws.awsSecretAccessKey | string | `nil` | Credentials for AWS stuff. | -| global.aws.enabled | bool | `false` | Set to true if deploying to AWS. Controls ingress annotations. | -| global.ddEnabled | bool | `false` | Whether Datadog is enabled. | -| global.dev | bool | `true` | Whether the deployment is for development purposes. | -| global.dictionaryUrl | string | `"https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json"` | URL of the data dictionary. | -| global.dispatcherJobNum | int | `"10"` | Number of dispatcher jobs. | -| global.environment | string | `"default"` | Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. | -| global.externalSecrets | map | `{"deploy":false,"separateSecretStore":false}` | External Secrets settings. | -| global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any wts secrets you have deployed. | -| global.externalSecrets.separateSecretStore | string | `false` | Will deploy a separate External Secret Store for this service. | -| global.hostname | string | `"localhost"` | Hostname for the deployment. | -| global.kubeBucket | string | `"kube-gen3"` | S3 bucket name for Kubernetes manifest files. | -| global.logsBucket | string | `"logs-gen3"` | S3 bucket name for log files. | -| global.minAvialable | int | `1` | The minimum amount of pods that are available at all times if the PDB is deployed. | -| global.netPolicy | bool | `true` | Whether network policies are enabled. | -| global.pdb | bool | `false` | If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. | -| global.portalApp | string | `"gitops"` | Portal application name. | -| global.postgres.dbCreate | bool | `true` | Whether the database should be created. | -| global.postgres.externalSecret | string | `""` | Name of external secret. Disabled if empty | -| global.postgres.master | map | `{"host":null,"password":null,"port":"5432","username":"postgres"}` | Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres | -| global.postgres.master.host | string | `nil` | hostname of postgres server | -| global.postgres.master.password | string | `nil` | password for superuser in postgres. This is used to create or restore databases | -| global.postgres.master.port | string | `"5432"` | Port for Postgres. | -| global.postgres.master.username | string | `"postgres"` | username of superuser in postgres. This is used to create or restore databases | -| global.publicDataSets | bool | `true` | Whether public datasets are enabled. | -| global.revproxyArn | string | `"arn:aws:acm:us-east-1:123456:certificate"` | ARN of the reverse proxy certificate. | -| global.tierAccessLevel | string | `"libre"` | Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` | -| hostname | string | `nil` | Hostname for the deployment. | -| image | map | `{"pullPolicy":"Always","repository":"quay.io/cdis/workspace-token-service","tag":"feat_wts_internalfence"}` | Docker image information. | -| image.pullPolicy | string | `"Always"` | Docker pull policy. | -| image.repository | string | `"quay.io/cdis/workspace-token-service"` | Docker repository. | -| image.tag | string | `"feat_wts_internalfence"` | Overrides the image tag whose default is the chart appVersion. | -| imagePullSecrets | list | `[]` | Docker image pull secrets. | -| nameOverride | string | `""` | Override the name of the chart. | -| nodeSelector | map | `{}` | Node Selector for the pods | -| oidc_client_id | string | `nil` | Id for the OIDC client. | -| oidc_client_secret | string | `nil` | Secret for the OIDC client. | -| partOf | string | `"Authentication"` | Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. | -| podAnnotations | map | `{}` | Annotations to add to the pod. | -| podSecurityContext | map | `{}` | Security context for the pod | -| postgres | map | `{"database":null,"dbCreate":null,"dbRestore":false,"host":null,"password":null,"port":"5432","separate":false,"username":null}` | Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you | -| postgres.database | string | `nil` | Database name for postgres. This is a service override, defaults to - | -| postgres.dbCreate | bool | `nil` | Whether the database should be created. Default to global.postgres.dbCreate | -| postgres.host | string | `nil` | Hostname for postgres server. This is a service override, defaults to global.postgres.host | -| postgres.password | string | `nil` | Password for Postgres. Will be autogenerated if left empty. | -| postgres.port | string | `"5432"` | Port for Postgres. | -| postgres.separate | string | `false` | Will create a Database for the individual service to help with developing it. | -| postgres.username | string | `nil` | Username for postgres. This is a service override, defaults to - | -| postgresql | map | `{"primary":{"persistence":{"enabled":false}}}` | Postgresql subchart settings if deployed separately option is set to "true". Disable persistence by default so we can spin up and down ephemeral environments | -| release | string | `"production"` | Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". | -| replicaCount | int | `1` | Number of replicas for the deployment. | -| resources | map | `{"limits":{"cpu":0.5,"memory":"512Mi"},"requests":{"cpu":0.2,"memory":"120Mi"}}` | Resource requests and limits for the containers in the pod | -| resources.limits | map | `{"cpu":0.5,"memory":"512Mi"}` | The maximum amount of resources that the container is allowed to use | -| resources.limits.cpu | string | `0.5` | The maximum amount of CPU the container can use | -| resources.limits.memory | string | `"512Mi"` | The maximum amount of memory the container can use | -| resources.requests | map | `{"cpu":0.2,"memory":"120Mi"}` | The amount of resources that the container requests | -| resources.requests.cpu | string | `0.2` | The amount of CPU requested | -| resources.requests.memory | string | `"120Mi"` | The amount of memory requested | -| roleName | string | `"workspace-token-service"` | Name of the role to be used for the role binding. | -| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"external_oidc":null}` | Values for wts secret and keys for External Secrets. | -| secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. | -| secrets.awsSecretAccessKey | str | `nil` | AWS secret access key ID. Overrides global key. | -| securityContext | map | `{}` | Security context for the containers in the pod | -| selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | -| service | map | `{"httpPort":80,"httpsPort":443,"type":"ClusterIP"}` | Configuration for the service | -| service.httpPort | int | `80` | Port on which the service is exposed | -| service.httpsPort | int | `443` | Secure port on which the service is exposed | -| service.type | string | `"ClusterIP"` | Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". | -| serviceAccount | map | `{"annotations":{},"create":true,"name":""}` | Service account to use or create. | -| serviceAccount.annotations | map | `{}` | Annotations to add to the service account. | -| serviceAccount.create | bool | `true` | Specifies whether a service account should be created. | -| serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | -| tolerations | list | `[]` | Tolerations for the pods | - diff --git a/helm/wts/templates/NOTES.txt b/helm/wts/templates/NOTES.txt deleted file mode 100644 index c1e7e1aef..000000000 --- a/helm/wts/templates/NOTES.txt +++ /dev/null @@ -1 +0,0 @@ -{{ .Chart.Name }} has been deployed successfully. diff --git a/helm/wts/templates/_helpers.tpl b/helm/wts/templates/_helpers.tpl deleted file mode 100644 index a8a094c0a..000000000 --- a/helm/wts/templates/_helpers.tpl +++ /dev/null @@ -1,81 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "wts.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "wts.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "wts.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "wts.labels" -}} -{{- if .Values.commonLabels }} - {{- with .Values.commonLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.commonLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "wts.selectorLabels" -}} -{{- if .Values.selectorLabels }} - {{- with .Values.selectorLabels }} - {{- toYaml . }} - {{- end }} -{{- else }} - {{- (include "common.selectorLabels" .)}} -{{- end }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "wts.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "wts.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} - - -{{/* - Postgres Password lookup -*/}} -{{- define "wts.postgres.password" -}} -{{- $localpass := (lookup "v1" "Secret" "postgres" "postgres-postgresql" ) -}} -{{- if $localpass }} -{{- default (index $localpass.data "postgres-password" | b64dec) }} -{{- else }} -{{- default .Values.postgres.password }} -{{- end }} -{{- end }} diff --git a/helm/wts/templates/aws-config.yaml b/helm/wts/templates/aws-config.yaml deleted file mode 100644 index 398770d36..000000000 --- a/helm/wts/templates/aws-config.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if or (.Values.secrets.awsSecretAccessKey) (.Values.global.aws.awsSecretAccessKey ) }} -{{ include "common.awsconfig" . }} -{{- end -}} \ No newline at end of file diff --git a/helm/wts/templates/db-init.yaml b/helm/wts/templates/db-init.yaml deleted file mode 100644 index d99ca1b2e..000000000 --- a/helm/wts/templates/db-init.yaml +++ /dev/null @@ -1,6 +0,0 @@ -{{ include "common.db-secret" . }} ---- -{{ include "common.db_setup_job" . }} ---- -{{ include "common.db_setup_sa" . }} ---- diff --git a/helm/wts/templates/deployment.yaml b/helm/wts/templates/deployment.yaml deleted file mode 100644 index e3f20dec1..000000000 --- a/helm/wts/templates/deployment.yaml +++ /dev/null @@ -1,209 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: wts-deployment - labels: - {{- include "wts.labels" . | nindent 4 }} - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 4 }} - {{- end }} - annotations: - gen3.io/network-ingress: "mariner" -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "wts.selectorLabels" . | nindent 6 }} - release: {{ .Values.release }} - revisionHistoryLimit: 2 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "wts.selectorLabels" . | nindent 8 }} - public: "yes" - netnolimit: "yes" - userhelper: "yes" - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogLabels" . | nindent 8 }} - {{- end }} - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - wts - topologyKey: "kubernetes.io/hostname" - terminationGracePeriodSeconds: 10 - volumes: - - name: wts-secret - secret: - secretName: "wts-g3auto" - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: workspace-token-service - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - volumeMounts: - - name: "wts-secret" - readOnly: true - mountPath: "/var/www/wts/appcreds.json" - subPath: appcreds.json - ports: - - name: http - containerPort: 80 - protocol: TCP - livenessProbe: - httpGet: - path: /_status - port: 80 - failureThreshold: 10 - initialDelaySeconds: 5 - readinessProbe: - httpGet: - path: /_status - port: 80 - env: - {{- if .Values.global.ddEnabled }} - {{- include "common.datadogEnvVar" . | nindent 11 }} - {{- end }} - - name: OIDC_CLIENT_ID - valueFrom: - secretKeyRef: - name: wts-oidc-client - key: client_id - optional: false - - name: OIDC_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: wts-oidc-client - key: client_secret - optional: false - - name: SECRET_CONFIG - value: "/var/www/wts/appcreds.json" - - name: AUTH_PLUGINS - value: k8s - - name: PGHOST - valueFrom: - secretKeyRef: - name: {{ .Chart.Name }}-dbcreds - key: host - optional: false - - name: PGUSER - valueFrom: - secretKeyRef: - name: {{ .Chart.Name }}-dbcreds - key: username - optional: false - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: {{ .Chart.Name }}-dbcreds - key: password - optional: false - - name: PGDB - valueFrom: - secretKeyRef: - name: {{ .Chart.Name }}-dbcreds - key: database - optional: false - - name: DBREADY - valueFrom: - secretKeyRef: - name: {{ .Chart.Name }}-dbcreds - key: dbcreated - optional: false - - name: SQLALCHEMY_DATABASE_URI - value: postgresql://$(PGUSER):$(PGPASSWORD)@$(PGHOST):5432/$(PGDB) - resources: - {{- toYaml .Values.resources | nindent 12 }} - initContainers: - - name: wts-db-migrate - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - volumeMounts: - - name: "wts-secret" - readOnly: true - mountPath: "/var/www/wts/appcreds.json" - subPath: appcreds.json - env: - - name: PGHOST - valueFrom: - secretKeyRef: - name: {{ .Chart.Name }}-dbcreds - key: host - optional: false - - name: PGUSER - valueFrom: - secretKeyRef: - name: {{ .Chart.Name }}-dbcreds - key: username - optional: false - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: {{ .Chart.Name }}-dbcreds - key: password - optional: false - - name: PGDB - valueFrom: - secretKeyRef: - name: {{ .Chart.Name }}-dbcreds - key: database - optional: false - - name: DBREADY - valueFrom: - secretKeyRef: - name: {{ .Chart.Name }}-dbcreds - key: dbcreated - optional: false - - name: SQLALCHEMY_DATABASE_URI - value: postgresql://$(PGUSER):$(PGPASSWORD)@$(PGHOST):5432/$(PGDB) - - name: SECRET_CONFIG - value: "/var/www/wts/appcreds.json" - resources: - {{- toYaml .Values.resources | nindent 12 }} - command: ["/bin/sh"] - args: - - "-c" - - | - if hash alembic 2>/dev/null; then - echo "Running DB migration" - cd /wts - alembic upgrade head - else - echo "Alembic not installed - not running DB migration" - fi - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - diff --git a/helm/wts/templates/external-secret.yaml b/helm/wts/templates/external-secret.yaml deleted file mode 100644 index 70c278fea..000000000 --- a/helm/wts/templates/external-secret.yaml +++ /dev/null @@ -1 +0,0 @@ -{{ include "common.externalSecret.db" . }} \ No newline at end of file diff --git a/helm/wts/templates/pdb.yaml b/helm/wts/templates/pdb.yaml deleted file mode 100644 index 2ef2de13d..000000000 --- a/helm/wts/templates/pdb.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if and .Values.global.pdb (gt (int .Values.replicaCount) 1) }} -{{ include "common.pod_disruption_budget" . }} -{{- end }} \ No newline at end of file diff --git a/helm/wts/templates/role.yaml b/helm/wts/templates/role.yaml deleted file mode 100644 index a0610bb4c..000000000 --- a/helm/wts/templates/role.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# TODO: name this so it can work in multiple namespaces -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "wts.fullname" . }} -rules: -- apiGroups: [""] - resources: ["namespaces", "pods"] - verbs: ["get", "list"] diff --git a/helm/wts/templates/rolebinding.yaml b/helm/wts/templates/rolebinding.yaml deleted file mode 100644 index 4c824d7fa..000000000 --- a/helm/wts/templates/rolebinding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - #WTS_BINDING - name: wts-role-binding-{{ $.Release.Namespace }} - namespace: {{ $.Release.Namespace }} -subjects: -- kind: ServiceAccount - name: {{ .Values.roleName }} - #CURRENT_NAMESPACE - namespace: {{ $.Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ include "wts.fullname" . }} - apiGroup: "" diff --git a/helm/wts/templates/secret-store.yaml b/helm/wts/templates/secret-store.yaml deleted file mode 100644 index 771c7760d..000000000 --- a/helm/wts/templates/secret-store.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{ if .Values.global.externalSecrets.separateSecretStore }} -{{ include "common.secretstore" . }} -{{- end }} \ No newline at end of file diff --git a/helm/wts/templates/secret.yaml b/helm/wts/templates/secret.yaml deleted file mode 100644 index 5cff98153..000000000 --- a/helm/wts/templates/secret.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: wts-g3auto -type: Opaque -stringData: -#TODO: Fill in external_oidc in appcreds.json from values.yaml - appcreds.json: | - { - "wts_base_url": "https://{{ .Values.global.hostname }}/wts/", - "encryption_key": "{{ .Values.encpryption_key | default (randAlphaNum 32 | b64enc) }}", - "secret_key": "{{ .Values.secret_key | default (randAlphaNum 32 | b64enc) }}", - "fence_base_url": "https://{{ .Values.global.hostname }}/user", - "external_oidc": {{ default (.Values.secrets.external_oidc | toJson) "[]" }} - } ---- -apiVersion: v1 -kind: Secret -metadata: - name: wts-oidc-client \ No newline at end of file diff --git a/helm/wts/templates/service.yaml b/helm/wts/templates/service.yaml deleted file mode 100644 index 66efd2cea..000000000 --- a/helm/wts/templates/service.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: workspace-token-service - labels: - {{- include "wts.labels" . | nindent 4 }} -spec: - selector: - app: wts - type: {{ .Values.service.type }} - ports: - - protocol: TCP - port: {{ .Values.service.httpPort }} - targetPort: {{ .Values.service.httpPort }} - name: http - - protocol: TCP - port: {{ .Values.service.httpsPort }} - targetPort: {{ .Values.service.httpsPort }} - name: https \ No newline at end of file diff --git a/helm/wts/templates/serviceaccount.yaml b/helm/wts/templates/serviceaccount.yaml deleted file mode 100644 index 85b63b11a..000000000 --- a/helm/wts/templates/serviceaccount.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Values.roleName }} - labels: - {{- include "wts.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: wts-secrets-role -rules: -- apiGroups: [""] # "" indicates the core API group - resources: ["secrets"] - verbs: ["*"] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: wts-secrets ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: wts-secrets-rolebinding -subjects: -- kind: ServiceAccount - name: wts-secrets - apiGroup: "" -roleRef: - kind: Role - name: wts-secrets-role - apiGroup: "" diff --git a/helm/wts/templates/tests/test-connection.yaml b/helm/wts/templates/tests/test-connection.yaml deleted file mode 100644 index 250d8c479..000000000 --- a/helm/wts/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "wts-test-connection" - labels: - {{- include "wts.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['workspace-token-service:80/_status'] - restartPolicy: Never diff --git a/helm/wts/templates/wts-oidc.yaml b/helm/wts/templates/wts-oidc.yaml deleted file mode 100644 index 769a3c47d..000000000 --- a/helm/wts/templates/wts-oidc.yaml +++ /dev/null @@ -1,112 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: wts-oidc-job -spec: - template: - metadata: - labels: - app: gen3job - spec: - restartPolicy: OnFailure - serviceAccountName: wts-secrets - volumes: - - name: config-volume - secret: - secretName: "fence-config" - - name: shared-volume - emptyDir: {} - initContainers: - - name: wait-for-fence - image: curlimages/curl:latest - command: ["/bin/sh","-c"] - args: ["while [ $(curl -sw '%{http_code}' http://fence-service -o /dev/null) -ne 200 ]; do sleep 5; echo 'Waiting for fence...'; done"] - containers: - - name: fence-client - # TODO: Make this configurable - image: "quay.io/cdis/fence:master" - imagePullPolicy: {{ .Values.image.pullPolicy }} - # TODO: ADD RESOURCES - # resources: - command: ["/bin/bash"] - args: - - "-c" - - | - export WTS_HOSTNAME={{ .Values.global.hostname }} - echo "Trying to generate WTS client for hostname : $WTS_HOSTNAME..." - - - - secrets=$(fence-create client-create --client wts --urls "https://${WTS_HOSTNAME}/wts/oauth2/authorize" --username wts --auto-approve | tail -1) - if [[ ! $secrets =~ (\'(.*)\', \'(.*)\') ]]; then - echo "Failed generating oidc client for workspace token service: $secrets" - exit 1 - fi - client_id="${BASH_REMATCH[2]}" - client_secret="${BASH_REMATCH[3]}" - echo client_id: $client_id - echo client_secret: $client_secret - echo -n $client_id > /shared/client_id - echo -n $client_secret > /shared/client_secret - env: - - name: PGHOST - valueFrom: - secretKeyRef: - name: fence-dbcreds - key: host - optional: false - - name: PGUSER - valueFrom: - secretKeyRef: - name: fence-dbcreds - key: username - optional: false - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: fence-dbcreds - key: password - optional: false - - name: PGDB - valueFrom: - secretKeyRef: - name: fence-dbcreds - key: database - optional: false - - name: DB - value: postgresql://$(PGUSER):$(PGPASSWORD)@$(PGHOST):5432/$(PGDB) - - name: PYTHONPATH - value: /var/www/fence - - name: FENCE_PUBLIC_CONFIG - valueFrom: - configMapKeyRef: - name: manifest-fence - key: fence-config-public.yaml - optional: true - volumeMounts: - - name: "shared-volume" - mountPath: "/shared" - - name: "config-volume" - readOnly: true - mountPath: "/var/www/fence/fence-config.yaml" - subPath: fence-config.yaml - - name: kubectl - #TODO: Make this configurable? - image: bitnami/kubectl:latest - volumeMounts: - - name: "shared-volume" - mountPath: "/shared" - command: ["/bin/bash"] - args: - - "-c" - - | - echo "waiting for /shared/client_id" - while [ ! -e /shared/client_id ] - do - echo "..." - sleep 5 - done - echo "Updating k8s secret wts-oidc-client" - CLIENT_ID=$(cat /shared/client_id | base64) - CLIENT_SECRET=$(cat /shared/client_secret | base64) - kubectl patch secret/wts-oidc-client --patch="{\"data\":{\"client_secret\":\"${CLIENT_SECRET}\", \"client_id\":\"${CLIENT_ID}\"}}" \ No newline at end of file diff --git a/helm/wts/values.yaml b/helm/wts/values.yaml deleted file mode 100644 index d4e10223f..000000000 --- a/helm/wts/values.yaml +++ /dev/null @@ -1,255 +0,0 @@ -# Default values for wts. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - - -# Global configuration -global: - # -- (map) AWS configuration - aws: - # -- (bool) Set to true if deploying to AWS. Controls ingress annotations. - enabled: false - # -- (string) Credentials for AWS stuff. - awsAccessKeyId: - # -- (string) Credentials for AWS stuff. - awsSecretAccessKey: - # -- (bool) Whether the deployment is for development purposes. - dev: true - - postgres: - # -- (bool) Whether the database should be created. - dbCreate: true - # -- (string) Name of external secret. Disabled if empty - externalSecret: "" - # -- (map) Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres - master: - # -- (string) hostname of postgres server - host: - # -- (string) username of superuser in postgres. This is used to create or restore databases - username: postgres - # -- (string) password for superuser in postgres. This is used to create or restore databases - password: - # -- (string) Port for Postgres. - port: "5432" - # -- (string) Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. - environment: default - # -- (string) Hostname for the deployment. - hostname: localhost - # -- (string) ARN of the reverse proxy certificate. - revproxyArn: arn:aws:acm:us-east-1:123456:certificate - # -- (string) URL of the data dictionary. - dictionaryUrl: https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json - # -- (string) Portal application name. - portalApp: gitops - # -- (string) S3 bucket name for Kubernetes manifest files. - kubeBucket: kube-gen3 - # -- (string) S3 bucket name for log files. - logsBucket: logs-gen3 - # -- (bool) Whether public datasets are enabled. - publicDataSets: true - # -- (string) Access level for tiers. acceptable values for `tier_access_level` are: `libre`, `regular` and `private`. If omitted, by default common will be treated as `private` - tierAccessLevel: libre - # -- (bool) Whether network policies are enabled. - netPolicy: true - # -- (int) Number of dispatcher jobs. - dispatcherJobNum: "10" - # -- (bool) Whether Datadog is enabled. - ddEnabled: false - # -- (bool) If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. - pdb: false - # -- (int) The minimum amount of pods that are available at all times if the PDB is deployed. - minAvialable: 1 - # -- (map) External Secrets settings. - externalSecrets: - # -- (bool) Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any wts secrets you have deployed. - deploy: false - # -- (string) Will deploy a separate External Secret Store for this service. - separateSecretStore: false - -# -- (map) External Secrets settings. -externalSecrets: - # -- (string) Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" - dbcreds: - -# -- (map) Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you -postgres: - # (bool) Whether the database should be restored from s3. Default to global.postgres.dbRestore - dbRestore: false - # -- (bool) Whether the database should be created. Default to global.postgres.dbCreate - dbCreate: - # -- (string) Hostname for postgres server. This is a service override, defaults to global.postgres.host - host: - # -- (string) Database name for postgres. This is a service override, defaults to - - database: - # -- (string) Username for postgres. This is a service override, defaults to - - username: - # -- (string) Port for Postgres. - port: "5432" - # -- (string) Password for Postgres. Will be autogenerated if left empty. - password: - # -- (string) Will create a Database for the individual service to help with developing it. - separate: false - -# -- (map) Postgresql subchart settings if deployed separately option is set to "true". -# Disable persistence by default so we can spin up and down ephemeral environments -postgresql: - primary: - persistence: - enabled: false - -# -- (int) Number of replicas for the deployment. -replicaCount: 1 - -# -- (map) Docker image information. -image: - # -- (string) Docker repository. - repository: quay.io/cdis/workspace-token-service - # -- (string) Docker pull policy. - pullPolicy: Always - # -- (string) Overrides the image tag whose default is the chart appVersion. - tag: "feat_wts_internalfence" - -# -- (list) Docker image pull secrets. -imagePullSecrets: [] - -# -- (string) Override the name of the chart. -nameOverride: "" - -# -- (string) Override the full name of the deployment. -fullnameOverride: "" - -# -- (string) Hostname for the deployment. -hostname: -# -- (string) Id for the OIDC client. -oidc_client_id: -# -- (string) Secret for the OIDC client. -oidc_client_secret: - -# -- (map) Service account to use or create. -serviceAccount: - # -- (bool) Specifies whether a service account should be created. - create: true - # -- (map) Annotations to add to the service account. - annotations: {} - # -- (string) The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -# -- (map) Annotations to add to the pod. -podAnnotations: {} - -# -- (map) Security context for the pod -podSecurityContext: {} - # fsGroup: 2000 - -# -- (map) Security context for the containers in the pod -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -# -- (map) Configuration for the service -service: - # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". - type: ClusterIP - # -- (int) Port on which the service is exposed - httpPort: 80 - # -- (int) Secure port on which the service is exposed - httpsPort: 443 - -# -- (map) Resource requests and limits for the containers in the pod -resources: - # -- (map) The amount of resources that the container requests - requests: - # -- (string) The amount of CPU requested - cpu: 0.2 - # -- (string) The amount of memory requested - memory: 120Mi - # -- (map) The maximum amount of resources that the container is allowed to use - limits: - # -- (string) The maximum amount of CPU the container can use - cpu: 0.5 - # -- (string) The maximum amount of memory the container can use - memory: 512Mi - -# -- (map) Configuration for autoscaling the number of replicas -autoscaling: - # -- (bool) Whether autoscaling is enabled or not - enabled: false - # -- (int) The minimum number of replicas to scale down to - minReplicas: 1 - # -- (int) The maximum number of replicas to scale up to - maxReplicas: 100 - # -- (int) The target CPU utilization percentage for autoscaling - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -# -- (map) Node Selector for the pods -nodeSelector: {} - -# -- (list) Tolerations for the pods -tolerations: [] - -# -- (map) Affinity to use for the deployment. -affinity: {} - -# -- (string) Name of the role to be used for the role binding. -roleName: workspace-token-service - -# -- (map) Values for wts secret and keys for External Secrets. -secrets: - # -- (str) AWS access key ID. Overrides global key. - awsAccessKeyId: - # -- (str) AWS secret access key ID. Overrides global key. - awsSecretAccessKey: - external_oidc: - # - base_url: - # oidc_client_id: - # oidc_client_secret: - # login_options: - # example-google-login: - # name: - # params: - # idp: - - # wts-g3auto: - # appcreds.json: | - # { - # "wts_base_url": "https://HOSTNAME.REPLACE/wts/", - # "encryption_key": "ENCRYPTION KEY HERE", - # "secret_key": "SECRET KEY HERE", - # "fence_base_url": "https://HOSTNAME.REPLACE/user/", - # "oidc_client_id": "OIDC_CLIENT_ID.REPLACE", - # "oidc_client_sucrut": "OIDC_CLIENT_SECRET.REPLACE", - # "external_oidc": [] - # } - # dbcreds.json: | - # { - # "db_host": "FENCE_DB_HOST.REPLACE", - # "db_username": "wts_default", - # "db_passwurd": "WTS_DB_PWD.REPLACE", - # "db_database": "wts_default" - # } - -# Values to determine the labels that are used for the deployment, pod, etc. -# -- (string) Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". -release: "production" -# -- (string) Valid options are "true" or "false". If invalid option is set- the value will default to "false". -criticalService: "true" -# -- (string) Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. -partOf: "Authentication" -# -- (map) Will completely override the selectorLabels defined in the common chart's _label_setup.tpl -selectorLabels: -# -- (map) Will completely override the commonLabels defined in the common chart's _label_setup.tpl -commonLabels: - -# Values to configure datadog if ddEnabled is set to "true". -# -- (bool) If enabled, the Datadog Agent will automatically inject Datadog-specific metadata into your application logs. -datadogLogsInjection: true -# -- (bool) If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. -datadogProfilingEnabled: true -# -- (int) A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. -datadogTraceSampleRate: 1