Upgrade TRE - tvstre-prod-main-v0.27.X - Continued

.devcontainer/Dockerfile

@@ -24,7 +24,7 @@ ARG NODE_VERSION="lts/*"
 RUN su $USERNAME -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"
 
 # Install terraform
-ARG TERRAFORM_VERSION="1.9.8"
+ARG TERRAFORM_VERSION="1.14.3"
 COPY .devcontainer/scripts/terraform.sh /tmp/
 RUN bash /tmp/terraform.sh "${TERRAFORM_VERSION}" /usr/bin
 
@@ -33,10 +33,10 @@ COPY .devcontainer/scripts/docker-client.sh /tmp/
 RUN /tmp/docker-client.sh $USERNAME
 
 # Install Docker
-ARG DOCKER_CE_VERSION="5:27.4.1-1~debian.12~bookworm"
-ARG DOCKER_CE_CLI_VERSION="5:27.4.1-1~debian.12~bookworm"
-ARG DOCKER_COMPOSE_PLUGIN_VERSION="2.32.1-1~debian.12~bookworm"
-ARG DOCKER_CONTAINERD_VERSION="1.7.24-1"
+ARG DOCKER_CE_VERSION="5:29.1.3-1~debian.12~bookworm"
+ARG DOCKER_CE_CLI_VERSION="5:29.1.3-1~debian.12~bookworm"
+ARG DOCKER_CONTAINERD_VERSION="2.2.1-1~debian.12~bookworm"
+ARG DOCKER_COMPOSE_PLUGIN_VERSION="5.0.0-1~debian.12~bookworm"
 RUN apt-get update && apt-get install -y ca-certificates curl gnupg lsb-release --no-install-recommends \
     && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \
     && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" \
@@ -54,10 +54,10 @@ RUN if [ "${INTERACTIVE}" = "true" ]; then \
 
 # Install Porter, updates should be propagated to the RP.
 ARG PORTER_HOME_V1=/home/$USERNAME/.porter/
-ARG PORTER_VERSION=v1.2.1
-ARG PORTER_TERRAFORM_MIXIN_VERSION=v1.0.5
-ARG PORTER_AZ_MIXIN_VERSION=v1.0.4
-ARG PORTER_AZURE_PLUGIN_VERSION=v1.2.3
+ARG PORTER_VERSION=v1.4.0
+ARG PORTER_TERRAFORM_MIXIN_VERSION=v1.0.8
+ARG PORTER_AZ_MIXIN_VERSION=v1.0.7
+ARG PORTER_AZURE_PLUGIN_VERSION=v1.2.4
 COPY .devcontainer/scripts/porter-v1.sh /tmp/
 RUN export PORTER_VERSION=${PORTER_VERSION} \
     PORTER_TERRAFORM_MIXIN_VERSION=${PORTER_TERRAFORM_MIXIN_VERSION} \
@@ -66,37 +66,26 @@ RUN export PORTER_VERSION=${PORTER_VERSION} \
     PORTER_HOME=${PORTER_HOME_V1} \
     && /tmp/porter-v1.sh
 
-ENV PATH ${PORTER_HOME_V1}:$PATH
+ENV PATH=${PORTER_HOME_V1}:$PATH
 
 # Install azure-cli
-ARG AZURE_CLI_VERSION=2.67.0-1~bookworm
+ARG AZURE_CLI_VERSION=2.81.0-1~bookworm
 COPY .devcontainer/scripts/azure-cli.sh /tmp/
 RUN export AZURE_CLI_VERSION=${AZURE_CLI_VERSION} \
     && /tmp/azure-cli.sh
 
-ARG YQ_VERSION="v4.44.6"
+ARG YQ_VERSION="v4.49.2"
 RUN curl -L --fail -o /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \
     && chmod +x /usr/local/bin/yq
 
-# Install AzCopy
-RUN curl -L --fail -o /tmp/azcopy.tar.gz "https://aka.ms/downloadazcopy-v10-linux" \
-  && tar -xvzf /tmp/azcopy.tar.gz -C /tmp \
-  && cp /tmp/azcopy_linux_amd64_*/azcopy /usr/local/bin/azcopy \
-  && chmod +x /usr/local/bin/azcopy
-
 ARG PAJV_VERSION="1.2.0"
-RUN npm install -g pajv@${PAJV_VERSION} \
-    && npm install -g yarn@1.22.22
+RUN npm install -g pajv@${PAJV_VERSION}
 
 # Install git - required for terraform's git modules
 RUN if [ "${INTERACTIVE}" = "false" ]; then \
 apt-get update && apt-get install --no-install-recommends -y git \
 && apt-get clean -y && rm -rf /var/lib/apt/lists/* ; fi
 
-# Override pip version
-ARG PIP_VERSION=24.2  # Override pip version
-RUN pip3 --no-cache-dir install pip==${PIP_VERSION} && pip3 config set global.disable-pip-version-check true
-
 USER $USERNAME
 
 # Save command line history
@@ -109,9 +98,13 @@ RUN echo "export HISTFILE=$HOME/commandhistory/.bash_history" >> "$HOME/.bashrc"
 COPY ./.devcontainer/scripts/gh.sh /tmp/
 RUN if [ "${INTERACTIVE}" = "true" ]; then /tmp/gh.sh; fi
 
+# Build x86-64 docker images by default
+ENV DOCKER_DEFAULT_PLATFORM=amd64
+
 # Install AzureTRE OSS
 ARG UPSTREAM_REPO
 ARG UPSTREAM_REPO_VERSION
+ARG GITHUB_TOKEN
 ENV AZURETRE_HOME=/home/$USERNAME/AzureTRE
 COPY .devcontainer/scripts/install-azure-tre-oss.sh .devcontainer/devcontainer.json /tmp/
 # hadolint ignore=DL3004
@@ -128,4 +121,4 @@ WORKDIR /"${AZURETRE_HOME}"/cli
 RUN make install-cli && echo -e "\n# Set up tre completion\nsource <(_TRE_COMPLETE=bash_source tre)" >> ~/.bashrc
 
 # Build x86-64 docker images by default
-ENV DOCKER_DEFAULT_PLATFORM=amd64
+ENV DOCKER_DEFAULT_PLATFORM=amd64

.devcontainer/devcontainer.json

@@ -18,7 +18,7 @@
       "DOCKER_GROUP_ID": "${localEnv:DOCKER_GROUP_ID}",
       "INTERACTIVE": "true",
       "UPSTREAM_REPO": "OxBRCInformatics/AzureTRE",
-      "UPSTREAM_REPO_VERSION": "tvstre-prod-main-v0.25.12",
+      "UPSTREAM_REPO_VERSION": "tvstre-prod-main-v0.27.3",
       "GITHUB_TOKEN": ""
     }
   },

.github/actions/devcontainer_run_command/action.yml

@@ -9,8 +9,14 @@ inputs:
   DEVCONTAINER_TAG:
     description: "The container label to use when running the command."
     required: true
-  AZURE_CREDENTIALS:
-    description: "Credentials to access Azure."
+  AZURE_CLIENT_ID:
+    description: "The Azure Client ID for OIDC authentication"
+    required: true
+  AZURE_TENANT_ID:
+    description: "The Azure Tenant ID"
+    required: true
+  AZURE_SUBSCRIPTION_ID:
+    description: "The Azure Subscription ID"
     required: true
   AZURE_ENVIRONMENT:
     description: "Azure Cloud Environment"
@@ -194,7 +200,9 @@ runs:
       uses: azure/login@v2
       if: contains(inputs.COMMAND, 'make bootstrap') != true
       with:
-        creds: ${{ inputs.AZURE_CREDENTIALS }}
+        client-id: ${{ inputs.AZURE_CLIENT_ID }}
+        tenant-id: ${{ inputs.AZURE_TENANT_ID }}
+        subscription-id: ${{ inputs.AZURE_SUBSCRIPTION_ID }}
         environment: "${{ env.AZURE_ENVIRONMENT }}"
 
     - name: ACR Login
@@ -242,6 +250,7 @@ runs:
           "type=bind,src=${{ github.workspace }},dst=/workspaces/tre" \
           -v /var/run/docker.sock:/var/run/docker.sock \
           -v "./command.sh:/workspaces/tre/command.sh" \
+          -v "$HOME/.azure:/home/vscode/.azure" \
           --workdir /workspaces/tre \
           --user vscode \
           -e TF_INPUT="0" \
@@ -255,10 +264,12 @@ runs:
           -e LOCATION="${{ inputs.LOCATION }}" \
           -e TF_VAR_location="${{ inputs.LOCATION }}" \
           -e RESOURCE_LOCATION="${{ inputs.LOCATION }}" \
-          -e ARM_CLIENT_ID="${{ fromJSON(inputs.AZURE_CREDENTIALS).clientId }}" \
-          -e ARM_CLIENT_SECRET="${{ fromJSON(inputs.AZURE_CREDENTIALS).clientSecret }}" \
-          -e ARM_TENANT_ID="${{ fromJSON(inputs.AZURE_CREDENTIALS).tenantId }}" \
-          -e ARM_SUBSCRIPTION_ID="${{ fromJSON(inputs.AZURE_CREDENTIALS).subscriptionId }}" \
+          -e ARM_CLIENT_ID="${{ inputs.AZURE_CLIENT_ID }}" \
+          -e ARM_TENANT_ID="${{ inputs.AZURE_TENANT_ID }}" \
+          -e ARM_SUBSCRIPTION_ID="${{ inputs.AZURE_SUBSCRIPTION_ID }}" \
+          -e ARM_USE_OIDC="true" \
+          -e ACTIONS_ID_TOKEN_REQUEST_URL \
+          -e ACTIONS_ID_TOKEN_REQUEST_TOKEN \
           -e TF_VAR_terraform_state_container_name="${{ (inputs.TERRAFORM_STATE_CONTAINER_NAME != ''
             && inputs.TERRAFORM_STATE_CONTAINER_NAME) || 'tfstate' }}" \
           -e TF_VAR_mgmt_storage_account_name="${{ inputs.MGMT_STORAGE_ACCOUNT_NAME }}" \
@@ -270,7 +281,7 @@ runs:
           -e TF_VAR_api_client_secret="${{ inputs.API_CLIENT_SECRET }}" \
           -e TF_VAR_application_admin_client_id="${{ inputs.APPLICATION_ADMIN_CLIENT_ID }}" \
           -e TF_VAR_application_admin_client_secret="${{ inputs.APPLICATION_ADMIN_CLIENT_SECRET }}" \
-          -e TF_VAR_arm_subscription_id="${{ fromJSON(inputs.AZURE_CREDENTIALS).subscriptionId }}" \
+          -e TF_VAR_arm_subscription_id="${{ inputs.AZURE_SUBSCRIPTION_ID }}" \
           -e TF_VAR_enable_swagger="${{ (inputs.ENABLE_SWAGGER != ''
             && inputs.ENABLE_SWAGGER) || 'false' }}" \
           -e SWAGGER_UI_CLIENT_ID="${{ inputs.SWAGGER_UI_CLIENT_ID }}" \

.github/workflows/build_validation_develop.yml

@@ -27,7 +27,7 @@ jobs:
           persist-credentials: false
       - uses: hashicorp/setup-terraform@v3
         with:
-          terraform_version: "1.9.8"
+          terraform_version: "1.14.3"
       - name: Terraform format check
         run: terraform fmt -check -recursive
 

.github/workflows/clean_validation_envs.yml

@@ -1,26 +1,28 @@
 ---
 name: Clean Validation Environments
 
-on:  # yamllint disable-line rule:truthy
+on: # yamllint disable-line rule:truthy
+  schedule:
+    # Every 2 hours
+    - cron: "0 */1 * * *"
   workflow_dispatch:
-    inputs:
-      environment:
-        description: The environment to run this workflow in
-        type: environment
-        default: Dev
-        required: true
 
+permissions:
+  contents: read
 
 jobs:
   clean:
     name: Clean
     runs-on: ubuntu-latest
-    if: |
-      github.ref != 'refs/heads/main'
-    environment: Dev
+    environment: CICD
     timeout-minutes: 30
+    permissions:
+      id-token: write
+      contents: read
+      pull-requests: read # For checking PRs
+      actions: read # For checking workflow runs
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           # This is CRITICAL since we're making decisions based on branch existence
           fetch-depth: 0
@@ -29,14 +31,16 @@ jobs:
       - name: Azure Login
         uses: azure/login@v2
         with:
-          creds: ${{ secrets.AZURE_CREDENTIALS }}
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
           environment: ${{ (secrets.AZURE_ENVIRONMENT != '' && secrets.AZURE_ENVIRONMENT) || 'AzureCloud' }}
 
       - name: Run clean
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           MAIN_TRE_ID: ${{ secrets.TRE_ID }}
           GITHUB_REPOSITORY: ${{ github.repository}}
-          BRANCH_LAST_ACTIVITY_IN_HOURS_FOR_STOP: 2
+          BRANCH_LAST_ACTIVITY_IN_HOURS_FOR_STOP: 4
           BRANCH_LAST_ACTIVITY_IN_HOURS_FOR_DESTROY: 48
-        run: .github/scripts/clean_ci_validation_envs.sh
+        run: devops/scripts/clean_ci_validation_envs.sh

.github/workflows/deploy_tre.yml

@@ -1,12 +1,7 @@
 ---
 name: Deploy Azure TRE
 # This workflow is the integration build run for pushes to the main branch
-# It also runs on a schedule, serving as the nightly build
-
 on:  # yamllint disable-line rule:truthy
-  schedule:
-    # midnight every day https://crontab.guru/#0_0_*_*_*
-    - cron: "0 0 * * *"
   push:
     branches: [main]
   workflow_dispatch:
@@ -26,6 +21,7 @@ jobs:
     if: github.ref == 'refs/heads/main'
     uses: ./.github/workflows/deploy_tre_reusable.yml
     permissions:
+      id-token: write
       checks: write
       contents: read
       pull-requests: write
@@ -38,9 +34,11 @@ jobs:
       E2E_TESTS_NUMBER_PROCESSES: 1
       DEVCONTAINER_TAG: "latest"
     secrets:
+      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
       AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }}
       ACR_NAME: ${{ secrets.ACR_NAME }}
-      AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
       API_CLIENT_ID: ${{ secrets.API_CLIENT_ID }}
       API_CLIENT_SECRET: ${{ secrets.API_CLIENT_SECRET }}
       APPLICATION_ADMIN_CLIENT_ID: ${{ secrets.APPLICATION_ADMIN_CLIENT_ID }}

.github/workflows/deploy_tre_branch.yml

@@ -59,6 +59,7 @@ jobs:
     needs: [prepare-not-main]
     uses: ./.github/workflows/deploy_tre_reusable.yml
     permissions:
+      id-token: write
       checks: write
       contents: read
       pull-requests: write
@@ -70,9 +71,11 @@ jobs:
       E2E_TESTS_NUMBER_PROCESSES: ${{ fromJSON(github.event.inputs.e2eProcesses) }}
       DEVCONTAINER_TAG: ${{ needs.prepare-not-main.outputs.refid }}
     secrets:
+      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
       AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }}
       ACR_NAME: ${{ format('tre{0}', needs.prepare-not-main.outputs.refid) }}
-      AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
       API_CLIENT_ID: ${{ secrets.API_CLIENT_ID }}
       API_CLIENT_SECRET: ${{ secrets.API_CLIENT_SECRET }}
       APPLICATION_ADMIN_CLIENT_ID: ${{ secrets.APPLICATION_ADMIN_CLIENT_ID }}