diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index ec3e945..f31c8ac 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -24,7 +24,7 @@ ARG NODE_VERSION="lts/*" RUN su $USERNAME -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1" # Install terraform -ARG TERRAFORM_VERSION="1.9.8" +ARG TERRAFORM_VERSION="1.14.3" COPY .devcontainer/scripts/terraform.sh /tmp/ RUN bash /tmp/terraform.sh "${TERRAFORM_VERSION}" /usr/bin @@ -33,10 +33,10 @@ COPY .devcontainer/scripts/docker-client.sh /tmp/ RUN /tmp/docker-client.sh $USERNAME # Install Docker -ARG DOCKER_CE_VERSION="5:27.4.1-1~debian.12~bookworm" -ARG DOCKER_CE_CLI_VERSION="5:27.4.1-1~debian.12~bookworm" -ARG DOCKER_COMPOSE_PLUGIN_VERSION="2.32.1-1~debian.12~bookworm" -ARG DOCKER_CONTAINERD_VERSION="1.7.24-1" +ARG DOCKER_CE_VERSION="5:29.1.3-1~debian.12~bookworm" +ARG DOCKER_CE_CLI_VERSION="5:29.1.3-1~debian.12~bookworm" +ARG DOCKER_CONTAINERD_VERSION="2.2.1-1~debian.12~bookworm" +ARG DOCKER_COMPOSE_PLUGIN_VERSION="5.0.0-1~debian.12~bookworm" RUN apt-get update && apt-get install -y ca-certificates curl gnupg lsb-release --no-install-recommends \ && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \ && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" \ @@ -54,10 +54,10 @@ RUN if [ "${INTERACTIVE}" = "true" ]; then \ # Install Porter, updates should be propagated to the RP. ARG PORTER_HOME_V1=/home/$USERNAME/.porter/ -ARG PORTER_VERSION=v1.2.1 -ARG PORTER_TERRAFORM_MIXIN_VERSION=v1.0.5 -ARG PORTER_AZ_MIXIN_VERSION=v1.0.4 -ARG PORTER_AZURE_PLUGIN_VERSION=v1.2.3 +ARG PORTER_VERSION=v1.4.0 +ARG PORTER_TERRAFORM_MIXIN_VERSION=v1.0.8 +ARG PORTER_AZ_MIXIN_VERSION=v1.0.7 +ARG PORTER_AZURE_PLUGIN_VERSION=v1.2.4 COPY .devcontainer/scripts/porter-v1.sh /tmp/ RUN export PORTER_VERSION=${PORTER_VERSION} \ PORTER_TERRAFORM_MIXIN_VERSION=${PORTER_TERRAFORM_MIXIN_VERSION} \ @@ -66,37 +66,26 @@ RUN export PORTER_VERSION=${PORTER_VERSION} \ PORTER_HOME=${PORTER_HOME_V1} \ && /tmp/porter-v1.sh -ENV PATH ${PORTER_HOME_V1}:$PATH +ENV PATH=${PORTER_HOME_V1}:$PATH # Install azure-cli -ARG AZURE_CLI_VERSION=2.67.0-1~bookworm +ARG AZURE_CLI_VERSION=2.81.0-1~bookworm COPY .devcontainer/scripts/azure-cli.sh /tmp/ RUN export AZURE_CLI_VERSION=${AZURE_CLI_VERSION} \ && /tmp/azure-cli.sh -ARG YQ_VERSION="v4.44.6" +ARG YQ_VERSION="v4.49.2" RUN curl -L --fail -o /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \ && chmod +x /usr/local/bin/yq -# Install AzCopy -RUN curl -L --fail -o /tmp/azcopy.tar.gz "https://aka.ms/downloadazcopy-v10-linux" \ - && tar -xvzf /tmp/azcopy.tar.gz -C /tmp \ - && cp /tmp/azcopy_linux_amd64_*/azcopy /usr/local/bin/azcopy \ - && chmod +x /usr/local/bin/azcopy - ARG PAJV_VERSION="1.2.0" -RUN npm install -g pajv@${PAJV_VERSION} \ - && npm install -g yarn@1.22.22 +RUN npm install -g pajv@${PAJV_VERSION} # Install git - required for terraform's git modules RUN if [ "${INTERACTIVE}" = "false" ]; then \ apt-get update && apt-get install --no-install-recommends -y git \ && apt-get clean -y && rm -rf /var/lib/apt/lists/* ; fi -# Override pip version -ARG PIP_VERSION=24.2 # Override pip version -RUN pip3 --no-cache-dir install pip==${PIP_VERSION} && pip3 config set global.disable-pip-version-check true - USER $USERNAME # Save command line history @@ -109,9 +98,13 @@ RUN echo "export HISTFILE=$HOME/commandhistory/.bash_history" >> "$HOME/.bashrc" COPY ./.devcontainer/scripts/gh.sh /tmp/ RUN if [ "${INTERACTIVE}" = "true" ]; then /tmp/gh.sh; fi +# Build x86-64 docker images by default +ENV DOCKER_DEFAULT_PLATFORM=amd64 + # Install AzureTRE OSS ARG UPSTREAM_REPO ARG UPSTREAM_REPO_VERSION +ARG GITHUB_TOKEN ENV AZURETRE_HOME=/home/$USERNAME/AzureTRE COPY .devcontainer/scripts/install-azure-tre-oss.sh .devcontainer/devcontainer.json /tmp/ # hadolint ignore=DL3004 @@ -128,4 +121,4 @@ WORKDIR /"${AZURETRE_HOME}"/cli RUN make install-cli && echo -e "\n# Set up tre completion\nsource <(_TRE_COMPLETE=bash_source tre)" >> ~/.bashrc # Build x86-64 docker images by default -ENV DOCKER_DEFAULT_PLATFORM=amd64 +ENV DOCKER_DEFAULT_PLATFORM=amd64 \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 1635194..da74ce0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -18,7 +18,7 @@ "DOCKER_GROUP_ID": "${localEnv:DOCKER_GROUP_ID}", "INTERACTIVE": "true", "UPSTREAM_REPO": "OxBRCInformatics/AzureTRE", - "UPSTREAM_REPO_VERSION": "tvstre-prod-main-v0.25.12", + "UPSTREAM_REPO_VERSION": "tvstre-prod-main-v0.27.3", "GITHUB_TOKEN": "" } }, diff --git a/.github/actions/devcontainer_run_command/action.yml b/.github/actions/devcontainer_run_command/action.yml index ef17627..15aea18 100644 --- a/.github/actions/devcontainer_run_command/action.yml +++ b/.github/actions/devcontainer_run_command/action.yml @@ -9,8 +9,14 @@ inputs: DEVCONTAINER_TAG: description: "The container label to use when running the command." required: true - AZURE_CREDENTIALS: - description: "Credentials to access Azure." + AZURE_CLIENT_ID: + description: "The Azure Client ID for OIDC authentication" + required: true + AZURE_TENANT_ID: + description: "The Azure Tenant ID" + required: true + AZURE_SUBSCRIPTION_ID: + description: "The Azure Subscription ID" required: true AZURE_ENVIRONMENT: description: "Azure Cloud Environment" @@ -194,7 +200,9 @@ runs: uses: azure/login@v2 if: contains(inputs.COMMAND, 'make bootstrap') != true with: - creds: ${{ inputs.AZURE_CREDENTIALS }} + client-id: ${{ inputs.AZURE_CLIENT_ID }} + tenant-id: ${{ inputs.AZURE_TENANT_ID }} + subscription-id: ${{ inputs.AZURE_SUBSCRIPTION_ID }} environment: "${{ env.AZURE_ENVIRONMENT }}" - name: ACR Login @@ -242,6 +250,7 @@ runs: "type=bind,src=${{ github.workspace }},dst=/workspaces/tre" \ -v /var/run/docker.sock:/var/run/docker.sock \ -v "./command.sh:/workspaces/tre/command.sh" \ + -v "$HOME/.azure:/home/vscode/.azure" \ --workdir /workspaces/tre \ --user vscode \ -e TF_INPUT="0" \ @@ -255,10 +264,12 @@ runs: -e LOCATION="${{ inputs.LOCATION }}" \ -e TF_VAR_location="${{ inputs.LOCATION }}" \ -e RESOURCE_LOCATION="${{ inputs.LOCATION }}" \ - -e ARM_CLIENT_ID="${{ fromJSON(inputs.AZURE_CREDENTIALS).clientId }}" \ - -e ARM_CLIENT_SECRET="${{ fromJSON(inputs.AZURE_CREDENTIALS).clientSecret }}" \ - -e ARM_TENANT_ID="${{ fromJSON(inputs.AZURE_CREDENTIALS).tenantId }}" \ - -e ARM_SUBSCRIPTION_ID="${{ fromJSON(inputs.AZURE_CREDENTIALS).subscriptionId }}" \ + -e ARM_CLIENT_ID="${{ inputs.AZURE_CLIENT_ID }}" \ + -e ARM_TENANT_ID="${{ inputs.AZURE_TENANT_ID }}" \ + -e ARM_SUBSCRIPTION_ID="${{ inputs.AZURE_SUBSCRIPTION_ID }}" \ + -e ARM_USE_OIDC="true" \ + -e ACTIONS_ID_TOKEN_REQUEST_URL \ + -e ACTIONS_ID_TOKEN_REQUEST_TOKEN \ -e TF_VAR_terraform_state_container_name="${{ (inputs.TERRAFORM_STATE_CONTAINER_NAME != '' && inputs.TERRAFORM_STATE_CONTAINER_NAME) || 'tfstate' }}" \ -e TF_VAR_mgmt_storage_account_name="${{ inputs.MGMT_STORAGE_ACCOUNT_NAME }}" \ @@ -270,7 +281,7 @@ runs: -e TF_VAR_api_client_secret="${{ inputs.API_CLIENT_SECRET }}" \ -e TF_VAR_application_admin_client_id="${{ inputs.APPLICATION_ADMIN_CLIENT_ID }}" \ -e TF_VAR_application_admin_client_secret="${{ inputs.APPLICATION_ADMIN_CLIENT_SECRET }}" \ - -e TF_VAR_arm_subscription_id="${{ fromJSON(inputs.AZURE_CREDENTIALS).subscriptionId }}" \ + -e TF_VAR_arm_subscription_id="${{ inputs.AZURE_SUBSCRIPTION_ID }}" \ -e TF_VAR_enable_swagger="${{ (inputs.ENABLE_SWAGGER != '' && inputs.ENABLE_SWAGGER) || 'false' }}" \ -e SWAGGER_UI_CLIENT_ID="${{ inputs.SWAGGER_UI_CLIENT_ID }}" \ diff --git a/.github/workflows/build_validation_develop.yml b/.github/workflows/build_validation_develop.yml index 9a01b8c..1fc435a 100644 --- a/.github/workflows/build_validation_develop.yml +++ b/.github/workflows/build_validation_develop.yml @@ -27,7 +27,7 @@ jobs: persist-credentials: false - uses: hashicorp/setup-terraform@v3 with: - terraform_version: "1.9.8" + terraform_version: "1.14.3" - name: Terraform format check run: terraform fmt -check -recursive diff --git a/.github/workflows/clean_validation_envs.yml b/.github/workflows/clean_validation_envs.yml index 565869b..e4acf4e 100644 --- a/.github/workflows/clean_validation_envs.yml +++ b/.github/workflows/clean_validation_envs.yml @@ -1,26 +1,28 @@ --- name: Clean Validation Environments -on: # yamllint disable-line rule:truthy +on: # yamllint disable-line rule:truthy + schedule: + # Every 2 hours + - cron: "0 */1 * * *" workflow_dispatch: - inputs: - environment: - description: The environment to run this workflow in - type: environment - default: Dev - required: true +permissions: + contents: read jobs: clean: name: Clean runs-on: ubuntu-latest - if: | - github.ref != 'refs/heads/main' - environment: Dev + environment: CICD timeout-minutes: 30 + permissions: + id-token: write + contents: read + pull-requests: read # For checking PRs + actions: read # For checking workflow runs steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: # This is CRITICAL since we're making decisions based on branch existence fetch-depth: 0 @@ -29,7 +31,9 @@ jobs: - name: Azure Login uses: azure/login@v2 with: - creds: ${{ secrets.AZURE_CREDENTIALS }} + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} environment: ${{ (secrets.AZURE_ENVIRONMENT != '' && secrets.AZURE_ENVIRONMENT) || 'AzureCloud' }} - name: Run clean @@ -37,6 +41,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} MAIN_TRE_ID: ${{ secrets.TRE_ID }} GITHUB_REPOSITORY: ${{ github.repository}} - BRANCH_LAST_ACTIVITY_IN_HOURS_FOR_STOP: 2 + BRANCH_LAST_ACTIVITY_IN_HOURS_FOR_STOP: 4 BRANCH_LAST_ACTIVITY_IN_HOURS_FOR_DESTROY: 48 - run: .github/scripts/clean_ci_validation_envs.sh + run: devops/scripts/clean_ci_validation_envs.sh diff --git a/.github/workflows/deploy_tre.yml b/.github/workflows/deploy_tre.yml index 7540f29..ecb0f5c 100644 --- a/.github/workflows/deploy_tre.yml +++ b/.github/workflows/deploy_tre.yml @@ -1,12 +1,7 @@ --- name: Deploy Azure TRE # This workflow is the integration build run for pushes to the main branch -# It also runs on a schedule, serving as the nightly build - on: # yamllint disable-line rule:truthy - schedule: - # midnight every day https://crontab.guru/#0_0_*_*_* - - cron: "0 0 * * *" push: branches: [main] workflow_dispatch: @@ -26,6 +21,7 @@ jobs: if: github.ref == 'refs/heads/main' uses: ./.github/workflows/deploy_tre_reusable.yml permissions: + id-token: write checks: write contents: read pull-requests: write @@ -38,9 +34,11 @@ jobs: E2E_TESTS_NUMBER_PROCESSES: 1 DEVCONTAINER_TAG: "latest" secrets: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }} ACR_NAME: ${{ secrets.ACR_NAME }} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} API_CLIENT_ID: ${{ secrets.API_CLIENT_ID }} API_CLIENT_SECRET: ${{ secrets.API_CLIENT_SECRET }} APPLICATION_ADMIN_CLIENT_ID: ${{ secrets.APPLICATION_ADMIN_CLIENT_ID }} diff --git a/.github/workflows/deploy_tre_branch.yml b/.github/workflows/deploy_tre_branch.yml index f652ca6..c1b8603 100644 --- a/.github/workflows/deploy_tre_branch.yml +++ b/.github/workflows/deploy_tre_branch.yml @@ -59,6 +59,7 @@ jobs: needs: [prepare-not-main] uses: ./.github/workflows/deploy_tre_reusable.yml permissions: + id-token: write checks: write contents: read pull-requests: write @@ -70,9 +71,11 @@ jobs: E2E_TESTS_NUMBER_PROCESSES: ${{ fromJSON(github.event.inputs.e2eProcesses) }} DEVCONTAINER_TAG: ${{ needs.prepare-not-main.outputs.refid }} secrets: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }} ACR_NAME: ${{ format('tre{0}', needs.prepare-not-main.outputs.refid) }} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} API_CLIENT_ID: ${{ secrets.API_CLIENT_ID }} API_CLIENT_SECRET: ${{ secrets.API_CLIENT_SECRET }} APPLICATION_ADMIN_CLIENT_ID: ${{ secrets.APPLICATION_ADMIN_CLIENT_ID }} diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml index 2327479..4cca2c6 100644 --- a/.github/workflows/deploy_tre_reusable.yml +++ b/.github/workflows/deploy_tre_reusable.yml @@ -95,7 +95,13 @@ on: # yamllint disable-line rule:truthy CI_CACHE_ACR_NAME: description: "" required: false - AZURE_CREDENTIALS: + AZURE_CLIENT_ID: + description: "" + required: true + AZURE_TENANT_ID: + description: "" + required: true + AZURE_SUBSCRIPTION_ID: description: "" required: true ENCRYPTION_KV_NAME: @@ -123,6 +129,7 @@ jobs: name: Deploy Management runs-on: ubuntu-latest permissions: + id-token: write checks: write contents: read environment: ${{ inputs.environmentName }} @@ -185,8 +192,14 @@ jobs: if [ "${{ secrets.TRE_ID }}" == '' ]; then echo "Missing secret: TRE_ID" && exit 1 fi - if [ "${{ secrets.AZURE_CREDENTIALS }}" == '' ]; then - echo "Missing secret: AZURE_CREDENTIALS" && exit 1 + if [ "${{ secrets.AZURE_CLIENT_ID }}" == '' ]; then + echo "Missing secret: AZURE_CLIENT_ID" && exit 1 + fi + if [ "${{ secrets.AZURE_TENANT_ID }}" == '' ]; then + echo "Missing secret: AZURE_TENANT_ID" && exit 1 + fi + if [ "${{ secrets.AZURE_SUBSCRIPTION_ID }}" == '' ]; then + echo "Missing secret: AZURE_SUBSCRIPTION_ID" && exit 1 fi if [ "${{ inputs.DEVCONTAINER_TAG }}" == '' ]; then echo "Missing input: DEVCONTAINER_TAG" && exit 1 @@ -218,7 +231,9 @@ jobs: - name: Azure Login uses: azure/login@v2 with: - creds: ${{ secrets.AZURE_CREDENTIALS }} + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} environment: ${{ (vars.AZURE_ENVIRONMENT != '' && vars.AZURE_ENVIRONMENT) || 'AzureCloud' }} - name: ACR Login @@ -258,7 +273,9 @@ jobs: COMMAND: "make bootstrap mgmt-deploy" DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} TRE_ID: ${{ secrets.TRE_ID }} LOCATION: ${{ vars.LOCATION }} @@ -320,7 +337,9 @@ jobs: COMMAND: "make ${{ matrix.target }}" DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} ACR_NAME: ${{ secrets.ACR_NAME }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} @@ -349,7 +368,9 @@ jobs: COMMAND: "make tre-start" DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} TRE_ID: ${{ secrets.TRE_ID }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} @@ -378,7 +399,9 @@ jobs: COMMAND: "TF_VAR_ci_git_ref=${{ inputs.ciGitRef }} TF_LOG=${{ vars.TF_LOG }} make deploy-core" DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} API_CLIENT_ID: "${{ secrets.API_CLIENT_ID }}" AAD_TENANT_ID: "${{ secrets.AAD_TENANT_ID }}" @@ -419,7 +442,9 @@ jobs: COMMAND: "make api-healthcheck" DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} TRE_ID: ${{ secrets.TRE_ID }} LOCATION: ${{ vars.LOCATION }} @@ -492,7 +517,9 @@ jobs: # Exit with the last status code (exit \$ec) DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} ACR_NAME: ${{ secrets.ACR_NAME }} @@ -548,7 +575,9 @@ jobs: # Exit with the last status code (exit \$ec) DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} ACR_NAME: ${{ secrets.ACR_NAME }} @@ -604,7 +633,9 @@ jobs: (exit \$ec) DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} ACR_NAME: ${{ secrets.ACR_NAME }} API_CLIENT_ID: "${{ secrets.API_CLIENT_ID }}" @@ -677,7 +708,9 @@ jobs: (exit \$ec) DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} ACR_NAME: ${{ secrets.ACR_NAME }} API_CLIENT_ID: "${{ secrets.API_CLIENT_ID }}" @@ -738,7 +771,9 @@ jobs: (exit \$ec) DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} ACR_NAME: ${{ secrets.ACR_NAME }} API_CLIENT_ID: "${{ secrets.API_CLIENT_ID }}" @@ -776,7 +811,9 @@ jobs: COMMAND: "make deploy-shared-service DIR=\\${AZURETRE_HOME}/templates/shared_services/firewall/ BUNDLE_TYPE=shared_service" DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} API_CLIENT_ID: "${{ secrets.API_CLIENT_ID }}" AAD_TENANT_ID: "${{ secrets.AAD_TENANT_ID }}" @@ -794,7 +831,9 @@ jobs: COMMAND: "make db-migrate" DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} API_CLIENT_ID: "${{ secrets.API_CLIENT_ID }}" AAD_TENANT_ID: "${{ secrets.AAD_TENANT_ID }}" @@ -832,7 +871,9 @@ jobs: COMMAND: "make build-and-deploy-ui" DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} API_CLIENT_ID: "${{ secrets.API_CLIENT_ID }}" AAD_TENANT_ID: "${{ secrets.AAD_TENANT_ID }}" diff --git a/.github/workflows/lets_encrypt.yml b/.github/workflows/lets_encrypt.yml index f1c8bf5..ce0088a 100644 --- a/.github/workflows/lets_encrypt.yml +++ b/.github/workflows/lets_encrypt.yml @@ -1,78 +1,81 @@ --- name: Renew Lets Encrypt Certificates -on: # yamllint disable-line rule:truthy - schedule: - # 3am each month https://crontab.guru/#0_3_1_*_* - - cron: "0 3 1 * *" +on: # yamllint disable-line rule:truthy + # schedule: + # # 3am each month https://crontab.guru/#0_3_1_*_* + # - cron: "0 3 1 * *" workflow_dispatch: + inputs: + environment: + description: The environment to run this workflow in + type: environment + default: CICD + required: true # This will prevent multiple runs of this entire workflow. # We should NOT cancel in progress runs as that can destabilize the environment. concurrency: letsencrypt +permissions: + id-token: write + contents: read + env: USE_ENV_VARS_NOT_FILES: true - TF_INPUT: 0 # interactive is off - TF_IN_AUTOMATION: 1 # Run in headless mode + TF_INPUT: 0 # interactive is off + TF_IN_AUTOMATION: 1 # Run in headless mode jobs: renew_letsencrypt_certs: name: Renew Lets Encrypt Certificates runs-on: ubuntu-latest - environment: CICD + environment: ${{ github.event.inputs.environment || 'CICD' }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: persist-credentials: false + - name: Install specific Azure CLI version + run: | + AZ_DIST=$(lsb_release -cs) + AZ_VER=2.81.0 + + apt-cache policy azure-cli + sudo apt-get install azure-cli="${AZ_VER}-1~${AZ_DIST}" --allow-downgrades + az version + - name: Install Terraform - uses: hashicorp/setup-terraform@v2 + uses: hashicorp/setup-terraform@v3 with: - terraform_version: 1.4.5 + terraform_version: 1.14.3 terraform_wrapper: false - # - name: Renew Certificates - # shell: bash - # env: - # ARM_CLIENT_ID: ${{ fromJSON(secrets.AZURE_CREDENTIALS).clientId }} - # ARM_CLIENT_SECRET: ${{ fromJSON(secrets.AZURE_CREDENTIALS).clientSecret }} - # ARM_SUBSCRIPTION_ID: ${{ fromJSON(secrets.AZURE_CREDENTIALS).tenantId }} - # ARM_TENANT_ID: ${{ fromJSON(secrets.AZURE_CREDENTIALS).subscriptionId }} - # TRE_ID: ${{ secrets.TRE_ID }} - # TERRAFORM_STATE_CONTAINER_NAME: - # ${{ secrets.TERRAFORM_STATE_CONTAINER_NAME && secrets.TERRAFORM_STATE_CONTAINER_NAME || 'tfstate' }} - # MGMT_RESOURCE_GROUP_NAME: ${{ secrets.MGMT_RESOURCE_GROUP_NAME }} - # MGMT_STORAGE_ACCOUNT_NAME: ${{ secrets.MGMT_STORAGE_ACCOUNT_NAME }} - # run: | - # sudo apt-get install -y python3 python3-venv libaugeas0 \ - # && python3 -m venv /opt/certbot/ \ - # && /opt/certbot/bin/pip install --upgrade pip \ - # && /opt/certbot/bin/pip install certbot - # make letsencrypt + - name: Azure Login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + environment: ${{ (vars.AZURE_ENVIRONMENT != '' && vars.AZURE_ENVIRONMENT) || 'AzureCloud' }} - name: Renew Certificates - uses: ./.github/actions/devcontainer_run_command - with: - COMMAND: "sudo apt-get install -y python3 python3-venv libaugeas0 \ + shell: bash + env: + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + AZURE_ENVIRONMENT: ${{ (vars.AZURE_ENVIRONMENT != '' && vars.AZURE_ENVIRONMENT) || 'AzureCloud' }} + TRE_ID: ${{ secrets.TRE_ID }} + TF_VAR_terraform_state_container_name: ${{ secrets.TERRAFORM_STATE_CONTAINER_NAME || 'tfstate' }} + TF_VAR_mgmt_resource_group_name: ${{ secrets.MGMT_RESOURCE_GROUP_NAME }} + TF_VAR_mgmt_storage_account_name: ${{ secrets.MGMT_STORAGE_ACCOUNT_NAME }} + CUSTOM_DOMAIN: ${{ secrets.CUSTOM_DOMAIN }} + PRIVATE_AGENT_SUBNET_ID: ${{ secrets.PRIVATE_AGENT_SUBNET_ID }} + run: | + sudo apt-get install -y python3 python3-venv libaugeas0 \ && python3 -m venv /opt/certbot/ \ && /opt/certbot/bin/pip install --upgrade pip \ && /opt/certbot/bin/pip install certbot - make letsencrypt" - DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} - CI_CACHE_ACR_NAME: ${{ secrets.ACR_NAME}} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} - AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} - ACR_NAME: ${{ secrets.ACR_NAME }} - API_CLIENT_ID: "${{ secrets.API_CLIENT_ID }}" - AAD_TENANT_ID: "${{ secrets.AAD_TENANT_ID }}" - TEST_APP_ID: "${{ secrets.TEST_APP_ID }}" - TEST_ACCOUNT_CLIENT_ID: "${{ secrets.TEST_ACCOUNT_CLIENT_ID }}" - TEST_ACCOUNT_CLIENT_SECRET: "${{ secrets.TEST_ACCOUNT_CLIENT_SECRET }}" - TRE_ID: ${{ secrets.TRE_ID }} - LOCATION: ${{ vars.LOCATION }} - BUNDLE_TYPE: ${{ matrix.BUNDLE_TYPE }} - TERRAFORM_STATE_CONTAINER_NAME: ${{ vars.TERRAFORM_STATE_CONTAINER_NAME }} - MGMT_RESOURCE_GROUP_NAME: ${{ secrets.MGMT_RESOURCE_GROUP_NAME }} - MGMT_STORAGE_ACCOUNT_NAME: ${{ secrets.MGMT_STORAGE_ACCOUNT_NAME }} + make letsencrypt diff --git a/.github/workflows/register_tre_bundle.yml b/.github/workflows/register_tre_bundle.yml index 16b86df..a6e478d 100644 --- a/.github/workflows/register_tre_bundle.yml +++ b/.github/workflows/register_tre_bundle.yml @@ -83,8 +83,8 @@ jobs: if [ "${{ secrets.TRE_ID }}" == '' ]; then echo "Missing secret: TRE_ID" && exit 1 fi - if [ "${{ secrets.AZURE_CREDENTIALS }}" == '' ]; then - echo "Missing secret: AZURE_CREDENTIALS" && exit 1 + if [ "${{ secrets.AZURE_CLIENT_ID }}" == '' ]; then + echo "Missing secret: AZURE_CLIENT_ID" && exit 1 fi # if bundle_name is not set, exit with error @@ -127,7 +127,9 @@ jobs: - name: Azure Login uses: azure/login@v2 with: - creds: ${{ secrets.AZURE_CREDENTIALS }} + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} environment: ${{ (vars.AZURE_ENVIRONMENT != '' && vars.AZURE_ENVIRONMENT) || 'AzureCloud' }} - name: ACR Login @@ -166,7 +168,9 @@ jobs: COMMAND: "make bootstrap mgmt-deploy" DEVCONTAINER_TAG: latest CI_CACHE_ACR_NAME: ${{ secrets.ACR_NAME}} - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} TRE_ID: ${{ secrets.TRE_ID }} LOCATION: ${{ vars.LOCATION }} @@ -222,6 +226,7 @@ jobs: checks: write contents: read pull-requests: write + id-token: write steps: - name: Checkout repository uses: actions/checkout@v4 @@ -231,7 +236,9 @@ jobs: with: COMMAND: "make bundle BUNDLE=${{ inputs.bundle_name }} BUNDLE_TYPE=${{ inputs.bundle_type }} WORKSPACE_SERVICE=${{ inputs.workspace_service_name }}" DEVCONTAINER_TAG: 'latest' - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} CI_CACHE_ACR_NAME: ${{ secrets.ACR_NAME}} ACR_NAME: ${{ secrets.ACR_NAME }} diff --git a/README.md b/README.md index 85f2bab..22bba31 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ To work with devcontainers you will need: | v0.20.0 | Increase size of Nexus VM, Add option to configure auto shutdown for Linux VM | 18-03-2025 | | *(not recorded)* | *placeholder — versions between v0.20.0 and v0.25.12 not documented* | — | | v0.25.12 | Upgraded ahead of schedule to pick up `az login -u` → `--client-id` change required for v0.27.0, plus other early v0.27.0 changes including Nexus PyPI/CRAN fix | 01-04-2025 | -| v0.27.0 | v0.26.0 skipped — went straight from v0.25.12 to v0.27.0 to avoid repeating the cherry-pick exercise from last time. Includes: GitHub Actions federated credential migration, Nexus fix reapplied post-merge | *(in progress)* | +| v0.27.1 | v0.26.0 skipped — went straight from v0.25.12 to v0.27.0 to avoid repeating the cherry-pick exercise from last time. Includes: GitHub Actions federated credential migration, Nexus fix reapplied post-merge. Increased to v0.27.1 due to required fix on core repo | *(in progress)* | ## Contributing diff --git a/config.sample.yaml b/config.sample.yaml index 1d27f4a..6e528e1 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -102,7 +102,7 @@ ui_config: # Footer text shown in the bottom left hand corner of the TRE portal ui_footer_text: "Azure Trusted Research Environment" -developer_settings: +# developer_settings: # Locks will not be added to stateful resources so they can be easily removed # stateful_resources_locked: false # TRE Core Key Vault purge protection will be disabled so it can be reused upon deletion diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm-ouh2/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm-ouh2/porter.yaml index a2cf93f..7dd956b 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm-ouh2/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm-ouh2/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-linuxvm-ouh2 -version: 1.2.3 +version: 1.2.4 description: "An Azure TRE User Resource Template for Guacamole (Linux)" dockerfile: Dockerfile.tmpl registry: azuretre @@ -109,7 +109,7 @@ parameters: default: "" type: string description: "Time of day to shutdown the VM (HHmm format)" - pattern: "^([01]?[0-9]|2[0-3])[0-5][0-9]?$" + pattern: "^$|^([01]?[0-9]|2[0-3])[0-5][0-9]?$" - name: shutdown_timezone type: string description: "Timezone for the shutdown schedule" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm-ouh2/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm-ouh2/porter.yaml index 1463dc7..47017da 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm-ouh2/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm-ouh2/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-windowsvm-ouh2 -version: 1.2.2 +version: 1.2.3 description: "An Azure TRE User Resource Template for Guacamole (Windows)" dockerfile: Dockerfile.tmpl registry: azuretre @@ -154,7 +154,7 @@ parameters: default: "" type: string description: "Time of day to shutdown the VM (HHmm format)" - pattern: "^([01]?[0-9]|2[0-3])[0-5][0-9]?$" + pattern: "^$|^([01]?[0-9]|2[0-3])[0-5][0-9]?$" - name: shutdown_timezone type: string description: "Timezone for the shutdown schedule"