diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7cbd1219a9..97d09a0558 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 0a80b038bcc0d76b2f26c7f22062942de75779e6 + 1574a0ce35761b7ce5e783074cc2f9567d278396 diff --git a/eng/common/Get-GitHubAppToken.ps1 b/eng/common/Get-GitHubAppToken.ps1 new file mode 100644 index 0000000000..ec005e487c --- /dev/null +++ b/eng/common/Get-GitHubAppToken.ps1 @@ -0,0 +1,208 @@ +# Mints a short-lived GitHub App installation access token by signing a JWT +# with an RSA private key (RS256). The signed JWT is exchanged with the GitHub +# API for a token scoped to a single installation. +# +# Requirements: +# - A GitHub App ID and PEM private key stored as Azure Key Vault secrets. +# - The federated Azure service connection running this script must have +# `Get` access to those two secrets. +# - The App must be installed on the target organization/account +# (`InstallationOwner`) with the permissions/repositories it needs. +# +# Installation tokens (ghs_*) are exempt from the enterprise classic-PAT +# lifetime policy, which is why this replaces the long-lived PAT. + +[CmdletBinding()] +param( + # Name of the Key Vault holding the GitHub App credentials. + [Parameter(Mandatory = $true)] + [string] $KeyVaultName, + + # Secret Manager projection containing the GitHub App ID. + [Parameter(Mandatory = $true)] + [string] $AppIdSecretName, + + # Secret Manager projection containing the PEM private key. + [Parameter(Mandatory = $true)] + [string] $AppPrivateKeySecretName, + + # Login of the organization or user account whose installation we should + # mint the token for (e.g. `dotnet`, `microsoft`). + [Parameter(Mandatory = $true)] + [string] $InstallationOwner, + + # Optional Azure DevOps pipeline variable name to set with the installation + # token (marked as a secret). When not specified, the token is written to + # stdout instead. + [Parameter(Mandatory = $false)] + [string] $OutputVariableName +) +$ErrorActionPreference = 'Stop' +$PSNativeCommandUseErrorActionPreference = $true + +. $PSScriptRoot\pipeline-logging-functions.ps1 + +if ($KeyVaultName -notmatch '^[A-Za-z][A-Za-z0-9-]{1,22}[A-Za-z0-9]$' -or $KeyVaultName.Contains('--')) { + Write-PipelineTelemetryError -Category 'Build' -Message "KeyVaultName '$KeyVaultName' is not a valid Azure Key Vault name." + exit 1 +} + +function ConvertTo-Base64Url([byte[]] $bytes) { + return [Convert]::ToBase64String($bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_') +} + +$previousNativeCommandErrorPreference = $PSNativeCommandUseErrorActionPreference +try { + # Azure CLI can emit non-fatal Python warnings to stderr. + $PSNativeCommandUseErrorActionPreference = $false + $keyVaultAccessToken = az account get-access-token ` + --resource https://vault.azure.net ` + --query accessToken ` + --output tsv ` + --only-show-errors + $tokenExitCode = $LASTEXITCODE +} +catch { + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to acquire an Azure Key Vault access token: $_" + exit 1 +} +finally { + $PSNativeCommandUseErrorActionPreference = $previousNativeCommandErrorPreference +} +if ($tokenExitCode -ne 0 -or [string]::IsNullOrWhiteSpace($keyVaultAccessToken)) { + Write-PipelineTelemetryError -Category 'Build' -Message "'az account get-access-token' exited with code $tokenExitCode while acquiring an Azure Key Vault access token." + exit 1 +} + +function Get-KeyVaultSecret([string] $SecretName) { + # Use the data-plane REST API because `az keyvault secret show` can fail + # with Errno 22 on hosted Windows agents when reading these projections. + $escapedSecretName = [Uri]::EscapeDataString($SecretName) + $secretUri = "https://$KeyVaultName.vault.azure.net/secrets/$escapedSecretName`?api-version=7.4" + try { + $response = Invoke-RestMethod ` + -Uri $secretUri ` + -Headers @{ Authorization = "Bearer $keyVaultAccessToken" } ` + -Method Get + } + catch { + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to read secret '$SecretName' from vault '$KeyVaultName': $_. Verify the secret exists and the service connection has 'Key Vault Secrets User' access to it." + exit 1 + } + if ([string]::IsNullOrWhiteSpace($response.value)) { + Write-PipelineTelemetryError -Category 'Build' -Message "Secret '$SecretName' in vault '$KeyVaultName' is empty." + exit 1 + } + return [string] $response.value +} + +Write-Host "Reading GitHub App credentials from vault '$KeyVaultName'..." +$appId = Get-KeyVaultSecret $AppIdSecretName +$privateKey = Get-KeyVaultSecret $AppPrivateKeySecretName + +# Build JWT header and payload. Use [ordered] hashtables so JSON +# serialization is deterministic. +$jwtHeader = [ordered]@{ + alg = 'RS256' + typ = 'JWT' +} +$now = [System.DateTimeOffset]::UtcNow +$jwtPayload = [ordered]@{ + iat = $now.AddMinutes(-1).ToUnixTimeSeconds() + exp = $now.AddMinutes(5).ToUnixTimeSeconds() + iss = $appId +} + +$headerEncoded = ConvertTo-Base64Url ([System.Text.Encoding]::UTF8.GetBytes(($jwtHeader | ConvertTo-Json -Compress))) +$payloadEncoded = ConvertTo-Base64Url ([System.Text.Encoding]::UTF8.GetBytes(($jwtPayload | ConvertTo-Json -Compress))) +$signingInput = "$headerEncoded.$payloadEncoded" + +$sha256 = [System.Security.Cryptography.SHA256]::Create() +try { + $digestBytes = $sha256.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($signingInput)) +} +finally { + $sha256.Dispose() +} + +Write-Host 'Signing JWT with the GitHub App private key...' +$rsa = [System.Security.Cryptography.RSA]::Create() +try { + $rsa.ImportFromPem($privateKey) + $signatureBytes = $rsa.SignHash( + $digestBytes, + [System.Security.Cryptography.HashAlgorithmName]::SHA256, + [System.Security.Cryptography.RSASignaturePadding]::Pkcs1) + $signatureUrl = ConvertTo-Base64Url $signatureBytes +} +catch { + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to sign the GitHub App JWT with the supplied private key: $_" + exit 1 +} +finally { + $rsa.Dispose() +} +$jwt = "$signingInput.$signatureUrl" + +$headers = @{ + Authorization = "Bearer $jwt" + 'X-GitHub-Api-Version' = '2022-11-28' + Accept = 'application/vnd.github+json' + 'User-Agent' = 'dotnet-arcade-onelocbuild' +} + +Write-Host "Looking up installation for '$InstallationOwner'..." +try { + $installations = [System.Collections.Generic.List[object]]::new() + $page = 1 + do { + $pageResponse = Invoke-RestMethod ` + -Uri "https://api.github.com/app/installations?per_page=100&page=$page" ` + -Headers $headers ` + -Method Get + $pageInstallationCount = 0 + foreach ($installation in $pageResponse) { + $installations.Add($installation) + $pageInstallationCount++ + } + $page++ + } while ($pageInstallationCount -eq 100) +} +catch { + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to list GitHub App installations: $_. The signed JWT may be invalid or the App ID may be incorrect." + exit 1 +} +$matchingInstallations = @($installations | Where-Object { $_.account.login -ieq $InstallationOwner }) +if ($matchingInstallations.Count -eq 0) { + $found = ($installations | ForEach-Object { $_.account.login }) -join ', ' + Write-PipelineTelemetryError -Category 'Build' -Message "No installation found for '$InstallationOwner'. App is installed on: $found" + exit 1 +} +if ($matchingInstallations.Count -ne 1) { + $matchingIds = ($matchingInstallations | ForEach-Object { $_.id }) -join ', ' + Write-PipelineTelemetryError -Category 'Build' -Message "Found multiple installations for '$InstallationOwner': $matchingIds" + exit 1 +} +$installation = $matchingInstallations[0] +Write-Host "Using installation $($installation.id) for '$($installation.account.login)'." + +try { + $tokenResponse = Invoke-RestMethod ` + -Uri "https://api.github.com/app/installations/$($installation.id)/access_tokens" ` + -Headers $headers ` + -Method Post ` + -ContentType 'application/json' +} +catch { + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to mint an installation access token for '$InstallationOwner' (installation $($installation.id)): $_" + exit 1 +} + +Write-Host "Got installation token for '$InstallationOwner' (expires $($tokenResponse.expires_at))." +if ($OutputVariableName) { + Write-Host "Setting pipeline variable '$OutputVariableName'." + Write-Host "##vso[task.setvariable variable=$OutputVariableName;issecret=true]$($tokenResponse.token)" +} +else { + Write-Host $tokenResponse.token -ForegroundColor Green +} diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 58002808bc..9efd17273a 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -11,9 +11,13 @@ # condition: eq(variables['Agent.OS'], 'Windows_NT') # inputs: # filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1 -# arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $Env:Token +# arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config # env: -# Token: $(dn-bot-dnceng-artifact-feeds-rw) +# Token: $(InternalFeedToken) +# +# Note: This logic is abstracted into enable-internal-sources.yml, which uses +# NuGetAuthenticate or a WIF-backed service connection. Prefer that template +# over calling this script directly. # # Note that the NuGetAuthenticate task should be called after SetupNugetSources. # This ensures that: @@ -25,12 +29,14 @@ [CmdletBinding()] param ( [Parameter(Mandatory = $true)][string]$ConfigFile, - $Password + # Keep the legacy name as an alias while callers migrate secrets to the Token environment variable. + [Alias("Password")]$Credential ) $ErrorActionPreference = "Stop" Set-StrictMode -Version 2.0 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$feedCredential = if ($env:Token) { $env:Token } else { $Credential } # This script only consumes helper functions from tools.ps1 to configure NuGet feeds. # Skip importing configure-toolset.ps1 so that repo-specific toolset setup (e.g. acquiring @@ -40,14 +46,14 @@ $disableConfigureToolsetImport = $true . $PSScriptRoot\tools.ps1 # Adds or enables the package source with the given name -function AddOrEnablePackageSource($sources, $disabledPackageSources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { - if ($disabledPackageSources -eq $null -or -not (EnableInternalPackageSource -DisabledPackageSources $disabledPackageSources -Creds $creds -PackageSourceName $SourceName)) { - AddPackageSource -Sources $sources -SourceName $SourceName -SourceEndPoint $SourceEndPoint -Creds $creds -Username $userName -pwd $Password +function AddOrEnablePackageSource($sources, $disabledPackageSources, $SourceName, $SourceEndPoint, $creds, $Username, $credential) { + if ($disabledPackageSources -eq $null -or -not (EnableInternalPackageSource -DisabledPackageSources $disabledPackageSources -Creds $creds -PackageSourceName $SourceName -Credential $credential)) { + AddPackageSource -Sources $sources -SourceName $SourceName -SourceEndPoint $SourceEndPoint -Creds $creds -Username $Username -credential $credential } } # Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $credential) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) @@ -63,13 +69,13 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern Write-Host "Package source $SourceName already present and enabled." } - AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd + AddCredential -Creds $creds -Source $SourceName -Username $Username -credential $credential } # Add a credential node for the specified source -function AddCredential($creds, $source, $username, $pwd) { +function AddCredential($creds, $source, $username, $credential) { # If no cred supplied, don't do anything. - if (!$pwd) { + if (!$credential) { return; } @@ -104,19 +110,19 @@ function AddCredential($creds, $source, $username, $pwd) { $sourceElement.AppendChild($passwordElement) | Out-Null } - $passwordElement.SetAttribute("value", $pwd) + $passwordElement.SetAttribute("value", $credential) } # Enable all darc-int package sources. -function EnableMaestroInternalPackageSources($DisabledPackageSources, $Creds) { +function EnableMaestroInternalPackageSources($DisabledPackageSources, $Creds, $Credential) { $maestroInternalSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") ForEach ($DisabledPackageSource in $maestroInternalSources) { - EnableInternalPackageSource -DisabledPackageSources $DisabledPackageSources -Creds $Creds -PackageSourceName $DisabledPackageSource.key + EnableInternalPackageSource -DisabledPackageSources $DisabledPackageSources -Creds $Creds -PackageSourceName $DisabledPackageSource.key -Credential $Credential } } # Enables an internal package source by name, if found. Returns true if the package source was found and enabled, false otherwise. -function EnableInternalPackageSource($DisabledPackageSources, $Creds, $PackageSourceName) { +function EnableInternalPackageSource($DisabledPackageSources, $Creds, $PackageSourceName, $Credential) { $DisabledPackageSource = $DisabledPackageSources.SelectSingleNode("add[@key='$PackageSourceName']") if ($DisabledPackageSource) { Write-Host "Enabling internal source '$($DisabledPackageSource.key)'." @@ -124,7 +130,7 @@ function EnableInternalPackageSource($DisabledPackageSources, $Creds, $PackageSo # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries $DisabledPackageSources.RemoveChild($DisabledPackageSource) - AddCredential -Creds $creds -Source $DisabledPackageSource.Key -Username $userName -pwd $Password + AddCredential -Creds $creds -Source $DisabledPackageSource.Key -Username $userName -credential $credential return $true } return $false @@ -149,7 +155,7 @@ if ($sources -eq $null) { $creds = $null $feedSuffix = "v3/index.json" -if ($Password) { +if ($feedCredential) { $feedSuffix = "v2" # Looks for a node. Create it if none is found. $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") @@ -165,16 +171,16 @@ $userName = "dn-bot" $disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources") if ($disabledSources -ne $null) { Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node" - EnableMaestroInternalPackageSources -DisabledPackageSources $disabledSources -Creds $creds + EnableMaestroInternalPackageSources -DisabledPackageSources $disabledSources -Creds $creds -Credential $feedCredential } -$dotnetVersions = @('5','6','7','8','9','10') +$dotnetVersions = @('5','6','7','8','9','10','11') foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") if ($dotnetSource -ne $null) { - AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password - AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password + AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -credential $feedCredential + AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/$feedSuffix" -Creds $creds -Username $userName -credential $feedCredential } } diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index 67e7e0942c..d4c66a98e1 100644 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -24,7 +24,9 @@ # This logic is also abstracted into enable-internal-sources.yml. ConfigFile=$1 -CredToken=$2 +# Prefer the environment variable so credentials do not appear in process arguments. +# Retain the positional argument as a compatibility fallback for existing callers. +CredToken=${Token:-$2} NL='\n' TB=' ' @@ -167,7 +169,7 @@ if [ "$?" == "0" ]; then EnableMaestroInternalPackageSources fi -DotNetVersions=('5' '6' '7' '8' '9' '10') +DotNetVersions=('5' '6' '7' '8' '9' '10' '11') for DotNetVersion in ${DotNetVersions[@]} ; do FeedPrefix="dotnet${DotNetVersion}"; diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 4b4f6b0923..fee2f83991 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -8,6 +8,7 @@ Param( [bool] $warnAsError = $true, [string] $warnNotAsError = '', [bool] $nodeReuse = $true, + [bool][Alias('mt')]$msbuildMultiThreaded = $false, [switch] $buildCheck = $false, [switch][Alias('r')]$restore, [switch] $deployDeps, @@ -23,6 +24,7 @@ Param( [switch] $clean, [switch][Alias('pb')]$productBuild, [switch]$fromVMR, + [switch]$disablePipelineSetResult, [switch][Alias('bl')]$binaryLog, [string][Alias('bln')]$binaryLogName = '', [switch][Alias('nobl')]$excludeCIBinarylog, @@ -78,8 +80,10 @@ function Print-Usage() { Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" Write-Host " -nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" + Write-Host " -msbuildMultiThreaded Sets MSBuild's multi-threaded mode, i.e. the -mt switch ('1' or '0') (short: -mt)" Write-Host " -buildCheck Sets /check msbuild parameter" Write-Host " -fromVMR Set when building from within the VMR" + Write-Host " -disablePipelineSetResult Set to disable masking the actual exit code in the pipeline when the build fails" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." @@ -173,7 +177,10 @@ try { if (-not $excludeCIBinarylog) { $binaryLog = $true } - $nodeReuse = $false + # Node reuse isn't used on CI unless it was explicitly requested via -nodeReuse. + if (-not $PSBoundParameters.ContainsKey('nodeReuse')) { + $nodeReuse = $false + } } if (-not [string]::IsNullOrEmpty($binaryLogName)) { diff --git a/eng/common/build.sh b/eng/common/build.sh index 719ee4b587..f65b048aa8 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -40,12 +40,15 @@ usage() echo " --projects Project or solution file(s) to build" echo " --ci Set when running on CI server" echo " --excludeCIBinarylog Don't output binary log (short: -nobl)" + echo " --pipelinesLog Promote msbuild errors/warnings to Azure Pipelines timeline issues; defaults to on in CI (short: -pl)" echo " --prepareMachine Prepare machine for CI run, clean up processes after build" echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" + echo " --msbuildMultiThreaded Sets MSBuild's multi-threaded mode, i.e. the -mt switch ('true' or 'false') (short: --mt)" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" echo " --warnNotAsError Sets a semi-colon delimited list of warning codes that should not be treated as errors" echo " --buildCheck Sets /check msbuild parameter" echo " --fromVMR Set when building from within the VMR" + echo " --disablePipelineSetResult Set to disable masking the actual exit code in the pipeline when the build fails" echo "" echo "Command line arguments not listed above are passed thru to msbuild." echo "Arguments can also be passed in with a single hyphen." @@ -68,6 +71,7 @@ build=false source_build=false product_build=false from_vmr=false +disable_pipeline_set_result=false rebuild=false test=false integration_test=false @@ -81,11 +85,14 @@ clean=false warn_as_error=true warn_not_as_error='' -node_reuse=true +# Empty means "not specified"; tools.sh defaults these to on for local builds and off on CI. +node_reuse='' +msbuild_multi_threaded='' build_check=false binary_log=false binary_log_name='' exclude_ci_binary_log=false +pipelines_log=false projects='' configuration='' @@ -124,6 +131,9 @@ while [[ $# -gt 0 ]]; do -excludecibinarylog|-nobl) exclude_ci_binary_log=true ;; + -pipelineslog|-pl) + pipelines_log=true + ;; -restore|-r) restore=true ;; @@ -152,6 +162,9 @@ while [[ $# -gt 0 ]]; do -fromvmr|-from-vmr) from_vmr=true ;; + -disablepipelinesetresult|-disable-pipeline-set-result) + disable_pipeline_set_result=true + ;; -test|-t) test=true ;; @@ -189,6 +202,10 @@ while [[ $# -gt 0 ]]; do node_reuse=$2 shift ;; + -msbuildmultithreaded|-mt) + msbuild_multi_threaded=$2 + shift + ;; -buildcheck) build_check=true ;; @@ -213,7 +230,7 @@ if [[ -z "$configuration" ]]; then fi if [[ "$ci" == true ]]; then - node_reuse=false + pipelines_log=true if [[ "$exclude_ci_binary_log" == false ]]; then binary_log=true fi @@ -237,7 +254,7 @@ function Build { properties+=("/p:Projects=$projects") fi - local bl="" + local bl=() if [[ "$binary_log" == true ]]; then local binary_log_path="" if [[ -z "$binary_log_name" ]]; then @@ -249,7 +266,7 @@ function Build { fi mkdir -p "$(dirname "$binary_log_path")" - bl="/bl:\"$binary_log_path\"" + bl=("/bl:$binary_log_path") fi local check="" @@ -257,8 +274,8 @@ function Build { check="/check" fi - MSBuild $_InitializeToolset \ - $bl \ + MSBuild "$_InitializeToolset" \ + ${bl[@]+"${bl[@]}"} \ $check \ /p:Configuration=$configuration \ /p:RepoRoot="$repo_root" \ @@ -282,7 +299,7 @@ function Build { if [[ "$clean" == true ]]; then if [ -d "$artifacts_dir" ]; then - rm -rf $artifacts_dir + rm -rf "$artifacts_dir" echo "Artifacts directory deleted." fi exit 0 diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index a8162c5116..53bbf74927 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -26,6 +26,11 @@ parameters: type: string default: '' +# Whether failures in the monitor job should allow the pipeline to continue. +- name: continueOnError + type: boolean + default: false + # NuGet package id of the Helix job monitor tool. - name: toolPackageId type: string @@ -57,6 +62,43 @@ parameters: type: number default: 30 +# Maximum number of work items whose results may be downloaded, parsed, and +# uploaded concurrently. +- name: testResultUploadParallelism + type: number + default: 48 + +# When 'true' (the default), Helix work items that exit 0 but have failed AzDO test results +# are treated as failed: they count toward the monitor's exit code and are resubmitted by a +# later invocation's retry pass. Set to 'false' to fall back to exit-code-only outcomes. +# Forwarded as --fail-on-failed-tests. +- name: failWorkItemsWithFailedTests + type: boolean + default: true + +# When true, allow the monitor to succeed when this stage produces no Helix jobs in any attempt. +# Forwarded as --allow-no-helix-jobs. +- name: allowNoHelixJobs + type: boolean + default: false + +# When true, test results are reported to Azure DevOps using the fully qualified test name +# (Namespace.Type.Method) as the stable automatedTestName and the visible title is qualified as +# well (--use-fully-qualified-test-name). Opt-in because it changes AzDO test identity and display; +# primarily useful for frameworks like MSTest whose display name is only the method name. +- name: useFullyQualifiedTestName + type: boolean + default: false + +# Controls per-test output attachments. Defaults to Failed. +- name: testResultAttachmentMode + type: string + default: Failed + values: + - Failed + - All + - None + # Advanced: optional pipeline artifact (produced earlier in this run) that contains the tool # nupkg. When set, the artifact is downloaded and the tool is installed from the nupkg into # a local tool-path; this bypasses the repo's .config/dotnet-tools.json manifest and is @@ -81,6 +123,7 @@ jobs: - job: HelixJobMonitor displayName: Monitor Helix Jobs timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + continueOnError: ${{ parameters.continueOnError }} ${{ if ne(length(parameters.dependsOn), 0) }}: dependsOn: ${{ parameters.dependsOn }} ${{ if ne(parameters.condition, '') }}: @@ -88,9 +131,11 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) + os: linux demands: ImageOverride -equals build.azurelinux.3.amd64.open ${{ else }}: name: $(DncEngInternalBuildPool) + os: linux demands: ImageOverride -equals build.azurelinux.3.amd64 steps: - checkout: self @@ -168,23 +213,36 @@ jobs: set -euo pipefail toolArgs=( - --helix-base-uri '${{ parameters.helixBaseUri }}' - --polling-interval-seconds '${{ parameters.pollingIntervalSeconds }}' - --max-wait-minutes "$((${{ parameters.timeoutInMinutes }} - 5))" # Set the tool's timeout slightly lower than the Azure DevOps job timeout to allow it to exit gracefully. - --stage-name '$(System.StageName)' + --helix-base-uri '${{ parameters.helixBaseUri }}' + --polling-interval-seconds '${{ parameters.pollingIntervalSeconds }}' + --fail-on-failed-tests '${{ parameters.failWorkItemsWithFailedTests }}' + --allow-no-helix-jobs '${{ parameters.allowNoHelixJobs }}' + --use-fully-qualified-test-name '${{ parameters.useFullyQualifiedTestName }}' + --max-wait-minutes "$((${{ parameters.timeoutInMinutes }} - 5))" # Set the tool's timeout slightly lower than the Azure DevOps job timeout to allow it to exit gracefully. + --stage-name '$(System.StageName)' + --stage-attempt '$(System.StageAttempt)' + --job-attempt '$(System.JobAttempt)' + --test-result-upload-parallelism '${{ parameters.testResultUploadParallelism }}' ) organization='${{ parameters.organization }}' repository='${{ parameters.repository }}' + testResultAttachmentMode='${{ parameters.testResultAttachmentMode }}' # Fall back to Azure DevOps-provided environment variables when the caller did not # supply organization / repository explicitly. BUILD_REPOSITORY_NAME is typically - # 'owner/repo' for GitHub-backed builds. + # 'owner/repo' for GitHub-backed builds and 'owner-repo' for internal builds. if [ -z "$organization" ] || [ -z "$repository" ]; then buildRepoName="${BUILD_REPOSITORY_NAME:-}" if [ -n "$buildRepoName" ] && [[ "$buildRepoName" == */* ]]; then repoOwner="${buildRepoName%%/*}" repoName="${buildRepoName#*/}" + elif [ -n "$buildRepoName" ] && [[ "$buildRepoName" == *-* ]]; then + repoOwner="${buildRepoName%%-*}" + repoName="${buildRepoName#*-}" + fi + + if [ -n "${repoOwner:-}" ] && [ -n "${repoName:-}" ]; then if [ -z "$organization" ]; then organization="$repoOwner"; fi if [ -z "$repository" ]; then repository="$repoName"; fi fi @@ -192,6 +250,9 @@ jobs: if [ -n "$organization" ]; then toolArgs+=( --organization "$organization" ); fi if [ -n "$repository" ]; then toolArgs+=( --repository "$repository" ); fi + if [ -n "$testResultAttachmentMode" ]; then + toolArgs+=( --test-result-attachment-mode "$testResultAttachmentMode" ) + fi # Build.Reason and Build.SourceBranch are required to derive the Helix source filter # the same way the Helix SDK submitter does (PR -> 'pr', internal -> 'official', diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml index cb60f52978..2716ecd18f 100644 --- a/eng/common/core-templates/job/job.yml +++ b/eng/common/core-templates/job/job.yml @@ -28,6 +28,7 @@ parameters: enablePublishTestResults: false enablePublishing: false enableBuildRetry: false + enableAstred: false mergeTestResults: false testRunTitle: '' testResultsFormat: '' @@ -119,6 +120,12 @@ jobs: - name: ${{ pair.key }} value: ${{ pair.value }} + - ${{ if and(eq(parameters.enableAstred, true), eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - name: MSBUILDDEBUGENGINE + value: 1 + - name: MSBUILDDEBUGPATH + value: $(Build.ArtifactStagingDirectory)/AstredCapture/binlogs + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: DotNet-HelixApi-Access @@ -236,3 +243,8 @@ jobs: condition: always() - ${{ each step in parameters.artifactPublishSteps }}: - ${{ step }} + + - ${{ if and(eq(parameters.enableAstred, true), eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/common/core-templates/steps/astred-artifacts.yml + parameters: + binlogDir: $(MSBUILDDEBUGPATH) diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml index 86ea9f6350..b772dc5788 100644 --- a/eng/common/core-templates/job/onelocbuild.yml +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -5,8 +5,14 @@ parameters: # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool pool: '' - CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex - GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + # Project-scoped WIF service connection for Ceapex feed authentication. + CeapexServiceConnection: 'dnceng-onelocbuild-ceapex' + + # GitHub App authentication for the OneLoc check-in PR. + GitHubAppServiceConnection: 'dnceng-oneloc-githubapp' + GitHubAppKeyVaultName: 'EngKeyVault' + GitHubAppIdSecretName: 'oneloc-localization-app-app-id' + GitHubAppPrivateKeySecretName: 'oneloc-localization-app-app-private-key' SourcesDirectory: $(System.DefaultWorkingDirectory) CreatePr: true @@ -34,7 +40,6 @@ jobs: displayName: OneLocBuild${{ parameters.JobNameSuffix }} variables: - - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat - name: _GenerateLocProjectArguments value: -SourcesDirectory ${{ parameters.SourcesDirectory }} -LanguageSet "${{ parameters.LanguageSet }}" @@ -65,6 +70,10 @@ jobs: steps: - ${{ if eq(parameters.is1ESPipeline, '') }}: - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + - ${{ if notIn(variables['System.TeamProject'], 'internal', 'DevDiv') }}: + - 'OneLocBuild is supported only in dnceng/internal and DevDiv/DevDiv.': error + - ${{ if eq(parameters.CeapexServiceConnection, '') }}: + - 'CeapexServiceConnection must identify a WIF service connection.': error - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: - task: Powershell@2 @@ -74,6 +83,29 @@ jobs: displayName: Generate LocProject.json condition: ${{ parameters.condition }} + # Acquire a short-lived Entra token for Ceapex feed access. + - template: /eng/common/templates/steps/get-federated-access-token.yml + parameters: + federatedServiceConnection: ${{ parameters.CeapexServiceConnection }} + outputVariableName: 'CeapexEntraToken' + condition: ${{ parameters.condition }} + + # Mint a short-lived GitHub App installation token for the loc check-in PR. + - ${{ if eq(parameters.RepoType, 'gitHub') }}: + - template: /eng/common/core-templates/steps/get-github-app-token.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + ${{ if and(eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.GitHubAppServiceConnection, 'dnceng-oneloc-githubapp')) }}: + azureSubscription: 'devdiv-oneloc-githubapp' + ${{ else }}: + azureSubscription: ${{ parameters.GitHubAppServiceConnection }} + keyVaultName: ${{ parameters.GitHubAppKeyVaultName }} + appIdSecretName: ${{ parameters.GitHubAppIdSecretName }} + appPrivateKeySecretName: ${{ parameters.GitHubAppPrivateKeySecretName }} + installationOwner: ${{ parameters.GitHubOrg }} + outputVariableName: 'GitHubAppInstallationToken' + condition: ${{ parameters.condition }} + - task: OneLocBuild@2 displayName: OneLocBuild env: @@ -89,10 +121,10 @@ jobs: isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} isShouldReusePrSelected: ${{ parameters.ReusePr }} packageSourceAuth: patAuth - patVariable: ${{ parameters.CeapexPat }} + patVariable: $(CeapexEntraToken) ${{ if eq(parameters.RepoType, 'gitHub') }}: repoType: ${{ parameters.RepoType }} - gitHubPatVariable: "${{ parameters.GithubPat }}" + gitHubPatVariable: "$(GitHubAppInstallationToken)" ${{ if ne(parameters.MirrorRepo, '') }}: isMirrorRepoSelected: true gitHubOrganization: ${{ parameters.GitHubOrg }} diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml index 700f771146..330225ae09 100644 --- a/eng/common/core-templates/job/publish-build-assets.yml +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -58,8 +58,6 @@ jobs: parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: Publish-Build-Assets - - group: AzureDevOps-Artifact-Feeds-Pats - name: runCodesignValidationInjection value: false # unconditional - needed for logs publishing (redactor tool version) @@ -122,9 +120,6 @@ jobs: # Populate internal runtime variables. - template: /eng/common/templates/steps/enable-internal-sources.yml - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - parameters: - legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) - template: /eng/common/templates/steps/enable-internal-runtimes.yml diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index bac6ac5faa..b0dc8f1706 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -12,6 +12,7 @@ jobs: - job: SourceIndexStage1 dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} + continueOnError: true variables: - name: BinlogPath value: ${{ parameters.binlogPath }} @@ -38,9 +39,11 @@ jobs: - ${{ each preStep in parameters.preSteps }}: - ${{ preStep }} - - script: ${{ parameters.sourceIndexBuildCommand }} - displayName: Build Repository + - ${{ if ne(parameters.sourceIndexBuildCommand, '') }}: + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository - template: /eng/common/core-templates/steps/source-index-stage1-publish.yml parameters: binLogPath: ${{ parameters.binLogPath }} + runAsPublic: ${{ parameters.runAsPublic }} diff --git a/eng/common/core-templates/post-build/common-variables.yml b/eng/common/core-templates/post-build/common-variables.yml index db298ae16b..a3a8480e25 100644 --- a/eng/common/core-templates/post-build/common-variables.yml +++ b/eng/common/core-templates/post-build/common-variables.yml @@ -1,6 +1,4 @@ variables: - - group: Publish-Build-Assets - # Whether the build is internal or not - name: IsInternalBuild value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml index 8aa86e3049..6dcee6664d 100644 --- a/eng/common/core-templates/post-build/post-build.yml +++ b/eng/common/core-templates/post-build/post-build.yml @@ -236,6 +236,7 @@ stages: StageLabel: 'Validation' JobLabel: 'Signing' BinlogToolVersion: $(BinlogToolVersion) + enableInternalRuntimes: false # SourceLink validation has been removed — the underlying CLI tool # (targeting netcoreapp2.1) has not functioned for years. @@ -295,8 +296,6 @@ stages: # Populate internal runtime variables. - template: /eng/common/templates/steps/enable-internal-sources.yml - parameters: - legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) - template: /eng/common/templates/steps/enable-internal-runtimes.yml diff --git a/eng/common/core-templates/stages/renovate.yml b/eng/common/core-templates/stages/renovate.yml index edab281825..cfa9683794 100644 --- a/eng/common/core-templates/stages/renovate.yml +++ b/eng/common/core-templates/stages/renovate.yml @@ -81,6 +81,8 @@ resources: extends: template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: + settings: + networkIsolationPolicy: Permissive pool: ${{ parameters.pool }} sdl: sourceAnalysisPool: ${{ parameters.sdlPool }} diff --git a/eng/common/core-templates/steps/astred-artifacts.yml b/eng/common/core-templates/steps/astred-artifacts.yml new file mode 100644 index 0000000000..b914082f58 --- /dev/null +++ b/eng/common/core-templates/steps/astred-artifacts.yml @@ -0,0 +1,103 @@ +# Astred footer for producing and uploading a portable digest. +# The calling job must configure its header before any build steps run: +# MSBUILDDEBUGENGINE=1 +# MSBUILDDEBUGPATH= +parameters: +- name: sourcesPath + type: string + default: $(Build.SourcesDirectory) +- name: binlogDir + type: string + default: $(Build.ArtifactStagingDirectory)/AstredCapture/binlogs +- name: capturePath + type: string + default: $(Build.ArtifactStagingDirectory)/AstredCapture + +steps: +- task: AstredInstaller@0 + displayName: Install Astred CLI + inputs: + Version: '2.14.1' + FeedUrl: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet-internal-FoSSE/nuget/v3/index.json' + +- pwsh: | + $ErrorActionPreference = 'Continue' + $apjOut = Join-Path $env:ASTRED_CAPTURE_PATH 'apj' + New-Item -ItemType Directory -Force -Path $apjOut | Out-Null + + $binlogs = Get-ChildItem -Path $env:ASTRED_BINLOG_DIR -Recurse -Force -Filter *.binlog ` + -ErrorAction SilentlyContinue + if (-not $binlogs) { + Write-Host "##vso[task.logissue type=warning]No binlogs found in $env:ASTRED_BINLOG_DIR. Check the calling job's Astred header configuration for MSBUILDDEBUGENGINE / MSBuildDebugEngine and MSBUILDDEBUGPATH." + exit 0 + } + + $project = Join-Path $apjOut '.astred.project.json' + $binlogPaths = @($binlogs.FullName) + Write-Host "astproj: processing $($binlogPaths.Count) binlog(s)" + astred astproj -nofolders @binlogPaths "-o:$project" + if ($LASTEXITCODE -ne 0) { + Write-Host "##vso[task.logissue type=warning]astproj failed (exit $LASTEXITCODE)" + } + displayName: Generate Astred Project Files + workingDirectory: ${{ parameters.sourcesPath }} + env: + ASTRED_BINLOG_DIR: ${{ parameters.binlogDir }} + ASTRED_CAPTURE_PATH: ${{ parameters.capturePath }} + condition: succeededOrFailed() + continueOnError: true + +- pwsh: | + $ErrorActionPreference = 'Continue' + $apjDir = Join-Path $env:ASTRED_CAPTURE_PATH 'apj' + $uploadRoot = Join-Path $env:ASTRED_CAPTURE_PATH 'upload' + $project = Join-Path $apjDir '.astred.project.json' + + if (-not (Test-Path $project)) { + Write-Host "##vso[task.logissue type=warning]No Astred project file was produced." + exit 0 + } + + $digest = Join-Path $apjDir '.astred.digest.zip' + Remove-Item $digest -ErrorAction SilentlyContinue + astred "-repo:$env:ASTRED_SOURCES_PATH" "-project:$project" -digest + if ($LASTEXITCODE -eq 0 -and (Test-Path $digest)) { + $commitTimeText = & git -C $env:ASTRED_SOURCES_PATH show -s --format=%cI $env:BUILD_SOURCEVERSION + if ($LASTEXITCODE -ne 0) { + throw "Could not read the commit timestamp for $env:BUILD_SOURCEVERSION." + } + + $commitTime = [DateTimeOffset]::Parse( + $commitTimeText.Trim(), + [Globalization.CultureInfo]::InvariantCulture) + $eventFolder = '{0}_{1}' -f ` + $commitTime.UtcDateTime.ToString('yyyy-MM-ddTHH-mm-ssZ'), ` + $env:BUILD_SOURCEVERSION + $targetDir = Join-Path (Join-Path $uploadRoot 'AST') $eventFolder + $target = Join-Path $targetDir '.astred.digest.zip' + New-Item -ItemType Directory -Force -Path $targetDir | Out-Null + Move-Item $digest $target -Force + Write-Host "Prepared Astred digest: $target" + Write-Host "##vso[task.setvariable variable=ASTRED_DIGEST_READY]true" + } + elseif ($LASTEXITCODE -ne 0) { + Write-Host "##vso[task.logissue type=warning]Digest generation failed for $project (exit $LASTEXITCODE)" + Remove-Item $digest -ErrorAction SilentlyContinue + } + else { + Write-Host "##vso[task.logissue type=warning]Digest not produced for $project" + } + displayName: Package Portable Astred Digests + env: + ASTRED_SOURCES_PATH: ${{ parameters.sourcesPath }} + ASTRED_CAPTURE_PATH: ${{ parameters.capturePath }} + condition: succeededOrFailed() + continueOnError: true + +- task: UploadAstred@0 + displayName: Upload Digest to Astred + condition: and(succeededOrFailed(), eq(variables['ASTRED_DIGEST_READY'], 'true')) + inputs: + SourcePath: ${{ parameters.capturePath }}/upload + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/common/core-templates/steps/enable-internal-sources.yml b/eng/common/core-templates/steps/enable-internal-sources.yml index 51af9a0170..843cdff782 100644 --- a/eng/common/core-templates/steps/enable-internal-sources.yml +++ b/eng/common/core-templates/steps/enable-internal-sources.yml @@ -19,7 +19,7 @@ steps: displayName: Setup Internal Feeds inputs: filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $Env:Token + arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config env: Token: ${{ parameters.legacyCredential }} - task: Bash@3 @@ -28,7 +28,7 @@ steps: inputs: targetType: inline script: | - "$(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh" "$(System.DefaultWorkingDirectory)/NuGet.config" "$Token" + "$(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh" "$(System.DefaultWorkingDirectory)/NuGet.config" env: Token: ${{ parameters.legacyCredential }} # If running on dnceng (internal project), just use the default behavior for NuGetAuthenticate. @@ -58,13 +58,17 @@ steps: displayName: Setup Internal Feeds inputs: filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $(dnceng-artifacts-feeds-read-access-token) + arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config + env: + Token: $(dnceng-artifacts-feeds-read-access-token) - task: Bash@3 condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) displayName: Setup Internal Feeds inputs: filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh - arguments: $(System.DefaultWorkingDirectory)/NuGet.config $(dnceng-artifacts-feeds-read-access-token) + arguments: $(System.DefaultWorkingDirectory)/NuGet.config + env: + Token: $(dnceng-artifacts-feeds-read-access-token) # This is required in certain scenarios to install the ADO credential provider. # It installed by default in some msbuild invocations (e.g. VS msbuild), but needs to be installed for others # (e.g. dotnet msbuild). diff --git a/eng/common/core-templates/steps/get-github-app-token.yml b/eng/common/core-templates/steps/get-github-app-token.yml new file mode 100644 index 0000000000..3eeb5a4c1b --- /dev/null +++ b/eng/common/core-templates/steps/get-github-app-token.yml @@ -0,0 +1,72 @@ +# Mints a short-lived GitHub App installation access token by signing a JWT +# with an RSA private key (RS256). The JWT is exchanged with the GitHub API +# for a token scoped to a single installation. +# +# Requirements (per GitHub App you want to authenticate as): +# - A GitHub App ID and PEM private key stored as Azure Key Vault secrets. +# - The Azure service connection passed via `azureSubscription` must have +# `Get` access to those two secrets. +# - The App must be installed on the target organization/account +# (`installationOwner`) with the permissions/repositories you need. +# +# Output: a secret pipeline variable named ${{ parameters.outputVariableName }} +# containing the installation access token. Token lifetime is ~1 hour and is +# automatically scrubbed from logs. Installation tokens are exempt from the +# enterprise classic-PAT lifetime policy. + +parameters: +# Azure DevOps service connection (federated) that can read the App credentials. +- name: azureSubscription + type: string + +# Name of the Key Vault holding Secret Manager's github-app-secret projections. +- name: keyVaultName + type: string + +- name: appIdSecretName + type: string + +- name: appPrivateKeySecretName + type: string + +# Login of the organization or user account whose installation we should +# mint the token for (e.g. `dotnet`, `microsoft`). +- name: installationOwner + type: string + +# Name of the pipeline variable that will receive the installation token. +- name: outputVariableName + type: string + +- name: is1ESPipeline + type: boolean + +- name: stepName + type: string + default: getGitHubAppInstallationToken + +- name: condition + type: string + default: '' + +- name: displayName + type: string + default: Get GitHub App installation token + +steps: +- task: AzureCLI@2 + displayName: ${{ parameters.displayName }} + name: ${{ parameters.stepName }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + inputs: + azureSubscription: ${{ parameters.azureSubscription }} + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + & "$(System.DefaultWorkingDirectory)/eng/common/Get-GitHubAppToken.ps1" ` + -KeyVaultName '${{ parameters.keyVaultName }}' ` + -AppIdSecretName '${{ parameters.appIdSecretName }}' ` + -AppPrivateKeySecretName '${{ parameters.appPrivateKeySecretName }}' ` + -InstallationOwner '${{ parameters.installationOwner }}' ` + -OutputVariableName '${{ parameters.outputVariableName }}' diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml index 2731e48cce..244fef0890 100644 --- a/eng/common/core-templates/steps/publish-logs.yml +++ b/eng/common/core-templates/steps/publish-logs.yml @@ -5,6 +5,7 @@ parameters: # A default - in case value from eng/common/core-templates/post-build/common-variables.yml is not passed BinlogToolVersion: '1.0.11' is1ESPipeline: false + enableInternalRuntimes: true steps: - task: Powershell@2 @@ -25,16 +26,20 @@ steps: # Sensitive data can as well be added to $(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt' # If the file exists - sensitive data for redaction will be sourced from it # (single entry per line, lines starting with '# ' are considered comments and skipped) - arguments: -InputPath '$(System.DefaultWorkingDirectory)/PostBuildLogs' - -BinlogToolVersion '${{parameters.BinlogToolVersion}}' - -TokensFilePath '$(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt' - -runtimeSourceFeed https://ci.dot.net/internal - -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' - '$(publishing-dnceng-devdiv-code-r-build-re)' - '$(dn-bot-all-orgs-artifact-feeds-rw)' - '$(akams-client-id)' - '$(System.AccessToken)' - ${{parameters.CustomSensitiveDataList}} + ${{ if and(eq(parameters.enableInternalRuntimes, true), ne(variables['System.TeamProject'], 'public')) }}: + arguments: -InputPath '$(System.DefaultWorkingDirectory)/PostBuildLogs' + -BinlogToolVersion '${{parameters.BinlogToolVersion}}' + -TokensFilePath '$(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt' + -runtimeSourceFeed https://ci.dot.net/internal + -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' + '$(System.AccessToken)' + ${{parameters.CustomSensitiveDataList}} + ${{ else }}: + arguments: -InputPath '$(System.DefaultWorkingDirectory)/PostBuildLogs' + -BinlogToolVersion '${{parameters.BinlogToolVersion}}' + -TokensFilePath '$(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt' + '$(System.AccessToken)' + ${{parameters.CustomSensitiveDataList}} continueOnError: true condition: always() @@ -57,4 +62,3 @@ steps: condition: always() retryCountOnTaskFailure: 10 # for any files being locked isProduction: false # logs are non-production artifacts - diff --git a/eng/common/core-templates/steps/send-to-helix.yml b/eng/common/core-templates/steps/send-to-helix.yml index 68fa739c4a..ec7a200039 100644 --- a/eng/common/core-templates/steps/send-to-helix.yml +++ b/eng/common/core-templates/steps/send-to-helix.yml @@ -10,6 +10,7 @@ parameters: HelixConfiguration: '' # optional -- additional property attached to a job HelixPreCommands: '' # optional -- commands to run before Helix work item execution HelixPostCommands: '' # optional -- commands to run after Helix work item execution + UseHelixMonitor: false # optional -- true will submit Helix jobs configured for the standalone Helix Job Monitor (results are reported/waited on out-of-band; this step will not wait, and WaitForWorkItemCompletion will be overridden) WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects @@ -31,7 +32,15 @@ parameters: continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + - powershell: > + $(Build.SourcesDirectory)\eng\common\msbuild.ps1 + $(Build.SourcesDirectory)/${{ parameters.HelixProjectPath }} + /restore + /p:TreatWarningsAsErrors=false + /p:EnableHelixJobMonitor=${{ parameters.UseHelixMonitor }} + ${{ parameters.HelixProjectArguments }} + /t:Test + /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog displayName: ${{ parameters.DisplayNamePrefix }} (Windows) env: BuildConfig: $(_BuildConfig) @@ -61,7 +70,15 @@ steps: SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + - script: > + $(Build.SourcesDirectory)/eng/common/msbuild.sh + $(Build.SourcesDirectory)/${{ parameters.HelixProjectPath }} + /restore + /p:TreatWarningsAsErrors=false + /p:EnableHelixJobMonitor=${{ parameters.UseHelixMonitor }} + ${{ parameters.HelixProjectArguments }} + /t:Test + /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog displayName: ${{ parameters.DisplayNamePrefix }} (Unix) env: BuildConfig: $(_BuildConfig) @@ -91,3 +108,4 @@ steps: SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} + diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index fdca622357..6d4173aaf7 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -1,7 +1,9 @@ parameters: - sourceIndexUploadPackageVersion: 2.0.0-20260521.2 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20260521.2 + runAsPublic: false + sourceIndexUploadPackageVersion: '2.0.0-20260521.2' + sourceIndexComplogPackageVersion: '*' sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexPublicPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json binlogPath: artifacts/log/Debug/Build.binlog steps: @@ -14,14 +16,17 @@ steps: workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools - displayName: "Source Index: Download netsourceindex Tools" + $(Agent.TempDirectory)/dotnet/dotnet tool install complog --version "${{parameters.sourceIndexComplogPackageVersion}}" --source ${{parameters.sourceIndexPublicPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version "${{parameters.sourceIndexUploadPackageVersion}}" --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: "Source Index: Download Tools" # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) -- script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i ${{parameters.BinlogPath}} -r $(System.DefaultWorkingDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output - displayName: "Source Index: Process Binlog into indexable sln" +- script: | + mkdir ".source-index/stage1output" + git rev-parse HEAD > .source-index/stage1output/hash + $(Agent.TempDirectory)/.source-index/tools/complog create ${{parameters.BinlogPath}} -o .source-index/stage1output/build.complog + displayName: "Source Index: Process Binlog into Complog" - ${{ if and(ne(parameters.runAsPublic, 'true'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: AzureCLI@2 diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 273cae651a..3fea306bc2 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -8,8 +8,8 @@ usage() echo "BuildArch can be: arm(default), arm64, loongarch64, ppc64le, riscv64, s390x, x64, x86" echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine" echo " for alpine can be specified with version: alpineX.YY or alpineedge" - echo " for FreeBSD can be: freebsd13, freebsd14" - echo " for OpenBSD can be: openbsd" + echo " for FreeBSD can be: freebsd14, freebsd15" + echo " for OpenBSD can be: openbsd7.8, openbsd7.9" echo " for illumos can be: illumos" echo " for Haiku can be: haiku." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" @@ -18,7 +18,10 @@ usage() echo "--skipsigcheck - optional, will skip package signature checks (allowing untrusted packages)." echo "--skipemulation - optional, will skip qemu and debootstrap requirement when building environment for debian based systems." echo "--use-mirror - optional, use mirror URL to fetch resources, when available." - echo "--jobs N - optional, restrict to N jobs." + echo "--ubuntu-repo - optional, override the Ubuntu apt repository base URL." + echo "--debian-repo - optional, override the Debian apt repository base URL." + echo "--alpine-repo - optional, override the Alpine Linux repository base URL." + echo "--jobs N (or --use-jobs N) - optional, restrict to N jobs." exit 1 } @@ -75,9 +78,9 @@ __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" -__FreeBSDBase="13.5-RELEASE" -__FreeBSDPkg="2.7.5" -__FreeBSDABI="13" +__FreeBSDBase="14.4-RELEASE" +__FreeBSDPkg="2.8.0" +__FreeBSDABI="14" __FreeBSDPackages="libunwind" __FreeBSDPackages+=" icu" __FreeBSDPackages+=" libinotify" @@ -88,8 +91,9 @@ __FreeBSDPackages+=" terminfo-db" __OpenBSDVersion="7.8" __OpenBSDPackages="heimdal-libs" __OpenBSDPackages+=" icu4c" -__OpenBSDPackages+=" inotify-tools" +__OpenBSDPackages+=" libinotify" __OpenBSDPackages+=" openssl" +__OpenBSDPackages+=" e2fsprogs" __IllumosPackages="icu" __IllumosPackages+=" mit-krb5" @@ -143,6 +147,9 @@ __KeyringFile="/usr/share/keyrings/ubuntu-archive-keyring.gpg" __SkipSigCheck=0 __SkipEmulation=0 __UseMirror=0 +__UbuntuRepoOverride= +__DebianRepoOverride= +__AlpineRepoOverride= __UnprocessedBuildArgs= while :; do @@ -180,17 +187,14 @@ while :; do __AlpineArch=loongarch64 __QEMUArch=loongarch64 __UbuntuArch=loong64 - __UbuntuSuites=unreleased __LLDB_Package="liblldb-19-dev" ;; riscv64) __BuildArch=riscv64 __AlpineArch=riscv64 - __AlpinePackages="${__AlpinePackages// lldb-dev/}" __QEMUArch=riscv64 __UbuntuArch=riscv64 - __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" - unset __LLDB_Package + __LLDB_Package="liblldb-19-dev" ;; ppc64le) __BuildArch=ppc64le @@ -284,6 +288,10 @@ while :; do __CodeName=noble __LLDB_Package="liblldb-19-dev" ;; + resolute) # Ubuntu 26.04 + __CodeName=resolute + __LLDB_Package="liblldb-21-dev" + ;; stretch) # Debian 9 __CodeName=stretch __LLDB_Package="liblldb-6.0-dev" @@ -324,7 +332,7 @@ while :; do # Debian-Ports architectures need different values case "$__UbuntuArch" in - amd64|arm64|armhf|i386|mips64el|ppc64el|riscv64|s390x) + amd64|arm64|armhf|i386|mips64el|ppc64el|riscv64|loong64|s390x) __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then @@ -358,20 +366,29 @@ while :; do __AlpineVersion="$__AlpineMajorVersion.$__AlpineMinorVersion" fi ;; - freebsd13) + freebsd14) __CodeName=freebsd __SkipUnmount=1 ;; - freebsd14) + freebsd15) __CodeName=freebsd - __FreeBSDBase="14.3-RELEASE" - __FreeBSDABI="14" + __FreeBSDBase="15.1-RELEASE" + __FreeBSDABI="15" __SkipUnmount=1 ;; openbsd) __CodeName=openbsd __SkipUnmount=1 ;; + openbsd7.8) + __CodeName=openbsd + __SkipUnmount=1 + ;; + openbsd7.9) + __CodeName=openbsd + __OpenBSDVersion="7.9" + __SkipUnmount=1 + ;; illumos) __CodeName=illumos __SkipUnmount=1 @@ -396,6 +413,31 @@ while :; do --use-mirror) __UseMirror=1 ;; + --ubuntu-repo|-ubuntu-repo) + shift + if [[ "$#" -le 0 ]]; then + echo "ERROR: --ubuntu-repo requires a URL argument." + usage + fi + __UbuntuRepoOverride="$1" + ;; + --debian-repo|-debian-repo) + shift + if [[ "$#" -le 0 ]]; then + echo "ERROR: --debian-repo requires a URL argument." + usage + fi + __DebianRepoOverride="$1" + ;; + --alpine-repo|-alpine-repo) + shift + if [[ "$#" -le 0 ]]; then + echo "ERROR: --alpine-repo requires a URL argument." + usage + fi + __AlpineRepoOverride="$1" + ;; + # Removed duplicate/invalid option handling block (was breaking case statement parsing). --use-jobs) shift MAXJOBS=$1 @@ -421,9 +463,12 @@ case "$__AlpineVersion" in elif [[ "$__AlpineArch" == "x86" ]]; then __AlpineVersion=3.17 # minimum version that supports lldb-dev __AlpinePackages+=" llvm15-libs" - elif [[ "$__AlpineArch" == "riscv64" || "$__AlpineArch" == "loongarch64" ]]; then + elif [[ "$__AlpineArch" == "loongarch64" ]]; then __AlpineVersion=3.21 # minimum version that supports lldb-dev __AlpinePackages+=" llvm19-libs" + elif [[ "$__AlpineArch" == "riscv64" ]]; then + __AlpineVersion=3.22 # lldb-dev requires 3.21+, but 3.22+ provides the newer linux-headers needed for RISC-V extension probes + __AlpinePackages+=" llvm20-libs" elif [[ -n "$__AlpineMajorVersion" ]]; then # use whichever alpine version is provided and select the latest toolchain libs __AlpineLlvmLibsLookup=1 @@ -445,6 +490,12 @@ if [[ -z "$__UbuntuRepo" ]]; then __UbuntuRepo="https://ports.ubuntu.com/" fi +if [[ -n "$__UbuntuRepoOverride" && "$__KeyringFile" == *ubuntu* ]]; then + __UbuntuRepo="$__UbuntuRepoOverride" +elif [[ -n "$__DebianRepoOverride" && "$__KeyringFile" == *debian* ]]; then + __UbuntuRepo="$__DebianRepoOverride" +fi + if [[ -n "$__LLVM_MajorVersion" ]]; then __UbuntuPackages+=" libclang-common-${__LLVM_MajorVersion}${__LLVM_MinorVersion:+.$__LLVM_MinorVersion}-dev" fi @@ -481,26 +532,33 @@ ensureDownloadTool() } if [[ "$__CodeName" == "alpine" ]]; then - __ApkToolsVersion=2.12.11 + __ApkToolsVersion=2.14.4-r1 __ApkToolsDir="$(mktemp -d)" __ApkKeysDir="$(mktemp -d)" arch="$(uname -m)" + __AlpineRepo="${__AlpineRepoOverride:-https://dl-cdn.alpinelinux.org/alpine}" ensureDownloadTool + __ApkToolsPackage="$__ApkToolsDir/apk-tools-static.apk" + __ApkToolsUrl="$__AlpineRepo/v3.20/main/$arch/apk-tools-static-$__ApkToolsVersion.apk" if [[ "$__hasWget" == 1 ]]; then - wget -P "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static" + wget -O "$__ApkToolsPackage" "$__ApkToolsUrl" else - curl -SLO --create-dirs --output-dir "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static" + curl -fSL -o "$__ApkToolsPackage" "$__ApkToolsUrl" fi + if [[ "$arch" == "x86_64" ]]; then - __ApkToolsSHA512SUM="53e57b49230da07ef44ee0765b9592580308c407a8d4da7125550957bb72cb59638e04f8892a18b584451c8d841d1c7cb0f0ab680cc323a3015776affaa3be33" + __ApkToolsSHA512SUM="b1b3cc382aa0ec26a2c24b742701a1f9885d0678365f9aea15d3d005926b06ecc802659cec8a7deba2717af99c19c708a17c23e1f0f07742268ee5be5400eb9e" elif [[ "$arch" == "aarch64" ]]; then - __ApkToolsSHA512SUM="9e2b37ecb2b56c05dad23d379be84fd494c14bd730b620d0d576bda760588e1f2f59a7fcb2f2080577e0085f23a0ca8eadd993b4e61c2ab29549fdb71969afd0" + __ApkToolsSHA512SUM="61f9a636c5ac4e96e7a3f69fd65e60fc57b3ec8b23619c4df86f59b89e71d1309b3e406388945bdf0dd9168dac22df376943a70ff3efa179e5687e586f825fb0" else - echo "WARNING: add missing hash for your host architecture. To find the value, use: 'find /tmp -name apk.static -exec sha512sum {} \;'" + >&2 echo "ERROR: Unsupported apk-tools-static host architecture '$arch'." + exit 1 fi - echo "$__ApkToolsSHA512SUM $__ApkToolsDir/apk.static" | sha512sum -c + echo "$__ApkToolsSHA512SUM $__ApkToolsPackage" | sha512sum -c + tar -xzf "$__ApkToolsPackage" -C "$__ApkToolsDir" --strip-components=1 sbin/apk.static + rm "$__ApkToolsPackage" chmod +x "$__ApkToolsDir/apk.static" if [[ "$__AlpineVersion" == "edge" ]]; then @@ -529,15 +587,15 @@ if [[ "$__CodeName" == "alpine" ]]; then # initialize DB # shellcheck disable=SC2086 "$__ApkToolsDir/apk.static" \ - -X "https://dl-cdn.alpinelinux.org/alpine/$version/main" \ - -X "https://dl-cdn.alpinelinux.org/alpine/$version/community" \ + -X "$__AlpineRepo/$version/main" \ + -X "$__AlpineRepo/$version/community" \ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" --initdb add if [[ "$__AlpineLlvmLibsLookup" == 1 ]]; then # shellcheck disable=SC2086 __AlpinePackages+=" $("$__ApkToolsDir/apk.static" \ - -X "https://dl-cdn.alpinelinux.org/alpine/$version/main" \ - -X "https://dl-cdn.alpinelinux.org/alpine/$version/community" \ + -X "$__AlpineRepo/$version/main" \ + -X "$__AlpineRepo/$version/community" \ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \ search 'llvm*-libs' | grep -E '^llvm' | sort | tail -1 | sed 's/-[^-]*//2g')" fi @@ -545,8 +603,8 @@ if [[ "$__CodeName" == "alpine" ]]; then # install all packages in one go # shellcheck disable=SC2086 "$__ApkToolsDir/apk.static" \ - -X "https://dl-cdn.alpinelinux.org/alpine/$version/main" \ - -X "https://dl-cdn.alpinelinux.org/alpine/$version/community" \ + -X "$__AlpineRepo/$version/main" \ + -X "$__AlpineRepo/$version/community" \ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" $__NoEmulationArg \ add $__AlpinePackages diff --git a/eng/common/cross/install-debs.py b/eng/common/cross/install-debs.py index 20ca770a1e..1d1dfabf7d 100644 --- a/eng/common/cross/install-debs.py +++ b/eng/common/cross/install-debs.py @@ -121,10 +121,14 @@ async def fetch_release_file(session, mirror, suite, keyring): await download_file(session, release_gpg_url, release_gpg_file.name) print("Verifying signature of Release with Release.gpg.") - verify_command = ["gpg"] + # Use gpgv rather than gpg for verification. gpgv verifies a detached + # signature against a fixed keyring without involving gpg-agent or + # keyboxd, which makes it robust on hosts running GnuPG 2.4+ (e.g. Azure + # Linux) where "gpg --keyring" routes through keyboxd and can fail. + verify_command = ["gpgv"] if keyring: verify_command += ["--keyring", keyring] - verify_command += ["--verify", release_gpg_file.name, release_file.name] + verify_command += [release_gpg_file.name, release_file.name] result = subprocess.run(verify_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if result.returncode != 0: diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 99d6dfe82d..70b71395e3 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -59,9 +59,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu") endif() elseif(FREEBSD) - set(triple "aarch64-unknown-freebsd12") + set(TOOLCHAIN "aarch64-unknown-freebsd14") elseif(OPENBSD) - set(triple "aarch64-unknown-openbsd") + set(TOOLCHAIN "aarch64-unknown-openbsd") endif() elseif(TARGET_ARCH_NAME STREQUAL "armel") set(CMAKE_SYSTEM_PROCESSOR armv7l) @@ -87,6 +87,8 @@ elseif(TARGET_ARCH_NAME STREQUAL "ppc64le") set(CMAKE_SYSTEM_PROCESSOR ppc64le) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/powerpc64le-alpine-linux-musl) set(TOOLCHAIN "powerpc64le-alpine-linux-musl") + elseif(FREEBSD) + set(TOOLCHAIN "powerpc64le-unknown-freebsd14") else() set(TOOLCHAIN "powerpc64le-linux-gnu") endif() @@ -117,9 +119,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64") set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu") endif() elseif(FREEBSD) - set(triple "x86_64-unknown-freebsd12") + set(TOOLCHAIN "x86_64-unknown-freebsd14") elseif(OPENBSD) - set(triple "x86_64-unknown-openbsd") + set(TOOLCHAIN "x86_64-unknown-openbsd") elseif(ILLUMOS) set(TOOLCHAIN "x86_64-illumos") elseif(HAIKU) @@ -160,8 +162,6 @@ if(TIZEN) find_toolchain_dir("${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") endif() - message(STATUS "TIZEN_TOOLCHAIN_PATH set to: ${TIZEN_TOOLCHAIN_PATH}") - include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++) include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++/${TIZEN_TOOLCHAIN}) endif() @@ -206,9 +206,9 @@ if(ANDROID) include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake) elseif(FREEBSD OR OPENBSD) # we cross-compile by instructing clang - set(CMAKE_C_COMPILER_TARGET ${triple}) - set(CMAKE_CXX_COMPILER_TARGET ${triple}) - set(CMAKE_ASM_COMPILER_TARGET ${triple}) + set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) + set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) + set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1 index 50ae627376..b6d45f2bdc 100644 --- a/eng/common/dotnet-install.ps1 +++ b/eng/common/dotnet-install.ps1 @@ -4,13 +4,16 @@ Param( [string] $architecture = '', [string] $version = 'Latest', [string] $runtime = 'dotnet', + [string] $dotnetPath = '', [string] $RuntimeSourceFeed = '', [string] $RuntimeSourceFeedKey = '' ) . $PSScriptRoot\tools.ps1 -if (-not [string]::IsNullOrEmpty($env:DOTNET_GLOBAL_INSTALL_DIR)) { +if (-not [string]::IsNullOrEmpty($dotnetPath)) { + $dotnetRoot = $dotnetPath +} elseif (-not [string]::IsNullOrEmpty($env:DOTNET_GLOBAL_INSTALL_DIR)) { $dotnetRoot = $env:DOTNET_GLOBAL_INSTALL_DIR } else { $dotnetRoot = Join-Path $RepoRoot '.dotnet' diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index 1cb3f5abac..58a7e6f384 100644 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -16,6 +16,7 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" version='Latest' architecture='' runtime='dotnet' +dotnetPath='' runtimeSourceFeed='' runtimeSourceFeedKey='' while [[ $# -gt 0 ]]; do @@ -33,6 +34,10 @@ while [[ $# -gt 0 ]]; do shift runtime="$1" ;; + -dotnetpath) + shift + dotnetPath="$1" + ;; -runtimesourcefeed) shift runtimeSourceFeed="$1" @@ -80,7 +85,9 @@ case $cpuname in ;; esac -if [[ -n "${DOTNET_GLOBAL_INSTALL_DIR:-}" ]]; then +if [[ -n "${dotnetPath:-}" ]]; then + dotnetRoot="$dotnetPath" +elif [[ -n "${DOTNET_GLOBAL_INSTALL_DIR:-}" ]]; then dotnetRoot="$DOTNET_GLOBAL_INSTALL_DIR" else dotnetRoot="${repo_root}.dotnet" diff --git a/eng/common/dotnet.ps1 b/eng/common/dotnet.ps1 index 45e5676c9e..ce4ea40730 100644 --- a/eng/common/dotnet.ps1 +++ b/eng/common/dotnet.ps1 @@ -8,4 +8,5 @@ $dotnetRoot = InitializeDotNetCli -install:$true if ($args.count -gt 0) { $env:DOTNET_NOLOGO=1 & "$dotnetRoot\dotnet.exe" $args + ExitWithExitCode $LASTEXITCODE } diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh index 3e6a8d6acf..4de1a04fee 100644 --- a/eng/common/init-tools-native.sh +++ b/eng/common/init-tools-native.sh @@ -60,8 +60,8 @@ while (($# > 0)); do echo " - (default) %USERPROFILE%/.netcoreeng/native" echo "" echo " --clean Switch specifying not to install anything, but cleanup native asset folders" - echo " --donotabortonfailure Switch specifiying whether to abort native tools installation on failure" - echo " --donotdisplaywarnings Switch specifiying whether to display warnings during native tools installation on failure" + echo " --donotabortonfailure Switch specifying whether to abort native tools installation on failure" + echo " --donotdisplaywarnings Switch specifying whether to display warnings during native tools installation on failure" echo " --force Clean and then install tools" echo " --help Print help and exit" echo "" @@ -83,7 +83,7 @@ function ReadGlobalJsonNativeTools { # KEY="" VALUE="" # followed by a null byte. # - # bash: read line with null byte delimeter and push to array (for later `eval`uation). + # bash: read line with null byte delimiter and push to array (for later `eval`uation). while IFS= read -rd '' line; do native_assets+=("$line") diff --git a/eng/common/loc/P22DotNetHtmlLocalization.lss b/eng/common/loc/P22DotNetHtmlLocalization.lss index 5d892d6193..c810350e58 100644 --- a/eng/common/loc/P22DotNetHtmlLocalization.lss +++ b/eng/common/loc/P22DotNetHtmlLocalization.lss @@ -16,7 +16,7 @@ -