Improve PR failure annotations

eng/pipelines/templates/jobs/analyze.yml

@@ -38,6 +38,12 @@ jobs:
 
   - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
 
+  - task: PowerShell@2
+    displayName: Validate pipeline diagnostic parsers
+    inputs:
+      pwsh: true
+      filePath: $(Build.SourcesDirectory)/eng/scripts/tests/Test-Diagnostics.ps1
+
   # Always skip audit. Audit requires contact with restricted endpoints and is
   # excluded from use in internal builds. Instead, teams must check on the state
   # of the Audit in GitHub Actions.
@@ -53,6 +59,13 @@ jobs:
         -Audit:$false
         -Deny
 
-  - template: /eng/common/pipelines/templates/steps/check-spelling.yml
-    parameters:
-      ContinueOnError: false
+  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+    - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml
+
+    - task: PowerShell@2
+      displayName: Check spelling (cspell)
+      condition: and(succeeded(), ne(variables['Skip.SpellCheck'],'true'))
+      inputs:
+        pwsh: true
+        filePath: $(Build.SourcesDirectory)/eng/scripts/Check-Spelling.ps1
+        arguments: -ExitWithError

eng/scripts/Analyze-Code.ps1

@@ -63,31 +63,62 @@ $packagesToAnalyze = Get-CargoSelectedPackages `
   -PackageInfoDirectory $packageInfoPath
 $workspaceManifestPath = [System.IO.Path]::Combine($RepoRoot, 'Cargo.toml')
 $packageArgs = if ($PackageName -or $ManifestDir) {
-  '--package ' + ($packagesToAnalyze.name -join ' --package ')
+  @($packagesToAnalyze.name | ForEach-Object { '--package'; $_ })
 }
+$packageArgsString = $packageArgs -join ' '
 
 if ($Audit) {
   Invoke-LoggedCommand "cargo audit" -GroupOutput
 }
 
-Invoke-LoggedCommand "cargo check --manifest-path sdk/core/azure_core/Cargo.toml $packageArgs --all-features --all-targets --keep-going" -GroupOutput
+[void](Invoke-CargoCommandWithDiagnostics `
+  -ArgumentList (@(
+    'check',
+    '--manifest-path',
+    'sdk/core/azure_core/Cargo.toml'
+  ) + $packageArgs + @(
+    '--all-features',
+    '--all-targets',
+    '--keep-going'
+  )) `
+  -GroupOutput)
 
 if ($packageArgs) {
-  Invoke-LoggedCommand "cargo fmt --manifest-path '$workspaceManifestPath' $packageArgs -- --check" -GroupOutput
+  Invoke-LoggedCommand "cargo fmt --manifest-path '$workspaceManifestPath' $packageArgsString -- --check" -GroupOutput
 }
 else {
   Invoke-LoggedCommand "cargo fmt --manifest-path '$workspaceManifestPath' --all -- --check" -GroupOutput
 }
 
 Invoke-LoggedCommand "taplo format --check"
 
-Invoke-LoggedCommand "cargo clippy --manifest-path '$workspaceManifestPath' $packageArgs --all-features --all-targets --keep-going --no-deps" -GroupOutput
+[void](Invoke-CargoCommandWithDiagnostics `
+  -ArgumentList (@(
+    'clippy',
+    '--manifest-path',
+    $workspaceManifestPath
+  ) + $packageArgs + @(
+    '--all-features',
+    '--all-targets',
+    '--keep-going',
+    '--no-deps'
+  )) `
+  -GroupOutput)
 
 if ($Deny) {
   Invoke-LoggedCommand "cargo deny --manifest-path '$workspaceManifestPath' --all-features check bans licenses sources" -GroupOutput
 }
 
-Invoke-LoggedCommand "cargo doc --manifest-path '$workspaceManifestPath' $packageArgs --no-deps --all-features" -GroupOutput
+[void](Invoke-CargoCommandWithDiagnostics `
+  -ArgumentList (@(
+    'doc',
+    '--manifest-path',
+    $workspaceManifestPath
+  ) + $packageArgs + @(
+    '--no-deps',
+    '--all-features'
+  )) `
+  -GroupOutput)
 
 # Verify package dependencies and keywords
 $verifyDependenciesScript = ([System.IO.Path]::Combine($RepoRoot, 'eng', 'scripts', 'verify-dependencies.rs'))

eng/scripts/Build-Crates.ps1

@@ -29,5 +29,13 @@ else {
 }
 
 foreach ($path in $manifestPath) {
-  Invoke-LoggedCommand "cargo build --manifest-path '$path' --keep-going --all-features" -GroupOutput
+  [void](Invoke-CargoCommandWithDiagnostics `
+    -ArgumentList @(
+      'build',
+      '--manifest-path',
+      $path,
+      '--keep-going',
+      '--all-features'
+    ) `
+    -GroupOutput)
 }

eng/scripts/Check-Spelling.ps1

@@ -0,0 +1,85 @@
+#!/usr/bin/env pwsh
+
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+#Requires -Version 7.0
+[CmdletBinding()]
+param(
+  [string]$CspellConfigPath = ([System.IO.Path]::Combine($PSScriptRoot, '..', '..', '.vscode', 'cspell.json')),
+  [string]$SpellCheckRoot = ([System.IO.Path]::Combine($PSScriptRoot, '..', '..')),
+  [switch]$ExitWithError,
+  [string]$SourceCommittish = $env:SYSTEM_PULLREQUEST_SOURCECOMMITID,
+  [string]$TargetCommittish = ("origin/$($env:SYSTEM_PULLREQUEST_TARGETBRANCH)" -replace 'refs/heads/')
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+. ([System.IO.Path]::Combine($PSScriptRoot, '..', 'common', 'scripts', 'common.ps1'))
+. ([System.IO.Path]::Combine($PSScriptRoot, 'shared', 'common.ps1'))
+
+if (!(Test-Path -Path $CspellConfigPath -PathType Leaf)) {
+  Write-PipelineIssue -Type error -Message "Could not locate CSpell config file '$CspellConfigPath'."
+  exit 1
+}
+
+$getChangedFilesScript = ([System.IO.Path]::Combine($PSScriptRoot, '..', 'common', 'scripts', 'get-changedfiles.ps1'))
+$invokeCspellScript = ([System.IO.Path]::Combine($PSScriptRoot, '..', 'common', 'spelling', 'Invoke-Cspell.ps1'))
+$changedFiles = @(
+  & $getChangedFilesScript `
+    -SourceCommittish $SourceCommittish `
+    -TargetCommittish $TargetCommittish |
+    ForEach-Object { Resolve-Path -Path $_ }
+)
+
+Write-Host "Git detected $($changedFiles.Count) changed file(s). Files checked by CSpell may exclude files according to cspell.json."
+if ($changedFiles.Count -eq 0) {
+  Write-Host 'No changes detected.'
+  exit 0
+}
+
+$spellingOutput = @(
+  & $invokeCspellScript `
+    -CSpellConfigPath $CspellConfigPath `
+    -SpellCheckRoot $SpellCheckRoot `
+    -FileList $changedFiles.Path
+)
+$cspellExitCode = $LASTEXITCODE
+$issueBudget = New-PipelineIssueBudget
+$parsedIssues = 0
+
+foreach ($line in $spellingOutput) {
+  $text = "$line"
+  Write-Host $text
+  $issue = ConvertFrom-CSpellIssue $text
+  if ($issue) {
+    Write-BudgetedPipelineIssue `
+      -Budget $issueBudget `
+      -Type $(if ($ExitWithError) { 'error' } else { 'warning' }) `
+      -Message $issue.Message `
+      -SourcePath $issue.SourcePath `
+      -LineNumber $issue.LineNumber `
+      -ColumnNumber $issue.ColumnNumber `
+      -Code 'cspell'
+    $parsedIssues++
+  }
+}
+
+Complete-PipelineIssueBudget $issueBudget
+
+if ($parsedIssues -gt 0) {
+  Write-Host 'Spelling errors detected. To correct false positives or learn about spell checking, see https://aka.ms/azsdk/engsys/spellcheck.'
+  if ($ExitWithError) {
+    exit 1
+  }
+}
+elseif ($cspellExitCode -ne 0) {
+  Write-PipelineIssue -Type error -Message "CSpell exited with code $cspellExitCode. This may indicate a configuration or tool failure."
+  exit $cspellExitCode
+}
+else {
+  Write-Host 'No spelling errors detected.'
+}
+
+exit 0

eng/scripts/Test-Packages.ps1

@@ -22,36 +22,43 @@ $cargoFeatureArgs = if ($FeatureSet -eq 'All') { @('--all-features') } else { @(
 # Helper function to run cargo test, capturing JSON output only when the active
 # toolchain supports `--format json -Z unstable-options`.
 function Invoke-CargoTest (
-  [string]$TestParams,
+  [string[]]$TestParams,
   [string]$PackageName,
   [string]$ManifestPath,
-  [string]$OutputFile
+  [string]$OutputFile,
+  [switch]$DisableJsonOutput
 ) {
   Write-Host "Running tests for $PackageName"
-  $commandParts = @('cargo', 'test', $TestParams, '--manifest-path', $ManifestPath) + $cargoFeatureArgs + @('--no-fail-fast')
-  $command = $commandParts -join ' '
+  $commandArgs = @('test') + $TestParams + @('--manifest-path', $ManifestPath) + $cargoFeatureArgs + @('--no-fail-fast')
+  $captureJson = $usesJsonTestOutput -and !$DisableJsonOutput
 
-  if ($usesJsonTestOutput) {
-    $result = Invoke-LoggedCommand `
-      "$command -- --format json -Z unstable-options" `
+  if ($captureJson) {
+    $result = Invoke-CargoCommandWithDiagnostics `
+      -ArgumentList ($commandArgs + @('--', '--format', 'json', '-Z', 'unstable-options')) `
       -GroupOutput `
-      -DoNotExitOnFailedExitCode
+      -DoNotExitOnFailedExitCode `
+      -ParseJsonTestOutput `
+      -TestOutputFile $OutputFile
 
     LogGroupStart 'Test result JSON'
-    $result | Tee-Object -FilePath $OutputFile
+    Get-Content $OutputFile | Write-Host
     LogGroupEnd
   }
   else {
-    Invoke-LoggedCommand $command -GroupOutput -DoNotExitOnFailedExitCode
+    $result = Invoke-CargoCommandWithDiagnostics `
+      -ArgumentList $commandArgs `
+      -GroupOutput `
+      -DoNotExitOnFailedExitCode `
+      -ParseHumanTestOutput
   }
 
-  if ($LASTEXITCODE) {
+  if ($result.ExitCode) {
     $message = "Tests failed for $PackageName."
-    if ($usesJsonTestOutput) {
+    if ($captureJson) {
       $message += " For more information see the pipeline Tests tab."
     }
-    LogError $message
-    exit $LASTEXITCODE
+    Write-Host $message
+    exit $result.ExitCode
   }
 }
 
@@ -113,29 +120,33 @@ foreach ($package in $packagesToTest) {
 
   Write-Host "`n`nTesting package: '$($package.Name)'`n"
 
-  $buildCommand = (@('cargo', 'build') + $cargoFeatureArgs + @('--keep-going')) -join ' '
-  Invoke-LoggedCommand $buildCommand -GroupOutput
+  [void](Invoke-CargoCommandWithDiagnostics `
+    -ArgumentList (@('build') + $cargoFeatureArgs + @('--keep-going')) `
+    -GroupOutput)
   Write-Host "`n`n"
 
   $manifestPath = [System.IO.Path]::Combine($packageDirectory, 'Cargo.toml')
   $timestamp = Get-Date -Format "yyyyMMdd-HHmmss-fff"
 
   $docTestOutput = ([System.IO.Path]::Combine($testResultsDir, "$($package.Name)-doctest-$timestamp.json"))
   Invoke-CargoTest `
-    -TestParams "--doc" `
+    -TestParams @('--doc') `
     -PackageName $package.Name `
     -ManifestPath $manifestPath `
     -OutputFile $docTestOutput
 
   $allTargetsOutput = ([System.IO.Path]::Combine($testResultsDir, "$($package.Name)-alltargets-$timestamp.json"))
   Invoke-CargoTest `
-    -TestParams "--lib --bins --tests --examples" `
+    -TestParams @('--lib', '--bins', '--tests', '--examples') `
     -PackageName $package.Name `
     -ManifestPath $manifestPath `
     -OutputFile $allTargetsOutput
 
-  $benchCommand = (@('cargo', 'test', '--benches', '--manifest-path', $manifestPath) + $cargoFeatureArgs + @('--no-fail-fast')) -join ' '
-  Invoke-LoggedCommand $benchCommand -GroupOutput
+  Invoke-CargoTest `
+    -TestParams @('--benches') `
+    -PackageName $package.Name `
+    -ManifestPath $manifestPath `
+    -DisableJsonOutput
 
   $cleanupScript = ([System.IO.Path]::Combine($packageDirectory, 'Test-Cleanup.ps1'))
   if (Test-Path $cleanupScript) {