diff --git a/source/Calamari.AzureResourceGroup/AzureResourceGroupOperator.cs b/source/Calamari.AzureResourceGroup/AzureResourceGroupOperator.cs index 5504cd868..fab8c85bc 100644 --- a/source/Calamari.AzureResourceGroup/AzureResourceGroupOperator.cs +++ b/source/Calamari.AzureResourceGroup/AzureResourceGroupOperator.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -9,7 +11,6 @@ using Azure.ResourceManager.Resources.Models; using Calamari.Common.Plumbing.Logging; using Calamari.Common.Plumbing.Variables; -using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Polly; using Polly.Timeout; @@ -48,19 +49,27 @@ public async Task> CreateDeployment(Resource } } - public async Task PollForCompletionWithTimeout(ArmOperation deploymentOperation, IVariables variables) + public async Task PollForCompletionWithTimeout(ArmOperation deploymentOperation, + ResourceGroupResource resourceGroupResource, + string deploymentName, + IVariables variables) { var pollingTimeout = GetPollingTimeout(variables); var asyncResourceGroupPollingTimeoutPolicy = Policy.TimeoutAsync(pollingTimeout, TimeoutStrategy.Optimistic); - await asyncResourceGroupPollingTimeoutPolicy.ExecuteAsync(ct => Poll(deploymentOperation, ct), CancellationToken.None); + await asyncResourceGroupPollingTimeoutPolicy.ExecuteAsync(ct => Poll(deploymentOperation, resourceGroupResource, deploymentName, ct), CancellationToken.None); } - public async Task PollForCompletion(ArmOperation deploymentOperation) + public async Task PollForCompletion(ArmOperation deploymentOperation, + ResourceGroupResource resourceGroupResource, + string deploymentName) { - await Poll(deploymentOperation, CancellationToken.None); + await Poll(deploymentOperation, resourceGroupResource, deploymentName, CancellationToken.None); } - async Task Poll(ArmOperation deploymentOperation, CancellationToken cancellationToken) + async Task Poll(ArmOperation deploymentOperation, + ResourceGroupResource resourceGroupResource, + string deploymentName, + CancellationToken cancellationToken) { log.Info("Polling for deployment completion..."); try @@ -69,9 +78,15 @@ async Task Poll(ArmOperation deploymentOperation, Cancell var response = await deploymentOperation.WaitForCompletionAsync(delayStrategy, cancellationToken); log.Info($"Deployment completed with status: {response.Value?.Data.Properties?.ProvisioningState}"); } - catch + catch (RequestFailedException ex) + { + var enhancedMessage = await TryEnhanceDeploymentError(resourceGroupResource, deploymentName, ex); + log.Error(enhancedMessage); + throw; + } + catch (Exception ex) { - log.Error("Error polling for deployment completion"); + log.Error($"Error polling for deployment completion: {ex.Message}"); throw; } } @@ -98,13 +113,118 @@ async Task LogOperationResults(ArmOperation operation) sb.AppendLine($"Status: {properties.StatusCode}"); sb.AppendLine($"Provisioning State: {properties.ProvisioningState}"); if (properties.StatusMessage != null) - sb.AppendLine($"Status Message: {JsonConvert.SerializeObject(properties.StatusMessage)}"); + sb.AppendLine($"Status Message: {FormatStatusMessage(properties.StatusMessage)}"); sb.Append(" \n"); } log.Info(sb.ToString()); } + async Task TryEnhanceDeploymentError(ResourceGroupResource resourceGroupResource, + string deploymentName, + RequestFailedException originalException) + { + var baseMessage = $"Error polling for deployment completion: {originalException.Message}"; + try + { + log.Verbose($"Attempting to retrieve detailed operation information for failed deployment '{deploymentName}'..."); + + ArmDeploymentResource? deploymentResource = null; + try + { + var deploymentResponse = await resourceGroupResource.GetArmDeploymentAsync(deploymentName); + if (deploymentResponse.HasValue) + deploymentResource = deploymentResponse.Value; + } + catch (Exception ex) + { + log.Verbose($"Could not retrieve deployment resource for error detail: {ex.Message}"); + } + + if (deploymentResource == null) + return baseMessage; + + var failedOperations = new List(); + var totalOperations = 0; + + await foreach (var op in deploymentResource.GetDeploymentOperationsAsync()) + { + totalOperations++; + var properties = op.Properties; + + if (properties?.ProvisioningState == "Failed") + { + var resourceName = properties.TargetResource?.ResourceName ?? "Unknown Resource"; + var resourceType = properties.TargetResource?.ResourceType ?? "Unknown Type"; + + var failureDetail = $"\n [FAILED] {resourceType} '{resourceName}'"; + + if (properties.StatusMessage != null) + { + var errorInfo = ExtractAzureErrorInfo(properties.StatusMessage); + if (!string.IsNullOrWhiteSpace(errorInfo)) + failureDetail += $"\n Error: {errorInfo}"; + } + + if (properties.Timestamp.HasValue) + failureDetail += $"\n Failed at: {properties.Timestamp.Value.ToLocalTime():yyyy-MM-dd HH:mm:ss}"; + + failedOperations.Add(failureDetail); + } + } + + log.Verbose($"Found {totalOperations} total operations, {failedOperations.Count} failed"); + + if (failedOperations.Any()) + { + return $"{baseMessage}\n\n" + + $"FAILED AZURE RESOURCES ({failedOperations.Count} of {totalOperations} operations failed):" + + string.Join("", failedOperations) + + "\n\nFor full details check Azure Portal > Resource Groups > Deployments, " + + "or see https://aka.ms/arm-deployment-operations for troubleshooting guidance."; + } + + if (totalOperations > 0) + { + return $"{baseMessage}\n\n" + + $"Found {totalOperations} deployment operations but none were marked as failed. " + + "Check the Azure Portal for detailed deployment status."; + } + + return baseMessage; + } + catch (Exception enhancementEx) + { + log.Verbose($"Failed to retrieve detailed deployment error information: {enhancementEx.Message}"); + return baseMessage; + } + } + + static string ExtractAzureErrorInfo(StatusMessage statusMessage) + { + var error = statusMessage.Error; + if (error == null) + return string.Empty; + + if (!string.IsNullOrWhiteSpace(error.Code) && !string.IsNullOrWhiteSpace(error.Message)) + return $"[{error.Code}] {error.Message}"; + if (!string.IsNullOrWhiteSpace(error.Message)) + return error.Message; + if (!string.IsNullOrWhiteSpace(error.Code)) + return error.Code; + + return string.Empty; + } + + static string FormatStatusMessage(StatusMessage statusMessage) + { + var errorInfo = ExtractAzureErrorInfo(statusMessage); + if (!string.IsNullOrWhiteSpace(errorInfo)) + return errorInfo; + + return statusMessage.ToString() ?? string.Empty; + } + void CaptureOutputs(string? outputsJson, IVariables variables) { if (string.IsNullOrWhiteSpace(outputsJson)) diff --git a/source/Calamari.AzureResourceGroup/Bicep/DeployBicepTemplateBehaviour.cs b/source/Calamari.AzureResourceGroup/Bicep/DeployBicepTemplateBehaviour.cs index e5c4da7f7..2be3e5cea 100644 --- a/source/Calamari.AzureResourceGroup/Bicep/DeployBicepTemplateBehaviour.cs +++ b/source/Calamari.AzureResourceGroup/Bicep/DeployBicepTemplateBehaviour.cs @@ -50,7 +50,7 @@ public async Task Execute(RunningDeployment context) deploymentMode, template, parameters); - await resourceGroupOperator.PollForCompletion(deploymentOperation); + await resourceGroupOperator.PollForCompletion(deploymentOperation, resourceGroup, armDeploymentName); await resourceGroupOperator.FinalizeDeployment(deploymentOperation, context.Variables); } diff --git a/source/Calamari.AzureResourceGroup/DeployAzureResourceGroupBehaviour.cs b/source/Calamari.AzureResourceGroup/DeployAzureResourceGroupBehaviour.cs index 21a2423b9..be0f6d0b5 100644 --- a/source/Calamari.AzureResourceGroup/DeployAzureResourceGroupBehaviour.cs +++ b/source/Calamari.AzureResourceGroup/DeployAzureResourceGroupBehaviour.cs @@ -71,7 +71,7 @@ public async Task Execute(RunningDeployment context) deploymentMode, template, parameters); - await azureResourceGroupOperator.PollForCompletionWithTimeout(deploymentOperation, variables); + await azureResourceGroupOperator.PollForCompletionWithTimeout(deploymentOperation, resourceGroupResource, deploymentName, variables); await azureResourceGroupOperator.FinalizeDeployment(deploymentOperation, variables); } } \ No newline at end of file