From 7c70d3aa6584a4f094fced8851b40bd838b634b3 Mon Sep 17 00:00:00 2001 From: Bec Callow Date: Thu, 18 Jun 2026 10:01:09 +1000 Subject: [PATCH 1/4] Handle pending helm updates and installs --- .../HelmUpgradeWithKOSConvention.cs | 69 ++++++++++++++++++- .../Kubernetes/Integration/HelmCli.cs | 48 ++++++++++++- 2 files changed, 113 insertions(+), 4 deletions(-) diff --git a/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs b/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs index 7ff9ee13ad..106d224a12 100644 --- a/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs +++ b/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs @@ -53,14 +53,22 @@ public void Install(RunningDeployment deployment) kubectl.SetKubectl(); + // GetCurrentRevision returns null when the release doesn't exist yet; in that case + // there's nothing to recover from, so we skip the status check entirely. var currentRevisionNumber = helmCli.GetCurrentRevision(releaseName); + if (currentRevisionNumber != null && CheckAndHandleStuckRelease(helmCli, releaseName)) + { + // Re-read revision after recovery so newRevisionNumber reflects the post-rollback state. + // Skipped on the happy path (no recovery ran) since the revision cannot have changed. + currentRevisionNumber = helmCli.GetCurrentRevision(releaseName); + } var newRevisionNumber = (currentRevisionNumber ?? 0) + 1; //This is used to cancel KOS when the helm upgrade has completed //It does not cancel the get manifest var helmInstallCompletedCts = new CancellationTokenSource(); - + //This is used to cancel the get manifest when the helm install fails (and we are still trying to retrieve the manifest) var helmInstallErrorCts = new CancellationTokenSource(); @@ -71,7 +79,7 @@ public void Install(RunningDeployment deployment) valueSourcesParser, helmCli, namespaceResolver); - + executor.ExecuteHelmUpgrade(deployment, releaseName, newRevisionNumber, helmInstallCompletedCts, helmInstallErrorCts); }); @@ -105,5 +113,62 @@ string GetReleaseName(IVariables variables) log.Info($"Using Release Name {releaseName}"); return releaseName; } + + // Returns true if a recovery action was attempted (indicating the revision number may have changed). + bool CheckAndHandleStuckRelease(HelmCli helmCli, string releaseName) + { + var status = helmCli.GetReleaseStatus(releaseName); + + if (status == null) + return false; + + log.Info($"Release {releaseName} current status: {status}"); + + // Handle problematic states that could be left from cancelled deployments + switch (status.ToLowerInvariant()) + { + case "pending-install": + // No prior successful revision exists, so rollback is not possible. Uninstall the + // stuck release so the next upgrade --install can start cleanly. + log.Warn($"Release {releaseName} is stuck in {status} state, likely from a cancelled first install. Uninstalling to recover..."); + try + { + var uninstallResult = helmCli.Uninstall(releaseName); + if (uninstallResult.ExitCode == 0) + log.Info($"Successfully uninstalled stuck release {releaseName}"); + else + log.Warn($"Uninstall had non-zero exit code but continuing: {uninstallResult.ExitCode}"); + } + catch (Exception ex) + { + log.Warn($"Failed to uninstall release {releaseName}: {ex.Message}. Continuing with deployment..."); + } + return true; + + case "pending-upgrade": + log.Warn($"Release {releaseName} is stuck in {status} state, likely from a cancelled deployment. Rolling back to recover..."); + try + { + var rollbackResult = helmCli.Rollback(releaseName); + if (rollbackResult.ExitCode == 0) + log.Info($"Successfully rolled back release {releaseName}"); + else + log.Warn($"Rollback had non-zero exit code but continuing: {rollbackResult.ExitCode}"); + } + catch (Exception ex) + { + log.Warn($"Failed to rollback release {releaseName}: {ex.Message}. Continuing with deployment..."); + } + return true; + + case "failed": + log.Info($"Release {releaseName} is in failed state. Helm upgrade --install should handle this automatically."); + return false; + + default: + log.Verbose($"Release {releaseName} status: {status} - proceeding with deployment"); + return false; + } + } } } \ No newline at end of file diff --git a/source/Calamari/Kubernetes/Integration/HelmCli.cs b/source/Calamari/Kubernetes/Integration/HelmCli.cs index ed66704f48..fccb786b22 100644 --- a/source/Calamari/Kubernetes/Integration/HelmCli.cs +++ b/source/Calamari/Kubernetes/Integration/HelmCli.cs @@ -107,6 +107,51 @@ public SemanticVersion GetParsedExecutableVersion() return metadata.revision; } + public string? GetReleaseStatus(string releaseName) + { + var result = ExecuteCommandAndReturnOutput("status", releaseName, "-o json", NamespaceArg()); + + if (result.Result.ExitCode != 0) + { + // Log any error output so auth/RBAC/network failures are visible rather than silently returning null + var errorOutput = result.Output.MergeInfoLogs(); + if (!string.IsNullOrWhiteSpace(errorOutput)) + log.Verbose($"helm status returned exit code {result.Result.ExitCode}: {errorOutput}"); + return null; + } + + var json = result.Output.MergeInfoLogs(); + var status = JsonConvert.DeserializeAnonymousType(json, + new + { + info = new + { + status = "" + } + }); + + return status?.info?.status; + } + + public CommandResult Rollback(string releaseName, int? revision = null) + { + var args = new List { "rollback", releaseName }; + + if (revision.HasValue) + args.Add(revision.Value.ToString()); + + args.Add(NamespaceArg()); + + var result = ExecuteCommandAndLogOutput(args); + return result; + } + + public CommandResult Uninstall(string releaseName) + { + var args = new List { "uninstall", releaseName, NamespaceArg() }; + return ExecuteCommandAndLogOutput(args); + } + public string GetManifest(string releaseName, int revisionNumber) { var result = ExecuteCommandAndReturnOutput("get", "manifest", releaseName, $"--revision {revisionNumber}", NamespaceArg()); @@ -125,8 +170,7 @@ public CommandResult Upgrade(string releaseName, string packagePath, IEnumerable buildArgs.AddRange(upgradeArgs); buildArgs.Add(NamespaceArg()); - //properly quote the release name and package path (consistent with previous code) - buildArgs.Add($"\"{releaseName}\""); + buildArgs.Add(releaseName); buildArgs.Add($"\"{packagePath}\""); var result = ExecuteCommandAndLogOutput(buildArgs); From f011729ceb6b3a256889fa3c20748b44666214a4 Mon Sep 17 00:00:00 2001 From: Bec Callow Date: Mon, 29 Jun 2026 14:18:03 +1000 Subject: [PATCH 2/4] Add tests --- .../Helm/HelmUpgradeWithKOSConventionTests.cs | 277 ++++++++++++++++++ .../Conventions/Helm/HelmUpgradeExecutor.cs | 28 ++ .../HelmUpgradeWithKOSConvention.cs | 92 ++---- 3 files changed, 324 insertions(+), 73 deletions(-) create mode 100644 source/Calamari.Tests/KubernetesFixtures/Conventions/Helm/HelmUpgradeWithKOSConventionTests.cs diff --git a/source/Calamari.Tests/KubernetesFixtures/Conventions/Helm/HelmUpgradeWithKOSConventionTests.cs b/source/Calamari.Tests/KubernetesFixtures/Conventions/Helm/HelmUpgradeWithKOSConventionTests.cs new file mode 100644 index 0000000000..fba8ea135a --- /dev/null +++ b/source/Calamari.Tests/KubernetesFixtures/Conventions/Helm/HelmUpgradeWithKOSConventionTests.cs @@ -0,0 +1,277 @@ +using System.Collections.Generic; +using System.IO; +using Calamari.Common.Commands; +using Calamari.Common.Features.Packages; +using Calamari.Common.Features.Processes; +using Calamari.Common.Plumbing.Deployment; +using Calamari.Common.Plumbing.FileSystem; +using Calamari.Common.Plumbing.Variables; +using Calamari.Kubernetes; +using Calamari.Kubernetes.Conventions; +using Calamari.Kubernetes.Helm; +using Calamari.Kubernetes.Integration; +using Calamari.Kubernetes.ResourceStatus; +using Calamari.Testing.Helpers; +using Calamari.Tests.Fixtures.Integration.FileSystem; +using FluentAssertions; +using NSubstitute; +using NUnit.Framework; + +namespace Calamari.Tests.KubernetesFixtures.Conventions.Helm +{ + [TestFixture] + public class HelmUpgradeWithKOSConventionTests + { + const string ReleaseName = "my-release"; + + readonly ICalamariFileSystem fileSystem = TestCalamariPhysicalFileSystem.GetPhysicalFileSystem(); + ICommandLineRunner commandLineRunner; + InMemoryLog log; + string tempDirectory; + + string ChartDirectory => Path.Combine(tempDirectory, "chart"); + + [SetUp] + public void SetUp() + { + log = new InMemoryLog(); + commandLineRunner = Substitute.For(); + tempDirectory = fileSystem.CreateTemporaryDirectory(); + + Directory.CreateDirectory(ChartDirectory); + File.WriteAllText(Path.Combine(ChartDirectory, "Chart.yaml"), "apiVersion: v2\nname: test\nversion: 0.1.0"); + + SetupKubectlMocks(); + SetupHelmVersionMock(); + SetupHelmUpgradeMock(); + SetupHelmRollbackMock(); + SetupHelmUninstallMock(); + } + + [TearDown] + public void TearDown() + { + fileSystem.DeleteDirectory(tempDirectory, FailureOptions.IgnoreFailure); + } + + [Test] + public void WhenReleaseIsPendingUpgrade_RollsBackBeforeUpgrade() + { + SetupHelmGetMetadataMock(revision: 2); + SetupHelmStatusMock("pending-upgrade"); + + RunInstall(); + + commandLineRunner.Received().Execute(Arg.Is(i => i.Arguments.Contains("rollback") && i.Arguments.Contains(ReleaseName))); + } + + [Test] + public void WhenReleaseIsPendingInstall_UninstallsBeforeUpgrade() + { + SetupHelmGetMetadataMock(revision: 1); + SetupHelmStatusMock("pending-install"); + + RunInstall(); + + commandLineRunner.Received().Execute(Arg.Is(i => i.Arguments.Contains("uninstall") && i.Arguments.Contains(ReleaseName))); + } + + [Test] + public void WhenReleaseIsDeployed_DoesNotRollbackOrUninstall() + { + SetupHelmGetMetadataMock(revision: 3); + SetupHelmStatusMock("deployed"); + + RunInstall(); + + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("rollback"))); + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("uninstall"))); + } + + [Test] + public void WhenReleaseIsFailed_DoesNotRollbackOrUninstall() + { + SetupHelmGetMetadataMock(revision: 2); + SetupHelmStatusMock("failed"); + + RunInstall(); + + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("rollback"))); + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("uninstall"))); + } + + [Test] + public void WhenReleaseDoesNotExist_DoesNotCheckStatusOrRollbackOrUninstall() + { + SetupHelmGetMetadataToReturnNotFound(); + + RunInstall(); + + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("status") && i.Arguments.Contains(ReleaseName))); + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("rollback"))); + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("uninstall"))); + } + + [Test] + public void WhenUpgradeSucceeds_ChecksStatusButDoesNotRollbackOrUninstall() + { + SetupHelmGetMetadataMock(revision: 3); + SetupHelmStatusMock("deployed"); + + RunInstall(); + + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("rollback"))); + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("uninstall"))); + } + + [Test] + public void WhenRollbackFails_LogsWarningAndContinuesToUpgrade() + { + SetupHelmGetMetadataMock(revision: 2); + SetupHelmStatusMock("pending-upgrade"); + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("rollback"))) + .Returns(new CommandResult("helm rollback", 1)); + + RunInstall(); + + log.MessagesWarnFormatted.Should().Contain(msg => msg.Contains(ReleaseName) && msg.Contains("pending-upgrade")); + log.MessagesWarnFormatted.Should().Contain(msg => msg.Contains("non-zero exit code")); + commandLineRunner.Received().Execute(Arg.Is(i => i.Arguments.Contains("upgrade"))); + } + + [Test] + public void WhenUninstallFails_LogsWarningAndContinuesToUpgrade() + { + SetupHelmGetMetadataMock(revision: 1); + SetupHelmStatusMock("pending-install"); + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("uninstall"))) + .Returns(new CommandResult("helm uninstall", 1)); + + RunInstall(); + + log.MessagesWarnFormatted.Should().Contain(msg => msg.Contains(ReleaseName) && msg.Contains("pending-install")); + log.MessagesWarnFormatted.Should().Contain(msg => msg.Contains("non-zero exit code")); + commandLineRunner.Received().Execute(Arg.Is(i => i.Arguments.Contains("upgrade"))); + } + + void RunInstall() + { + var variables = CreateVariables(); + var deployment = CreateRunningDeployment(variables); + var convention = CreateConvention(deployment); + convention.Install(deployment); + } + + void SetupKubectlMocks() + { + // Return kubectl path from where/which + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("kubectl"))) + .Returns(info => + { + var invocation = (CommandLineInvocation)info[0]; + invocation.AdditionalInvocationOutputSink?.WriteInfo("kubectl"); + return new CommandResult("where kubectl", 0); + }); + + // Return success for kubectl version --client + commandLineRunner.Execute(Arg.Is(i => i.Executable == "kubectl")) + .Returns(new CommandResult("kubectl version", 0)); + } + + void SetupHelmVersionMock() + { + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("version") && i.Arguments.Contains("--client"))) + .Returns(info => + { + var invocation = (CommandLineInvocation)info[0]; + invocation.AdditionalInvocationOutputSink?.WriteInfo("v3.14.0"); + return new CommandResult("helm version", 0); + }); + } + + void SetupHelmGetMetadataMock(int revision) + { + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("get") && i.Arguments.Contains("metadata"))) + .Returns(info => + { + var invocation = (CommandLineInvocation)info[0]; + invocation.AdditionalInvocationOutputSink?.WriteInfo($"{{\"revision\":{revision}}}"); + return new CommandResult("helm get metadata", 0); + }); + } + + void SetupHelmGetMetadataToReturnNotFound() + { + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("get") && i.Arguments.Contains("metadata"))) + .Returns(new CommandResult("helm get metadata", 1)); + } + + void SetupHelmStatusMock(string status) + { + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("status") && i.Arguments.Contains(ReleaseName))) + .Returns(info => + { + var invocation = (CommandLineInvocation)info[0]; + invocation.AdditionalInvocationOutputSink?.WriteInfo($"{{\"info\":{{\"status\":\"{status}\"}}}}"); + return new CommandResult("helm status", 0); + }); + } + + void SetupHelmRollbackMock() + { + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("rollback"))) + .Returns(new CommandResult("helm rollback", 0)); + } + + void SetupHelmUninstallMock() + { + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("uninstall"))) + .Returns(new CommandResult("helm uninstall", 0)); + } + + void SetupHelmUpgradeMock() + { + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("upgrade"))) + .Returns(new CommandResult("helm upgrade", 0)); + } + + CalamariVariables CreateVariables() + { + var variables = new CalamariVariables + { + [SpecialVariables.Helm.ReleaseName] = ReleaseName, + [PackageVariables.Output.InstallationDirectoryPath] = ChartDirectory, + // --dry-run bypasses the manifest reporting path in HelmManifestAndStatusReporter + [SpecialVariables.Helm.AdditionalArguments] = "--dry-run" + }; + return variables; + } + + RunningDeployment CreateRunningDeployment(CalamariVariables variables) + { + return new RunningDeployment(variables, new Dictionary()) + { + CurrentDirectoryProvider = DeploymentWorkingDirectory.StagingDirectory, + StagingDirectory = tempDirectory + }; + } + + HelmUpgradeWithKOSConvention CreateConvention(RunningDeployment deployment) + { + var kubectl = new Kubectl(deployment.Variables, log, commandLineRunner, tempDirectory, deployment.EnvironmentVariables); + var templateValueSourcesParser = new HelmTemplateValueSourcesParser(fileSystem, log); + var namespaceResolver = Substitute.For(); + var statusReporter = Substitute.For(); + var manifestReporter = Substitute.For(); + + return new HelmUpgradeWithKOSConvention(log, + commandLineRunner, + fileSystem, + templateValueSourcesParser, + statusReporter, + manifestReporter, + namespaceResolver, + kubectl); + } + } +} diff --git a/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs b/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs index 0550dc2c3e..a68388d7d8 100644 --- a/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs +++ b/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs @@ -5,6 +5,7 @@ using System.Threading; using Calamari.Common.Commands; using Calamari.Common.Features.Packages; +using Calamari.Common.Features.Processes; using Calamari.Common.FeatureToggles; using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; @@ -70,6 +71,33 @@ public void ExecuteHelmUpgrade(RunningDeployment deployment, installCompletedCts.Cancel(); } + // Checks for a stuck pending release and recovers by uninstalling or rolling back. + // Returns the correct newRevisionNumber to use for the upgrade. + public int RecoverFromPendingRelease(string releaseName, int expectedRevisionNumber) + { + var status = helmCli.GetReleaseStatus(releaseName); + + switch (status?.ToLowerInvariant()) + { + case "pending-install": + log.Warn($"Release {releaseName} is stuck in {status} state, likely from a cancelled first install. Uninstalling to recover..."); + var uninstallResult = helmCli.Uninstall(releaseName); + if (uninstallResult.ExitCode != 0) + log.Warn($"Uninstall returned non-zero exit code {uninstallResult.ExitCode}. Continuing with upgrade..."); + return 1; + + case "pending-upgrade": + log.Warn($"Release {releaseName} is stuck in {status} state, likely from a cancelled deployment. Rolling back to recover..."); + var rollbackResult = helmCli.Rollback(releaseName); + if (rollbackResult.ExitCode != 0) + log.Warn($"Rollback returned non-zero exit code {rollbackResult.ExitCode}. Continuing with upgrade..."); + return expectedRevisionNumber; + + default: + return expectedRevisionNumber; + } + } + void SetAppliedResourcesOutputVariable(RunningDeployment deployment, string releaseName, int revisionNumber) { string manifest = null; diff --git a/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs b/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs index 106d224a12..e7a5a2503e 100644 --- a/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs +++ b/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs @@ -53,17 +53,17 @@ public void Install(RunningDeployment deployment) kubectl.SetKubectl(); - // GetCurrentRevision returns null when the release doesn't exist yet; in that case - // there's nothing to recover from, so we skip the status check entirely. var currentRevisionNumber = helmCli.GetCurrentRevision(releaseName); - if (currentRevisionNumber != null && CheckAndHandleStuckRelease(helmCli, releaseName)) - { - // Re-read revision after recovery so newRevisionNumber reflects the post-rollback state. - // Skipped on the happy path (no recovery ran) since the revision cannot have changed. - currentRevisionNumber = helmCli.GetCurrentRevision(releaseName); - } - var newRevisionNumber = (currentRevisionNumber ?? 0) + 1; + var executor = new HelmUpgradeExecutor(log, fileSystem, valueSourcesParser, helmCli, namespaceResolver); + + var expectedRevisionNumber = (currentRevisionNumber ?? 0) + 1; + + // If a release exists and is stuck in a pending state from a previous cancelled deployment, + // recover before starting the upgrade so both tasks receive the correct revision number. + var newRevisionNumber = currentRevisionNumber != null + ? executor.RecoverFromPendingRelease(releaseName, expectedRevisionNumber) + : expectedRevisionNumber; //This is used to cancel KOS when the helm upgrade has completed //It does not cancel the get manifest @@ -74,12 +74,6 @@ public void Install(RunningDeployment deployment) var helmUpgradeTask = Task.Run(() => { - var executor = new HelmUpgradeExecutor(log, - fileSystem, - valueSourcesParser, - helmCli, - namespaceResolver); - executor.ExecuteHelmUpgrade(deployment, releaseName, newRevisionNumber, helmInstallCompletedCts, helmInstallErrorCts); }); @@ -90,13 +84,21 @@ public void Install(RunningDeployment deployment) await runner.StartBackgroundMonitoringAndReporting(deployment, releaseName, newRevisionNumber, - helmInstallCompletedCts.Token, + helmInstallCompletedCts.Token, helmInstallErrorCts.Token); }, helmInstallCompletedCts.Token); //we run both the helm upgrade and the manifest & status in parallel - Task.WhenAll(helmUpgradeTask, manifestAndStatusCheckTask).GetAwaiter().GetResult(); + // TaskCanceledException from the manifest task is expected when the upgrade completes before + // monitoring starts (helmInstallCompletedCts is already cancelled); treat it as success. + try + { + Task.WhenAll(helmUpgradeTask, manifestAndStatusCheckTask).GetAwaiter().GetResult(); + } + catch (OperationCanceledException) when (helmInstallCompletedCts.IsCancellationRequested && !helmInstallErrorCts.IsCancellationRequested) + { + } } string GetReleaseName(IVariables variables) @@ -114,61 +116,5 @@ string GetReleaseName(IVariables variables) return releaseName; } - // Returns true if a recovery action was attempted (indicating the revision number may have changed). - bool CheckAndHandleStuckRelease(HelmCli helmCli, string releaseName) - { - var status = helmCli.GetReleaseStatus(releaseName); - - if (status == null) - return false; - - log.Info($"Release {releaseName} current status: {status}"); - - // Handle problematic states that could be left from cancelled deployments - switch (status.ToLowerInvariant()) - { - case "pending-install": - // No prior successful revision exists, so rollback is not possible. Uninstall the - // stuck release so the next upgrade --install can start cleanly. - log.Warn($"Release {releaseName} is stuck in {status} state, likely from a cancelled first install. Uninstalling to recover..."); - try - { - var uninstallResult = helmCli.Uninstall(releaseName); - if (uninstallResult.ExitCode == 0) - log.Info($"Successfully uninstalled stuck release {releaseName}"); - else - log.Warn($"Uninstall had non-zero exit code but continuing: {uninstallResult.ExitCode}"); - } - catch (Exception ex) - { - log.Warn($"Failed to uninstall release {releaseName}: {ex.Message}. Continuing with deployment..."); - } - return true; - - case "pending-upgrade": - log.Warn($"Release {releaseName} is stuck in {status} state, likely from a cancelled deployment. Rolling back to recover..."); - try - { - var rollbackResult = helmCli.Rollback(releaseName); - if (rollbackResult.ExitCode == 0) - log.Info($"Successfully rolled back release {releaseName}"); - else - log.Warn($"Rollback had non-zero exit code but continuing: {rollbackResult.ExitCode}"); - } - catch (Exception ex) - { - log.Warn($"Failed to rollback release {releaseName}: {ex.Message}. Continuing with deployment..."); - } - return true; - - case "failed": - log.Info($"Release {releaseName} is in failed state. Helm upgrade --install should handle this automatically."); - return false; - - default: - log.Verbose($"Release {releaseName} status: {status} - proceeding with deployment"); - return false; - } - } } } \ No newline at end of file From 7aa287e0e5d7e639ca3af3321bb5bff06b1ec5d8 Mon Sep 17 00:00:00 2001 From: Bec Callow Date: Tue, 30 Jun 2026 15:55:54 +1000 Subject: [PATCH 3/4] Remove separate release status check --- .../Helm/HelmUpgradeWithKOSConventionTests.cs | 51 ++++++------------- .../Conventions/Helm/HelmUpgradeExecutor.cs | 4 +- .../HelmUpgradeWithKOSConvention.cs | 19 ++----- .../Kubernetes/Integration/HelmCli.cs | 47 +++-------------- 4 files changed, 30 insertions(+), 91 deletions(-) diff --git a/source/Calamari.Tests/KubernetesFixtures/Conventions/Helm/HelmUpgradeWithKOSConventionTests.cs b/source/Calamari.Tests/KubernetesFixtures/Conventions/Helm/HelmUpgradeWithKOSConventionTests.cs index fba8ea135a..f571e8426e 100644 --- a/source/Calamari.Tests/KubernetesFixtures/Conventions/Helm/HelmUpgradeWithKOSConventionTests.cs +++ b/source/Calamari.Tests/KubernetesFixtures/Conventions/Helm/HelmUpgradeWithKOSConventionTests.cs @@ -57,8 +57,7 @@ public void TearDown() [Test] public void WhenReleaseIsPendingUpgrade_RollsBackBeforeUpgrade() { - SetupHelmGetMetadataMock(revision: 2); - SetupHelmStatusMock("pending-upgrade"); + SetupHelmGetMetadataMock(revision: 2, status: "pending-upgrade"); RunInstall(); @@ -68,8 +67,7 @@ public void WhenReleaseIsPendingUpgrade_RollsBackBeforeUpgrade() [Test] public void WhenReleaseIsPendingInstall_UninstallsBeforeUpgrade() { - SetupHelmGetMetadataMock(revision: 1); - SetupHelmStatusMock("pending-install"); + SetupHelmGetMetadataMock(revision: 1, status: "pending-install"); RunInstall(); @@ -79,8 +77,7 @@ public void WhenReleaseIsPendingInstall_UninstallsBeforeUpgrade() [Test] public void WhenReleaseIsDeployed_DoesNotRollbackOrUninstall() { - SetupHelmGetMetadataMock(revision: 3); - SetupHelmStatusMock("deployed"); + SetupHelmGetMetadataMock(revision: 3, status: "deployed"); RunInstall(); @@ -91,8 +88,7 @@ public void WhenReleaseIsDeployed_DoesNotRollbackOrUninstall() [Test] public void WhenReleaseIsFailed_DoesNotRollbackOrUninstall() { - SetupHelmGetMetadataMock(revision: 2); - SetupHelmStatusMock("failed"); + SetupHelmGetMetadataMock(revision: 2, status: "failed"); RunInstall(); @@ -107,16 +103,14 @@ public void WhenReleaseDoesNotExist_DoesNotCheckStatusOrRollbackOrUninstall() RunInstall(); - commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("status") && i.Arguments.Contains(ReleaseName))); commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("rollback"))); commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("uninstall"))); } [Test] - public void WhenUpgradeSucceeds_ChecksStatusButDoesNotRollbackOrUninstall() + public void WhenUpgradeSucceeds_DoesNotRollbackOrUninstall() { - SetupHelmGetMetadataMock(revision: 3); - SetupHelmStatusMock("deployed"); + SetupHelmGetMetadataMock(revision: 3, status: "deployed"); RunInstall(); @@ -127,8 +121,7 @@ public void WhenUpgradeSucceeds_ChecksStatusButDoesNotRollbackOrUninstall() [Test] public void WhenRollbackFails_LogsWarningAndContinuesToUpgrade() { - SetupHelmGetMetadataMock(revision: 2); - SetupHelmStatusMock("pending-upgrade"); + SetupHelmGetMetadataMock(revision: 2, status: "pending-upgrade"); commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("rollback"))) .Returns(new CommandResult("helm rollback", 1)); @@ -142,8 +135,7 @@ public void WhenRollbackFails_LogsWarningAndContinuesToUpgrade() [Test] public void WhenUninstallFails_LogsWarningAndContinuesToUpgrade() { - SetupHelmGetMetadataMock(revision: 1); - SetupHelmStatusMock("pending-install"); + SetupHelmGetMetadataMock(revision: 1, status: "pending-install"); commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("uninstall"))) .Returns(new CommandResult("helm uninstall", 1)); @@ -180,22 +172,22 @@ void SetupKubectlMocks() void SetupHelmVersionMock() { - commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("version") && i.Arguments.Contains("--client"))) + commandLineRunner.Execute(Arg.Is(i => i.Executable == "helm" && i.Arguments.Contains("version") && i.Arguments.Contains("--client"))) .Returns(info => - { - var invocation = (CommandLineInvocation)info[0]; - invocation.AdditionalInvocationOutputSink?.WriteInfo("v3.14.0"); - return new CommandResult("helm version", 0); - }); + { + var invocation = (CommandLineInvocation)info[0]; + invocation.AdditionalInvocationOutputSink?.WriteInfo("v3.14.0"); + return new CommandResult("helm version", 0); + }); } - void SetupHelmGetMetadataMock(int revision) + void SetupHelmGetMetadataMock(int revision, string status) { commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("get") && i.Arguments.Contains("metadata"))) .Returns(info => { var invocation = (CommandLineInvocation)info[0]; - invocation.AdditionalInvocationOutputSink?.WriteInfo($"{{\"revision\":{revision}}}"); + invocation.AdditionalInvocationOutputSink?.WriteInfo($"{{\"revision\":{revision},\"status\":\"{status}\"}}"); return new CommandResult("helm get metadata", 0); }); } @@ -206,17 +198,6 @@ void SetupHelmGetMetadataToReturnNotFound() .Returns(new CommandResult("helm get metadata", 1)); } - void SetupHelmStatusMock(string status) - { - commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("status") && i.Arguments.Contains(ReleaseName))) - .Returns(info => - { - var invocation = (CommandLineInvocation)info[0]; - invocation.AdditionalInvocationOutputSink?.WriteInfo($"{{\"info\":{{\"status\":\"{status}\"}}}}"); - return new CommandResult("helm status", 0); - }); - } - void SetupHelmRollbackMock() { commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("rollback"))) diff --git a/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs b/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs index a68388d7d8..f41b9782f5 100644 --- a/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs +++ b/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs @@ -73,10 +73,8 @@ public void ExecuteHelmUpgrade(RunningDeployment deployment, // Checks for a stuck pending release and recovers by uninstalling or rolling back. // Returns the correct newRevisionNumber to use for the upgrade. - public int RecoverFromPendingRelease(string releaseName, int expectedRevisionNumber) + public int RecoverFromPendingRelease(string releaseName, string status, int expectedRevisionNumber) { - var status = helmCli.GetReleaseStatus(releaseName); - switch (status?.ToLowerInvariant()) { case "pending-install": diff --git a/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs b/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs index e7a5a2503e..600b5bd530 100644 --- a/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs +++ b/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs @@ -53,16 +53,16 @@ public void Install(RunningDeployment deployment) kubectl.SetKubectl(); - var currentRevisionNumber = helmCli.GetCurrentRevision(releaseName); + var currentMetadata = helmCli.GetCurrentReleaseMetadata(releaseName); var executor = new HelmUpgradeExecutor(log, fileSystem, valueSourcesParser, helmCli, namespaceResolver); - var expectedRevisionNumber = (currentRevisionNumber ?? 0) + 1; + var expectedRevisionNumber = (currentMetadata?.Revision ?? 0) + 1; // If a release exists and is stuck in a pending state from a previous cancelled deployment, // recover before starting the upgrade so both tasks receive the correct revision number. - var newRevisionNumber = currentRevisionNumber != null - ? executor.RecoverFromPendingRelease(releaseName, expectedRevisionNumber) + var newRevisionNumber = currentMetadata != null + ? executor.RecoverFromPendingRelease(releaseName, currentMetadata.Value.Status, expectedRevisionNumber) : expectedRevisionNumber; //This is used to cancel KOS when the helm upgrade has completed @@ -90,15 +90,7 @@ await runner.StartBackgroundMonitoringAndReporting(deployment, helmInstallCompletedCts.Token); //we run both the helm upgrade and the manifest & status in parallel - // TaskCanceledException from the manifest task is expected when the upgrade completes before - // monitoring starts (helmInstallCompletedCts is already cancelled); treat it as success. - try - { - Task.WhenAll(helmUpgradeTask, manifestAndStatusCheckTask).GetAwaiter().GetResult(); - } - catch (OperationCanceledException) when (helmInstallCompletedCts.IsCancellationRequested && !helmInstallErrorCts.IsCancellationRequested) - { - } + Task.WhenAll(helmUpgradeTask, manifestAndStatusCheckTask).GetAwaiter().GetResult(); } string GetReleaseName(IVariables variables) @@ -115,6 +107,5 @@ string GetReleaseName(IVariables variables) log.Info($"Using Release Name {releaseName}"); return releaseName; } - } } \ No newline at end of file diff --git a/source/Calamari/Kubernetes/Integration/HelmCli.cs b/source/Calamari/Kubernetes/Integration/HelmCli.cs index fccb786b22..ef98efdeb2 100644 --- a/source/Calamari/Kubernetes/Integration/HelmCli.cs +++ b/source/Calamari/Kubernetes/Integration/HelmCli.cs @@ -85,52 +85,20 @@ public SemanticVersion GetParsedExecutableVersion() return SemVerFactory.CreateVersion(vStripped); } - public int? GetCurrentRevision(string releaseName) + public (int Revision, string Status)? GetCurrentReleaseMetadata(string releaseName) { var result = ExecuteCommandAndReturnOutput("get", "metadata", releaseName, "-o json", NamespaceArg()); - //if we get _any_ error back, assume it probably hasn't been installed yet if (result.Result.ExitCode != 0) - return null; // - - //parse the output - var json = result.Output.MergeInfoLogs(); - var metadata = JsonConvert.DeserializeAnonymousType(json, - new - { - //we only care about parsing the revision - revision = 0 - }); - - //the next revision - return metadata.revision; - } - - public string? GetReleaseStatus(string releaseName) - { - var result = ExecuteCommandAndReturnOutput("status", releaseName, "-o json", NamespaceArg()); - - if (result.Result.ExitCode != 0) - { - // Log any error output so auth/RBAC/network failures are visible rather than silently returning null - var errorOutput = result.Output.MergeInfoLogs(); - if (!string.IsNullOrWhiteSpace(errorOutput)) - log.Verbose($"helm status returned exit code {result.Result.ExitCode}: {errorOutput}"); return null; - } + //parse the output var json = result.Output.MergeInfoLogs(); - var status = JsonConvert.DeserializeAnonymousType(json, - new - { - info = new - { - status = "" - } - }); - - return status?.info?.status; + var metadata = JsonConvert.DeserializeAnonymousType(json, new { revision = 0, status = string.Empty }); + + // Only revision and status are required for now + return (metadata.revision, metadata.status); } public CommandResult Rollback(string releaseName, int? revision = null) @@ -170,7 +138,8 @@ public CommandResult Upgrade(string releaseName, string packagePath, IEnumerable buildArgs.AddRange(upgradeArgs); buildArgs.Add(NamespaceArg()); - buildArgs.Add(releaseName); + //properly quote the release name and package path (consistent with previous code) + buildArgs.Add($"\"{releaseName}\""); buildArgs.Add($"\"{packagePath}\""); var result = ExecuteCommandAndLogOutput(buildArgs); From 18c021ef77402d9ea798c1c8177e23eb47d9a6a6 Mon Sep 17 00:00:00 2001 From: Bec Callow Date: Tue, 30 Jun 2026 17:03:55 +1000 Subject: [PATCH 4/4] Fix revision number --- .../Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs b/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs index f41b9782f5..bccb56f3f6 100644 --- a/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs +++ b/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs @@ -5,7 +5,6 @@ using System.Threading; using Calamari.Common.Commands; using Calamari.Common.Features.Packages; -using Calamari.Common.Features.Processes; using Calamari.Common.FeatureToggles; using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; @@ -82,6 +81,7 @@ public int RecoverFromPendingRelease(string releaseName, string status, int expe var uninstallResult = helmCli.Uninstall(releaseName); if (uninstallResult.ExitCode != 0) log.Warn($"Uninstall returned non-zero exit code {uninstallResult.ExitCode}. Continuing with upgrade..."); + // Uninstall resets the revision number return 1; case "pending-upgrade": @@ -89,7 +89,8 @@ public int RecoverFromPendingRelease(string releaseName, string status, int expe var rollbackResult = helmCli.Rollback(releaseName); if (rollbackResult.ExitCode != 0) log.Warn($"Rollback returned non-zero exit code {rollbackResult.ExitCode}. Continuing with upgrade..."); - return expectedRevisionNumber; + // Rollback creates a new revision, so the subsequent upgrade will be one higher than expected. + return expectedRevisionNumber + 1; default: return expectedRevisionNumber;