Handle pending helm updates and installs

source/Calamari.Tests/KubernetesFixtures/Conventions/Helm/HelmUpgradeWithKOSConventionTests.cs

@@ -0,0 +1,258 @@
+using System.Collections.Generic;
+using System.IO;
+using Calamari.Common.Commands;
+using Calamari.Common.Features.Packages;
+using Calamari.Common.Features.Processes;
+using Calamari.Common.Plumbing.Deployment;
+using Calamari.Common.Plumbing.FileSystem;
+using Calamari.Common.Plumbing.Variables;
+using Calamari.Kubernetes;
+using Calamari.Kubernetes.Conventions;
+using Calamari.Kubernetes.Helm;
+using Calamari.Kubernetes.Integration;
+using Calamari.Kubernetes.ResourceStatus;
+using Calamari.Testing.Helpers;
+using Calamari.Tests.Fixtures.Integration.FileSystem;
+using FluentAssertions;
+using NSubstitute;
+using NUnit.Framework;
+
+namespace Calamari.Tests.KubernetesFixtures.Conventions.Helm
+{
+    [TestFixture]
+    public class HelmUpgradeWithKOSConventionTests
+    {
+        const string ReleaseName = "my-release";
+
+        readonly ICalamariFileSystem fileSystem = TestCalamariPhysicalFileSystem.GetPhysicalFileSystem();
+        ICommandLineRunner commandLineRunner;
+        InMemoryLog log;
+        string tempDirectory;
+
+        string ChartDirectory => Path.Combine(tempDirectory, "chart");
+
+        [SetUp]
+        public void SetUp()
+        {
+            log = new InMemoryLog();
+            commandLineRunner = Substitute.For<ICommandLineRunner>();
+            tempDirectory = fileSystem.CreateTemporaryDirectory();
+
+            Directory.CreateDirectory(ChartDirectory);
+            File.WriteAllText(Path.Combine(ChartDirectory, "Chart.yaml"), "apiVersion: v2\nname: test\nversion: 0.1.0");
+
+            SetupKubectlMocks();
+            SetupHelmVersionMock();
+            SetupHelmUpgradeMock();
+            SetupHelmRollbackMock();
+            SetupHelmUninstallMock();
+        }
+
+        [TearDown]
+        public void TearDown()
+        {
+            fileSystem.DeleteDirectory(tempDirectory, FailureOptions.IgnoreFailure);
+        }
+
+        [Test]
+        public void WhenReleaseIsPendingUpgrade_RollsBackBeforeUpgrade()
+        {
+            SetupHelmGetMetadataMock(revision: 2, status: "pending-upgrade");
+
+            RunInstall();
+
+            commandLineRunner.Received().Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("rollback") && i.Arguments.Contains(ReleaseName)));
+        }
+
+        [Test]
+        public void WhenReleaseIsPendingInstall_UninstallsBeforeUpgrade()
+        {
+            SetupHelmGetMetadataMock(revision: 1, status: "pending-install");
+
+            RunInstall();
+
+            commandLineRunner.Received().Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("uninstall") && i.Arguments.Contains(ReleaseName)));
+        }
+
+        [Test]
+        public void WhenReleaseIsDeployed_DoesNotRollbackOrUninstall()
+        {
+            SetupHelmGetMetadataMock(revision: 3, status: "deployed");
+
+            RunInstall();
+
+            commandLineRunner.DidNotReceive().Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("rollback")));
+            commandLineRunner.DidNotReceive().Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("uninstall")));
+        }
+
+        [Test]
+        public void WhenReleaseIsFailed_DoesNotRollbackOrUninstall()
+        {
+            SetupHelmGetMetadataMock(revision: 2, status: "failed");
+
+            RunInstall();
+
+            commandLineRunner.DidNotReceive().Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("rollback")));
+            commandLineRunner.DidNotReceive().Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("uninstall")));
+        }
+
+        [Test]
+        public void WhenReleaseDoesNotExist_DoesNotCheckStatusOrRollbackOrUninstall()
+        {
+            SetupHelmGetMetadataToReturnNotFound();
+
+            RunInstall();
+
+            commandLineRunner.DidNotReceive().Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("rollback")));
+            commandLineRunner.DidNotReceive().Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("uninstall")));
+        }
+
+        [Test]
+        public void WhenUpgradeSucceeds_DoesNotRollbackOrUninstall()
+        {
+            SetupHelmGetMetadataMock(revision: 3, status: "deployed");
+
+            RunInstall();
+
+            commandLineRunner.DidNotReceive().Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("rollback")));
+            commandLineRunner.DidNotReceive().Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("uninstall")));
+        }
+
+        [Test]
+        public void WhenRollbackFails_LogsWarningAndContinuesToUpgrade()
+        {
+            SetupHelmGetMetadataMock(revision: 2, status: "pending-upgrade");
+            commandLineRunner.Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("rollback")))
+                             .Returns(new CommandResult("helm rollback", 1));
+
+            RunInstall();
+
+            log.MessagesWarnFormatted.Should().Contain(msg => msg.Contains(ReleaseName) && msg.Contains("pending-upgrade"));
+            log.MessagesWarnFormatted.Should().Contain(msg => msg.Contains("non-zero exit code"));
+            commandLineRunner.Received().Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("upgrade")));
+        }
+
+        [Test]
+        public void WhenUninstallFails_LogsWarningAndContinuesToUpgrade()
+        {
+            SetupHelmGetMetadataMock(revision: 1, status: "pending-install");
+            commandLineRunner.Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("uninstall")))
+                             .Returns(new CommandResult("helm uninstall", 1));
+
+            RunInstall();
+
+            log.MessagesWarnFormatted.Should().Contain(msg => msg.Contains(ReleaseName) && msg.Contains("pending-install"));
+            log.MessagesWarnFormatted.Should().Contain(msg => msg.Contains("non-zero exit code"));
+            commandLineRunner.Received().Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("upgrade")));
+        }
+
+        void RunInstall()
+        {
+            var variables = CreateVariables();
+            var deployment = CreateRunningDeployment(variables);
+            var convention = CreateConvention(deployment);
+            convention.Install(deployment);
+        }
+
+        void SetupKubectlMocks()
+        {
+            // Return kubectl path from where/which
+            commandLineRunner.Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("kubectl")))
+                             .Returns(info =>
+                             {
+                                 var invocation = (CommandLineInvocation)info[0];
+                                 invocation.AdditionalInvocationOutputSink?.WriteInfo("kubectl");
+                                 return new CommandResult("where kubectl", 0);
+                             });
+
+            // Return success for kubectl version --client
+            commandLineRunner.Execute(Arg.Is<CommandLineInvocation>(i => i.Executable == "kubectl"))
+                             .Returns(new CommandResult("kubectl version", 0));
+        }
+
+        void SetupHelmVersionMock()
+        {
+            commandLineRunner.Execute(Arg.Is<CommandLineInvocation>(i => i.Executable == "helm" && i.Arguments.Contains("version") && i.Arguments.Contains("--client")))
+                             .Returns(info =>
+                                      {
+                                          var invocation = (CommandLineInvocation)info[0];
+                                          invocation.AdditionalInvocationOutputSink?.WriteInfo("v3.14.0");
+                                          return new CommandResult("helm version", 0);
+                                      });
+        }
+
+        void SetupHelmGetMetadataMock(int revision, string status)
+        {
+            commandLineRunner.Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("get") && i.Arguments.Contains("metadata")))
+                             .Returns(info =>
+                             {
+                                 var invocation = (CommandLineInvocation)info[0];
+                                 invocation.AdditionalInvocationOutputSink?.WriteInfo($"{{\"revision\":{revision},\"status\":\"{status}\"}}");
+                                 return new CommandResult("helm get metadata", 0);
+                             });
+        }
+
+        void SetupHelmGetMetadataToReturnNotFound()
+        {
+            commandLineRunner.Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("get") && i.Arguments.Contains("metadata")))
+                             .Returns(new CommandResult("helm get metadata", 1));
+        }
+
+        void SetupHelmRollbackMock()
+        {
+            commandLineRunner.Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("rollback")))
+                             .Returns(new CommandResult("helm rollback", 0));
+        }
+
+        void SetupHelmUninstallMock()
+        {
+            commandLineRunner.Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("uninstall")))
+                             .Returns(new CommandResult("helm uninstall", 0));
+        }
+
+        void SetupHelmUpgradeMock()
+        {
+            commandLineRunner.Execute(Arg.Is<CommandLineInvocation>(i => i.Arguments.Contains("upgrade")))
+                             .Returns(new CommandResult("helm upgrade", 0));
+        }
+
+        CalamariVariables CreateVariables()
+        {
+            var variables = new CalamariVariables
+            {
+                [SpecialVariables.Helm.ReleaseName] = ReleaseName,
+                [PackageVariables.Output.InstallationDirectoryPath] = ChartDirectory,
+                // --dry-run bypasses the manifest reporting path in HelmManifestAndStatusReporter
+                [SpecialVariables.Helm.AdditionalArguments] = "--dry-run"
+            };
+            return variables;
+        }
+
+        RunningDeployment CreateRunningDeployment(CalamariVariables variables)
+        {
+            return new RunningDeployment(variables, new Dictionary<string, string>())
+            {
+                CurrentDirectoryProvider = DeploymentWorkingDirectory.StagingDirectory,
+                StagingDirectory = tempDirectory
+            };
+        }
+
+        HelmUpgradeWithKOSConvention CreateConvention(RunningDeployment deployment)
+        {
+            var kubectl = new Kubectl(deployment.Variables, log, commandLineRunner, tempDirectory, deployment.EnvironmentVariables);
+            var templateValueSourcesParser = new HelmTemplateValueSourcesParser(fileSystem, log);
+            var namespaceResolver = Substitute.For<IKubernetesManifestNamespaceResolver>();
+            var statusReporter = Substitute.For<IResourceStatusReportExecutor>();
+            var manifestReporter = Substitute.For<IManifestReporter>();
+
+            return new HelmUpgradeWithKOSConvention(log,
+                                                    commandLineRunner,
+                                                    fileSystem,
+                                                    templateValueSourcesParser,
+                                                    statusReporter,
+                                                    manifestReporter,
+                                                    namespaceResolver,
+                                                    kubectl);
+        }
+    }
+}

source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs

@@ -70,6 +70,33 @@ public void ExecuteHelmUpgrade(RunningDeployment deployment,
             installCompletedCts.Cancel();
         }
 
+        // Checks for a stuck pending release and recovers by uninstalling or rolling back.
+        // Returns the correct newRevisionNumber to use for the upgrade.
+        public int RecoverFromPendingRelease(string releaseName, string status, int expectedRevisionNumber)
+        {
+            switch (status?.ToLowerInvariant())
+            {
+                case "pending-install":
+                    log.Warn($"Release {releaseName} is stuck in {status} state, likely from a cancelled first install. Uninstalling to recover...");
+                    var uninstallResult = helmCli.Uninstall(releaseName);
+                    if (uninstallResult.ExitCode != 0)
+                        log.Warn($"Uninstall returned non-zero exit code {uninstallResult.ExitCode}. Continuing with upgrade...");
+                    // Uninstall resets the revision number
+                    return 1;
+
+                case "pending-upgrade":
+                    log.Warn($"Release {releaseName} is stuck in {status} state, likely from a cancelled deployment. Rolling back to recover...");
+                    var rollbackResult = helmCli.Rollback(releaseName);
+                    if (rollbackResult.ExitCode != 0)
+                        log.Warn($"Rollback returned non-zero exit code {rollbackResult.ExitCode}. Continuing with upgrade...");
+                    // Rollback creates a new revision, so the subsequent upgrade will be one higher than expected.
+                    return expectedRevisionNumber + 1;
+
+                default:
+                    return expectedRevisionNumber;
+            }
+        }
+
         void SetAppliedResourcesOutputVariable(RunningDeployment deployment, string releaseName, int revisionNumber)
         {
             string manifest = null;

source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs

@@ -53,25 +53,27 @@ public void Install(RunningDeployment deployment)
 
             kubectl.SetKubectl();
 
-            var currentRevisionNumber = helmCli.GetCurrentRevision(releaseName);
+            var currentMetadata = helmCli.GetCurrentReleaseMetadata(releaseName);
 
-            var newRevisionNumber = (currentRevisionNumber ?? 0) + 1;
+            var executor = new HelmUpgradeExecutor(log, fileSystem, valueSourcesParser, helmCli, namespaceResolver);
+
+            var expectedRevisionNumber = (currentMetadata?.Revision ?? 0) + 1;
+
+            // If a release exists and is stuck in a pending state from a previous cancelled deployment,
+            // recover before starting the upgrade so both tasks receive the correct revision number.
+            var newRevisionNumber = currentMetadata != null
+                                        ? executor.RecoverFromPendingRelease(releaseName, currentMetadata.Value.Status, expectedRevisionNumber)
+                                        : expectedRevisionNumber;
 
             //This is used to cancel KOS when the helm upgrade has completed
             //It does not cancel the get manifest
             var helmInstallCompletedCts = new CancellationTokenSource();
-            
+
             //This is used to cancel the get manifest when the helm install fails (and we are still trying to retrieve the manifest)
             var helmInstallErrorCts = new CancellationTokenSource();
 
             var helmUpgradeTask = Task.Run(() =>
                                            {
-                                               var executor = new HelmUpgradeExecutor(log,
-                                                                                      fileSystem,
-                                                                                      valueSourcesParser,
-                                                                                      helmCli,
-                                                                                      namespaceResolver);
-
                                                executor.ExecuteHelmUpgrade(deployment, releaseName, newRevisionNumber, helmInstallCompletedCts, helmInstallErrorCts);
                                            });
 
@@ -82,7 +84,7 @@ public void Install(RunningDeployment deployment)
                                                           await runner.StartBackgroundMonitoringAndReporting(deployment,
                                                                                releaseName,
                                                                                newRevisionNumber,
-                                                                               helmInstallCompletedCts.Token, 
+                                                                               helmInstallCompletedCts.Token,
                                                                                helmInstallErrorCts.Token);
                                                       },
                                                       helmInstallCompletedCts.Token);

source/Calamari/Kubernetes/Integration/HelmCli.cs

@@ -85,26 +85,39 @@ public SemanticVersion GetParsedExecutableVersion()
             return SemVerFactory.CreateVersion(vStripped);
         }
 
-        public int? GetCurrentRevision(string releaseName)
+        public (int Revision, string Status)? GetCurrentReleaseMetadata(string releaseName)
         {
             var result = ExecuteCommandAndReturnOutput("get", "metadata", releaseName, "-o json", NamespaceArg());
 
-
             //if we get _any_ error back, assume it probably hasn't been installed yet
             if (result.Result.ExitCode != 0)
-                return null; //
-            
+                return null;
+
             //parse the output
             var json = result.Output.MergeInfoLogs();
-            var metadata = JsonConvert.DeserializeAnonymousType(json,
-                                                                new
-                                                                {
-                                                                    //we only care about parsing the revision
-                                                                    revision = 0
-                                                                });
-
-            //the next revision 
-            return metadata.revision;
+            var metadata = JsonConvert.DeserializeAnonymousType(json, new { revision = 0, status = string.Empty });
+
+            // Only revision and status are required for now
+            return (metadata.revision, metadata.status);
+        }
+
+        public CommandResult Rollback(string releaseName, int? revision = null)
+        {
+            var args = new List<string> { "rollback", releaseName };
+
+            if (revision.HasValue)
+                args.Add(revision.Value.ToString());
+
+            args.Add(NamespaceArg());
+
+            var result = ExecuteCommandAndLogOutput(args);
+            return result;
+        }
+
+        public CommandResult Uninstall(string releaseName)
+        {
+            var args = new List<string> { "uninstall", releaseName, NamespaceArg() };
+            return ExecuteCommandAndLogOutput(args);
         }
 
         public string GetManifest(string releaseName, int revisionNumber)