diff --git a/source/Calamari.Tests/KubernetesFixtures/Conventions/Helm/HelmUpgradeWithKOSConventionTests.cs b/source/Calamari.Tests/KubernetesFixtures/Conventions/Helm/HelmUpgradeWithKOSConventionTests.cs new file mode 100644 index 000000000..f571e8426 --- /dev/null +++ b/source/Calamari.Tests/KubernetesFixtures/Conventions/Helm/HelmUpgradeWithKOSConventionTests.cs @@ -0,0 +1,258 @@ +using System.Collections.Generic; +using System.IO; +using Calamari.Common.Commands; +using Calamari.Common.Features.Packages; +using Calamari.Common.Features.Processes; +using Calamari.Common.Plumbing.Deployment; +using Calamari.Common.Plumbing.FileSystem; +using Calamari.Common.Plumbing.Variables; +using Calamari.Kubernetes; +using Calamari.Kubernetes.Conventions; +using Calamari.Kubernetes.Helm; +using Calamari.Kubernetes.Integration; +using Calamari.Kubernetes.ResourceStatus; +using Calamari.Testing.Helpers; +using Calamari.Tests.Fixtures.Integration.FileSystem; +using FluentAssertions; +using NSubstitute; +using NUnit.Framework; + +namespace Calamari.Tests.KubernetesFixtures.Conventions.Helm +{ + [TestFixture] + public class HelmUpgradeWithKOSConventionTests + { + const string ReleaseName = "my-release"; + + readonly ICalamariFileSystem fileSystem = TestCalamariPhysicalFileSystem.GetPhysicalFileSystem(); + ICommandLineRunner commandLineRunner; + InMemoryLog log; + string tempDirectory; + + string ChartDirectory => Path.Combine(tempDirectory, "chart"); + + [SetUp] + public void SetUp() + { + log = new InMemoryLog(); + commandLineRunner = Substitute.For(); + tempDirectory = fileSystem.CreateTemporaryDirectory(); + + Directory.CreateDirectory(ChartDirectory); + File.WriteAllText(Path.Combine(ChartDirectory, "Chart.yaml"), "apiVersion: v2\nname: test\nversion: 0.1.0"); + + SetupKubectlMocks(); + SetupHelmVersionMock(); + SetupHelmUpgradeMock(); + SetupHelmRollbackMock(); + SetupHelmUninstallMock(); + } + + [TearDown] + public void TearDown() + { + fileSystem.DeleteDirectory(tempDirectory, FailureOptions.IgnoreFailure); + } + + [Test] + public void WhenReleaseIsPendingUpgrade_RollsBackBeforeUpgrade() + { + SetupHelmGetMetadataMock(revision: 2, status: "pending-upgrade"); + + RunInstall(); + + commandLineRunner.Received().Execute(Arg.Is(i => i.Arguments.Contains("rollback") && i.Arguments.Contains(ReleaseName))); + } + + [Test] + public void WhenReleaseIsPendingInstall_UninstallsBeforeUpgrade() + { + SetupHelmGetMetadataMock(revision: 1, status: "pending-install"); + + RunInstall(); + + commandLineRunner.Received().Execute(Arg.Is(i => i.Arguments.Contains("uninstall") && i.Arguments.Contains(ReleaseName))); + } + + [Test] + public void WhenReleaseIsDeployed_DoesNotRollbackOrUninstall() + { + SetupHelmGetMetadataMock(revision: 3, status: "deployed"); + + RunInstall(); + + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("rollback"))); + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("uninstall"))); + } + + [Test] + public void WhenReleaseIsFailed_DoesNotRollbackOrUninstall() + { + SetupHelmGetMetadataMock(revision: 2, status: "failed"); + + RunInstall(); + + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("rollback"))); + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("uninstall"))); + } + + [Test] + public void WhenReleaseDoesNotExist_DoesNotCheckStatusOrRollbackOrUninstall() + { + SetupHelmGetMetadataToReturnNotFound(); + + RunInstall(); + + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("rollback"))); + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("uninstall"))); + } + + [Test] + public void WhenUpgradeSucceeds_DoesNotRollbackOrUninstall() + { + SetupHelmGetMetadataMock(revision: 3, status: "deployed"); + + RunInstall(); + + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("rollback"))); + commandLineRunner.DidNotReceive().Execute(Arg.Is(i => i.Arguments.Contains("uninstall"))); + } + + [Test] + public void WhenRollbackFails_LogsWarningAndContinuesToUpgrade() + { + SetupHelmGetMetadataMock(revision: 2, status: "pending-upgrade"); + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("rollback"))) + .Returns(new CommandResult("helm rollback", 1)); + + RunInstall(); + + log.MessagesWarnFormatted.Should().Contain(msg => msg.Contains(ReleaseName) && msg.Contains("pending-upgrade")); + log.MessagesWarnFormatted.Should().Contain(msg => msg.Contains("non-zero exit code")); + commandLineRunner.Received().Execute(Arg.Is(i => i.Arguments.Contains("upgrade"))); + } + + [Test] + public void WhenUninstallFails_LogsWarningAndContinuesToUpgrade() + { + SetupHelmGetMetadataMock(revision: 1, status: "pending-install"); + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("uninstall"))) + .Returns(new CommandResult("helm uninstall", 1)); + + RunInstall(); + + log.MessagesWarnFormatted.Should().Contain(msg => msg.Contains(ReleaseName) && msg.Contains("pending-install")); + log.MessagesWarnFormatted.Should().Contain(msg => msg.Contains("non-zero exit code")); + commandLineRunner.Received().Execute(Arg.Is(i => i.Arguments.Contains("upgrade"))); + } + + void RunInstall() + { + var variables = CreateVariables(); + var deployment = CreateRunningDeployment(variables); + var convention = CreateConvention(deployment); + convention.Install(deployment); + } + + void SetupKubectlMocks() + { + // Return kubectl path from where/which + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("kubectl"))) + .Returns(info => + { + var invocation = (CommandLineInvocation)info[0]; + invocation.AdditionalInvocationOutputSink?.WriteInfo("kubectl"); + return new CommandResult("where kubectl", 0); + }); + + // Return success for kubectl version --client + commandLineRunner.Execute(Arg.Is(i => i.Executable == "kubectl")) + .Returns(new CommandResult("kubectl version", 0)); + } + + void SetupHelmVersionMock() + { + commandLineRunner.Execute(Arg.Is(i => i.Executable == "helm" && i.Arguments.Contains("version") && i.Arguments.Contains("--client"))) + .Returns(info => + { + var invocation = (CommandLineInvocation)info[0]; + invocation.AdditionalInvocationOutputSink?.WriteInfo("v3.14.0"); + return new CommandResult("helm version", 0); + }); + } + + void SetupHelmGetMetadataMock(int revision, string status) + { + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("get") && i.Arguments.Contains("metadata"))) + .Returns(info => + { + var invocation = (CommandLineInvocation)info[0]; + invocation.AdditionalInvocationOutputSink?.WriteInfo($"{{\"revision\":{revision},\"status\":\"{status}\"}}"); + return new CommandResult("helm get metadata", 0); + }); + } + + void SetupHelmGetMetadataToReturnNotFound() + { + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("get") && i.Arguments.Contains("metadata"))) + .Returns(new CommandResult("helm get metadata", 1)); + } + + void SetupHelmRollbackMock() + { + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("rollback"))) + .Returns(new CommandResult("helm rollback", 0)); + } + + void SetupHelmUninstallMock() + { + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("uninstall"))) + .Returns(new CommandResult("helm uninstall", 0)); + } + + void SetupHelmUpgradeMock() + { + commandLineRunner.Execute(Arg.Is(i => i.Arguments.Contains("upgrade"))) + .Returns(new CommandResult("helm upgrade", 0)); + } + + CalamariVariables CreateVariables() + { + var variables = new CalamariVariables + { + [SpecialVariables.Helm.ReleaseName] = ReleaseName, + [PackageVariables.Output.InstallationDirectoryPath] = ChartDirectory, + // --dry-run bypasses the manifest reporting path in HelmManifestAndStatusReporter + [SpecialVariables.Helm.AdditionalArguments] = "--dry-run" + }; + return variables; + } + + RunningDeployment CreateRunningDeployment(CalamariVariables variables) + { + return new RunningDeployment(variables, new Dictionary()) + { + CurrentDirectoryProvider = DeploymentWorkingDirectory.StagingDirectory, + StagingDirectory = tempDirectory + }; + } + + HelmUpgradeWithKOSConvention CreateConvention(RunningDeployment deployment) + { + var kubectl = new Kubectl(deployment.Variables, log, commandLineRunner, tempDirectory, deployment.EnvironmentVariables); + var templateValueSourcesParser = new HelmTemplateValueSourcesParser(fileSystem, log); + var namespaceResolver = Substitute.For(); + var statusReporter = Substitute.For(); + var manifestReporter = Substitute.For(); + + return new HelmUpgradeWithKOSConvention(log, + commandLineRunner, + fileSystem, + templateValueSourcesParser, + statusReporter, + manifestReporter, + namespaceResolver, + kubectl); + } + } +} diff --git a/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs b/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs index 0550dc2c3..bccb56f3f 100644 --- a/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs +++ b/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs @@ -70,6 +70,33 @@ public void ExecuteHelmUpgrade(RunningDeployment deployment, installCompletedCts.Cancel(); } + // Checks for a stuck pending release and recovers by uninstalling or rolling back. + // Returns the correct newRevisionNumber to use for the upgrade. + public int RecoverFromPendingRelease(string releaseName, string status, int expectedRevisionNumber) + { + switch (status?.ToLowerInvariant()) + { + case "pending-install": + log.Warn($"Release {releaseName} is stuck in {status} state, likely from a cancelled first install. Uninstalling to recover..."); + var uninstallResult = helmCli.Uninstall(releaseName); + if (uninstallResult.ExitCode != 0) + log.Warn($"Uninstall returned non-zero exit code {uninstallResult.ExitCode}. Continuing with upgrade..."); + // Uninstall resets the revision number + return 1; + + case "pending-upgrade": + log.Warn($"Release {releaseName} is stuck in {status} state, likely from a cancelled deployment. Rolling back to recover..."); + var rollbackResult = helmCli.Rollback(releaseName); + if (rollbackResult.ExitCode != 0) + log.Warn($"Rollback returned non-zero exit code {rollbackResult.ExitCode}. Continuing with upgrade..."); + // Rollback creates a new revision, so the subsequent upgrade will be one higher than expected. + return expectedRevisionNumber + 1; + + default: + return expectedRevisionNumber; + } + } + void SetAppliedResourcesOutputVariable(RunningDeployment deployment, string releaseName, int revisionNumber) { string manifest = null; diff --git a/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs b/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs index 7ff9ee13a..600b5bd53 100644 --- a/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs +++ b/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs @@ -53,25 +53,27 @@ public void Install(RunningDeployment deployment) kubectl.SetKubectl(); - var currentRevisionNumber = helmCli.GetCurrentRevision(releaseName); + var currentMetadata = helmCli.GetCurrentReleaseMetadata(releaseName); - var newRevisionNumber = (currentRevisionNumber ?? 0) + 1; + var executor = new HelmUpgradeExecutor(log, fileSystem, valueSourcesParser, helmCli, namespaceResolver); + + var expectedRevisionNumber = (currentMetadata?.Revision ?? 0) + 1; + + // If a release exists and is stuck in a pending state from a previous cancelled deployment, + // recover before starting the upgrade so both tasks receive the correct revision number. + var newRevisionNumber = currentMetadata != null + ? executor.RecoverFromPendingRelease(releaseName, currentMetadata.Value.Status, expectedRevisionNumber) + : expectedRevisionNumber; //This is used to cancel KOS when the helm upgrade has completed //It does not cancel the get manifest var helmInstallCompletedCts = new CancellationTokenSource(); - + //This is used to cancel the get manifest when the helm install fails (and we are still trying to retrieve the manifest) var helmInstallErrorCts = new CancellationTokenSource(); var helmUpgradeTask = Task.Run(() => { - var executor = new HelmUpgradeExecutor(log, - fileSystem, - valueSourcesParser, - helmCli, - namespaceResolver); - executor.ExecuteHelmUpgrade(deployment, releaseName, newRevisionNumber, helmInstallCompletedCts, helmInstallErrorCts); }); @@ -82,7 +84,7 @@ public void Install(RunningDeployment deployment) await runner.StartBackgroundMonitoringAndReporting(deployment, releaseName, newRevisionNumber, - helmInstallCompletedCts.Token, + helmInstallCompletedCts.Token, helmInstallErrorCts.Token); }, helmInstallCompletedCts.Token); diff --git a/source/Calamari/Kubernetes/Integration/HelmCli.cs b/source/Calamari/Kubernetes/Integration/HelmCli.cs index ed66704f4..ef98efdeb 100644 --- a/source/Calamari/Kubernetes/Integration/HelmCli.cs +++ b/source/Calamari/Kubernetes/Integration/HelmCli.cs @@ -85,26 +85,39 @@ public SemanticVersion GetParsedExecutableVersion() return SemVerFactory.CreateVersion(vStripped); } - public int? GetCurrentRevision(string releaseName) + public (int Revision, string Status)? GetCurrentReleaseMetadata(string releaseName) { var result = ExecuteCommandAndReturnOutput("get", "metadata", releaseName, "-o json", NamespaceArg()); - //if we get _any_ error back, assume it probably hasn't been installed yet if (result.Result.ExitCode != 0) - return null; // - + return null; + //parse the output var json = result.Output.MergeInfoLogs(); - var metadata = JsonConvert.DeserializeAnonymousType(json, - new - { - //we only care about parsing the revision - revision = 0 - }); - - //the next revision - return metadata.revision; + var metadata = JsonConvert.DeserializeAnonymousType(json, new { revision = 0, status = string.Empty }); + + // Only revision and status are required for now + return (metadata.revision, metadata.status); + } + + public CommandResult Rollback(string releaseName, int? revision = null) + { + var args = new List { "rollback", releaseName }; + + if (revision.HasValue) + args.Add(revision.Value.ToString()); + + args.Add(NamespaceArg()); + + var result = ExecuteCommandAndLogOutput(args); + return result; + } + + public CommandResult Uninstall(string releaseName) + { + var args = new List { "uninstall", releaseName, NamespaceArg() }; + return ExecuteCommandAndLogOutput(args); } public string GetManifest(string releaseName, int revisionNumber)