diff --git a/source/Calamari.Tests/ArgoCD/Commands/Conventions/UpdateArgoCDAppImagesInstallConventionTest.cs b/source/Calamari.Tests/ArgoCD/Commands/Conventions/UpdateArgoCDAppImagesInstallConventionTest.cs index 0a23ccac50..ed86daabb2 100644 --- a/source/Calamari.Tests/ArgoCD/Commands/Conventions/UpdateArgoCDAppImagesInstallConventionTest.cs +++ b/source/Calamari.Tests/ArgoCD/Commands/Conventions/UpdateArgoCDAppImagesInstallConventionTest.cs @@ -1281,6 +1281,114 @@ public void MultiSource_OutOfScopeSource_IsNotTracked() source1.CommitSha.Should().HaveLength(40, "source 1 was updated and committed"); } + [Test] + public void MultipleApplications_SharingRepoAndBranch_AreClonedOnce_AndCommittedPerApplication() + { + // Arrange: two applications that point at the same repository + branch, each with its own outdated image. + var updater = CreateConvention(); + var runningDeployment = CreateRunningDeployment(("nginx", "index.docker.io/nginx:1.27.1")); + + var file1 = Path.Combine("app1", "deployment.yaml"); + var file2 = Path.Combine("app2", "deployment.yaml"); + originRepo.AddFilesToBranch(argoCDBranchName, [ + (file1, MakeDeploymentYaml("app1-deployment", "nginx:1.19")), + (file2, MakeDeploymentYaml("app2-deployment", "nginx:1.19")), + ]); + + customPropertiesLoader.Load() + .Returns(new ArgoCDCustomPropertiesDto( + [new ArgoCDGatewayDto(GatewayId, "Gateway1")], + [ + new ArgoCDApplicationDto(GatewayId, "App1", "argocd", "yaml1", "docker.io", "http://my-argo.com"), + new ArgoCDApplicationDto(GatewayId, "App2", "argocd", "yaml2", "docker.io", "http://my-argo.com"), + ], + [new GitCredentialDto(OriginUrl, "", "")])); + + argoCdApplicationManifestParser.ParseManifest("yaml1").Returns(BuildDirectoryApp("App1", "app1")); + argoCdApplicationManifestParser.ParseManifest("yaml2").Returns(BuildDirectoryApp("App2", "app2")); + + var getResults = CaptureReporterResults(); + + // Act + updater.Install(runningDeployment); + + // Assert + using var scope = new AssertionScope(); + + // The shared repository is cloned exactly once, even though two applications use it. + log.StandardOut.Count(m => m.Contains("Cloning repository")).Should().Be(1, "the shared repository should only be cloned once"); + + var results = getResults(); + results.Should().HaveCount(2); + var app1 = results.Single(r => r.ApplicationName.Value == "argocd/App1"); + var app2 = results.Single(r => r.ApplicationName.Value == "argocd/App2"); + + var app1Sha = app1.TrackedSourceDetails.Single().CommitSha; + var app2Sha = app2.TrackedSourceDetails.Single().CommitSha; + app1Sha.Should().HaveLength(40, "App1 had an outdated image and was committed"); + app2Sha.Should().HaveLength(40, "App2 had an outdated image and was committed"); + app1Sha.Should().NotBe(app2Sha, "each application is committed separately within the shared clone"); + + // Both applications' changes reached the remote. + originRepo.ReadFileFromBranch(argoCDBranchName, file1).Should().Contain("nginx:1.27.1"); + originRepo.ReadFileFromBranch(argoCDBranchName, file2).Should().Contain("nginx:1.27.1"); + } + + [Test] + public void MultipleApplications_SameRepoDifferentBranches_AreClonedOnce_AndCheckedOutPerBranch() + { + // Arrange: two applications in the same repository but targeting different branches. + var updater = CreateConvention(); + var runningDeployment = CreateRunningDeployment(("nginx", "index.docker.io/nginx:1.27.1")); + + var mainBranch = RepositoryHelpers.MainBranchName; + var file = Path.Combine("app", "deployment.yaml"); + originRepo.AddFilesToBranch(argoCDBranchName, [(file, MakeDeploymentYaml("dev-deployment", "nginx:1.19"))]); + originRepo.AddFilesToBranch(mainBranch, [(file, MakeDeploymentYaml("main-deployment", "nginx:1.19"))]); + + customPropertiesLoader.Load() + .Returns(new ArgoCDCustomPropertiesDto( + [new ArgoCDGatewayDto(GatewayId, "Gateway1")], + [ + new ArgoCDApplicationDto(GatewayId, "App1", "argocd", "yaml1", "docker.io", "http://my-argo.com"), + new ArgoCDApplicationDto(GatewayId, "App2", "argocd", "yaml2", "docker.io", "http://my-argo.com"), + ], + [new GitCredentialDto(OriginUrl, "", "")])); + + argoCdApplicationManifestParser.ParseManifest("yaml1").Returns(BuildDirectoryApp("App1", "app", ArgoCDBranchFriendlyName)); + argoCdApplicationManifestParser.ParseManifest("yaml2").Returns(BuildDirectoryApp("App2", "app", mainBranch.ToFriendlyName())); + + var getResults = CaptureReporterResults(); + + // Act + updater.Install(runningDeployment); + + // Assert + using var scope = new AssertionScope(); + + log.StandardOut.Count(m => m.Contains("Cloning repository")).Should().Be(1, "the shared repository should only be cloned once across both branches"); + log.MessagesVerboseFormatted.Should().Contain(m => m.Contains($"Checking out '{argoCDBranchName.Value}'")); + log.MessagesVerboseFormatted.Should().Contain(m => m.Contains($"Checking out '{mainBranch.Value}'")); + + var results = getResults(); + results.Should().HaveCount(2); + results.Should().OnlyContain(r => r.Updated, "both applications had an outdated image on their respective branch"); + + originRepo.ReadFileFromBranch(argoCDBranchName, file).Should().Contain("nginx:1.27.1"); + originRepo.ReadFileFromBranch(mainBranch, file).Should().Contain("nginx:1.27.1"); + } + + Application BuildDirectoryApp(string name, string path, string targetRevision = ArgoCDBranchFriendlyName) + => new ArgoCDApplicationBuilder() + .WithName(name).WithNamespace("argocd") + .WithAnnotations(new Dictionary + { + [ArgoCDConstants.Annotations.OctopusProjectAnnotationKey(null)] = ProjectSlug, + [ArgoCDConstants.Annotations.OctopusEnvironmentAnnotationKey(null)] = EnvironmentSlug, + }) + .WithSource(new ApplicationSource { OriginalRepoUrl = OriginUrl, Path = path, TargetRevision = targetRevision }, SourceTypeConstants.Directory) + .Build(); + static string MakeDeploymentYaml(string name, params string[] images) { var containerName = (string image) => image.Split('/').Last().Split(':').First(); diff --git a/source/Calamari.Tests/ArgoCD/Conventions/FileUpdateResultTests.cs b/source/Calamari.Tests/ArgoCD/Conventions/FileUpdateResultTests.cs new file mode 100644 index 0000000000..fa785ffb52 --- /dev/null +++ b/source/Calamari.Tests/ArgoCD/Conventions/FileUpdateResultTests.cs @@ -0,0 +1,38 @@ +using Calamari.ArgoCD.Conventions.UpdateImageTag; +using FluentAssertions; +using NUnit.Framework; +using Octopus.Calamari.Contracts.ArgoCD; + +namespace Calamari.Tests.ArgoCD.Conventions +{ + [TestFixture] + public class FileUpdateResultTests + { + [Test] + public void Merge_CombinesImagesReplacedPatchedAndRemovedFiles() + { + var first = new FileUpdateResult(["nginx:1.27.1"], [new FileHash("a.yaml", "h1")], [new FileJsonPatch("a.yaml", "patchA")], ["removedA.yaml"]); + var second = new FileUpdateResult(["redis:7.0", "nginx:1.27.1"], [new FileHash("b.yaml", "h2")], [new FileJsonPatch("b.yaml", "patchB")], ["removedB.yaml"]); + + var merged = FileUpdateResult.Merge([first, second]); + + merged.UpdatedImages.Should().BeEquivalentTo(["nginx:1.27.1", "redis:7.0"], "duplicate images are de-duplicated"); + merged.ReplacedFiles.Should().BeEquivalentTo([new FileHash("a.yaml", "h1"), new FileHash("b.yaml", "h2")]); + merged.PatchedFiles.Should().BeEquivalentTo([new FileJsonPatch("a.yaml", "patchA"), new FileJsonPatch("b.yaml", "patchB")]); + merged.FilesRemoved.Should().BeEquivalentTo(["removedA.yaml", "removedB.yaml"]); + merged.HasChanges().Should().BeTrue(); + } + + [Test] + public void Merge_OfEmptyResults_HasNoChanges() + { + var merged = FileUpdateResult.Merge([FileUpdateResult.EmptyFileUpdateResult, FileUpdateResult.EmptyFileUpdateResult]); + + merged.HasChanges().Should().BeFalse(); + merged.UpdatedImages.Should().BeEmpty(); + merged.ReplacedFiles.Should().BeEmpty(); + merged.PatchedFiles.Should().BeEmpty(); + merged.FilesRemoved.Should().BeEmpty(); + } + } +} diff --git a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationSourceUpdater.cs b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationSourceFactory.cs similarity index 50% rename from source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationSourceUpdater.cs rename to source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationSourceFactory.cs index 94bd2a5e22..06ad50c448 100644 --- a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationSourceUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationSourceFactory.cs @@ -1,5 +1,5 @@ +using Calamari.ArgoCD.Conventions.UpdateImageTag; using Calamari.ArgoCD.Domain; -using Calamari.ArgoCD.Git; using Calamari.ArgoCD.Models; using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; @@ -7,37 +7,30 @@ namespace Calamari.ArgoCD.Conventions.ManifestTemplating; -public class ApplicationSourceUpdater +// Determines whether a source is in scope for this deployment and builds the file updater (which copies the +// templated package files into the source path). Clone/commit/push is handled by the GroupedRepositoryProcessor. +public class ApplicationSourceFactory { readonly Application applicationFromYaml; readonly DeploymentScope deploymentScope; - readonly RepositoryAdapter repositoryAdapter; readonly ArgoCommitToGitConfig deploymentConfig; readonly IPackageRelativeFile[] packageFiles; - readonly ArgoCDGatewayDto gateway; readonly ILog log; readonly ICalamariFileSystem fileSystem; - readonly ArgoCDOutputVariablesWriter outputVariablesWriter; - public ApplicationSourceUpdater(Application applicationFromYaml, - ArgoCDGatewayDto gateway, + public ApplicationSourceFactory(Application applicationFromYaml, DeploymentScope deploymentScope, ArgoCommitToGitConfig deploymentConfig, IPackageRelativeFile[] packageFiles, ILog log, - ICalamariFileSystem fileSystem, - ArgoCDOutputVariablesWriter outputVariablesWriter, - RepositoryAdapter repositoryAdapter) + ICalamariFileSystem fileSystem) { this.applicationFromYaml = applicationFromYaml; this.deploymentScope = deploymentScope; this.deploymentConfig = deploymentConfig; this.packageFiles = packageFiles; - this.gateway = gateway; this.log = log; this.fileSystem = fileSystem; - this.outputVariablesWriter = outputVariablesWriter; - this.repositoryAdapter = repositoryAdapter; } public bool IsAppInScope(ApplicationSourceWithMetadata sourceWithMetadata) @@ -50,28 +43,11 @@ public bool IsAppInScope(ApplicationSourceWithMetadata sourceWithMetadata) return deploymentScope.Matches(annotatedScope); } - public ManifestUpdateResult ProcessSource(ApplicationSourceWithMetadata sourceWithMetadata) + public ISourceUpdater CreateSourceUpdater(ApplicationSourceWithMetadata sourceWithMetadata) { var applicationSource = sourceWithMetadata.Source; - log.Info($"Writing files to repository '{applicationSource.OriginalRepoUrl}' for '{applicationFromYaml.Metadata.Name}'"); + applicationFromYaml.Metadata.Annotations.TryGetValue(ArgoCDConstants.Annotations.OctopusPathAnnotationKey(applicationSource.Name.ToApplicationSourceName()), out var pathOverrideFromAnnotation); - applicationFromYaml.Metadata.Annotations.TryGetValue(ArgoCDConstants.Annotations.OctopusPathAnnotationKey(applicationSource.Name.ToApplicationSourceName()), out var pathOverrideFromAnnotation); - - var sourceUpdater = new CopyTemplatesSourceUpdater(packageFiles, log, fileSystem, deploymentConfig.PurgeOutputDirectory, pathOverrideFromAnnotation); - - var sourceUpdateResult = repositoryAdapter.Process(sourceWithMetadata, sourceUpdater); - - outputVariablesWriter.WriteSourceUpdateResultOutputWhenPushResultExists(gateway.Name, - NamespacedApplicationName.Create(applicationFromYaml.Metadata.Name, applicationFromYaml.Metadata.Namespace), - sourceWithMetadata.Index, - sourceUpdateResult); - - if (sourceUpdateResult.PushResult is not null) - { - return new ManifestUpdateResult(true, sourceUpdateResult.PushResult.CommitSha, sourceUpdateResult.PushResult.CommitTimestamp, sourceUpdateResult.ReplacedFiles); - } - - log.Info("No changes were committed"); - return new ManifestUpdateResult(false, null, null, sourceUpdateResult.ReplacedFiles); + return new CopyTemplatesSourceUpdater(packageFiles, log, fileSystem, deploymentConfig.PurgeOutputDirectory, pathOverrideFromAnnotation); } } diff --git a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs index fef9aa5d8f..224310aa92 100644 --- a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Linq; +using Calamari.ArgoCD.Conventions.UpdateImageTag; using Calamari.ArgoCD.Domain; using Calamari.ArgoCD.Git; using Calamari.ArgoCD.Models; @@ -11,25 +12,22 @@ namespace Calamari.ArgoCD.Conventions.ManifestTemplating; public class ApplicationUpdater { - readonly AuthenticatingRepositoryFactory repositoryFactory; readonly DeploymentScope deploymentScope; readonly ArgoCommitToGitConfig deploymentConfig; readonly ILog log; readonly ICalamariFileSystem fileSystem; readonly IArgoCDApplicationManifestParser argoCdApplicationManifestParser; - readonly ICommitMessageGenerator commitMessageGenerator; readonly ArgoCDOutputVariablesWriter outputVariablesWriter; readonly IPackageRelativeFile[] packageFiles; - - public ApplicationUpdater(AuthenticatingRepositoryFactory repositoryFactory, DeploymentScope deploymentScope, ArgoCommitToGitConfig deploymentConfig, ILog log, + public ApplicationUpdater(DeploymentScope deploymentScope, + ArgoCommitToGitConfig deploymentConfig, + ILog log, ICalamariFileSystem fileSystem, IArgoCDApplicationManifestParser argoCdApplicationManifestParser, ArgoCDOutputVariablesWriter outputVariablesWriter, - IPackageRelativeFile[] packageFiles, - ICommitMessageGenerator commitMessageGenerator) + IPackageRelativeFile[] packageFiles) { - this.repositoryFactory = repositoryFactory; this.deploymentScope = deploymentScope; this.deploymentConfig = deploymentConfig; this.log = log; @@ -37,67 +35,68 @@ public ApplicationUpdater(AuthenticatingRepositoryFactory repositoryFactory, Dep this.argoCdApplicationManifestParser = argoCdApplicationManifestParser; this.outputVariablesWriter = outputVariablesWriter; this.packageFiles = packageFiles; - this.commitMessageGenerator = commitMessageGenerator; } - - public ProcessApplicationResult ProcessApplication( - ArgoCDApplicationDto application, - ArgoCDGatewayDto gateway) + + // Phase 1: parse, validate, scope and build the set of source updates for an application. + public PlannedApplication Plan(ArgoCDApplicationDto application, ArgoCDGatewayDto gateway) { log.InfoFormat("Processing application {0}", application.Name); var applicationFromYaml = argoCdApplicationManifestParser.ParseManifest(application.Manifest); var containsMultipleSources = applicationFromYaml.Spec.Sources.Count > 1; - var applicationName = applicationFromYaml.Metadata.Name; LogWarningIfUpdatingMultipleSources(applicationFromYaml.Spec.Sources, applicationFromYaml.Metadata.Annotations, containsMultipleSources); - + ValidateApplication(applicationFromYaml); - var repositoryAdapter = new RepositoryAdapter(repositoryFactory, new RepositoryUpdater(deploymentConfig.CommitParameters, log, commitMessageGenerator)); - var sourceUpdater = new ApplicationSourceUpdater(applicationFromYaml, - gateway, - deploymentScope, - deploymentConfig, - packageFiles, - log, - fileSystem, - outputVariablesWriter, - repositoryAdapter); - - var trackedSourceUpdateResults = applicationFromYaml - .GetSourcesWithMetadata() - .Where(sourceUpdater.IsAppInScope) - .Select(applicationSource => new - { - UpdateResult = sourceUpdater.ProcessSource(applicationSource), - applicationSource - }) - .ToList(); + var sourceUpdater = new ApplicationSourceFactory(applicationFromYaml, deploymentScope, deploymentConfig, packageFiles, log, fileSystem); + + var plannedSources = applicationFromYaml.GetSourcesWithMetadata() + .Where(sourceUpdater.IsAppInScope) + .Select(source => new PlannedSource(source, new RepositorySourceUpdate(applicationFromYaml.NamespacedName, source, sourceUpdater.CreateSourceUpdater(source)))) + .ToList(); + + return new PlannedApplication(application, gateway, plannedSources, applicationFromYaml.Spec.Sources.Count); + } + + // Phase 3: turn the processed results back into a per-application result, writing per-source output variables. + public ProcessApplicationResult AssembleResult(PlannedApplication plan, IReadOnlyDictionary resultsByUpdate) + { + foreach (var plannedSource in plan.MatchingSources) + { + outputVariablesWriter.WriteSourceUpdateResultOutputWhenPushResultExists(plan.Gateway.Name, + plan.NamespacedName, + plannedSource.Source.Index, + resultsByUpdate[plannedSource.Update]); + } + + var trackedSourceDetails = plan.MatchingSources.Select(plannedSource => + { + var result = resultsByUpdate[plannedSource.Update]; + return new TrackedSourceDetail(result.PushResult?.CommitSha, result.PushResult?.CommitTimestamp, plannedSource.Source.Index, result.ReplacedFiles, []); + }) + .ToList(); //if we have links, use that to generate a link, otherwise just put the name there - var instanceLinks = application.InstanceWebUiUrl != null ? new ArgoCDInstanceLinks(application.InstanceWebUiUrl) : null; + var instanceLinks = plan.Application.InstanceWebUiUrl != null ? new ArgoCDInstanceLinks(plan.Application.InstanceWebUiUrl) : null; var linkifiedAppName = instanceLinks != null - ? log.FormatLink(instanceLinks.ApplicationDetails(applicationName, applicationFromYaml.Metadata.Namespace), applicationName) - : applicationName; - - var message = trackedSourceUpdateResults.Any(u => u.UpdateResult.Updated) - ? "Updated Application {0}" - : "Nothing to update for Application {0}"; + ? log.FormatLink(instanceLinks.ApplicationDetails(plan.ApplicationName, plan.ApplicationNamespace), plan.ApplicationName) + : plan.ApplicationName; - log.InfoFormat(message, linkifiedAppName); + var anyUpdated = plan.MatchingSources.Any(plannedSource => resultsByUpdate[plannedSource.Update].Updated); + log.InfoFormat(anyUpdated ? "Updated Application {0}" : "Nothing to update for Application {0}", linkifiedAppName); return new ProcessApplicationResult( - application.GatewayId, - NamespacedApplicationName.Create(applicationName, applicationFromYaml.Metadata.Namespace), - applicationFromYaml.Spec.Sources.Count, - applicationFromYaml.Spec.Sources.Count(s => deploymentScope.Matches(ScopingAnnotationReader.GetScopeForApplicationSource(s.Name.ToApplicationSourceName(), applicationFromYaml.Metadata.Annotations, containsMultipleSources))), - trackedSourceUpdateResults.Select(r => new TrackedSourceDetail(r.UpdateResult.CommitSha, r.UpdateResult.CommitTimestamp, r.applicationSource.Index, r.UpdateResult.ReplacedFiles, [])).ToList(), + plan.Application.GatewayId, + plan.NamespacedName, + plan.TotalSourceCount, + plan.MatchingSourceCount, + trackedSourceDetails, [], - trackedSourceUpdateResults.Where(r => r.UpdateResult.Updated).Select(r => r.applicationSource.Source.OriginalRepoUrl).ToHashSet()); + plan.MatchingSources.Where(plannedSource => resultsByUpdate[plannedSource.Update].Updated).Select(plannedSource => plannedSource.Source.Source.OriginalRepoUrl).ToHashSet()); } - + void LogWarningIfUpdatingMultipleSources( List sourcesToInspect, Dictionary applicationAnnotations, diff --git a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ManifestUpdateResult.cs b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ManifestUpdateResult.cs deleted file mode 100644 index 508436fd5e..0000000000 --- a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ManifestUpdateResult.cs +++ /dev/null @@ -1,8 +0,0 @@ -#nullable enable -using System; -using System.Collections.Generic; -using Octopus.Calamari.Contracts.ArgoCD; - -namespace Calamari.ArgoCD.Conventions.ManifestTemplating; - -public record ManifestUpdateResult(bool Updated, string? CommitSha, DateTimeOffset? CommitTimestamp, List ReplacedFiles); diff --git a/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs b/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs new file mode 100644 index 0000000000..28f720d8f7 --- /dev/null +++ b/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs @@ -0,0 +1,63 @@ +using System.Collections.Generic; +using Calamari.ArgoCD.Domain; +using Calamari.ArgoCD.Git; +using Octopus.Calamari.Contracts.ArgoCD; + +namespace Calamari.ArgoCD.Conventions +{ + // A single in-scope source of an application, paired with the work item that will write/commit/push it. + public record PlannedSource(ApplicationSourceWithMetadata Source, RepositorySourceUpdate Update); + + // The result of inspecting one application: its parsed manifest, the gateway it belongs to, and the + // in-scope sources that need processing. Built up-front for every application so that the sources can + // be grouped by repository/branch across applications before any cloning happens. + public class PlannedApplication + { + public PlannedApplication(ArgoCDApplicationDto application, + ArgoCDGatewayDto gateway, + IReadOnlyList sources, + int totalSourceCount) + { + Application = application; + Gateway = gateway; + MatchingSources = sources; + TotalSourceCount = totalSourceCount; + } + + public ArgoCDApplicationDto Application { get; } + public ArgoCDGatewayDto Gateway { get; } + public NamespacedApplicationName NamespacedName + { + get + { + return NamespacedApplicationName.Create(ApplicationName, ApplicationNamespace); + } + } + + public string ApplicationName + { + get + { + return Application.Name; + } + } + + public string ApplicationNamespace + { + get + { + return Application.KubernetesNamespace; + } + } + + public IReadOnlyList MatchingSources { get; } + public int TotalSourceCount { get; } + public int MatchingSourceCount + { + get + { + return MatchingSources.Count; + } + } + } +} diff --git a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs index 04fcc55a49..33bd894e90 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs @@ -69,20 +69,30 @@ public void Install(RunningDeployment deployment) var appUpdater = new ApplicationUpdater(deploymentScope, deploymentConfig, - authenticatingRepositoryFactory, log, fileSystem, argoCdApplicationManifestParser, - new ImageTagUpdateCommitMessageGenerator(deploymentConfig.CommitParameters.Description), outputVariablesWriter); - var applicationResults = argoProperties.Applications - .Select(application => - { - var gateway = argoProperties.Gateways.Single(g => g.Id == application.GatewayId); - return appUpdater.ProcessApplication(application, gateway); - }) - .ToList(); + // Phase 1: plan every application's in-scope sources. + var plans = argoProperties.Applications + .Select(application => + { + var gateway = argoProperties.Gateways.Single(g => g.Id == application.GatewayId); + return appUpdater.Plan(application, gateway); + }) + .ToList(); + + // Phase 2: process all sources, grouped so each repository is cloned once and each branch checked out once. + var commitMessageGenerator = new ImageTagUpdateCommitMessageGenerator(deploymentConfig.CommitParameters.Description); + var processor = new GroupedRepositoryProcessor(authenticatingRepositoryFactory, deploymentConfig.CommitParameters, commitMessageGenerator); + + var updates = plans.SelectMany(p => p.MatchingSources.Select(s => s.Update)).ToList(); + var results = processor.Process(updates); + var resultsByUpdate = updates.Zip(results, (update, result) => (update, result)).ToDictionary(x => x.update, x => x.result); + + // Phase 3: assemble per-application results (also writes per-source output variables). + var applicationResults = plans.Select(p => appUpdater.AssembleResult(p, resultsByUpdate)).ToList(); //if we are creating a pull request, we don't want to report files updated (as this will be passed down as output variables _with_ the PR info) diff --git a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDApplicationManifestsInstallConvention.cs b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDApplicationManifestsInstallConvention.cs index 00de914204..7731363bfb 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDApplicationManifestsInstallConvention.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDApplicationManifestsInstallConvention.cs @@ -74,23 +74,33 @@ public void Install(RunningDeployment deployment) log.LogApplicationCounts(deploymentScope, argoProperties.Applications); - var applicationUpdater = new ApplicationUpdater(authenticatingRepositoryFactory, - deploymentScope, + var applicationUpdater = new ApplicationUpdater(deploymentScope, deploymentConfig, log, fileSystem, argoCdApplicationManifestParser, outputVariablesWriter, - packageFiles, - new UserDefinedCommitMessageGenerator(deploymentConfig.CommitParameters.Description)); - - var applicationResults = argoProperties.Applications - .Select(application => - { - var gateway = argoProperties.Gateways.Single(g => g.Id == application.GatewayId); - return applicationUpdater.ProcessApplication(application, gateway); - }) - .ToList(); + packageFiles); + + // Phase 1: plan every application's in-scope sources. + var plans = argoProperties.Applications + .Select(application => + { + var gateway = argoProperties.Gateways.Single(g => g.Id == application.GatewayId); + return applicationUpdater.Plan(application, gateway); + }) + .ToList(); + + // Phase 2: process all sources, grouped so each repository is cloned once and each branch checked out once. + var commitMessageGenerator = new UserDefinedCommitMessageGenerator(deploymentConfig.CommitParameters.Description); + var processor = new GroupedRepositoryProcessor(authenticatingRepositoryFactory, deploymentConfig.CommitParameters, commitMessageGenerator); + + var updates = plans.SelectMany(p => p.MatchingSources.Select(s => s.Update)).ToList(); + var results = processor.Process(updates); + var resultsByUpdate = updates.Zip(results, (update, result) => (update, result)).ToDictionary(x => x.update, x => x.result); + + // Phase 3: assemble per-application results (also writes per-source output variables). + var applicationResults = plans.Select(p => applicationUpdater.AssembleResult(p, resultsByUpdate)).ToList(); reporter.ReportFilesUpdated(deploymentConfig.CommitParameters, applicationResults); diff --git a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationSourceUpdater.cs b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationSourceFactory.cs similarity index 72% rename from source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationSourceUpdater.cs rename to source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationSourceFactory.cs index 93b5ec6c5f..c03e63826e 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationSourceUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationSourceFactory.cs @@ -1,6 +1,5 @@ using System; using Calamari.ArgoCD.Domain; -using Calamari.ArgoCD.Git; using Calamari.ArgoCD.Models; using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; @@ -8,36 +7,29 @@ namespace Calamari.ArgoCD.Conventions.UpdateImageTag; -public class ApplicationSourceUpdater +// Determines whether a source is in scope for this deployment and builds the appropriate file updater for it. +// The actual clone/commit/push is handled centrally by the GroupedRepositoryProcessor. +public class ApplicationSourceFactory { readonly Application applicationFromYaml; - readonly ArgoCDGatewayDto gateway; - readonly RepositoryAdapter repositoryAdapter; readonly DeploymentScope deploymentScope; readonly UpdateArgoCDAppDeploymentConfig deploymentConfig; readonly ILog log; readonly string defaultRegistry; - readonly ArgoCDOutputVariablesWriter outputVariablesWriter; readonly ICalamariFileSystem fileSystem; - public ApplicationSourceUpdater(Application applicationFromYaml, - RepositoryAdapter repositoryAdapter, + public ApplicationSourceFactory(Application applicationFromYaml, DeploymentScope deploymentScope, UpdateArgoCDAppDeploymentConfig deploymentConfig, ILog log, - ArgoCDGatewayDto gateway, string defaultRegistry, - ArgoCDOutputVariablesWriter outputVariablesWriter, ICalamariFileSystem fileSystem) { this.applicationFromYaml = applicationFromYaml; - this.repositoryAdapter = repositoryAdapter; this.deploymentScope = deploymentScope; this.deploymentConfig = deploymentConfig; this.log = log; - this.gateway = gateway; this.defaultRegistry = defaultRegistry; - this.outputVariablesWriter = outputVariablesWriter; this.fileSystem = fileSystem; } @@ -52,21 +44,7 @@ public bool IsAppInScope(ApplicationSourceWithMetadata sourceWithMetadata) return deploymentScope.Matches(annotatedScope); } - public SourceUpdateResult ProcessSource(ApplicationSourceWithMetadata sourceWithMetadata) - { - var sourceUpdater = CreateSpecificUpdater(sourceWithMetadata); - - var sourceUpdateResult = repositoryAdapter.Process(sourceWithMetadata, sourceUpdater); - - outputVariablesWriter.WriteSourceUpdateResultOutputWhenPushResultExists(gateway.Name, - NamespacedApplicationName.Create(applicationFromYaml.Metadata.Name, applicationFromYaml.Metadata.Namespace), - sourceWithMetadata.Index, - sourceUpdateResult); - - return sourceUpdateResult; - } - - ISourceUpdater CreateSpecificUpdater(ApplicationSourceWithMetadata sourceWithMetadata) + public ISourceUpdater CreateSourceUpdater(ApplicationSourceWithMetadata sourceWithMetadata) { ISourceUpdater sourceUpdater; if (sourceWithMetadata.SourceType == SourceType.Directory) @@ -114,4 +92,4 @@ ISourceUpdater CreateSpecificUpdater(ApplicationSourceWithMetadata sourceWithMet return sourceUpdater; } -} \ No newline at end of file +} diff --git a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs index f5393a7316..cbe21f71b7 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Linq; using Calamari.ArgoCD.Domain; using Calamari.ArgoCD.Git; @@ -11,103 +12,108 @@ namespace Calamari.ArgoCD.Conventions.UpdateImageTag; public class ApplicationUpdater { - readonly AuthenticatingRepositoryFactory repositoryFactory; readonly DeploymentScope deploymentScope; readonly UpdateArgoCDAppDeploymentConfig deploymentConfig; readonly ILog log; readonly ICalamariFileSystem fileSystem; readonly IArgoCDApplicationManifestParser argoCdApplicationManifestParser; - readonly ICommitMessageGenerator commitMessageGenerator; readonly ArgoCDOutputVariablesWriter outputVariablesWriter; public ApplicationUpdater(DeploymentScope deploymentScope, UpdateArgoCDAppDeploymentConfig deploymentConfig, - AuthenticatingRepositoryFactory repositoryFactory, ILog log, ICalamariFileSystem fileSystem, IArgoCDApplicationManifestParser argoCdApplicationManifestParser, - ICommitMessageGenerator commitMessageGenerator, ArgoCDOutputVariablesWriter outputVariablesWriter) { this.deploymentScope = deploymentScope; this.deploymentConfig = deploymentConfig; - this.repositoryFactory = repositoryFactory; this.log = log; this.fileSystem = fileSystem; this.argoCdApplicationManifestParser = argoCdApplicationManifestParser; - this.commitMessageGenerator = commitMessageGenerator; this.outputVariablesWriter = outputVariablesWriter; } - - public ProcessApplicationResult ProcessApplication( - ArgoCDApplicationDto application, - ArgoCDGatewayDto gateway) + + // Phase 1: parse, validate, scope and build the set of source updates for an application. The updates are + // grouped (by repository/branch) across all applications and processed together by the GroupedRepositoryProcessor. + public PlannedApplication Plan(ArgoCDApplicationDto application, ArgoCDGatewayDto gateway) + { + log.InfoFormat("Processing application {0}", application.Name); + var applicationFromYaml = argoCdApplicationManifestParser.ParseManifest(application.Manifest); + var containsMultipleSources = applicationFromYaml.Spec.Sources.Count > 1; + + ValidateApplication(applicationFromYaml); + LogHelmAnnotationWarning(applicationFromYaml); + + var sourceUpdater = new ApplicationSourceFactory(applicationFromYaml, deploymentScope, deploymentConfig, log, application.DefaultRegistry, fileSystem); + + var plannedSources = applicationFromYaml.GetSourcesWithMetadata() + .Where(sourceUpdater.IsAppInScope) + .Select(source => new PlannedSource(source, new RepositorySourceUpdate(applicationFromYaml.NamespacedName, source, sourceUpdater.CreateSourceUpdater(source)))) + .ToList(); + + return new PlannedApplication(application, gateway, plannedSources, applicationFromYaml.Spec.Sources.Count); + } + + // Phase 3: turn the processed results back into a per-application result, writing per-source output variables. + public ProcessApplicationResult AssembleResult(PlannedApplication plan, IReadOnlyDictionary resultsByUpdate) + { + foreach (var plannedSource in plan.MatchingSources) { - log.InfoFormat("Processing application {0}", application.Name); - var applicationFromYaml = argoCdApplicationManifestParser.ParseManifest(application.Manifest); - var containsMultipleSources = applicationFromYaml.Spec.Sources.Count > 1; - var applicationName = applicationFromYaml.Metadata.Name; - - ValidateApplication(applicationFromYaml); - - LogHelmAnnotationWarning(applicationFromYaml); - - var repositoryAdapter = new RepositoryAdapter(repositoryFactory, new RepositoryUpdater(deploymentConfig.CommitParameters, log, commitMessageGenerator)); - var sourceUpdater = new ApplicationSourceUpdater(applicationFromYaml, repositoryAdapter, deploymentScope, deploymentConfig, log, gateway, application.DefaultRegistry, outputVariablesWriter, fileSystem); - - var appliedSourcesResults = applicationFromYaml.GetSourcesWithMetadata() - .Where(sourceUpdater.IsAppInScope) - .Select(applicationSource => new - { - UpdateResult = sourceUpdater.ProcessSource(applicationSource), - applicationSource, - }) - .ToList(); - - //if we have links, use that to generate a link, otherwise just put the name there - var instanceLinks = application.InstanceWebUiUrl != null ? new ArgoCDInstanceLinks(application.InstanceWebUiUrl) : null; - var linkifiedAppName = instanceLinks != null - ? log.FormatLink(instanceLinks.ApplicationDetails(applicationName, application.KubernetesNamespace), applicationName) - : applicationName; - - var message = appliedSourcesResults.Any(r => r.UpdateResult.Updated) - ? "Updated Application {0}" - : "Nothing to update for Application {0}"; - - log.InfoFormat(message, linkifiedAppName); - - return new ProcessApplicationResult( - application.GatewayId, - NamespacedApplicationName.Create(applicationName, applicationFromYaml.Metadata.Namespace), - applicationFromYaml.Spec.Sources.Count, - applicationFromYaml.Spec.Sources.Count(s => deploymentScope.Matches(ScopingAnnotationReader.GetScopeForApplicationSource(s.Name.ToApplicationSourceName(), applicationFromYaml.Metadata.Annotations, containsMultipleSources))), - appliedSourcesResults.Select(r => new TrackedSourceDetail(r.UpdateResult.PushResult?.CommitSha, r.UpdateResult.PushResult?.CommitTimestamp, r.applicationSource.Index, [], r.UpdateResult.PatchedFiles)).ToList(), - appliedSourcesResults.SelectMany(r => r.UpdateResult.ImagesUpdated).ToHashSet(), - appliedSourcesResults.Where(r => r.UpdateResult.Updated).Select(r => r.applicationSource.Source.OriginalRepoUrl).ToHashSet()); + outputVariablesWriter.WriteSourceUpdateResultOutputWhenPushResultExists(plan.Gateway.Name, + plan.NamespacedName, + plannedSource.Source.Index, + resultsByUpdate[plannedSource.Update]); } - void LogHelmAnnotationWarning(Application applicationFromYaml) - { - var imagesWithoutHelmValuePath = deploymentConfig.ImageReferences.Where(ir => ir.HelmReference.IsNullOrEmpty()).ToList(); + var trackedSourceDetails = plan.MatchingSources.Select(plannedSource => + { + var result = resultsByUpdate[plannedSource.Update]; + return new TrackedSourceDetail(result.PushResult?.CommitSha, result.PushResult?.CommitTimestamp, plannedSource.Source.Index, [], result.PatchedFiles); + }) + .ToList(); + + //if we have links, use that to generate a link, otherwise just put the name there + var instanceLinks = plan.Application.InstanceWebUiUrl != null ? new ArgoCDInstanceLinks(plan.Application.InstanceWebUiUrl) : null; + var linkifiedAppName = instanceLinks != null + ? log.FormatLink(instanceLinks.ApplicationDetails(plan.ApplicationName, plan.Application.KubernetesNamespace), plan.ApplicationName) + : plan.ApplicationName; + + var anyUpdated = plan.MatchingSources.Any(plannedSource => resultsByUpdate[plannedSource.Update].Updated); + log.InfoFormat(anyUpdated ? "Updated Application {0}" : "Nothing to update for Application {0}", linkifiedAppName); - var someButNotAllHaveHelmValuePath = (imagesWithoutHelmValuePath.Count > 0) && (imagesWithoutHelmValuePath.Count < deploymentConfig.ImageReferences.Count); - - if (someButNotAllHaveHelmValuePath && applicationFromYaml.GetSourcesWithMetadata().Any(src => src.SourceType == SourceType.Helm)) + return new ProcessApplicationResult( + plan.Application.GatewayId, + plan.NamespacedName, + plan.TotalSourceCount, + plan.MatchingSourceCount, + trackedSourceDetails, + plan.MatchingSources.SelectMany(plannedSource => resultsByUpdate[plannedSource.Update].ImagesUpdated).ToHashSet(), + plan.MatchingSources.Where(plannedSource => resultsByUpdate[plannedSource.Update].Updated).Select(plannedSource => plannedSource.Source.Source.OriginalRepoUrl).ToHashSet()); + } + + void LogHelmAnnotationWarning(Application applicationFromYaml) + { + var imagesWithoutHelmValuePath = deploymentConfig.ImageReferences.Where(ir => ir.HelmReference.IsNullOrEmpty()).ToList(); + + var someButNotAllHaveHelmValuePath = (imagesWithoutHelmValuePath.Count > 0) && (imagesWithoutHelmValuePath.Count < deploymentConfig.ImageReferences.Count); + + if (someButNotAllHaveHelmValuePath && applicationFromYaml.GetSourcesWithMetadata().Any(src => src.SourceType == SourceType.Helm)) + { + foreach (var image in imagesWithoutHelmValuePath) { - foreach (var image in imagesWithoutHelmValuePath) - { - log.Verbose($"{image.ContainerReference.FriendlyName()} will not be updated in helm sources, as no helm yaml path has been specified for it in the step configuration."); - } + log.Verbose($"{image.ContainerReference.FriendlyName()} will not be updated in helm sources, as no helm yaml path has been specified for it in the step configuration."); } } + } - void ValidateApplication(Application applicationFromYaml) - { - var validationResult = ValidationResult.Merge( - ApplicationValidator.ValidateSourceNames(applicationFromYaml), - ApplicationValidator.ValidateUnnamedAnnotationsInMultiSourceApplication(applicationFromYaml), - ApplicationValidator.ValidateSourceTypes(applicationFromYaml) - ); - validationResult.Action(log); - } -} \ No newline at end of file + void ValidateApplication(Application applicationFromYaml) + { + var validationResult = ValidationResult.Merge( + ApplicationValidator.ValidateSourceNames(applicationFromYaml), + ApplicationValidator.ValidateUnnamedAnnotationsInMultiSourceApplication(applicationFromYaml), + ApplicationValidator.ValidateSourceTypes(applicationFromYaml) + ); + validationResult.Action(log); + } +} diff --git a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/FileUpdateResult.cs b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/FileUpdateResult.cs index 80a49f09f6..a6524ded1d 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/FileUpdateResult.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/FileUpdateResult.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using Octopus.Calamari.Contracts.ArgoCD; namespace Calamari.ArgoCD.Conventions.UpdateImageTag; @@ -11,6 +12,18 @@ public bool HasChanges() { return ReplacedFiles.Count > 0 || PatchedFiles.Count > 0 || FilesRemoved.Length > 0; } + + // Combines the file changes from several sources so they can be described in a single commit + // (used when committing all of an application's sources in one repo+branch group together). + public static FileUpdateResult Merge(IEnumerable results) + { + var materialised = results.ToList(); + return new FileUpdateResult( + materialised.SelectMany(r => r.UpdatedImages).ToHashSet(), + materialised.SelectMany(r => r.ReplacedFiles).ToList(), + materialised.SelectMany(r => r.PatchedFiles).ToList(), + materialised.SelectMany(r => r.FilesRemoved).ToArray()); + } } diff --git a/source/Calamari/ArgoCD/Domain/Application.cs b/source/Calamari/ArgoCD/Domain/Application.cs index c3e6acf26c..8d3e5ef7ec 100644 --- a/source/Calamari/ArgoCD/Domain/Application.cs +++ b/source/Calamari/ArgoCD/Domain/Application.cs @@ -1,6 +1,6 @@ -using System; using System.Text.Json.Serialization; using Calamari.ArgoCD.Domain.Converters; +using Octopus.Calamari.Contracts.ArgoCD; namespace Calamari.ArgoCD.Domain { @@ -16,6 +16,7 @@ public class Application [JsonPropertyName("status")] [JsonConverter(typeof(ApplicationStatusConverter))] public ApplicationStatus Status { get; set; } = new ApplicationStatus(); - + + public NamespacedApplicationName NamespacedName => NamespacedApplicationName.Create(Metadata.Name, Metadata.Namespace); } } diff --git a/source/Calamari/ArgoCD/Git/GroupedRepositoryProcessor.cs b/source/Calamari/ArgoCD/Git/GroupedRepositoryProcessor.cs new file mode 100644 index 0000000000..d7a647e8e4 --- /dev/null +++ b/source/Calamari/ArgoCD/Git/GroupedRepositoryProcessor.cs @@ -0,0 +1,174 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using Calamari.ArgoCD.Conventions; +using Calamari.ArgoCD.Conventions.UpdateImageTag; +using Calamari.ArgoCD.Domain; +using Octopus.Calamari.Contracts.ArgoCD; + +namespace Calamari.ArgoCD.Git +{ + // A single application source that needs to be written into a Git repository and committed/pushed. + // The repository URL and target revision (the grouping keys) are derived from the source itself. + public class RepositorySourceUpdate + { + public RepositorySourceUpdate(NamespacedApplicationName applicationName, ApplicationSourceWithMetadata source, ISourceUpdater updater) + { + ApplicationName = applicationName; + Source = source; + Updater = updater; + } + + public NamespacedApplicationName ApplicationName { get; } + public ApplicationSourceWithMetadata Source { get; } + public ISourceUpdater Updater { get; } + + public string RepoUrl => Source.Source.OriginalRepoUrl; + public string TargetRevision => Source.Source.TargetRevision; + } + + // Processes a set of source updates by grouping them so that each repository is cloned once and each + // branch is checked out once. Within a repo+branch group, changes are committed once per application + // (direct push) and pushed in a single push; when raising pull requests, behaviour is unchanged - one + // branch, commit and pull request per source - while still reusing the shared clone and checkout. + // + // Failure handling is fail-fast: if an application genuinely fails to commit, the exception propagates, + // the clone is disposed (so no partial commits are pushed for that group), and the step aborts. + public class GroupedRepositoryProcessor + { + readonly AuthenticatingRepositoryFactory repositoryFactory; + readonly GitCommitParameters commitParameters; + readonly ICommitMessageGenerator commitMessageGenerator; + + public GroupedRepositoryProcessor(AuthenticatingRepositoryFactory repositoryFactory, + GitCommitParameters commitParameters, + ICommitMessageGenerator commitMessageGenerator) + { + this.repositoryFactory = repositoryFactory; + this.commitParameters = commitParameters; + this.commitMessageGenerator = commitMessageGenerator; + } + + // Returns a result for each update, in the same order the updates were provided. + public IReadOnlyList Process(IReadOnlyList updates) + { + var results = new SourceUpdateResult[updates.Count]; + var indexed = updates.Select((update, index) => (update, index)).ToList(); + + foreach (var repoGroup in indexed.GroupBy(x => x.update.RepoUrl)) + { + var branchGroups = repoGroup.GroupBy(x => x.update.TargetRevision).ToList(); + + // Clone once per repository. The clone fetches every branch, so each branch group below just + // checks out the branch it needs without re-cloning. + using var repository = repositoryFactory.CloneRepository(repoGroup.Key, branchGroups[0].Key); + + foreach (var branchGroup in branchGroups) + { + var reference = GitReference.CreateFromString(branchGroup.Key); + var items = branchGroup.ToList(); + + if (commitParameters.RequiresPr) + { + ProcessBranchAsPullRequests(repository, reference, items, results); + } + else + { + ProcessBranchAsDirectCommits(repository, reference, items, results); + } + } + } + + return results; + } + + void ProcessBranchAsDirectCommits(RepositoryWrapper repository, + GitReference reference, + List<(RepositorySourceUpdate update, int index)> items, + SourceUpdateResult[] results) + { + repository.CheckoutBranch(reference); + + var committedAnything = false; + + // Apply and commit one application at a time so each commit contains only that application's changes. + foreach (var applicationGroup in items.GroupBy(x => x.update.ApplicationName.Value)) + { + // Apply each source and record whether it actually changed the working tree. A source can + // produce a result without changing anything (e.g. an image already at the target tag), and + // such a source must not be attributed the application's commit. + var changedPaths = new HashSet(repository.GetChangedFilePaths()); + var applied = new List<(int index, FileUpdateResult fileResult, bool changedWorkingTree)>(); + foreach (var item in applicationGroup) + { + var fileResult = item.update.Updater.Process(item.update.Source, repository.WorkingDirectory); + var changedPathsNow = new HashSet(repository.GetChangedFilePaths()); + var changedWorkingTree = changedPathsNow.Except(changedPaths).Any(); + changedPaths = changedPathsNow; + applied.Add((item.index, fileResult, changedWorkingTree)); + } + + PushResult? applicationCommit = null; + if (applied.Any(a => a.changedWorkingTree)) + { + repository.StageAllChanges(); + var description = commitMessageGenerator.GenerateDescription(FileUpdateResult.Merge(applied.Select(a => a.fileResult))); + if (repository.CommitChanges(commitParameters.Summary, description)) + { + // Captured before the push. Pushes are serial and the clone is fresh, so a rebase-on-push + // (which would rewrite this SHA) is not expected; see RepositoryWrapper.PushChanges. + applicationCommit = repository.GetHeadCommitResult(); + committedAnything = true; + } + } + + foreach (var item in applied) + { + var pushResult = item.changedWorkingTree ? applicationCommit : null; + results[item.index] = new SourceUpdateResult(item.fileResult.UpdatedImages, pushResult, item.fileResult.ReplacedFiles, item.fileResult.PatchedFiles); + } + } + + // A single push carries every application's commit on this branch. + if (committedAnything) + { + repository.PushChanges(false, commitParameters.Summary, string.Empty, reference, commitParameters.PushRetryAttempts, CancellationToken.None) + .GetAwaiter() + .GetResult(); + } + } + + void ProcessBranchAsPullRequests(RepositoryWrapper repository, + GitReference reference, + List<(RepositorySourceUpdate update, int index)> items, + SourceUpdateResult[] results) + { + foreach (var item in items) + { + // Reset back to the remote tip so each pull request branch contains only this source's commit. + repository.CheckoutBranch(reference); + + var fileResult = item.update.Updater.Process(item.update.Source, repository.WorkingDirectory); + if (!fileResult.HasChanges()) + { + results[item.index] = new SourceUpdateResult(fileResult.UpdatedImages, null, fileResult.ReplacedFiles, fileResult.PatchedFiles); + continue; + } + + repository.StageAllChanges(); + var description = commitMessageGenerator.GenerateDescription(fileResult); + if (!repository.CommitChanges(commitParameters.Summary, description)) + { + results[item.index] = new SourceUpdateResult(fileResult.UpdatedImages, null, fileResult.ReplacedFiles, fileResult.PatchedFiles); + continue; + } + + var pushResult = repository.PushChanges(true, commitParameters.Summary, description, reference, commitParameters.PushRetryAttempts, CancellationToken.None) + .GetAwaiter() + .GetResult(); + results[item.index] = new SourceUpdateResult(fileResult.UpdatedImages, pushResult, fileResult.ReplacedFiles, fileResult.PatchedFiles); + } + } + } +} diff --git a/source/Calamari/ArgoCD/Git/RepositoryAdapter.cs b/source/Calamari/ArgoCD/Git/RepositoryAdapter.cs deleted file mode 100644 index b8abeddc99..0000000000 --- a/source/Calamari/ArgoCD/Git/RepositoryAdapter.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System; -using Calamari.ArgoCD.Conventions; -using Calamari.ArgoCD.Conventions.UpdateImageTag; -using Calamari.ArgoCD.Domain; - -namespace Calamari.ArgoCD.Git; - -public class RepositoryAdapter -{ - readonly AuthenticatingRepositoryFactory repositoryFactory; - readonly RepositoryUpdater repositoryUpdater; - - public RepositoryAdapter(AuthenticatingRepositoryFactory repositoryFactory, - RepositoryUpdater repositoryUpdater) - { - this.repositoryFactory = repositoryFactory; - this.repositoryUpdater = repositoryUpdater; - } - - public SourceUpdateResult Process(ApplicationSourceWithMetadata sourceWithMetadata, ISourceUpdater updater) - { - using var repository = repositoryFactory.CloneRepository(sourceWithMetadata.Source.OriginalRepoUrl, sourceWithMetadata.Source.TargetRevision); - var filesUpdated = updater.Process(sourceWithMetadata, repository.WorkingDirectory); - - if (filesUpdated.HasChanges()) - { - var pushResult = repositoryUpdater.PushToRemote(repository, GitReference.CreateFromString(sourceWithMetadata.Source.TargetRevision), filesUpdated); - return new SourceUpdateResult(filesUpdated.UpdatedImages, pushResult, filesUpdated.ReplacedFiles, filesUpdated.PatchedFiles); - } - - return new SourceUpdateResult([], null, filesUpdated.ReplacedFiles, filesUpdated.PatchedFiles); - } -} diff --git a/source/Calamari/ArgoCD/Git/RepositoryFactory.cs b/source/Calamari/ArgoCD/Git/RepositoryFactory.cs index 771c9b83b0..fdd23a1055 100644 --- a/source/Calamari/ArgoCD/Git/RepositoryFactory.cs +++ b/source/Calamari/ArgoCD/Git/RepositoryFactory.cs @@ -1,6 +1,5 @@ using System; using System.IO; -using System.Linq; using System.Threading; using Calamari.ArgoCD.Git.PullRequests; using Calamari.Common.Commands; @@ -58,16 +57,11 @@ public RepositoryWrapper CloneRepository(string repositoryName, IGitConnection g RepositoryWrapper CheckoutGitRepository(IGitConnection gitConnection, string checkoutPath) { - //if the branch name is head, then we just clone the default - //if it's not head, then clone the branch immediately - var options = gitConnection.GitReference is GitHead - ? new CloneOptions() - : new CloneOptions - { - //note: when cloning, libgit2sharp prepends "refs/remotes/origin/" to this value (so _must_ be a branch to succeed). - BranchName = (gitConnection.GitReference as GitBranchName)?.ToFriendlyName() - }; - + // Always clone with default options so the remote's default branch is checked out first. This lets + // the RepositoryWrapper capture the true default branch (for resolving 'HEAD' references) before we + // check out the requested reference, and leaves every remote branch available so the clone can be + // reused to check out additional branches without re-cloning. + var options = new CloneOptions(); options.FetchOptions.CredentialsProvider = gitConnection.ToLibGit2SharpCredentialHandler(); options.FetchOptions.CertificateCheck = gitConnection.ToLibGit2SharpCertificateCheckHandler(log); @@ -91,28 +85,6 @@ RepositoryWrapper CheckoutGitRepository(IGitConnection gitConnection, string che var repo = new Repository(repoPath); - try - { - //this is required to handle the issue around "HEAD" - var branchToCheckout = repo.GetBranchName(gitConnection.GitReference); - var remoteBranch = repo.Branches.First(f => f.IsRemote && f.UpstreamBranchCanonicalName == branchToCheckout.Value); - - log.VerboseFormat("Checking out '{0}' @ {1}", branchToCheckout, remoteBranch.Tip.Sha.Substring(0, 10)); - - //A local branch is required such that libgit2sharp can create "tracking" data - // libgit2sharp does not support pushing from a detached head - if (repo.Branches[branchToCheckout.Value] == null) - { - repo.CreateBranch(branchToCheckout.Value, remoteBranch.Tip); - } - - LibGit2Sharp.Commands.Checkout(repo, branchToCheckout.ToFriendlyName()); - } - catch (LibGit2SharpException e) - { - throw new CommandException($"Failed to checkout branch '{gitConnection.GitReference}' in repository at {gitConnection.Url}. Error: {e.Message}", e); - } - //TODO(tmm): Make this function (and all callers async). var gitVendorApiAdapter = gitConnection is HttpsGitConnection httpsGitConnection ? gitVendorPullRequestClientResolver.TryResolve(httpsGitConnection, log, CancellationToken.None).Result @@ -123,13 +95,26 @@ RepositoryWrapper CheckoutGitRepository(IGitConnection gitConnection, string che log.Verbose("Git is using SSH authentication, Git vendor functionality such as PR creation will not be available"); } - return new RepositoryWrapper(repo, - fileSystem, - checkoutPath, - log, - gitConnection, - gitVendorApiAdapter, - clock); + var repository = new RepositoryWrapper(repo, + fileSystem, + checkoutPath, + log, + gitConnection, + gitVendorApiAdapter, + clock); + + try + { + repository.CheckoutBranch(gitConnection.GitReference); + } + catch (Exception e) + { + repository.Dispose(); + // Preserve the original behaviour where requesting a reference that is not a branch surfaces as a clone failure. + throw new CommandException($"Failed to clone Git repository at {gitConnection.Url}. Are you sure this URL is a Git repository, and the reference is a branch?", e); + } + + return repository; } } } diff --git a/source/Calamari/ArgoCD/Git/RepositoryWrapper.cs b/source/Calamari/ArgoCD/Git/RepositoryWrapper.cs index ca7361ad91..70080bba34 100644 --- a/source/Calamari/ArgoCD/Git/RepositoryWrapper.cs +++ b/source/Calamari/ArgoCD/Git/RepositoryWrapper.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; @@ -36,8 +37,70 @@ public class RepositoryWrapper( readonly Identity repositoryIdentity = new("Octopus", "octopus@octopus.com"); + // Captured at construction (immediately after clone, before any checkout) so that 'HEAD' + // references can be resolved to the remote's default branch even after we have checked out + // a different branch on this (reused) clone. + readonly string defaultBranchCanonicalName = repository.Head.CanonicalName; + public string WorkingDirectory => repository.Info.WorkingDirectory; + // Checks out the requested reference, creating a local tracking branch if required, and hard-resets + // it to the remote tip. Safe to call repeatedly on a single clone to switch between branches (or to + // reset the current branch back to the remote tip between sources when raising pull requests). + public void CheckoutBranch(GitReference reference) + { + var branchToCheckout = reference is GitHead + ? new GitBranchName(defaultBranchCanonicalName) + : repository.GetBranchName(reference); + + var remoteBranch = repository.Branches.FirstOrDefault(f => f.IsRemote && f.UpstreamBranchCanonicalName == branchToCheckout.Value); + if (remoteBranch == null) + { + throw new CommandException($"Failed to checkout branch '{reference}' in repository at {connection.Url}. The reference could not be found as a branch on the remote."); + } + + try + { + log.VerboseFormat("Checking out '{0}' @ {1}", branchToCheckout, remoteBranch.Tip.Sha.Substring(0, 10)); + + //A local branch is required such that libgit2sharp can create "tracking" data + // libgit2sharp does not support pushing from a detached head + if (repository.Branches[branchToCheckout.Value] == null) + { + repository.CreateBranch(branchToCheckout.Value, remoteBranch.Tip); + } + + LibGit2Sharp.Commands.Checkout(repository, branchToCheckout.ToFriendlyName()); + // Ensure the local branch matches the remote tip. This matters when the clone is reused: + // a previous source may have left a local commit on this branch that must not leak into this one. + repository.Reset(ResetMode.Hard, remoteBranch.Tip); + } + catch (LibGit2SharpException e) + { + throw new CommandException($"Failed to checkout branch '{reference}' in repository at {connection.Url}. Error: {e.Message}", e); + } + } + + // Returns the current HEAD commit as a PushResult. Used to capture a per-application commit before + // a single push of all the application commits made on a branch. + public PushResult GetHeadCommitResult() + { + var commit = repository.Head.Tip; + return new PushResult(commit.Sha, commit.ShortSha(), commit.Author.When); + } + + // The set of files that currently differ from HEAD (modified, added, deleted, untracked). Used to + // detect whether a particular source actually changed anything: a source updater may produce a result + // (e.g. a computed image patch) that is identical to what is already committed, in which case it must + // not be attributed a commit. Mirrors git's "nothing to commit" behaviour at a per-source granularity. + public IReadOnlyCollection GetChangedFilePaths() + { + var status = repository.RetrieveStatus(new StatusOptions { IncludeUntracked = true, RecurseUntrackedDirs = true, IncludeIgnored = false }); + return status.Where(e => e.State != FileStatus.Unaltered && e.State != FileStatus.Ignored) + .Select(e => e.FilePath) + .ToList(); + } + // returns true if changes were made to the repository public bool CommitChanges(string summary, string description) {