From 64a10234833165b392984b500a776104ab731187 Mon Sep 17 00:00:00 2001 From: Noam Gal Date: Tue, 23 Jun 2026 15:46:13 +0300 Subject: [PATCH 1/6] Clone once per repo for Argo CD app updates Group sources across applications by repository and branch so each repo is cloned once and each branch checked out once. Direct pushes commit once per application and push once per branch group; pull request mode is unchanged. Co-Authored-By: Claude Opus 4.8 --- ...ateArgoCDAppImagesInstallConventionTest.cs | 108 +++++++++++ .../Conventions/FileUpdateResultTests.cs | 38 ++++ .../ApplicationSourceUpdater.cs | 38 +--- .../ManifestTemplating/ApplicationUpdater.cs | 99 +++++----- .../ManifestUpdateResult.cs | 8 - .../ArgoCD/Conventions/PlannedApplication.cs | 44 +++++ .../UpdateArgoCDAppImagesInstallConvention.cs | 28 ++- ...CDApplicationManifestsInstallConvention.cs | 34 ++-- .../ApplicationSourceUpdater.cs | 30 +-- .../UpdateImageTag/ApplicationUpdater.cs | 152 ++++++++------- .../UpdateImageTag/FileUpdateResult.cs | 13 ++ .../ArgoCD/Git/GroupedRepositoryProcessor.cs | 174 ++++++++++++++++++ .../Calamari/ArgoCD/Git/RepositoryAdapter.cs | 33 ---- .../Calamari/ArgoCD/Git/RepositoryFactory.cs | 65 +++---- .../Calamari/ArgoCD/Git/RepositoryWrapper.cs | 63 +++++++ 15 files changed, 649 insertions(+), 278 deletions(-) create mode 100644 source/Calamari.Tests/ArgoCD/Conventions/FileUpdateResultTests.cs delete mode 100644 source/Calamari/ArgoCD/Conventions/ManifestTemplating/ManifestUpdateResult.cs create mode 100644 source/Calamari/ArgoCD/Conventions/PlannedApplication.cs create mode 100644 source/Calamari/ArgoCD/Git/GroupedRepositoryProcessor.cs delete mode 100644 source/Calamari/ArgoCD/Git/RepositoryAdapter.cs diff --git a/source/Calamari.Tests/ArgoCD/Commands/Conventions/UpdateArgoCDAppImagesInstallConventionTest.cs b/source/Calamari.Tests/ArgoCD/Commands/Conventions/UpdateArgoCDAppImagesInstallConventionTest.cs index 0a23ccac50..ed86daabb2 100644 --- a/source/Calamari.Tests/ArgoCD/Commands/Conventions/UpdateArgoCDAppImagesInstallConventionTest.cs +++ b/source/Calamari.Tests/ArgoCD/Commands/Conventions/UpdateArgoCDAppImagesInstallConventionTest.cs @@ -1281,6 +1281,114 @@ public void MultiSource_OutOfScopeSource_IsNotTracked() source1.CommitSha.Should().HaveLength(40, "source 1 was updated and committed"); } + [Test] + public void MultipleApplications_SharingRepoAndBranch_AreClonedOnce_AndCommittedPerApplication() + { + // Arrange: two applications that point at the same repository + branch, each with its own outdated image. + var updater = CreateConvention(); + var runningDeployment = CreateRunningDeployment(("nginx", "index.docker.io/nginx:1.27.1")); + + var file1 = Path.Combine("app1", "deployment.yaml"); + var file2 = Path.Combine("app2", "deployment.yaml"); + originRepo.AddFilesToBranch(argoCDBranchName, [ + (file1, MakeDeploymentYaml("app1-deployment", "nginx:1.19")), + (file2, MakeDeploymentYaml("app2-deployment", "nginx:1.19")), + ]); + + customPropertiesLoader.Load() + .Returns(new ArgoCDCustomPropertiesDto( + [new ArgoCDGatewayDto(GatewayId, "Gateway1")], + [ + new ArgoCDApplicationDto(GatewayId, "App1", "argocd", "yaml1", "docker.io", "http://my-argo.com"), + new ArgoCDApplicationDto(GatewayId, "App2", "argocd", "yaml2", "docker.io", "http://my-argo.com"), + ], + [new GitCredentialDto(OriginUrl, "", "")])); + + argoCdApplicationManifestParser.ParseManifest("yaml1").Returns(BuildDirectoryApp("App1", "app1")); + argoCdApplicationManifestParser.ParseManifest("yaml2").Returns(BuildDirectoryApp("App2", "app2")); + + var getResults = CaptureReporterResults(); + + // Act + updater.Install(runningDeployment); + + // Assert + using var scope = new AssertionScope(); + + // The shared repository is cloned exactly once, even though two applications use it. + log.StandardOut.Count(m => m.Contains("Cloning repository")).Should().Be(1, "the shared repository should only be cloned once"); + + var results = getResults(); + results.Should().HaveCount(2); + var app1 = results.Single(r => r.ApplicationName.Value == "argocd/App1"); + var app2 = results.Single(r => r.ApplicationName.Value == "argocd/App2"); + + var app1Sha = app1.TrackedSourceDetails.Single().CommitSha; + var app2Sha = app2.TrackedSourceDetails.Single().CommitSha; + app1Sha.Should().HaveLength(40, "App1 had an outdated image and was committed"); + app2Sha.Should().HaveLength(40, "App2 had an outdated image and was committed"); + app1Sha.Should().NotBe(app2Sha, "each application is committed separately within the shared clone"); + + // Both applications' changes reached the remote. + originRepo.ReadFileFromBranch(argoCDBranchName, file1).Should().Contain("nginx:1.27.1"); + originRepo.ReadFileFromBranch(argoCDBranchName, file2).Should().Contain("nginx:1.27.1"); + } + + [Test] + public void MultipleApplications_SameRepoDifferentBranches_AreClonedOnce_AndCheckedOutPerBranch() + { + // Arrange: two applications in the same repository but targeting different branches. + var updater = CreateConvention(); + var runningDeployment = CreateRunningDeployment(("nginx", "index.docker.io/nginx:1.27.1")); + + var mainBranch = RepositoryHelpers.MainBranchName; + var file = Path.Combine("app", "deployment.yaml"); + originRepo.AddFilesToBranch(argoCDBranchName, [(file, MakeDeploymentYaml("dev-deployment", "nginx:1.19"))]); + originRepo.AddFilesToBranch(mainBranch, [(file, MakeDeploymentYaml("main-deployment", "nginx:1.19"))]); + + customPropertiesLoader.Load() + .Returns(new ArgoCDCustomPropertiesDto( + [new ArgoCDGatewayDto(GatewayId, "Gateway1")], + [ + new ArgoCDApplicationDto(GatewayId, "App1", "argocd", "yaml1", "docker.io", "http://my-argo.com"), + new ArgoCDApplicationDto(GatewayId, "App2", "argocd", "yaml2", "docker.io", "http://my-argo.com"), + ], + [new GitCredentialDto(OriginUrl, "", "")])); + + argoCdApplicationManifestParser.ParseManifest("yaml1").Returns(BuildDirectoryApp("App1", "app", ArgoCDBranchFriendlyName)); + argoCdApplicationManifestParser.ParseManifest("yaml2").Returns(BuildDirectoryApp("App2", "app", mainBranch.ToFriendlyName())); + + var getResults = CaptureReporterResults(); + + // Act + updater.Install(runningDeployment); + + // Assert + using var scope = new AssertionScope(); + + log.StandardOut.Count(m => m.Contains("Cloning repository")).Should().Be(1, "the shared repository should only be cloned once across both branches"); + log.MessagesVerboseFormatted.Should().Contain(m => m.Contains($"Checking out '{argoCDBranchName.Value}'")); + log.MessagesVerboseFormatted.Should().Contain(m => m.Contains($"Checking out '{mainBranch.Value}'")); + + var results = getResults(); + results.Should().HaveCount(2); + results.Should().OnlyContain(r => r.Updated, "both applications had an outdated image on their respective branch"); + + originRepo.ReadFileFromBranch(argoCDBranchName, file).Should().Contain("nginx:1.27.1"); + originRepo.ReadFileFromBranch(mainBranch, file).Should().Contain("nginx:1.27.1"); + } + + Application BuildDirectoryApp(string name, string path, string targetRevision = ArgoCDBranchFriendlyName) + => new ArgoCDApplicationBuilder() + .WithName(name).WithNamespace("argocd") + .WithAnnotations(new Dictionary + { + [ArgoCDConstants.Annotations.OctopusProjectAnnotationKey(null)] = ProjectSlug, + [ArgoCDConstants.Annotations.OctopusEnvironmentAnnotationKey(null)] = EnvironmentSlug, + }) + .WithSource(new ApplicationSource { OriginalRepoUrl = OriginUrl, Path = path, TargetRevision = targetRevision }, SourceTypeConstants.Directory) + .Build(); + static string MakeDeploymentYaml(string name, params string[] images) { var containerName = (string image) => image.Split('/').Last().Split(':').First(); diff --git a/source/Calamari.Tests/ArgoCD/Conventions/FileUpdateResultTests.cs b/source/Calamari.Tests/ArgoCD/Conventions/FileUpdateResultTests.cs new file mode 100644 index 0000000000..fa785ffb52 --- /dev/null +++ b/source/Calamari.Tests/ArgoCD/Conventions/FileUpdateResultTests.cs @@ -0,0 +1,38 @@ +using Calamari.ArgoCD.Conventions.UpdateImageTag; +using FluentAssertions; +using NUnit.Framework; +using Octopus.Calamari.Contracts.ArgoCD; + +namespace Calamari.Tests.ArgoCD.Conventions +{ + [TestFixture] + public class FileUpdateResultTests + { + [Test] + public void Merge_CombinesImagesReplacedPatchedAndRemovedFiles() + { + var first = new FileUpdateResult(["nginx:1.27.1"], [new FileHash("a.yaml", "h1")], [new FileJsonPatch("a.yaml", "patchA")], ["removedA.yaml"]); + var second = new FileUpdateResult(["redis:7.0", "nginx:1.27.1"], [new FileHash("b.yaml", "h2")], [new FileJsonPatch("b.yaml", "patchB")], ["removedB.yaml"]); + + var merged = FileUpdateResult.Merge([first, second]); + + merged.UpdatedImages.Should().BeEquivalentTo(["nginx:1.27.1", "redis:7.0"], "duplicate images are de-duplicated"); + merged.ReplacedFiles.Should().BeEquivalentTo([new FileHash("a.yaml", "h1"), new FileHash("b.yaml", "h2")]); + merged.PatchedFiles.Should().BeEquivalentTo([new FileJsonPatch("a.yaml", "patchA"), new FileJsonPatch("b.yaml", "patchB")]); + merged.FilesRemoved.Should().BeEquivalentTo(["removedA.yaml", "removedB.yaml"]); + merged.HasChanges().Should().BeTrue(); + } + + [Test] + public void Merge_OfEmptyResults_HasNoChanges() + { + var merged = FileUpdateResult.Merge([FileUpdateResult.EmptyFileUpdateResult, FileUpdateResult.EmptyFileUpdateResult]); + + merged.HasChanges().Should().BeFalse(); + merged.UpdatedImages.Should().BeEmpty(); + merged.ReplacedFiles.Should().BeEmpty(); + merged.PatchedFiles.Should().BeEmpty(); + merged.FilesRemoved.Should().BeEmpty(); + } + } +} diff --git a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationSourceUpdater.cs b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationSourceUpdater.cs index 94bd2a5e22..4d191b10e8 100644 --- a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationSourceUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationSourceUpdater.cs @@ -1,5 +1,5 @@ +using Calamari.ArgoCD.Conventions.UpdateImageTag; using Calamari.ArgoCD.Domain; -using Calamari.ArgoCD.Git; using Calamari.ArgoCD.Models; using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; @@ -7,37 +7,30 @@ namespace Calamari.ArgoCD.Conventions.ManifestTemplating; +// Determines whether a source is in scope for this deployment and builds the file updater (which copies the +// templated package files into the source path). Clone/commit/push is handled by the GroupedRepositoryProcessor. public class ApplicationSourceUpdater { readonly Application applicationFromYaml; readonly DeploymentScope deploymentScope; - readonly RepositoryAdapter repositoryAdapter; readonly ArgoCommitToGitConfig deploymentConfig; readonly IPackageRelativeFile[] packageFiles; - readonly ArgoCDGatewayDto gateway; readonly ILog log; readonly ICalamariFileSystem fileSystem; - readonly ArgoCDOutputVariablesWriter outputVariablesWriter; public ApplicationSourceUpdater(Application applicationFromYaml, - ArgoCDGatewayDto gateway, DeploymentScope deploymentScope, ArgoCommitToGitConfig deploymentConfig, IPackageRelativeFile[] packageFiles, ILog log, - ICalamariFileSystem fileSystem, - ArgoCDOutputVariablesWriter outputVariablesWriter, - RepositoryAdapter repositoryAdapter) + ICalamariFileSystem fileSystem) { this.applicationFromYaml = applicationFromYaml; this.deploymentScope = deploymentScope; this.deploymentConfig = deploymentConfig; this.packageFiles = packageFiles; - this.gateway = gateway; this.log = log; this.fileSystem = fileSystem; - this.outputVariablesWriter = outputVariablesWriter; - this.repositoryAdapter = repositoryAdapter; } public bool IsAppInScope(ApplicationSourceWithMetadata sourceWithMetadata) @@ -50,28 +43,11 @@ public bool IsAppInScope(ApplicationSourceWithMetadata sourceWithMetadata) return deploymentScope.Matches(annotatedScope); } - public ManifestUpdateResult ProcessSource(ApplicationSourceWithMetadata sourceWithMetadata) + public ISourceUpdater CreateSourceUpdater(ApplicationSourceWithMetadata sourceWithMetadata) { var applicationSource = sourceWithMetadata.Source; - log.Info($"Writing files to repository '{applicationSource.OriginalRepoUrl}' for '{applicationFromYaml.Metadata.Name}'"); + applicationFromYaml.Metadata.Annotations.TryGetValue(ArgoCDConstants.Annotations.OctopusPathAnnotationKey(applicationSource.Name.ToApplicationSourceName()), out var pathOverrideFromAnnotation); - applicationFromYaml.Metadata.Annotations.TryGetValue(ArgoCDConstants.Annotations.OctopusPathAnnotationKey(applicationSource.Name.ToApplicationSourceName()), out var pathOverrideFromAnnotation); - - var sourceUpdater = new CopyTemplatesSourceUpdater(packageFiles, log, fileSystem, deploymentConfig.PurgeOutputDirectory, pathOverrideFromAnnotation); - - var sourceUpdateResult = repositoryAdapter.Process(sourceWithMetadata, sourceUpdater); - - outputVariablesWriter.WriteSourceUpdateResultOutputWhenPushResultExists(gateway.Name, - NamespacedApplicationName.Create(applicationFromYaml.Metadata.Name, applicationFromYaml.Metadata.Namespace), - sourceWithMetadata.Index, - sourceUpdateResult); - - if (sourceUpdateResult.PushResult is not null) - { - return new ManifestUpdateResult(true, sourceUpdateResult.PushResult.CommitSha, sourceUpdateResult.PushResult.CommitTimestamp, sourceUpdateResult.ReplacedFiles); - } - - log.Info("No changes were committed"); - return new ManifestUpdateResult(false, null, null, sourceUpdateResult.ReplacedFiles); + return new CopyTemplatesSourceUpdater(packageFiles, log, fileSystem, deploymentConfig.PurgeOutputDirectory, pathOverrideFromAnnotation); } } diff --git a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs index fef9aa5d8f..4753173275 100644 --- a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Linq; +using Calamari.ArgoCD.Conventions.UpdateImageTag; using Calamari.ArgoCD.Domain; using Calamari.ArgoCD.Git; using Calamari.ArgoCD.Models; @@ -11,25 +12,22 @@ namespace Calamari.ArgoCD.Conventions.ManifestTemplating; public class ApplicationUpdater { - readonly AuthenticatingRepositoryFactory repositoryFactory; readonly DeploymentScope deploymentScope; readonly ArgoCommitToGitConfig deploymentConfig; readonly ILog log; readonly ICalamariFileSystem fileSystem; readonly IArgoCDApplicationManifestParser argoCdApplicationManifestParser; - readonly ICommitMessageGenerator commitMessageGenerator; readonly ArgoCDOutputVariablesWriter outputVariablesWriter; readonly IPackageRelativeFile[] packageFiles; - - public ApplicationUpdater(AuthenticatingRepositoryFactory repositoryFactory, DeploymentScope deploymentScope, ArgoCommitToGitConfig deploymentConfig, ILog log, + public ApplicationUpdater(DeploymentScope deploymentScope, + ArgoCommitToGitConfig deploymentConfig, + ILog log, ICalamariFileSystem fileSystem, IArgoCDApplicationManifestParser argoCdApplicationManifestParser, ArgoCDOutputVariablesWriter outputVariablesWriter, - IPackageRelativeFile[] packageFiles, - ICommitMessageGenerator commitMessageGenerator) + IPackageRelativeFile[] packageFiles) { - this.repositoryFactory = repositoryFactory; this.deploymentScope = deploymentScope; this.deploymentConfig = deploymentConfig; this.log = log; @@ -37,67 +35,72 @@ public ApplicationUpdater(AuthenticatingRepositoryFactory repositoryFactory, Dep this.argoCdApplicationManifestParser = argoCdApplicationManifestParser; this.outputVariablesWriter = outputVariablesWriter; this.packageFiles = packageFiles; - this.commitMessageGenerator = commitMessageGenerator; } - - public ProcessApplicationResult ProcessApplication( - ArgoCDApplicationDto application, - ArgoCDGatewayDto gateway) + + // Phase 1: parse, validate, scope and build the set of source updates for an application. + public PlannedApplication Plan(ArgoCDApplicationDto application, ArgoCDGatewayDto gateway) { log.InfoFormat("Processing application {0}", application.Name); var applicationFromYaml = argoCdApplicationManifestParser.ParseManifest(application.Manifest); var containsMultipleSources = applicationFromYaml.Spec.Sources.Count > 1; var applicationName = applicationFromYaml.Metadata.Name; + var namespacedName = NamespacedApplicationName.Create(applicationName, applicationFromYaml.Metadata.Namespace); LogWarningIfUpdatingMultipleSources(applicationFromYaml.Spec.Sources, applicationFromYaml.Metadata.Annotations, containsMultipleSources); - + ValidateApplication(applicationFromYaml); - var repositoryAdapter = new RepositoryAdapter(repositoryFactory, new RepositoryUpdater(deploymentConfig.CommitParameters, log, commitMessageGenerator)); - var sourceUpdater = new ApplicationSourceUpdater(applicationFromYaml, - gateway, - deploymentScope, - deploymentConfig, - packageFiles, - log, - fileSystem, - outputVariablesWriter, - repositoryAdapter); - - var trackedSourceUpdateResults = applicationFromYaml - .GetSourcesWithMetadata() - .Where(sourceUpdater.IsAppInScope) - .Select(applicationSource => new - { - UpdateResult = sourceUpdater.ProcessSource(applicationSource), - applicationSource - }) - .ToList(); + var sourceUpdater = new ApplicationSourceUpdater(applicationFromYaml, deploymentScope, deploymentConfig, packageFiles, log, fileSystem); + + var plannedSources = applicationFromYaml.GetSourcesWithMetadata() + .Where(sourceUpdater.IsAppInScope) + .Select(source => new PlannedSource(source, new RepositorySourceUpdate(namespacedName, source, sourceUpdater.CreateSourceUpdater(source)))) + .ToList(); + + var matchingSourceCount = applicationFromYaml.Spec.Sources.Count(s => deploymentScope.Matches(ScopingAnnotationReader.GetScopeForApplicationSource(s.Name.ToApplicationSourceName(), applicationFromYaml.Metadata.Annotations, containsMultipleSources))); + + return new PlannedApplication(application, gateway, applicationFromYaml, namespacedName, applicationName, plannedSources, applicationFromYaml.Spec.Sources.Count, matchingSourceCount); + } + + // Phase 3: turn the processed results back into a per-application result, writing per-source output variables. + public ProcessApplicationResult AssembleResult(PlannedApplication plan, IReadOnlyDictionary resultsByUpdate) + { + foreach (var plannedSource in plan.Sources) + { + outputVariablesWriter.WriteSourceUpdateResultOutputWhenPushResultExists(plan.Gateway.Name, + plan.NamespacedName, + plannedSource.Source.Index, + resultsByUpdate[plannedSource.Update]); + } + + var trackedSourceDetails = plan.Sources.Select(plannedSource => + { + var result = resultsByUpdate[plannedSource.Update]; + return new TrackedSourceDetail(result.PushResult?.CommitSha, result.PushResult?.CommitTimestamp, plannedSource.Source.Index, result.ReplacedFiles, []); + }) + .ToList(); //if we have links, use that to generate a link, otherwise just put the name there - var instanceLinks = application.InstanceWebUiUrl != null ? new ArgoCDInstanceLinks(application.InstanceWebUiUrl) : null; + var instanceLinks = plan.Application.InstanceWebUiUrl != null ? new ArgoCDInstanceLinks(plan.Application.InstanceWebUiUrl) : null; var linkifiedAppName = instanceLinks != null - ? log.FormatLink(instanceLinks.ApplicationDetails(applicationName, applicationFromYaml.Metadata.Namespace), applicationName) - : applicationName; - - var message = trackedSourceUpdateResults.Any(u => u.UpdateResult.Updated) - ? "Updated Application {0}" - : "Nothing to update for Application {0}"; + ? log.FormatLink(instanceLinks.ApplicationDetails(plan.ApplicationName, plan.ApplicationFromYaml.Metadata.Namespace), plan.ApplicationName) + : plan.ApplicationName; - log.InfoFormat(message, linkifiedAppName); + var anyUpdated = plan.Sources.Any(plannedSource => resultsByUpdate[plannedSource.Update].Updated); + log.InfoFormat(anyUpdated ? "Updated Application {0}" : "Nothing to update for Application {0}", linkifiedAppName); return new ProcessApplicationResult( - application.GatewayId, - NamespacedApplicationName.Create(applicationName, applicationFromYaml.Metadata.Namespace), - applicationFromYaml.Spec.Sources.Count, - applicationFromYaml.Spec.Sources.Count(s => deploymentScope.Matches(ScopingAnnotationReader.GetScopeForApplicationSource(s.Name.ToApplicationSourceName(), applicationFromYaml.Metadata.Annotations, containsMultipleSources))), - trackedSourceUpdateResults.Select(r => new TrackedSourceDetail(r.UpdateResult.CommitSha, r.UpdateResult.CommitTimestamp, r.applicationSource.Index, r.UpdateResult.ReplacedFiles, [])).ToList(), + plan.Application.GatewayId, + plan.NamespacedName, + plan.TotalSourceCount, + plan.MatchingSourceCount, + trackedSourceDetails, [], - trackedSourceUpdateResults.Where(r => r.UpdateResult.Updated).Select(r => r.applicationSource.Source.OriginalRepoUrl).ToHashSet()); + plan.Sources.Where(plannedSource => resultsByUpdate[plannedSource.Update].Updated).Select(plannedSource => plannedSource.Source.Source.OriginalRepoUrl).ToHashSet()); } - + void LogWarningIfUpdatingMultipleSources( List sourcesToInspect, Dictionary applicationAnnotations, diff --git a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ManifestUpdateResult.cs b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ManifestUpdateResult.cs deleted file mode 100644 index 508436fd5e..0000000000 --- a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ManifestUpdateResult.cs +++ /dev/null @@ -1,8 +0,0 @@ -#nullable enable -using System; -using System.Collections.Generic; -using Octopus.Calamari.Contracts.ArgoCD; - -namespace Calamari.ArgoCD.Conventions.ManifestTemplating; - -public record ManifestUpdateResult(bool Updated, string? CommitSha, DateTimeOffset? CommitTimestamp, List ReplacedFiles); diff --git a/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs b/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs new file mode 100644 index 0000000000..52b137b77c --- /dev/null +++ b/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs @@ -0,0 +1,44 @@ +using System.Collections.Generic; +using Calamari.ArgoCD.Domain; +using Calamari.ArgoCD.Git; +using Octopus.Calamari.Contracts.ArgoCD; + +namespace Calamari.ArgoCD.Conventions +{ + // A single in-scope source of an application, paired with the work item that will write/commit/push it. + public record PlannedSource(ApplicationSourceWithMetadata Source, RepositorySourceUpdate Update); + + // The result of inspecting one application: its parsed manifest, the gateway it belongs to, and the + // in-scope sources that need processing. Built up-front for every application so that the sources can + // be grouped by repository/branch across applications before any cloning happens. + public class PlannedApplication + { + public PlannedApplication(ArgoCDApplicationDto application, + ArgoCDGatewayDto gateway, + Application applicationFromYaml, + NamespacedApplicationName namespacedName, + string applicationName, + IReadOnlyList sources, + int totalSourceCount, + int matchingSourceCount) + { + Application = application; + Gateway = gateway; + ApplicationFromYaml = applicationFromYaml; + NamespacedName = namespacedName; + ApplicationName = applicationName; + Sources = sources; + TotalSourceCount = totalSourceCount; + MatchingSourceCount = matchingSourceCount; + } + + public ArgoCDApplicationDto Application { get; } + public ArgoCDGatewayDto Gateway { get; } + public Application ApplicationFromYaml { get; } + public NamespacedApplicationName NamespacedName { get; } + public string ApplicationName { get; } + public IReadOnlyList Sources { get; } + public int TotalSourceCount { get; } + public int MatchingSourceCount { get; } + } +} diff --git a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs index 04fcc55a49..6e03421aed 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs @@ -69,20 +69,30 @@ public void Install(RunningDeployment deployment) var appUpdater = new ApplicationUpdater(deploymentScope, deploymentConfig, - authenticatingRepositoryFactory, log, fileSystem, argoCdApplicationManifestParser, - new ImageTagUpdateCommitMessageGenerator(deploymentConfig.CommitParameters.Description), outputVariablesWriter); - var applicationResults = argoProperties.Applications - .Select(application => - { - var gateway = argoProperties.Gateways.Single(g => g.Id == application.GatewayId); - return appUpdater.ProcessApplication(application, gateway); - }) - .ToList(); + // Phase 1: plan every application's in-scope sources. + var plans = argoProperties.Applications + .Select(application => + { + var gateway = argoProperties.Gateways.Single(g => g.Id == application.GatewayId); + return appUpdater.Plan(application, gateway); + }) + .ToList(); + + // Phase 2: process all sources, grouped so each repository is cloned once and each branch checked out once. + var commitMessageGenerator = new ImageTagUpdateCommitMessageGenerator(deploymentConfig.CommitParameters.Description); + var processor = new GroupedRepositoryProcessor(authenticatingRepositoryFactory, deploymentConfig.CommitParameters, commitMessageGenerator); + + var updates = plans.SelectMany(p => p.Sources.Select(s => s.Update)).ToList(); + var results = processor.Process(updates); + var resultsByUpdate = updates.Zip(results, (update, result) => (update, result)).ToDictionary(x => x.update, x => x.result); + + // Phase 3: assemble per-application results (also writes per-source output variables). + var applicationResults = plans.Select(p => appUpdater.AssembleResult(p, resultsByUpdate)).ToList(); //if we are creating a pull request, we don't want to report files updated (as this will be passed down as output variables _with_ the PR info) diff --git a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDApplicationManifestsInstallConvention.cs b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDApplicationManifestsInstallConvention.cs index 00de914204..93df9d3a06 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDApplicationManifestsInstallConvention.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDApplicationManifestsInstallConvention.cs @@ -74,23 +74,33 @@ public void Install(RunningDeployment deployment) log.LogApplicationCounts(deploymentScope, argoProperties.Applications); - var applicationUpdater = new ApplicationUpdater(authenticatingRepositoryFactory, - deploymentScope, + var applicationUpdater = new ApplicationUpdater(deploymentScope, deploymentConfig, log, fileSystem, argoCdApplicationManifestParser, outputVariablesWriter, - packageFiles, - new UserDefinedCommitMessageGenerator(deploymentConfig.CommitParameters.Description)); - - var applicationResults = argoProperties.Applications - .Select(application => - { - var gateway = argoProperties.Gateways.Single(g => g.Id == application.GatewayId); - return applicationUpdater.ProcessApplication(application, gateway); - }) - .ToList(); + packageFiles); + + // Phase 1: plan every application's in-scope sources. + var plans = argoProperties.Applications + .Select(application => + { + var gateway = argoProperties.Gateways.Single(g => g.Id == application.GatewayId); + return applicationUpdater.Plan(application, gateway); + }) + .ToList(); + + // Phase 2: process all sources, grouped so each repository is cloned once and each branch checked out once. + var commitMessageGenerator = new UserDefinedCommitMessageGenerator(deploymentConfig.CommitParameters.Description); + var processor = new GroupedRepositoryProcessor(authenticatingRepositoryFactory, deploymentConfig.CommitParameters, commitMessageGenerator); + + var updates = plans.SelectMany(p => p.Sources.Select(s => s.Update)).ToList(); + var results = processor.Process(updates); + var resultsByUpdate = updates.Zip(results, (update, result) => (update, result)).ToDictionary(x => x.update, x => x.result); + + // Phase 3: assemble per-application results (also writes per-source output variables). + var applicationResults = plans.Select(p => applicationUpdater.AssembleResult(p, resultsByUpdate)).ToList(); reporter.ReportFilesUpdated(deploymentConfig.CommitParameters, applicationResults); diff --git a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationSourceUpdater.cs b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationSourceUpdater.cs index 93b5ec6c5f..3dd01528c4 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationSourceUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationSourceUpdater.cs @@ -1,6 +1,5 @@ using System; using Calamari.ArgoCD.Domain; -using Calamari.ArgoCD.Git; using Calamari.ArgoCD.Models; using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; @@ -8,36 +7,29 @@ namespace Calamari.ArgoCD.Conventions.UpdateImageTag; +// Determines whether a source is in scope for this deployment and builds the appropriate file updater for it. +// The actual clone/commit/push is handled centrally by the GroupedRepositoryProcessor. public class ApplicationSourceUpdater { readonly Application applicationFromYaml; - readonly ArgoCDGatewayDto gateway; - readonly RepositoryAdapter repositoryAdapter; readonly DeploymentScope deploymentScope; readonly UpdateArgoCDAppDeploymentConfig deploymentConfig; readonly ILog log; readonly string defaultRegistry; - readonly ArgoCDOutputVariablesWriter outputVariablesWriter; readonly ICalamariFileSystem fileSystem; public ApplicationSourceUpdater(Application applicationFromYaml, - RepositoryAdapter repositoryAdapter, DeploymentScope deploymentScope, UpdateArgoCDAppDeploymentConfig deploymentConfig, ILog log, - ArgoCDGatewayDto gateway, string defaultRegistry, - ArgoCDOutputVariablesWriter outputVariablesWriter, ICalamariFileSystem fileSystem) { this.applicationFromYaml = applicationFromYaml; - this.repositoryAdapter = repositoryAdapter; this.deploymentScope = deploymentScope; this.deploymentConfig = deploymentConfig; this.log = log; - this.gateway = gateway; this.defaultRegistry = defaultRegistry; - this.outputVariablesWriter = outputVariablesWriter; this.fileSystem = fileSystem; } @@ -52,21 +44,7 @@ public bool IsAppInScope(ApplicationSourceWithMetadata sourceWithMetadata) return deploymentScope.Matches(annotatedScope); } - public SourceUpdateResult ProcessSource(ApplicationSourceWithMetadata sourceWithMetadata) - { - var sourceUpdater = CreateSpecificUpdater(sourceWithMetadata); - - var sourceUpdateResult = repositoryAdapter.Process(sourceWithMetadata, sourceUpdater); - - outputVariablesWriter.WriteSourceUpdateResultOutputWhenPushResultExists(gateway.Name, - NamespacedApplicationName.Create(applicationFromYaml.Metadata.Name, applicationFromYaml.Metadata.Namespace), - sourceWithMetadata.Index, - sourceUpdateResult); - - return sourceUpdateResult; - } - - ISourceUpdater CreateSpecificUpdater(ApplicationSourceWithMetadata sourceWithMetadata) + public ISourceUpdater CreateSourceUpdater(ApplicationSourceWithMetadata sourceWithMetadata) { ISourceUpdater sourceUpdater; if (sourceWithMetadata.SourceType == SourceType.Directory) @@ -114,4 +92,4 @@ ISourceUpdater CreateSpecificUpdater(ApplicationSourceWithMetadata sourceWithMet return sourceUpdater; } -} \ No newline at end of file +} diff --git a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs index f5393a7316..a6ee505155 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Linq; using Calamari.ArgoCD.Domain; using Calamari.ArgoCD.Git; @@ -11,103 +12,112 @@ namespace Calamari.ArgoCD.Conventions.UpdateImageTag; public class ApplicationUpdater { - readonly AuthenticatingRepositoryFactory repositoryFactory; readonly DeploymentScope deploymentScope; readonly UpdateArgoCDAppDeploymentConfig deploymentConfig; readonly ILog log; readonly ICalamariFileSystem fileSystem; readonly IArgoCDApplicationManifestParser argoCdApplicationManifestParser; - readonly ICommitMessageGenerator commitMessageGenerator; readonly ArgoCDOutputVariablesWriter outputVariablesWriter; public ApplicationUpdater(DeploymentScope deploymentScope, UpdateArgoCDAppDeploymentConfig deploymentConfig, - AuthenticatingRepositoryFactory repositoryFactory, ILog log, ICalamariFileSystem fileSystem, IArgoCDApplicationManifestParser argoCdApplicationManifestParser, - ICommitMessageGenerator commitMessageGenerator, ArgoCDOutputVariablesWriter outputVariablesWriter) { this.deploymentScope = deploymentScope; this.deploymentConfig = deploymentConfig; - this.repositoryFactory = repositoryFactory; this.log = log; this.fileSystem = fileSystem; this.argoCdApplicationManifestParser = argoCdApplicationManifestParser; - this.commitMessageGenerator = commitMessageGenerator; this.outputVariablesWriter = outputVariablesWriter; } - - public ProcessApplicationResult ProcessApplication( - ArgoCDApplicationDto application, - ArgoCDGatewayDto gateway) + + // Phase 1: parse, validate, scope and build the set of source updates for an application. The updates are + // grouped (by repository/branch) across all applications and processed together by the GroupedRepositoryProcessor. + public PlannedApplication Plan(ArgoCDApplicationDto application, ArgoCDGatewayDto gateway) + { + log.InfoFormat("Processing application {0}", application.Name); + var applicationFromYaml = argoCdApplicationManifestParser.ParseManifest(application.Manifest); + var containsMultipleSources = applicationFromYaml.Spec.Sources.Count > 1; + var applicationName = applicationFromYaml.Metadata.Name; + var namespacedName = NamespacedApplicationName.Create(applicationName, applicationFromYaml.Metadata.Namespace); + + ValidateApplication(applicationFromYaml); + LogHelmAnnotationWarning(applicationFromYaml); + + var sourceUpdater = new ApplicationSourceUpdater(applicationFromYaml, deploymentScope, deploymentConfig, log, application.DefaultRegistry, fileSystem); + + var plannedSources = applicationFromYaml.GetSourcesWithMetadata() + .Where(sourceUpdater.IsAppInScope) + .Select(source => new PlannedSource(source, new RepositorySourceUpdate(namespacedName, source, sourceUpdater.CreateSourceUpdater(source)))) + .ToList(); + + var matchingSourceCount = applicationFromYaml.Spec.Sources.Count(s => deploymentScope.Matches(ScopingAnnotationReader.GetScopeForApplicationSource(s.Name.ToApplicationSourceName(), applicationFromYaml.Metadata.Annotations, containsMultipleSources))); + + return new PlannedApplication(application, gateway, applicationFromYaml, namespacedName, applicationName, plannedSources, applicationFromYaml.Spec.Sources.Count, matchingSourceCount); + } + + // Phase 3: turn the processed results back into a per-application result, writing per-source output variables. + public ProcessApplicationResult AssembleResult(PlannedApplication plan, IReadOnlyDictionary resultsByUpdate) + { + foreach (var plannedSource in plan.Sources) { - log.InfoFormat("Processing application {0}", application.Name); - var applicationFromYaml = argoCdApplicationManifestParser.ParseManifest(application.Manifest); - var containsMultipleSources = applicationFromYaml.Spec.Sources.Count > 1; - var applicationName = applicationFromYaml.Metadata.Name; - - ValidateApplication(applicationFromYaml); - - LogHelmAnnotationWarning(applicationFromYaml); - - var repositoryAdapter = new RepositoryAdapter(repositoryFactory, new RepositoryUpdater(deploymentConfig.CommitParameters, log, commitMessageGenerator)); - var sourceUpdater = new ApplicationSourceUpdater(applicationFromYaml, repositoryAdapter, deploymentScope, deploymentConfig, log, gateway, application.DefaultRegistry, outputVariablesWriter, fileSystem); - - var appliedSourcesResults = applicationFromYaml.GetSourcesWithMetadata() - .Where(sourceUpdater.IsAppInScope) - .Select(applicationSource => new - { - UpdateResult = sourceUpdater.ProcessSource(applicationSource), - applicationSource, - }) - .ToList(); - - //if we have links, use that to generate a link, otherwise just put the name there - var instanceLinks = application.InstanceWebUiUrl != null ? new ArgoCDInstanceLinks(application.InstanceWebUiUrl) : null; - var linkifiedAppName = instanceLinks != null - ? log.FormatLink(instanceLinks.ApplicationDetails(applicationName, application.KubernetesNamespace), applicationName) - : applicationName; - - var message = appliedSourcesResults.Any(r => r.UpdateResult.Updated) - ? "Updated Application {0}" - : "Nothing to update for Application {0}"; - - log.InfoFormat(message, linkifiedAppName); - - return new ProcessApplicationResult( - application.GatewayId, - NamespacedApplicationName.Create(applicationName, applicationFromYaml.Metadata.Namespace), - applicationFromYaml.Spec.Sources.Count, - applicationFromYaml.Spec.Sources.Count(s => deploymentScope.Matches(ScopingAnnotationReader.GetScopeForApplicationSource(s.Name.ToApplicationSourceName(), applicationFromYaml.Metadata.Annotations, containsMultipleSources))), - appliedSourcesResults.Select(r => new TrackedSourceDetail(r.UpdateResult.PushResult?.CommitSha, r.UpdateResult.PushResult?.CommitTimestamp, r.applicationSource.Index, [], r.UpdateResult.PatchedFiles)).ToList(), - appliedSourcesResults.SelectMany(r => r.UpdateResult.ImagesUpdated).ToHashSet(), - appliedSourcesResults.Where(r => r.UpdateResult.Updated).Select(r => r.applicationSource.Source.OriginalRepoUrl).ToHashSet()); + outputVariablesWriter.WriteSourceUpdateResultOutputWhenPushResultExists(plan.Gateway.Name, + plan.NamespacedName, + plannedSource.Source.Index, + resultsByUpdate[plannedSource.Update]); } - void LogHelmAnnotationWarning(Application applicationFromYaml) - { - var imagesWithoutHelmValuePath = deploymentConfig.ImageReferences.Where(ir => ir.HelmReference.IsNullOrEmpty()).ToList(); + var trackedSourceDetails = plan.Sources.Select(plannedSource => + { + var result = resultsByUpdate[plannedSource.Update]; + return new TrackedSourceDetail(result.PushResult?.CommitSha, result.PushResult?.CommitTimestamp, plannedSource.Source.Index, [], result.PatchedFiles); + }) + .ToList(); + + //if we have links, use that to generate a link, otherwise just put the name there + var instanceLinks = plan.Application.InstanceWebUiUrl != null ? new ArgoCDInstanceLinks(plan.Application.InstanceWebUiUrl) : null; + var linkifiedAppName = instanceLinks != null + ? log.FormatLink(instanceLinks.ApplicationDetails(plan.ApplicationName, plan.Application.KubernetesNamespace), plan.ApplicationName) + : plan.ApplicationName; - var someButNotAllHaveHelmValuePath = (imagesWithoutHelmValuePath.Count > 0) && (imagesWithoutHelmValuePath.Count < deploymentConfig.ImageReferences.Count); - - if (someButNotAllHaveHelmValuePath && applicationFromYaml.GetSourcesWithMetadata().Any(src => src.SourceType == SourceType.Helm)) + var anyUpdated = plan.Sources.Any(plannedSource => resultsByUpdate[plannedSource.Update].Updated); + log.InfoFormat(anyUpdated ? "Updated Application {0}" : "Nothing to update for Application {0}", linkifiedAppName); + + return new ProcessApplicationResult( + plan.Application.GatewayId, + plan.NamespacedName, + plan.TotalSourceCount, + plan.MatchingSourceCount, + trackedSourceDetails, + plan.Sources.SelectMany(plannedSource => resultsByUpdate[plannedSource.Update].ImagesUpdated).ToHashSet(), + plan.Sources.Where(plannedSource => resultsByUpdate[plannedSource.Update].Updated).Select(plannedSource => plannedSource.Source.Source.OriginalRepoUrl).ToHashSet()); + } + + void LogHelmAnnotationWarning(Application applicationFromYaml) + { + var imagesWithoutHelmValuePath = deploymentConfig.ImageReferences.Where(ir => ir.HelmReference.IsNullOrEmpty()).ToList(); + + var someButNotAllHaveHelmValuePath = (imagesWithoutHelmValuePath.Count > 0) && (imagesWithoutHelmValuePath.Count < deploymentConfig.ImageReferences.Count); + + if (someButNotAllHaveHelmValuePath && applicationFromYaml.GetSourcesWithMetadata().Any(src => src.SourceType == SourceType.Helm)) + { + foreach (var image in imagesWithoutHelmValuePath) { - foreach (var image in imagesWithoutHelmValuePath) - { - log.Verbose($"{image.ContainerReference.FriendlyName()} will not be updated in helm sources, as no helm yaml path has been specified for it in the step configuration."); - } + log.Verbose($"{image.ContainerReference.FriendlyName()} will not be updated in helm sources, as no helm yaml path has been specified for it in the step configuration."); } } + } - void ValidateApplication(Application applicationFromYaml) - { - var validationResult = ValidationResult.Merge( - ApplicationValidator.ValidateSourceNames(applicationFromYaml), - ApplicationValidator.ValidateUnnamedAnnotationsInMultiSourceApplication(applicationFromYaml), - ApplicationValidator.ValidateSourceTypes(applicationFromYaml) - ); - validationResult.Action(log); - } -} \ No newline at end of file + void ValidateApplication(Application applicationFromYaml) + { + var validationResult = ValidationResult.Merge( + ApplicationValidator.ValidateSourceNames(applicationFromYaml), + ApplicationValidator.ValidateUnnamedAnnotationsInMultiSourceApplication(applicationFromYaml), + ApplicationValidator.ValidateSourceTypes(applicationFromYaml) + ); + validationResult.Action(log); + } +} diff --git a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/FileUpdateResult.cs b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/FileUpdateResult.cs index 80a49f09f6..a6524ded1d 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/FileUpdateResult.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/FileUpdateResult.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using Octopus.Calamari.Contracts.ArgoCD; namespace Calamari.ArgoCD.Conventions.UpdateImageTag; @@ -11,6 +12,18 @@ public bool HasChanges() { return ReplacedFiles.Count > 0 || PatchedFiles.Count > 0 || FilesRemoved.Length > 0; } + + // Combines the file changes from several sources so they can be described in a single commit + // (used when committing all of an application's sources in one repo+branch group together). + public static FileUpdateResult Merge(IEnumerable results) + { + var materialised = results.ToList(); + return new FileUpdateResult( + materialised.SelectMany(r => r.UpdatedImages).ToHashSet(), + materialised.SelectMany(r => r.ReplacedFiles).ToList(), + materialised.SelectMany(r => r.PatchedFiles).ToList(), + materialised.SelectMany(r => r.FilesRemoved).ToArray()); + } } diff --git a/source/Calamari/ArgoCD/Git/GroupedRepositoryProcessor.cs b/source/Calamari/ArgoCD/Git/GroupedRepositoryProcessor.cs new file mode 100644 index 0000000000..9893c9b389 --- /dev/null +++ b/source/Calamari/ArgoCD/Git/GroupedRepositoryProcessor.cs @@ -0,0 +1,174 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using Calamari.ArgoCD.Conventions; +using Calamari.ArgoCD.Conventions.UpdateImageTag; +using Calamari.ArgoCD.Domain; +using Octopus.Calamari.Contracts.ArgoCD; + +namespace Calamari.ArgoCD.Git +{ + // A single application source that needs to be written into a Git repository and committed/pushed. + // The repository URL and target revision (the grouping keys) are derived from the source itself. + public class RepositorySourceUpdate + { + public RepositorySourceUpdate(NamespacedApplicationName applicationName, ApplicationSourceWithMetadata source, ISourceUpdater updater) + { + ApplicationName = applicationName; + Source = source; + Updater = updater; + } + + public NamespacedApplicationName ApplicationName { get; } + public ApplicationSourceWithMetadata Source { get; } + public ISourceUpdater Updater { get; } + + public string RepoUrl => Source.Source.OriginalRepoUrl; + public string TargetRevision => Source.Source.TargetRevision; + } + + // Processes a set of source updates by grouping them so that each repository is cloned once and each + // branch is checked out once. Within a repo+branch group, changes are committed once per application + // (direct push) and pushed in a single push; when raising pull requests, behaviour is unchanged - one + // branch, commit and pull request per source - while still reusing the shared clone and checkout. + // + // Failure handling is fail-fast: if an application genuinely fails to commit, the exception propagates, + // the clone is disposed (so no partial commits are pushed for that group), and the step aborts. + public class GroupedRepositoryProcessor + { + readonly AuthenticatingRepositoryFactory repositoryFactory; + readonly GitCommitParameters commitParameters; + readonly ICommitMessageGenerator commitMessageGenerator; + + public GroupedRepositoryProcessor(AuthenticatingRepositoryFactory repositoryFactory, + GitCommitParameters commitParameters, + ICommitMessageGenerator commitMessageGenerator) + { + this.repositoryFactory = repositoryFactory; + this.commitParameters = commitParameters; + this.commitMessageGenerator = commitMessageGenerator; + } + + // Returns a result for each update, in the same order the updates were provided. + public IReadOnlyList Process(IReadOnlyList updates) + { + var results = new SourceUpdateResult[updates.Count]; + var indexed = updates.Select((update, index) => (update, index)).ToList(); + + foreach (var repoGroup in indexed.GroupBy(x => x.update.RepoUrl)) + { + var branchGroups = repoGroup.GroupBy(x => x.update.TargetRevision).ToList(); + + // Clone once per repository. The clone fetches every branch, so each branch group below just + // checks out the branch it needs without re-cloning. + using var repository = repositoryFactory.CloneRepository(repoGroup.Key, branchGroups[0].Key); + + foreach (var branchGroup in branchGroups) + { + var reference = GitReference.CreateFromString(branchGroup.Key); + var items = branchGroup.ToList(); + + if (commitParameters.RequiresPr) + { + ProcessBranchAsPullRequests(repository, reference, items, results); + } + else + { + ProcessBranchAsDirectCommits(repository, reference, items, results); + } + } + } + + return results; + } + + void ProcessBranchAsDirectCommits(RepositoryWrapper repository, + GitReference reference, + List<(RepositorySourceUpdate update, int index)> items, + SourceUpdateResult[] results) + { + repository.CheckoutBranch(reference); + + var committedAnything = false; + + // Apply and commit one application at a time so each commit contains only that application's changes. + foreach (var applicationGroup in items.GroupBy(x => x.update.ApplicationName.Value)) + { + // Apply each source and record whether it actually changed the working tree. A source can + // produce a result without changing anything (e.g. an image already at the target tag), and + // such a source must not be attributed the application's commit. + var changedPaths = new HashSet(repository.GetChangedFilePaths()); + var applied = new List<(int index, FileUpdateResult fileResult, bool changedWorkingTree)>(); + foreach (var item in applicationGroup) + { + var fileResult = item.update.Updater.Process(item.update.Source, repository.WorkingDirectory); + var changedPathsNow = new HashSet(repository.GetChangedFilePaths()); + var changedWorkingTree = changedPathsNow.Except(changedPaths).Any(); + changedPaths = changedPathsNow; + applied.Add((item.index, fileResult, changedWorkingTree)); + } + + PushResult? applicationCommit = null; + if (applied.Any(a => a.changedWorkingTree)) + { + repository.StageAllChanges(); + var description = commitMessageGenerator.GenerateDescription(FileUpdateResult.Merge(applied.Select(a => a.fileResult))); + if (repository.CommitChanges(commitParameters.Summary, description)) + { + // Captured before the push. Pushes are serial and the clone is fresh, so a rebase-on-push + // (which would rewrite this SHA) is not expected; see RepositoryWrapper.PushChanges. + applicationCommit = repository.GetHeadCommitResult(); + committedAnything = true; + } + } + + foreach (var item in applied) + { + var pushResult = item.changedWorkingTree ? applicationCommit : null; + results[item.index] = new SourceUpdateResult(item.fileResult.UpdatedImages, pushResult, item.fileResult.ReplacedFiles, item.fileResult.PatchedFiles); + } + } + + // A single push carries every application's commit on this branch. + if (committedAnything) + { + repository.PushChanges(false, commitParameters.Summary, string.Empty, reference, CancellationToken.None) + .GetAwaiter() + .GetResult(); + } + } + + void ProcessBranchAsPullRequests(RepositoryWrapper repository, + GitReference reference, + List<(RepositorySourceUpdate update, int index)> items, + SourceUpdateResult[] results) + { + foreach (var item in items) + { + // Reset back to the remote tip so each pull request branch contains only this source's commit. + repository.CheckoutBranch(reference); + + var fileResult = item.update.Updater.Process(item.update.Source, repository.WorkingDirectory); + if (!fileResult.HasChanges()) + { + results[item.index] = new SourceUpdateResult(fileResult.UpdatedImages, null, fileResult.ReplacedFiles, fileResult.PatchedFiles); + continue; + } + + repository.StageAllChanges(); + var description = commitMessageGenerator.GenerateDescription(fileResult); + if (!repository.CommitChanges(commitParameters.Summary, description)) + { + results[item.index] = new SourceUpdateResult(fileResult.UpdatedImages, null, fileResult.ReplacedFiles, fileResult.PatchedFiles); + continue; + } + + var pushResult = repository.PushChanges(true, commitParameters.Summary, description, reference, CancellationToken.None) + .GetAwaiter() + .GetResult(); + results[item.index] = new SourceUpdateResult(fileResult.UpdatedImages, pushResult, fileResult.ReplacedFiles, fileResult.PatchedFiles); + } + } + } +} diff --git a/source/Calamari/ArgoCD/Git/RepositoryAdapter.cs b/source/Calamari/ArgoCD/Git/RepositoryAdapter.cs deleted file mode 100644 index b8abeddc99..0000000000 --- a/source/Calamari/ArgoCD/Git/RepositoryAdapter.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System; -using Calamari.ArgoCD.Conventions; -using Calamari.ArgoCD.Conventions.UpdateImageTag; -using Calamari.ArgoCD.Domain; - -namespace Calamari.ArgoCD.Git; - -public class RepositoryAdapter -{ - readonly AuthenticatingRepositoryFactory repositoryFactory; - readonly RepositoryUpdater repositoryUpdater; - - public RepositoryAdapter(AuthenticatingRepositoryFactory repositoryFactory, - RepositoryUpdater repositoryUpdater) - { - this.repositoryFactory = repositoryFactory; - this.repositoryUpdater = repositoryUpdater; - } - - public SourceUpdateResult Process(ApplicationSourceWithMetadata sourceWithMetadata, ISourceUpdater updater) - { - using var repository = repositoryFactory.CloneRepository(sourceWithMetadata.Source.OriginalRepoUrl, sourceWithMetadata.Source.TargetRevision); - var filesUpdated = updater.Process(sourceWithMetadata, repository.WorkingDirectory); - - if (filesUpdated.HasChanges()) - { - var pushResult = repositoryUpdater.PushToRemote(repository, GitReference.CreateFromString(sourceWithMetadata.Source.TargetRevision), filesUpdated); - return new SourceUpdateResult(filesUpdated.UpdatedImages, pushResult, filesUpdated.ReplacedFiles, filesUpdated.PatchedFiles); - } - - return new SourceUpdateResult([], null, filesUpdated.ReplacedFiles, filesUpdated.PatchedFiles); - } -} diff --git a/source/Calamari/ArgoCD/Git/RepositoryFactory.cs b/source/Calamari/ArgoCD/Git/RepositoryFactory.cs index 771c9b83b0..fdd23a1055 100644 --- a/source/Calamari/ArgoCD/Git/RepositoryFactory.cs +++ b/source/Calamari/ArgoCD/Git/RepositoryFactory.cs @@ -1,6 +1,5 @@ using System; using System.IO; -using System.Linq; using System.Threading; using Calamari.ArgoCD.Git.PullRequests; using Calamari.Common.Commands; @@ -58,16 +57,11 @@ public RepositoryWrapper CloneRepository(string repositoryName, IGitConnection g RepositoryWrapper CheckoutGitRepository(IGitConnection gitConnection, string checkoutPath) { - //if the branch name is head, then we just clone the default - //if it's not head, then clone the branch immediately - var options = gitConnection.GitReference is GitHead - ? new CloneOptions() - : new CloneOptions - { - //note: when cloning, libgit2sharp prepends "refs/remotes/origin/" to this value (so _must_ be a branch to succeed). - BranchName = (gitConnection.GitReference as GitBranchName)?.ToFriendlyName() - }; - + // Always clone with default options so the remote's default branch is checked out first. This lets + // the RepositoryWrapper capture the true default branch (for resolving 'HEAD' references) before we + // check out the requested reference, and leaves every remote branch available so the clone can be + // reused to check out additional branches without re-cloning. + var options = new CloneOptions(); options.FetchOptions.CredentialsProvider = gitConnection.ToLibGit2SharpCredentialHandler(); options.FetchOptions.CertificateCheck = gitConnection.ToLibGit2SharpCertificateCheckHandler(log); @@ -91,28 +85,6 @@ RepositoryWrapper CheckoutGitRepository(IGitConnection gitConnection, string che var repo = new Repository(repoPath); - try - { - //this is required to handle the issue around "HEAD" - var branchToCheckout = repo.GetBranchName(gitConnection.GitReference); - var remoteBranch = repo.Branches.First(f => f.IsRemote && f.UpstreamBranchCanonicalName == branchToCheckout.Value); - - log.VerboseFormat("Checking out '{0}' @ {1}", branchToCheckout, remoteBranch.Tip.Sha.Substring(0, 10)); - - //A local branch is required such that libgit2sharp can create "tracking" data - // libgit2sharp does not support pushing from a detached head - if (repo.Branches[branchToCheckout.Value] == null) - { - repo.CreateBranch(branchToCheckout.Value, remoteBranch.Tip); - } - - LibGit2Sharp.Commands.Checkout(repo, branchToCheckout.ToFriendlyName()); - } - catch (LibGit2SharpException e) - { - throw new CommandException($"Failed to checkout branch '{gitConnection.GitReference}' in repository at {gitConnection.Url}. Error: {e.Message}", e); - } - //TODO(tmm): Make this function (and all callers async). var gitVendorApiAdapter = gitConnection is HttpsGitConnection httpsGitConnection ? gitVendorPullRequestClientResolver.TryResolve(httpsGitConnection, log, CancellationToken.None).Result @@ -123,13 +95,26 @@ RepositoryWrapper CheckoutGitRepository(IGitConnection gitConnection, string che log.Verbose("Git is using SSH authentication, Git vendor functionality such as PR creation will not be available"); } - return new RepositoryWrapper(repo, - fileSystem, - checkoutPath, - log, - gitConnection, - gitVendorApiAdapter, - clock); + var repository = new RepositoryWrapper(repo, + fileSystem, + checkoutPath, + log, + gitConnection, + gitVendorApiAdapter, + clock); + + try + { + repository.CheckoutBranch(gitConnection.GitReference); + } + catch (Exception e) + { + repository.Dispose(); + // Preserve the original behaviour where requesting a reference that is not a branch surfaces as a clone failure. + throw new CommandException($"Failed to clone Git repository at {gitConnection.Url}. Are you sure this URL is a Git repository, and the reference is a branch?", e); + } + + return repository; } } } diff --git a/source/Calamari/ArgoCD/Git/RepositoryWrapper.cs b/source/Calamari/ArgoCD/Git/RepositoryWrapper.cs index ca7361ad91..70080bba34 100644 --- a/source/Calamari/ArgoCD/Git/RepositoryWrapper.cs +++ b/source/Calamari/ArgoCD/Git/RepositoryWrapper.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; @@ -36,8 +37,70 @@ public class RepositoryWrapper( readonly Identity repositoryIdentity = new("Octopus", "octopus@octopus.com"); + // Captured at construction (immediately after clone, before any checkout) so that 'HEAD' + // references can be resolved to the remote's default branch even after we have checked out + // a different branch on this (reused) clone. + readonly string defaultBranchCanonicalName = repository.Head.CanonicalName; + public string WorkingDirectory => repository.Info.WorkingDirectory; + // Checks out the requested reference, creating a local tracking branch if required, and hard-resets + // it to the remote tip. Safe to call repeatedly on a single clone to switch between branches (or to + // reset the current branch back to the remote tip between sources when raising pull requests). + public void CheckoutBranch(GitReference reference) + { + var branchToCheckout = reference is GitHead + ? new GitBranchName(defaultBranchCanonicalName) + : repository.GetBranchName(reference); + + var remoteBranch = repository.Branches.FirstOrDefault(f => f.IsRemote && f.UpstreamBranchCanonicalName == branchToCheckout.Value); + if (remoteBranch == null) + { + throw new CommandException($"Failed to checkout branch '{reference}' in repository at {connection.Url}. The reference could not be found as a branch on the remote."); + } + + try + { + log.VerboseFormat("Checking out '{0}' @ {1}", branchToCheckout, remoteBranch.Tip.Sha.Substring(0, 10)); + + //A local branch is required such that libgit2sharp can create "tracking" data + // libgit2sharp does not support pushing from a detached head + if (repository.Branches[branchToCheckout.Value] == null) + { + repository.CreateBranch(branchToCheckout.Value, remoteBranch.Tip); + } + + LibGit2Sharp.Commands.Checkout(repository, branchToCheckout.ToFriendlyName()); + // Ensure the local branch matches the remote tip. This matters when the clone is reused: + // a previous source may have left a local commit on this branch that must not leak into this one. + repository.Reset(ResetMode.Hard, remoteBranch.Tip); + } + catch (LibGit2SharpException e) + { + throw new CommandException($"Failed to checkout branch '{reference}' in repository at {connection.Url}. Error: {e.Message}", e); + } + } + + // Returns the current HEAD commit as a PushResult. Used to capture a per-application commit before + // a single push of all the application commits made on a branch. + public PushResult GetHeadCommitResult() + { + var commit = repository.Head.Tip; + return new PushResult(commit.Sha, commit.ShortSha(), commit.Author.When); + } + + // The set of files that currently differ from HEAD (modified, added, deleted, untracked). Used to + // detect whether a particular source actually changed anything: a source updater may produce a result + // (e.g. a computed image patch) that is identical to what is already committed, in which case it must + // not be attributed a commit. Mirrors git's "nothing to commit" behaviour at a per-source granularity. + public IReadOnlyCollection GetChangedFilePaths() + { + var status = repository.RetrieveStatus(new StatusOptions { IncludeUntracked = true, RecurseUntrackedDirs = true, IncludeIgnored = false }); + return status.Where(e => e.State != FileStatus.Unaltered && e.State != FileStatus.Ignored) + .Select(e => e.FilePath) + .ToList(); + } + // returns true if changes were made to the repository public bool CommitChanges(string summary, string description) { From da32dea7a8da324a92b1266cc4c209589cbf91aa Mon Sep 17 00:00:00 2001 From: Noam Gal Date: Thu, 25 Jun 2026 17:39:28 +0300 Subject: [PATCH 2/6] working on pr comments --- ...Updater.cs => ApplicationSourceFactory.cs} | 4 ++-- .../ManifestTemplating/ApplicationUpdater.cs | 8 +++---- .../ArgoCD/Conventions/PlannedApplication.cs | 21 +++++++++++++------ ...Updater.cs => ApplicationSourceFactory.cs} | 4 ++-- .../UpdateImageTag/ApplicationUpdater.cs | 8 +++---- source/Calamari/ArgoCD/Domain/Application.cs | 5 +++-- 6 files changed, 28 insertions(+), 22 deletions(-) rename source/Calamari/ArgoCD/Conventions/ManifestTemplating/{ApplicationSourceUpdater.cs => ApplicationSourceFactory.cs} (96%) rename source/Calamari/ArgoCD/Conventions/UpdateImageTag/{ApplicationSourceUpdater.cs => ApplicationSourceFactory.cs} (97%) diff --git a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationSourceUpdater.cs b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationSourceFactory.cs similarity index 96% rename from source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationSourceUpdater.cs rename to source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationSourceFactory.cs index 4d191b10e8..06ad50c448 100644 --- a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationSourceUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationSourceFactory.cs @@ -9,7 +9,7 @@ namespace Calamari.ArgoCD.Conventions.ManifestTemplating; // Determines whether a source is in scope for this deployment and builds the file updater (which copies the // templated package files into the source path). Clone/commit/push is handled by the GroupedRepositoryProcessor. -public class ApplicationSourceUpdater +public class ApplicationSourceFactory { readonly Application applicationFromYaml; readonly DeploymentScope deploymentScope; @@ -18,7 +18,7 @@ public class ApplicationSourceUpdater readonly ILog log; readonly ICalamariFileSystem fileSystem; - public ApplicationSourceUpdater(Application applicationFromYaml, + public ApplicationSourceFactory(Application applicationFromYaml, DeploymentScope deploymentScope, ArgoCommitToGitConfig deploymentConfig, IPackageRelativeFile[] packageFiles, diff --git a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs index 4753173275..83519453c4 100644 --- a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs @@ -43,8 +43,6 @@ public PlannedApplication Plan(ArgoCDApplicationDto application, ArgoCDGatewayDt log.InfoFormat("Processing application {0}", application.Name); var applicationFromYaml = argoCdApplicationManifestParser.ParseManifest(application.Manifest); var containsMultipleSources = applicationFromYaml.Spec.Sources.Count > 1; - var applicationName = applicationFromYaml.Metadata.Name; - var namespacedName = NamespacedApplicationName.Create(applicationName, applicationFromYaml.Metadata.Namespace); LogWarningIfUpdatingMultipleSources(applicationFromYaml.Spec.Sources, applicationFromYaml.Metadata.Annotations, @@ -52,16 +50,16 @@ public PlannedApplication Plan(ArgoCDApplicationDto application, ArgoCDGatewayDt ValidateApplication(applicationFromYaml); - var sourceUpdater = new ApplicationSourceUpdater(applicationFromYaml, deploymentScope, deploymentConfig, packageFiles, log, fileSystem); + var sourceUpdater = new ApplicationSourceFactory(applicationFromYaml, deploymentScope, deploymentConfig, packageFiles, log, fileSystem); var plannedSources = applicationFromYaml.GetSourcesWithMetadata() .Where(sourceUpdater.IsAppInScope) - .Select(source => new PlannedSource(source, new RepositorySourceUpdate(namespacedName, source, sourceUpdater.CreateSourceUpdater(source)))) + .Select(source => new PlannedSource(source, new RepositorySourceUpdate(applicationFromYaml.QualifiedName, source, sourceUpdater.CreateSourceUpdater(source)))) .ToList(); var matchingSourceCount = applicationFromYaml.Spec.Sources.Count(s => deploymentScope.Matches(ScopingAnnotationReader.GetScopeForApplicationSource(s.Name.ToApplicationSourceName(), applicationFromYaml.Metadata.Annotations, containsMultipleSources))); - return new PlannedApplication(application, gateway, applicationFromYaml, namespacedName, applicationName, plannedSources, applicationFromYaml.Spec.Sources.Count, matchingSourceCount); + return new PlannedApplication(application, gateway, applicationFromYaml, plannedSources, applicationFromYaml.Spec.Sources.Count, matchingSourceCount); } // Phase 3: turn the processed results back into a per-application result, writing per-source output variables. diff --git a/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs b/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs index 52b137b77c..efd8a63d95 100644 --- a/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs +++ b/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs @@ -16,8 +16,6 @@ public class PlannedApplication public PlannedApplication(ArgoCDApplicationDto application, ArgoCDGatewayDto gateway, Application applicationFromYaml, - NamespacedApplicationName namespacedName, - string applicationName, IReadOnlyList sources, int totalSourceCount, int matchingSourceCount) @@ -25,8 +23,6 @@ public PlannedApplication(ArgoCDApplicationDto application, Application = application; Gateway = gateway; ApplicationFromYaml = applicationFromYaml; - NamespacedName = namespacedName; - ApplicationName = applicationName; Sources = sources; TotalSourceCount = totalSourceCount; MatchingSourceCount = matchingSourceCount; @@ -35,8 +31,21 @@ public PlannedApplication(ArgoCDApplicationDto application, public ArgoCDApplicationDto Application { get; } public ArgoCDGatewayDto Gateway { get; } public Application ApplicationFromYaml { get; } - public NamespacedApplicationName NamespacedName { get; } - public string ApplicationName { get; } + public NamespacedApplicationName NamespacedName + { + get + { + return ApplicationFromYaml.QualifiedName; + } + } + + public string ApplicationName + { + get + { + return ApplicationFromYaml.Metadata.Name; + } + } public IReadOnlyList Sources { get; } public int TotalSourceCount { get; } public int MatchingSourceCount { get; } diff --git a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationSourceUpdater.cs b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationSourceFactory.cs similarity index 97% rename from source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationSourceUpdater.cs rename to source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationSourceFactory.cs index 3dd01528c4..c03e63826e 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationSourceUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationSourceFactory.cs @@ -9,7 +9,7 @@ namespace Calamari.ArgoCD.Conventions.UpdateImageTag; // Determines whether a source is in scope for this deployment and builds the appropriate file updater for it. // The actual clone/commit/push is handled centrally by the GroupedRepositoryProcessor. -public class ApplicationSourceUpdater +public class ApplicationSourceFactory { readonly Application applicationFromYaml; readonly DeploymentScope deploymentScope; @@ -18,7 +18,7 @@ public class ApplicationSourceUpdater readonly string defaultRegistry; readonly ICalamariFileSystem fileSystem; - public ApplicationSourceUpdater(Application applicationFromYaml, + public ApplicationSourceFactory(Application applicationFromYaml, DeploymentScope deploymentScope, UpdateArgoCDAppDeploymentConfig deploymentConfig, ILog log, diff --git a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs index a6ee505155..c0fe35c14a 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs @@ -41,22 +41,20 @@ public PlannedApplication Plan(ArgoCDApplicationDto application, ArgoCDGatewayDt log.InfoFormat("Processing application {0}", application.Name); var applicationFromYaml = argoCdApplicationManifestParser.ParseManifest(application.Manifest); var containsMultipleSources = applicationFromYaml.Spec.Sources.Count > 1; - var applicationName = applicationFromYaml.Metadata.Name; - var namespacedName = NamespacedApplicationName.Create(applicationName, applicationFromYaml.Metadata.Namespace); ValidateApplication(applicationFromYaml); LogHelmAnnotationWarning(applicationFromYaml); - var sourceUpdater = new ApplicationSourceUpdater(applicationFromYaml, deploymentScope, deploymentConfig, log, application.DefaultRegistry, fileSystem); + var sourceUpdater = new ApplicationSourceFactory(applicationFromYaml, deploymentScope, deploymentConfig, log, application.DefaultRegistry, fileSystem); var plannedSources = applicationFromYaml.GetSourcesWithMetadata() .Where(sourceUpdater.IsAppInScope) - .Select(source => new PlannedSource(source, new RepositorySourceUpdate(namespacedName, source, sourceUpdater.CreateSourceUpdater(source)))) + .Select(source => new PlannedSource(source, new RepositorySourceUpdate(applicationFromYaml.QualifiedName, source, sourceUpdater.CreateSourceUpdater(source)))) .ToList(); var matchingSourceCount = applicationFromYaml.Spec.Sources.Count(s => deploymentScope.Matches(ScopingAnnotationReader.GetScopeForApplicationSource(s.Name.ToApplicationSourceName(), applicationFromYaml.Metadata.Annotations, containsMultipleSources))); - return new PlannedApplication(application, gateway, applicationFromYaml, namespacedName, applicationName, plannedSources, applicationFromYaml.Spec.Sources.Count, matchingSourceCount); + return new PlannedApplication(application, gateway, applicationFromYaml, plannedSources, applicationFromYaml.Spec.Sources.Count, matchingSourceCount); } // Phase 3: turn the processed results back into a per-application result, writing per-source output variables. diff --git a/source/Calamari/ArgoCD/Domain/Application.cs b/source/Calamari/ArgoCD/Domain/Application.cs index c3e6acf26c..862baa18cf 100644 --- a/source/Calamari/ArgoCD/Domain/Application.cs +++ b/source/Calamari/ArgoCD/Domain/Application.cs @@ -1,6 +1,6 @@ -using System; using System.Text.Json.Serialization; using Calamari.ArgoCD.Domain.Converters; +using Octopus.Calamari.Contracts.ArgoCD; namespace Calamari.ArgoCD.Domain { @@ -16,6 +16,7 @@ public class Application [JsonPropertyName("status")] [JsonConverter(typeof(ApplicationStatusConverter))] public ApplicationStatus Status { get; set; } = new ApplicationStatus(); - + + public NamespacedApplicationName QualifiedName => NamespacedApplicationName.Create(Metadata.Name, Metadata.Namespace); } } From 81108a13eecdce1944e5df3bcf5dc90ff2dc98e0 Mon Sep 17 00:00:00 2001 From: Noam Gal Date: Thu, 25 Jun 2026 18:38:46 +0300 Subject: [PATCH 3/6] simplified PlannedApplication ctor --- .../ManifestTemplating/ApplicationUpdater.cs | 6 +++--- .../ArgoCD/Conventions/PlannedApplication.cs | 16 +++++++++++----- .../UpdateImageTag/ApplicationUpdater.cs | 4 ++-- source/Calamari/ArgoCD/Domain/Application.cs | 2 +- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs index 83519453c4..8f5024b195 100644 --- a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs @@ -54,12 +54,12 @@ public PlannedApplication Plan(ArgoCDApplicationDto application, ArgoCDGatewayDt var plannedSources = applicationFromYaml.GetSourcesWithMetadata() .Where(sourceUpdater.IsAppInScope) - .Select(source => new PlannedSource(source, new RepositorySourceUpdate(applicationFromYaml.QualifiedName, source, sourceUpdater.CreateSourceUpdater(source)))) + .Select(source => new PlannedSource(source, new RepositorySourceUpdate(applicationFromYaml.NamespacedName, source, sourceUpdater.CreateSourceUpdater(source)))) .ToList(); var matchingSourceCount = applicationFromYaml.Spec.Sources.Count(s => deploymentScope.Matches(ScopingAnnotationReader.GetScopeForApplicationSource(s.Name.ToApplicationSourceName(), applicationFromYaml.Metadata.Annotations, containsMultipleSources))); - return new PlannedApplication(application, gateway, applicationFromYaml, plannedSources, applicationFromYaml.Spec.Sources.Count, matchingSourceCount); + return new PlannedApplication(application, gateway, plannedSources, applicationFromYaml.Spec.Sources.Count, matchingSourceCount); } // Phase 3: turn the processed results back into a per-application result, writing per-source output variables. @@ -83,7 +83,7 @@ public ProcessApplicationResult AssembleResult(PlannedApplication plan, IReadOnl //if we have links, use that to generate a link, otherwise just put the name there var instanceLinks = plan.Application.InstanceWebUiUrl != null ? new ArgoCDInstanceLinks(plan.Application.InstanceWebUiUrl) : null; var linkifiedAppName = instanceLinks != null - ? log.FormatLink(instanceLinks.ApplicationDetails(plan.ApplicationName, plan.ApplicationFromYaml.Metadata.Namespace), plan.ApplicationName) + ? log.FormatLink(instanceLinks.ApplicationDetails(plan.ApplicationName, plan.ApplicationNamespace), plan.ApplicationName) : plan.ApplicationName; var anyUpdated = plan.Sources.Any(plannedSource => resultsByUpdate[plannedSource.Update].Updated); diff --git a/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs b/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs index efd8a63d95..563e845bf5 100644 --- a/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs +++ b/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs @@ -15,14 +15,12 @@ public class PlannedApplication { public PlannedApplication(ArgoCDApplicationDto application, ArgoCDGatewayDto gateway, - Application applicationFromYaml, IReadOnlyList sources, int totalSourceCount, int matchingSourceCount) { Application = application; Gateway = gateway; - ApplicationFromYaml = applicationFromYaml; Sources = sources; TotalSourceCount = totalSourceCount; MatchingSourceCount = matchingSourceCount; @@ -30,12 +28,11 @@ public PlannedApplication(ArgoCDApplicationDto application, public ArgoCDApplicationDto Application { get; } public ArgoCDGatewayDto Gateway { get; } - public Application ApplicationFromYaml { get; } public NamespacedApplicationName NamespacedName { get { - return ApplicationFromYaml.QualifiedName; + return NamespacedApplicationName.Create(ApplicationName, ApplicationNamespace); } } @@ -43,9 +40,18 @@ public string ApplicationName { get { - return ApplicationFromYaml.Metadata.Name; + return Application.Name; } } + + public string ApplicationNamespace + { + get + { + return Application.KubernetesNamespace; + } + } + public IReadOnlyList Sources { get; } public int TotalSourceCount { get; } public int MatchingSourceCount { get; } diff --git a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs index c0fe35c14a..3ac37d6efa 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs @@ -49,12 +49,12 @@ public PlannedApplication Plan(ArgoCDApplicationDto application, ArgoCDGatewayDt var plannedSources = applicationFromYaml.GetSourcesWithMetadata() .Where(sourceUpdater.IsAppInScope) - .Select(source => new PlannedSource(source, new RepositorySourceUpdate(applicationFromYaml.QualifiedName, source, sourceUpdater.CreateSourceUpdater(source)))) + .Select(source => new PlannedSource(source, new RepositorySourceUpdate(applicationFromYaml.NamespacedName, source, sourceUpdater.CreateSourceUpdater(source)))) .ToList(); var matchingSourceCount = applicationFromYaml.Spec.Sources.Count(s => deploymentScope.Matches(ScopingAnnotationReader.GetScopeForApplicationSource(s.Name.ToApplicationSourceName(), applicationFromYaml.Metadata.Annotations, containsMultipleSources))); - return new PlannedApplication(application, gateway, applicationFromYaml, plannedSources, applicationFromYaml.Spec.Sources.Count, matchingSourceCount); + return new PlannedApplication(application, gateway, plannedSources, applicationFromYaml.Spec.Sources.Count, matchingSourceCount); } // Phase 3: turn the processed results back into a per-application result, writing per-source output variables. diff --git a/source/Calamari/ArgoCD/Domain/Application.cs b/source/Calamari/ArgoCD/Domain/Application.cs index 862baa18cf..8d3e5ef7ec 100644 --- a/source/Calamari/ArgoCD/Domain/Application.cs +++ b/source/Calamari/ArgoCD/Domain/Application.cs @@ -17,6 +17,6 @@ public class Application [JsonConverter(typeof(ApplicationStatusConverter))] public ApplicationStatus Status { get; set; } = new ApplicationStatus(); - public NamespacedApplicationName QualifiedName => NamespacedApplicationName.Create(Metadata.Name, Metadata.Namespace); + public NamespacedApplicationName NamespacedName => NamespacedApplicationName.Create(Metadata.Name, Metadata.Namespace); } } From 41c689780e3b0140efcceee7681e888e2e77cb3f Mon Sep 17 00:00:00 2001 From: Noam Gal Date: Thu, 25 Jun 2026 18:47:22 +0300 Subject: [PATCH 4/6] simplified PlannedApplication ctor --- .../ManifestTemplating/ApplicationUpdater.cs | 4 +--- .../ArgoCD/Conventions/PlannedApplication.cs | 12 ++++++++---- .../Conventions/UpdateImageTag/ApplicationUpdater.cs | 4 +--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs index 8f5024b195..65af08317a 100644 --- a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs @@ -57,9 +57,7 @@ public PlannedApplication Plan(ArgoCDApplicationDto application, ArgoCDGatewayDt .Select(source => new PlannedSource(source, new RepositorySourceUpdate(applicationFromYaml.NamespacedName, source, sourceUpdater.CreateSourceUpdater(source)))) .ToList(); - var matchingSourceCount = applicationFromYaml.Spec.Sources.Count(s => deploymentScope.Matches(ScopingAnnotationReader.GetScopeForApplicationSource(s.Name.ToApplicationSourceName(), applicationFromYaml.Metadata.Annotations, containsMultipleSources))); - - return new PlannedApplication(application, gateway, plannedSources, applicationFromYaml.Spec.Sources.Count, matchingSourceCount); + return new PlannedApplication(application, gateway, plannedSources, applicationFromYaml.Spec.Sources.Count); } // Phase 3: turn the processed results back into a per-application result, writing per-source output variables. diff --git a/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs b/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs index 563e845bf5..e564bdf61e 100644 --- a/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs +++ b/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs @@ -16,14 +16,12 @@ public class PlannedApplication public PlannedApplication(ArgoCDApplicationDto application, ArgoCDGatewayDto gateway, IReadOnlyList sources, - int totalSourceCount, - int matchingSourceCount) + int totalSourceCount) { Application = application; Gateway = gateway; Sources = sources; TotalSourceCount = totalSourceCount; - MatchingSourceCount = matchingSourceCount; } public ArgoCDApplicationDto Application { get; } @@ -54,6 +52,12 @@ public string ApplicationNamespace public IReadOnlyList Sources { get; } public int TotalSourceCount { get; } - public int MatchingSourceCount { get; } + public int MatchingSourceCount + { + get + { + return Sources.Count; + } + } } } diff --git a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs index 3ac37d6efa..310ebe6568 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs @@ -52,9 +52,7 @@ public PlannedApplication Plan(ArgoCDApplicationDto application, ArgoCDGatewayDt .Select(source => new PlannedSource(source, new RepositorySourceUpdate(applicationFromYaml.NamespacedName, source, sourceUpdater.CreateSourceUpdater(source)))) .ToList(); - var matchingSourceCount = applicationFromYaml.Spec.Sources.Count(s => deploymentScope.Matches(ScopingAnnotationReader.GetScopeForApplicationSource(s.Name.ToApplicationSourceName(), applicationFromYaml.Metadata.Annotations, containsMultipleSources))); - - return new PlannedApplication(application, gateway, plannedSources, applicationFromYaml.Spec.Sources.Count, matchingSourceCount); + return new PlannedApplication(application, gateway, plannedSources, applicationFromYaml.Spec.Sources.Count); } // Phase 3: turn the processed results back into a per-application result, writing per-source output variables. From 0754dbe6189285f07a31df94ea9b81759d053fc1 Mon Sep 17 00:00:00 2001 From: Noam Gal Date: Thu, 25 Jun 2026 18:49:52 +0300 Subject: [PATCH 5/6] another small refactor (fix pr comment) --- .../ManifestTemplating/ApplicationUpdater.cs | 8 ++++---- .../Calamari/ArgoCD/Conventions/PlannedApplication.cs | 6 +++--- .../UpdateArgoCDAppImagesInstallConvention.cs | 2 +- ...pdateArgoCDApplicationManifestsInstallConvention.cs | 2 +- .../Conventions/UpdateImageTag/ApplicationUpdater.cs | 10 +++++----- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs index 65af08317a..224310aa92 100644 --- a/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/ManifestTemplating/ApplicationUpdater.cs @@ -63,7 +63,7 @@ public PlannedApplication Plan(ArgoCDApplicationDto application, ArgoCDGatewayDt // Phase 3: turn the processed results back into a per-application result, writing per-source output variables. public ProcessApplicationResult AssembleResult(PlannedApplication plan, IReadOnlyDictionary resultsByUpdate) { - foreach (var plannedSource in plan.Sources) + foreach (var plannedSource in plan.MatchingSources) { outputVariablesWriter.WriteSourceUpdateResultOutputWhenPushResultExists(plan.Gateway.Name, plan.NamespacedName, @@ -71,7 +71,7 @@ public ProcessApplicationResult AssembleResult(PlannedApplication plan, IReadOnl resultsByUpdate[plannedSource.Update]); } - var trackedSourceDetails = plan.Sources.Select(plannedSource => + var trackedSourceDetails = plan.MatchingSources.Select(plannedSource => { var result = resultsByUpdate[plannedSource.Update]; return new TrackedSourceDetail(result.PushResult?.CommitSha, result.PushResult?.CommitTimestamp, plannedSource.Source.Index, result.ReplacedFiles, []); @@ -84,7 +84,7 @@ public ProcessApplicationResult AssembleResult(PlannedApplication plan, IReadOnl ? log.FormatLink(instanceLinks.ApplicationDetails(plan.ApplicationName, plan.ApplicationNamespace), plan.ApplicationName) : plan.ApplicationName; - var anyUpdated = plan.Sources.Any(plannedSource => resultsByUpdate[plannedSource.Update].Updated); + var anyUpdated = plan.MatchingSources.Any(plannedSource => resultsByUpdate[plannedSource.Update].Updated); log.InfoFormat(anyUpdated ? "Updated Application {0}" : "Nothing to update for Application {0}", linkifiedAppName); return new ProcessApplicationResult( @@ -94,7 +94,7 @@ public ProcessApplicationResult AssembleResult(PlannedApplication plan, IReadOnl plan.MatchingSourceCount, trackedSourceDetails, [], - plan.Sources.Where(plannedSource => resultsByUpdate[plannedSource.Update].Updated).Select(plannedSource => plannedSource.Source.Source.OriginalRepoUrl).ToHashSet()); + plan.MatchingSources.Where(plannedSource => resultsByUpdate[plannedSource.Update].Updated).Select(plannedSource => plannedSource.Source.Source.OriginalRepoUrl).ToHashSet()); } void LogWarningIfUpdatingMultipleSources( diff --git a/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs b/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs index e564bdf61e..28f720d8f7 100644 --- a/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs +++ b/source/Calamari/ArgoCD/Conventions/PlannedApplication.cs @@ -20,7 +20,7 @@ public PlannedApplication(ArgoCDApplicationDto application, { Application = application; Gateway = gateway; - Sources = sources; + MatchingSources = sources; TotalSourceCount = totalSourceCount; } @@ -50,13 +50,13 @@ public string ApplicationNamespace } } - public IReadOnlyList Sources { get; } + public IReadOnlyList MatchingSources { get; } public int TotalSourceCount { get; } public int MatchingSourceCount { get { - return Sources.Count; + return MatchingSources.Count; } } } diff --git a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs index 6e03421aed..33bd894e90 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs @@ -87,7 +87,7 @@ public void Install(RunningDeployment deployment) var commitMessageGenerator = new ImageTagUpdateCommitMessageGenerator(deploymentConfig.CommitParameters.Description); var processor = new GroupedRepositoryProcessor(authenticatingRepositoryFactory, deploymentConfig.CommitParameters, commitMessageGenerator); - var updates = plans.SelectMany(p => p.Sources.Select(s => s.Update)).ToList(); + var updates = plans.SelectMany(p => p.MatchingSources.Select(s => s.Update)).ToList(); var results = processor.Process(updates); var resultsByUpdate = updates.Zip(results, (update, result) => (update, result)).ToDictionary(x => x.update, x => x.result); diff --git a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDApplicationManifestsInstallConvention.cs b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDApplicationManifestsInstallConvention.cs index 93df9d3a06..7731363bfb 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDApplicationManifestsInstallConvention.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDApplicationManifestsInstallConvention.cs @@ -95,7 +95,7 @@ public void Install(RunningDeployment deployment) var commitMessageGenerator = new UserDefinedCommitMessageGenerator(deploymentConfig.CommitParameters.Description); var processor = new GroupedRepositoryProcessor(authenticatingRepositoryFactory, deploymentConfig.CommitParameters, commitMessageGenerator); - var updates = plans.SelectMany(p => p.Sources.Select(s => s.Update)).ToList(); + var updates = plans.SelectMany(p => p.MatchingSources.Select(s => s.Update)).ToList(); var results = processor.Process(updates); var resultsByUpdate = updates.Zip(results, (update, result) => (update, result)).ToDictionary(x => x.update, x => x.result); diff --git a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs index 310ebe6568..cbe21f71b7 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateImageTag/ApplicationUpdater.cs @@ -58,7 +58,7 @@ public PlannedApplication Plan(ArgoCDApplicationDto application, ArgoCDGatewayDt // Phase 3: turn the processed results back into a per-application result, writing per-source output variables. public ProcessApplicationResult AssembleResult(PlannedApplication plan, IReadOnlyDictionary resultsByUpdate) { - foreach (var plannedSource in plan.Sources) + foreach (var plannedSource in plan.MatchingSources) { outputVariablesWriter.WriteSourceUpdateResultOutputWhenPushResultExists(plan.Gateway.Name, plan.NamespacedName, @@ -66,7 +66,7 @@ public ProcessApplicationResult AssembleResult(PlannedApplication plan, IReadOnl resultsByUpdate[plannedSource.Update]); } - var trackedSourceDetails = plan.Sources.Select(plannedSource => + var trackedSourceDetails = plan.MatchingSources.Select(plannedSource => { var result = resultsByUpdate[plannedSource.Update]; return new TrackedSourceDetail(result.PushResult?.CommitSha, result.PushResult?.CommitTimestamp, plannedSource.Source.Index, [], result.PatchedFiles); @@ -79,7 +79,7 @@ public ProcessApplicationResult AssembleResult(PlannedApplication plan, IReadOnl ? log.FormatLink(instanceLinks.ApplicationDetails(plan.ApplicationName, plan.Application.KubernetesNamespace), plan.ApplicationName) : plan.ApplicationName; - var anyUpdated = plan.Sources.Any(plannedSource => resultsByUpdate[plannedSource.Update].Updated); + var anyUpdated = plan.MatchingSources.Any(plannedSource => resultsByUpdate[plannedSource.Update].Updated); log.InfoFormat(anyUpdated ? "Updated Application {0}" : "Nothing to update for Application {0}", linkifiedAppName); return new ProcessApplicationResult( @@ -88,8 +88,8 @@ public ProcessApplicationResult AssembleResult(PlannedApplication plan, IReadOnl plan.TotalSourceCount, plan.MatchingSourceCount, trackedSourceDetails, - plan.Sources.SelectMany(plannedSource => resultsByUpdate[plannedSource.Update].ImagesUpdated).ToHashSet(), - plan.Sources.Where(plannedSource => resultsByUpdate[plannedSource.Update].Updated).Select(plannedSource => plannedSource.Source.Source.OriginalRepoUrl).ToHashSet()); + plan.MatchingSources.SelectMany(plannedSource => resultsByUpdate[plannedSource.Update].ImagesUpdated).ToHashSet(), + plan.MatchingSources.Where(plannedSource => resultsByUpdate[plannedSource.Update].Updated).Select(plannedSource => plannedSource.Source.Source.OriginalRepoUrl).ToHashSet()); } void LogHelmAnnotationWarning(Application applicationFromYaml) From 016c28f662daa213f2ac270860a7d2f0fea491cb Mon Sep 17 00:00:00 2001 From: Noam Gal Date: Thu, 25 Jun 2026 22:50:47 +0300 Subject: [PATCH 6/6] fixed rebase --- source/Calamari/ArgoCD/Git/GroupedRepositoryProcessor.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/Calamari/ArgoCD/Git/GroupedRepositoryProcessor.cs b/source/Calamari/ArgoCD/Git/GroupedRepositoryProcessor.cs index 9893c9b389..d7a647e8e4 100644 --- a/source/Calamari/ArgoCD/Git/GroupedRepositoryProcessor.cs +++ b/source/Calamari/ArgoCD/Git/GroupedRepositoryProcessor.cs @@ -133,7 +133,7 @@ void ProcessBranchAsDirectCommits(RepositoryWrapper repository, // A single push carries every application's commit on this branch. if (committedAnything) { - repository.PushChanges(false, commitParameters.Summary, string.Empty, reference, CancellationToken.None) + repository.PushChanges(false, commitParameters.Summary, string.Empty, reference, commitParameters.PushRetryAttempts, CancellationToken.None) .GetAwaiter() .GetResult(); } @@ -164,7 +164,7 @@ void ProcessBranchAsPullRequests(RepositoryWrapper repository, continue; } - var pushResult = repository.PushChanges(true, commitParameters.Summary, description, reference, CancellationToken.None) + var pushResult = repository.PushChanges(true, commitParameters.Summary, description, reference, commitParameters.PushRetryAttempts, CancellationToken.None) .GetAwaiter() .GetResult(); results[item.index] = new SourceUpdateResult(fileResult.UpdatedImages, pushResult, fileResult.ReplacedFiles, fileResult.PatchedFiles);