From 0d5a553640a8b48611a8b5fce41466a9e3a837e5 Mon Sep 17 00:00:00 2001 From: Pedro Sakuma Travi <39205549+pedrosakuma@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:01:14 +0000 Subject: [PATCH] perf: restore per-schema incremental granularity (#170) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/copilot-instructions.md | 6 +- CHANGELOG.md | 4 + .../Generators/UtilitiesCodeGenerator.cs | 30 ++- src/SbeCodeGenerator/SBESourceGenerator.cs | 242 +++++++++++------- .../IncrementalPipelineTests.cs | 119 +++++++++ 5 files changed, 299 insertions(+), 102 deletions(-) create mode 100644 tests/SbeCodeGenerator.Tests/IncrementalPipelineTests.cs diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index f3868b1..9bff096 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -32,15 +32,17 @@ This is a **Roslyn incremental source generator** (`IIncrementalGenerator`) that ``` XML schema (*.xml via AdditionalFiles) - → SBESourceGenerator (entry point, namespace derivation, SchemaContext creation) + → SBESourceGenerator (entry point, per-schema incremental pipeline, namespace derivation, SchemaContext creation) → TypesCodeGenerator (enums, types, composites, sets, derived constants on decimal composites) → MessagesCodeGenerator (messages, fields, groups, varData, per-message {Msg}VersionMap when multi-version) → DispatcherGenerator (per-schema ISbeMessageHandler + zero-cost SbeDispatcher.Dispatch) - → UtilitiesCodeGenerator (SpanReader, SpanWriter, endian helpers) + → UtilitiesCodeGenerator (SpanReader, SpanWriter, endian helpers; emitted once per runtime namespace in a separate incremental step) → ValidationGenerator (optional validation) → sourceContext.AddSource() per generated file ``` +Incremental behavior matters: keep schema `AdditionalText` inputs as per-item `IncrementalValuesProvider`s through to `RegisterSourceOutput`. Only collect genuinely shared/project-wide data (for example semantic-type registrations, or runtime-namespace dedup when emitting shared helpers). + Each generator implements `ICodeGenerator` and returns `IEnumerable<(string name, string content)>`. ### Key Abstractions diff --git a/CHANGELOG.md b/CHANGELOG.md index 37c3557..08316ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **Per-schema incremental invalidation for XML AdditionalFiles (#170)**: `SBESourceGenerator` no longer `.Collect()`s the entire schema set before generation. Each schema now flows through its own `RegisterSourceOutput` path, so editing one XML file only re-runs the generators for that schema. Runtime helpers (`SpanReader` / `SpanWriter`) are emitted in a separate namespace-deduplicated step to preserve shared helper generation without reintroducing whole-project schema invalidation. + ## [1.7.0] - 2026-04-30 ### Added diff --git a/src/SbeCodeGenerator/Generators/UtilitiesCodeGenerator.cs b/src/SbeCodeGenerator/Generators/UtilitiesCodeGenerator.cs index 084c886..6922a4e 100644 --- a/src/SbeCodeGenerator/Generators/UtilitiesCodeGenerator.cs +++ b/src/SbeCodeGenerator/Generators/UtilitiesCodeGenerator.cs @@ -17,16 +17,28 @@ internal class UtilitiesCodeGenerator : ICodeGenerator if (context.GeneratedRuntimeNamespaces.Add(runtimeNamespace)) { - // Generate SpanReader once per runtime namespace - StringBuilder sb = new StringBuilder(); - new SpanReaderGenerator(runtimeNamespace).AppendFileContent(sb); - yield return (context.CreateHintName(runtimeNamespace, "Runtime", "SpanReader"), sb.ToString()); - - // Generate SpanWriter once per runtime namespace - sb = new StringBuilder(); - new SpanWriterGenerator(runtimeNamespace).AppendFileContent(sb); - yield return (context.CreateHintName(runtimeNamespace, "Runtime", "SpanWriter"), sb.ToString()); + foreach (var item in GenerateRuntimeSources( + runtimeNamespace, + typeName => context.CreateHintName(runtimeNamespace, "Runtime", typeName))) + { + yield return item; + } } } + + internal static IEnumerable<(string name, string content)> GenerateRuntimeSources( + string runtimeNamespace, + Func createHintName) + { + // Generate SpanReader once per runtime namespace + StringBuilder sb = new StringBuilder(); + new SpanReaderGenerator(runtimeNamespace).AppendFileContent(sb); + yield return (createHintName("SpanReader"), sb.ToString()); + + // Generate SpanWriter once per runtime namespace + sb = new StringBuilder(); + new SpanWriterGenerator(runtimeNamespace).AppendFileContent(sb); + yield return (createHintName("SpanWriter"), sb.ToString()); + } } } diff --git a/src/SbeCodeGenerator/SBESourceGenerator.cs b/src/SbeCodeGenerator/SBESourceGenerator.cs index ba360d9..efcbfe1 100644 --- a/src/SbeCodeGenerator/SBESourceGenerator.cs +++ b/src/SbeCodeGenerator/SBESourceGenerator.cs @@ -12,6 +12,7 @@ using System.Linq; using System.Security.Cryptography; using System.Text; +using System.Threading; namespace SbeSourceGenerator { @@ -48,13 +49,28 @@ public void Initialize(IncrementalGeneratorInitializationContext initContext) } }); + var semanticRegistry = userAttributeResults + .Select(static (results, _) => BuildSemanticRegistry(results)); + // Stage 2: Combine with analyzer config options (for SbeAssumeHostEndianness hint) and the registry. - var combined = xmlSchemaFiles.Collect() + var combined = xmlSchemaFiles .Combine(initContext.AnalyzerConfigOptionsProvider) - .Combine(userAttributeResults); + .Combine(semanticRegistry) + .Select(static (input, _) => ( + Path: input.Left.Left.Path, + Schema: input.Left.Left, + Options: input.Left.Right, + SemanticRegistry: input.Right)) + .WithTrackingName("PerSchemaGeneration"); + + var runtimeNamespaces = xmlSchemaFiles + .Select(static (schemaFile, cancellationToken) => TryGetRuntimeNamespace(schemaFile, cancellationToken)) + .Collect() + .WithTrackingName("RuntimeNamespaceCollection"); // Stage 3: Register source generation with diagnostic support RegisterSourceGeneration(initContext, combined); + RegisterRuntimeGeneration(initContext, runtimeNamespaces); } /// @@ -69,20 +85,11 @@ private static IncrementalValuesProvider CollectXmlSchemaFiles(I /// Registers source output for each XML schema with diagnostic reporting. /// private static void RegisterSourceGeneration(IncrementalGeneratorInitializationContext initContext, - IncrementalValueProvider<((ImmutableArray Schemas, AnalyzerConfigOptionsProvider Options) Left, ImmutableArray UserRegs)> combined) + IncrementalValuesProvider<(string Path, AdditionalText Schema, AnalyzerConfigOptionsProvider Options, SemanticConverterRegistry SemanticRegistry)> combined) { initContext.RegisterSourceOutput(combined, (sourceContext, input) => { - var ((text, configOptions), userRegs) = input; - - if (text.IsDefaultOrEmpty) - return; - - // Build the semantic registry once per generation pass. - var userRegistrations = userRegs.IsDefaultOrEmpty - ? ImmutableArray.Empty - : userRegs.Where(r => r.Registration != null).Select(r => r.Registration!).ToImmutableArray(); - var semanticRegistry = SemanticConverterRegistry.Build(userRegistrations); + var (path, additionalText, configOptions, semanticRegistry) = input; // Read the optional SbeAssumeHostEndianness MSBuild property string? hostHint = null; @@ -92,115 +99,168 @@ private static void RegisterSourceGeneration(IncrementalGeneratorInitializationC hostHint = hintValue; } - var emittedRuntimeNamespaces = new HashSet(StringComparer.Ordinal); var emittedHintNames = new HashSet(StringComparer.Ordinal); - foreach (var additionalText in text) + try { - try - { - string path = additionalText.Path; - var xmlContent = additionalText.GetText(sourceContext.CancellationToken)?.ToString(); - if (string.IsNullOrEmpty(xmlContent)) - continue; + var xmlContent = additionalText.GetText(sourceContext.CancellationToken)?.ToString(); + if (string.IsNullOrEmpty(xmlContent)) + return; - var schema = SchemaReader.Parse(xmlContent!, sourceContext); + var schema = SchemaReader.Parse(xmlContent!, sourceContext); - string ns = GetNamespaceFromSchema(schema, path); - string schemaKey = CreateSchemaKey(path); + string ns = GetNamespaceFromSchema(schema, path); + string schemaKey = CreateSchemaKey(path); - // Create a per-schema context to hold mutable state (sharing runtime tracking) - var context = new SchemaContext(schemaKey, emittedRuntimeNamespaces); - context.SemanticConverters = semanticRegistry; + // Create a per-schema context to hold mutable state. + var context = new SchemaContext(schemaKey); + context.SemanticConverters = semanticRegistry; - if (!string.IsNullOrEmpty(schema.ByteOrder)) - { - context.ByteOrder = schema.ByteOrder; - } + if (!string.IsNullOrEmpty(schema.ByteOrder)) + { + context.ByteOrder = schema.ByteOrder; + } - context.EndianConversion = ComputeEndianConversion( - context.ByteOrder, hostHint, sourceContext, path); + context.EndianConversion = ComputeEndianConversion( + context.ByteOrder, hostHint, sourceContext, path); - if (!string.IsNullOrEmpty(schema.HeaderType)) - context.HeaderType = schema.HeaderType; + if (!string.IsNullOrEmpty(schema.HeaderType)) + context.HeaderType = schema.HeaderType; - // Use specialized generators to handle different categories - var typesGenerator = new TypesCodeGenerator(); - var messagesGenerator = new MessagesCodeGenerator(); - var utilitiesGenerator = new UtilitiesCodeGenerator(); - var validationGenerator = new ValidationGenerator(); + // Use specialized generators to handle different categories + var typesGenerator = new TypesCodeGenerator(); + var messagesGenerator = new MessagesCodeGenerator(); + var validationGenerator = new ValidationGenerator(); - var generators = new (string phase, ICodeGenerator gen)[] - { - ("types", typesGenerator), - ("messages", messagesGenerator), - ("dispatcher", new DispatcherGenerator()), - ("utilities", utilitiesGenerator), - ("validation", validationGenerator) - }; - - foreach (var (phase, gen) in generators) + var generators = new (string phase, ICodeGenerator gen)[] + { + ("types", typesGenerator), + ("messages", messagesGenerator), + ("dispatcher", new DispatcherGenerator()), + ("validation", validationGenerator) + }; + + foreach (var (phase, gen) in generators) + { + try { - try + foreach (var item in gen.Generate(ns, schema, context, sourceContext)) { - foreach (var item in gen.Generate(ns, schema, context, sourceContext)) + try { - try + if (!emittedHintNames.Add(item.name)) { - if (!emittedHintNames.Add(item.name)) + // Roslyn would throw ArgumentException on duplicate hintName, + // aborting the rest of the phase. Suppress and continue so a + // single duplicate doesn't cascade into thousands of CS0246s + // against partially-emitted files. + if (!sourceContext.CancellationToken.IsCancellationRequested) { - // Roslyn would throw ArgumentException on duplicate hintName, - // aborting the rest of the phase. Suppress and continue so a - // single duplicate doesn't cascade into thousands of CS0246s - // against partially-emitted files. - if (!sourceContext.CancellationToken.IsCancellationRequested) - { - sourceContext.ReportDiagnostic(Diagnostic.Create( - SbeDiagnostics.DuplicateGeneratedSource, - Location.None, - item.name, - phase)); - } - continue; + sourceContext.ReportDiagnostic(Diagnostic.Create( + SbeDiagnostics.DuplicateGeneratedSource, + Location.None, + item.name, + phase)); } - sourceContext.AddSource(item.name, item.content); - } - catch (Exception itemEx) when (!sourceContext.CancellationToken.IsCancellationRequested) - { - // Per-item failure must not derail subsequent items in the same phase. - sourceContext.ReportDiagnostic(Diagnostic.Create( - SbeDiagnostics.MalformedSchema, - Location.None, - path, - $"[{phase}] {item.name}: {itemEx.Message}")); + continue; } + sourceContext.AddSource(item.name, item.content); + } + catch (Exception itemEx) when (!sourceContext.CancellationToken.IsCancellationRequested) + { + // Per-item failure must not derail subsequent items in the same phase. + sourceContext.ReportDiagnostic(Diagnostic.Create( + SbeDiagnostics.MalformedSchema, + Location.None, + path, + $"[{phase}] {item.name}: {itemEx.Message}")); } - } - catch (Exception genEx) when (!sourceContext.CancellationToken.IsCancellationRequested) - { - sourceContext.ReportDiagnostic(Diagnostic.Create( - SbeDiagnostics.MalformedSchema, - Location.None, - path, - $"[{phase}] {genEx.Message}")); } } - } - catch (Exception ex) - { - if (!sourceContext.CancellationToken.IsCancellationRequested) + catch (Exception genEx) when (!sourceContext.CancellationToken.IsCancellationRequested) { sourceContext.ReportDiagnostic(Diagnostic.Create( SbeDiagnostics.MalformedSchema, Location.None, - additionalText.Path, - ex.Message)); + path, + $"[{phase}] {genEx.Message}")); } } } + catch (Exception ex) + { + if (!sourceContext.CancellationToken.IsCancellationRequested) + { + sourceContext.ReportDiagnostic(Diagnostic.Create( + SbeDiagnostics.MalformedSchema, + Location.None, + additionalText.Path, + ex.Message)); + } + } }); } + private static void RegisterRuntimeGeneration( + IncrementalGeneratorInitializationContext initContext, + IncrementalValueProvider> runtimeNamespaces) + { + initContext.RegisterSourceOutput(runtimeNamespaces, (sourceContext, namespaces) => + { + if (namespaces.IsDefaultOrEmpty) + return; + + var emittedRuntimeNamespaces = new HashSet(StringComparer.Ordinal); + foreach (var runtimeNamespace in namespaces) + { + if (string.IsNullOrWhiteSpace(runtimeNamespace)) + continue; + + var resolvedRuntimeNamespace = runtimeNamespace!; + if (!emittedRuntimeNamespaces.Add(resolvedRuntimeNamespace)) + continue; + + foreach (var item in UtilitiesCodeGenerator.GenerateRuntimeSources( + resolvedRuntimeNamespace, + typeName => CreateRuntimeHintName(resolvedRuntimeNamespace, typeName))) + { + sourceContext.AddSource(item.name, item.content); + } + } + }); + } + + private static SemanticConverterRegistry BuildSemanticRegistry(ImmutableArray userRegs) + { + var userRegistrations = userRegs.IsDefaultOrEmpty + ? ImmutableArray.Empty + : userRegs.Where(r => r.Registration != null).Select(r => r.Registration!).ToImmutableArray(); + + return SemanticConverterRegistry.Build(userRegistrations); + } + + private static string? TryGetRuntimeNamespace(AdditionalText additionalText, CancellationToken cancellationToken) + { + var xmlContent = additionalText.GetText(cancellationToken)?.ToString(); + if (string.IsNullOrEmpty(xmlContent)) + return null; + + try + { + var schema = SchemaReader.Parse(xmlContent!); + return GetNamespaceFromSchema(schema, additionalText.Path); + } + catch + { + return null; + } + } + + private static string CreateRuntimeHintName(string runtimeNamespace, string typeName) + { + return string.Concat(runtimeNamespace, "\\Runtime\\", typeName); + } + private static string CreateSchemaKey(string path) { string fileName = Path.GetFileNameWithoutExtension(path); diff --git a/tests/SbeCodeGenerator.Tests/IncrementalPipelineTests.cs b/tests/SbeCodeGenerator.Tests/IncrementalPipelineTests.cs new file mode 100644 index 0000000..32b2a31 --- /dev/null +++ b/tests/SbeCodeGenerator.Tests/IncrementalPipelineTests.cs @@ -0,0 +1,119 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Diagnostics; +using Microsoft.CodeAnalysis.Text; +using SbeSourceGenerator; +using SbeSourceGenerator.SemanticTypes; +using System.Collections.Immutable; +using System.Text; +using Xunit; + +namespace SbeCodeGenerator.Tests +{ + public class IncrementalPipelineTests + { + private sealed class InMemoryAdditionalText : AdditionalText + { + private readonly SourceText _text; + + public InMemoryAdditionalText(string path, string content) + { + Path = path; + _text = SourceText.From(content, Encoding.UTF8); + } + + public override string Path { get; } + + public override SourceText GetText(System.Threading.CancellationToken cancellationToken = default) => _text; + } + + [Fact] + public void Initialize_WhenOneOfTwoSchemasChanges_CachesTheUnchangedSchema() + { + var firstSchema = new InMemoryAdditionalText("first-schema.xml", CreateSchema("First.Schema", "Order", withExtraEnumValue: false)); + var secondSchema = new InMemoryAdditionalText("second-schema.xml", CreateSchema("Second.Schema", "Trade", withExtraEnumValue: false)); + var updatedFirstSchema = new InMemoryAdditionalText("first-schema.xml", CreateSchema("First.Schema", "Order", withExtraEnumValue: true)); + + var compilation = CSharpCompilation.Create( + "TestAssembly", + syntaxTrees: [], + references: + [ + MetadataReference.CreateFromFile(typeof(object).Assembly.Location), + ]); + + GeneratorDriver driver = CSharpGeneratorDriver.Create( + generators: [new SBESourceGenerator().AsSourceGenerator()], + additionalTexts: [firstSchema, secondSchema], + driverOptions: new GeneratorDriverOptions( + IncrementalGeneratorOutputKind.None, + trackIncrementalGeneratorSteps: true)); + + driver = driver.RunGenerators(compilation); + driver = driver.ReplaceAdditionalText(firstSchema, updatedFirstSchema); + driver = driver.RunGenerators(compilation); + + GeneratorRunResult generatorResult = Assert.Single(driver.GetRunResult().Results); + Assert.Null(generatorResult.Exception); + + Assert.True( + generatorResult.TrackedSteps.TryGetValue("PerSchemaGeneration", out var steps), + "Expected the per-schema generation step to be tracked."); + + var reasonsByPath = steps + .SelectMany(static step => step.Outputs) + .ToDictionary( + static output => ExtractPath(output.Value), + static output => output.Reason, + System.StringComparer.Ordinal); + + Assert.True( + reasonsByPath.TryGetValue("first-schema.xml", out var changedReason), + "Expected the changed schema to be present in the tracked outputs."); + Assert.True( + changedReason is IncrementalStepRunReason.Modified or IncrementalStepRunReason.New, + $"Expected first-schema.xml to be Modified or New, but was {changedReason}."); + + Assert.True( + reasonsByPath.TryGetValue("second-schema.xml", out var unchangedReason), + "Expected the unchanged schema to be present in the tracked outputs."); + Assert.True( + unchangedReason is IncrementalStepRunReason.Cached or IncrementalStepRunReason.Unchanged, + $"Expected second-schema.xml to be Cached or Unchanged, but was {unchangedReason}."); + } + + private static string ExtractPath(object value) + { + if (value is ValueTuple tracked) + return tracked.Item1; + + throw new Xunit.Sdk.XunitException($"Unexpected tracked output value type: {value.GetType().FullName}"); + } + + private static string CreateSchema(string package, string messageName, bool withExtraEnumValue) + { + string extraEnumValue = withExtraEnumValue ? Environment.NewLine + " 3" : string.Empty; + + return $@" + + + + + + + + + + + 1 + 2{extraEnumValue} + + + + + +"; + } + } +}