From 29964cbb67c93b0f9108e081b374eb31cad7f6ed Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 11 Apr 2026 17:07:18 -0300 Subject: [PATCH 1/9] impl done & tests --- RecordParser.Test/FileReaderTest.cs | 49 +++++++- .../VariableLengthReaderExtensions.cs | 107 +++++++++++++++++- 2 files changed, 152 insertions(+), 4 deletions(-) diff --git a/RecordParser.Test/FileReaderTest.cs b/RecordParser.Test/FileReaderTest.cs index 77da323..4e89ff1 100644 --- a/RecordParser.Test/FileReaderTest.cs +++ b/RecordParser.Test/FileReaderTest.cs @@ -155,7 +155,7 @@ public void Given_record_is_too_large_for_custom_buffer_size_then_exception_shou var records = reader.ReadRecords(parser, new() { HasHeader = true, - // BufferSize = bufferSize, + // BufferSize = bufferSize, }); foreach (var item in records) @@ -176,6 +176,7 @@ public void Given_record_is_too_large_for_custom_buffer_size_then_exception_shou public static string GetFilePath(string fileName) => Path.Combine(Directory.GetCurrentDirectory(), fileName); + public const string Header = "HEADER"; public static IEnumerable Given_quoted_csv_file_should_read_quoted_properly_theory(string file) { var fileNames = new[] @@ -206,7 +207,7 @@ public static IEnumerable Given_quoted_csv_file_should_read_quoted_pro } if (hasHeader) - fileBuilder.Insert(index: 0, "Id,Date,Name,Rate,Ranking" + Environment.NewLine); + fileBuilder.Insert(index: 0, Header + Environment.NewLine); if (blankLineAtEnd) fileBuilder.AppendLine(); @@ -568,5 +569,49 @@ public void Read_plain_text_of_fixed_length_file(string fileContent, bool parall result.Should().BeEquivalentTo(expected, cfg => cfg.WithStrictOrdering()); } + + [Theory] + [MemberData(nameof(Given_quoted_csv_file_should_read_quoted_properly_theory), new object[] { "AllFieldsQuotedCsv.csv" })] + public void Read_csv_file_with_header_should_automatic_bind_mapping(string fileContent, bool hasHeader, bool parallelProcessing, bool blankLineAtEnd, int repeat) + { + if (!hasHeader) + return; + + // Arrange + + fileContent = fileContent.Replace(Header, "id,name,age,birthday,gender,email,UNMAPPED,children"); + using var fileStream = fileContent.ToStream(); + using var streamReader = new StreamReader(fileStream, Encoding.UTF8); + + var expectedItems = new PersonComplete[] + { + new () { id = new Guid("ec9a8be9-a000-503b-adcf-7266804f1eb1"), name = "Lilly Bradley", age = 21, birthday = DateTime.Parse("11/16/1977"), gender = Gender.Male, email = "pak@witak.bf", children = true }, + new () { id = new Guid("63858071-cbb3-5abd-9f88-3dfd565cc4ab"), name = "Lucy Berry", age = 49, birthday = DateTime.Parse("11/12/1961"), gender = Gender.Female, email = "vanvo@ro.pk", children = false }, + new () { id = new Guid("203804f9-93e7-5510-8bb2-177296bafe6a"), name = "Frank Fox", age = 36, birthday = DateTime.Parse("3/19/1977"), gender = Gender.Male, email = "vav@ped.fj", children = true }, + new () { id = new Guid("a8af66fb-bad4-51eb-810c-bf3ca22337c6"), name = "Isabel Todd", age = 51, birthday = DateTime.Parse("9/16/1999"), gender = Gender.Female, email = "gu@or.bz", children = false }, + new () { id = new Guid("1a3d8a66-3e0c-50eb-99c1-a3926bce15ed"), name = $"Joseph {Environment.NewLine}Scott", age = 55, birthday = DateTime.Parse("10/26/1986"), gender = Gender.Male, email = "bup@vugeb.tt", children = false }, + new () { id = new Guid("aa7d4395-f10f-5776-9912-e3d86c4b9d3c"), name = "Gilbert Brooks", age = 56, birthday = DateTime.Parse("3/1/1956"), gender = Gender.Female, email = "epiju@ba.ly", children = true }, + new () { id = new Guid("1d25b811-4002-5744-ac40-93a50f2a442c"), name = "Louis \"Ronaldo\" Bennett", age = 25, birthday = DateTime.Parse("4/4/1967"), gender = Gender.Male, email = "ma@itrovive.tv", children = true }, + new () { id = new Guid("8e963ae5-a9ed-5572-b11c-566abc6a8a56"), name = "Norman Parker", age = 57, birthday = DateTime.Parse("4/17/1969"), gender = Gender.Male, email = "omi@hewepa.bw", children = true }, + new () { id = new Guid("4d373cfb-79e3-54ce-87ff-f2a08fde8f28"), name = "Gary Doyle", age = 20, birthday = DateTime.Parse("1/21/1958"), gender = Gender.Male, email = "orjohma@cabmofa.ps", children = true }, + new () { id = new Guid("5af00cdf-0758-5317-bcdf-c9a3337cc266"), name = "Bruce Silva", age = 39, birthday = DateTime.Parse("1/11/1968"), gender = Gender.Female, email = "ta@ovonib.ir", children = true }, + } + .Repeat(repeat); + + var readOptions = new VariableLengthReaderOptions + { + HasHeader = hasHeader, + ParallelismOptions = new() { Enabled = parallelProcessing }, + ContainsQuotedFields = true, + }; + + // Act + + var items = streamReader.ReadRecords(readOptions, skipMismatchedColumns: true); + + // Assert + + items.Should().BeEquivalentTo(expectedItems, cfg => cfg.WithStrictOrdering()); + } } } \ No newline at end of file diff --git a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs index c27bbd3..41d875f 100644 --- a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs +++ b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs @@ -1,13 +1,17 @@ -using RecordParser.Extensions.FileReader.RowReaders; +using RecordParser.Builders.Reader; +using RecordParser.Extensions.FileReader.RowReaders; using RecordParser.Parsers; using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Linq.Expressions; +using System.Reflection; using static RecordParser.Extensions.ReaderCommon; namespace RecordParser.Extensions { - public class VariableLengthReaderOptions + public record class VariableLengthReaderOptions { /// /// Indicates if there is a header record present in the reader's content. @@ -53,5 +57,104 @@ public static IEnumerable ReadRecords(this TextReader reader, IVariableLen ? ReadRecordsParallel(selector, func, options.HasHeader, parallelOptions) : ReadRecordsSequential(selector, func, options.HasHeader); } + + public static IEnumerable ReadRecords(this + TextReader reader, + VariableLengthReaderOptions options, + bool skipMismatchedColumns = false, + Action> action = null) + { + string header; + + if (!options.HasHeader || string.IsNullOrEmpty(header = reader.ReadLine())) + throw new InvalidOperationException("header is mandatory"); + + var separator = DetectDelimiter(header.AsMemory()); + var columns = header.Split(separator); + var parser = BuildParser(columns, separator, skipMismatchedColumns, action); + + return ReadRecords(reader, parser, options with { HasHeader = false }); + } + + private static IVariableLengthReader BuildParser(IEnumerable columns, string separator, bool skipMismatchedColumns, Action> action) + { + var builder = new VariableLengthReaderSequentialBuilder(); + foreach (var column in columns) + { + try + { + var expression = CreateExpression(column.Trim()); + builder.Map((dynamic)expression); + } + catch when (skipMismatchedColumns) + { + builder.Skip(1); + } + } + + action?.Invoke(builder); + + return builder.Build(separator); + } + + private static LambdaExpression CreateExpression(string propertyName) + { + var param = Expression.Parameter(typeof(T), "x"); + Expression body = param; + var parts = propertyName.Split('.'); + + foreach (var part in parts) + { + // starts in T, then loop recursively + var currentType = body.Type; + + var member = (MemberInfo) + currentType.GetProperty(part, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance) ?? + currentType.GetField(part, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + + if (member == null) + throw new ArgumentException($"Can not find property or field '{part}' in type '{currentType.Name}'."); + + body = Expression.MakeMemberAccess(body, member); + } + + return Expression.Lambda(body, param); + } + + internal static string DetectDelimiter(ReadOnlyMemory header) + { + var candidates = new string[] { ",", ";", "\t", "|", "||", "::", "@" }; + var headerSpan = header.Span; + var bestCandidate = ","; + var maxCount = 0; + + foreach (var candidate in candidates) + { + var currentCount = 0; + var candidateSpan = candidate.AsSpan(); + var candidateLen = candidateSpan.Length; + + if (candidateLen == 0 || candidateLen > headerSpan.Length) + continue; + + for (var i = 0; i <= headerSpan.Length - candidateLen; i++) + { + if (headerSpan.Slice(i, candidateLen).SequenceEqual(candidateSpan)) + { + currentCount++; + // avoid overlap + i += candidateLen - 1; + } + } + + if (currentCount > maxCount) + { + maxCount = currentCount; + bestCandidate = candidate; + } + } + + return bestCandidate; + } } } From 0c61accda00615e5d9bb10ef0a820a0a24181e47 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 13 Apr 2026 00:31:32 -0300 Subject: [PATCH 2/9] add condition csharp package for netstandard --- RecordParser/RecordParser.csproj | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/RecordParser/RecordParser.csproj b/RecordParser/RecordParser.csproj index 9e00fba..2de4a06 100644 --- a/RecordParser/RecordParser.csproj +++ b/RecordParser/RecordParser.csproj @@ -2,7 +2,7 @@ RecordParser - netstandard2.1;net6.0;net7.0;net8.0 + netstandard2.1;net8.0 latest Leandro Fernandes Vieira (leandromoh) @@ -35,7 +35,13 @@ <_Parameter1>RecordParser.Test - + + + + all + + + (readOptions, skipMismatchedColumns: false); + + // Assert + + items.Should().BeEquivalentTo(expected, cfg => cfg.WithStrictOrdering()); + } + [Fact] + public void Read_csv_file_with_header_should_automatic_bind_support_additional_configuration() + { // Arrange - fileContent = fileContent.Replace(Header, "id,name,age,birthday,gender,email,UNMAPPED,children"); - using var fileStream = fileContent.ToStream(); - using var streamReader = new StreamReader(fileStream, Encoding.UTF8); + var fileContent = $""" + AAA,BBB,CCC,DDD + 1,2,3,4 + 5,6,7,8 + 9,10,11,12 + 13,14,15,16 + 87,88,89,100 + 89,99,100,101 + 88,89,90,91 + """; - var expectedItems = new PersonComplete[] + var reader = new StringReader(fileContent); + var expected = new RegularCaseRecord[] { - new () { id = new Guid("ec9a8be9-a000-503b-adcf-7266804f1eb1"), name = "Lilly Bradley", age = 21, birthday = DateTime.Parse("11/16/1977"), gender = Gender.Male, email = "pak@witak.bf", children = true }, - new () { id = new Guid("63858071-cbb3-5abd-9f88-3dfd565cc4ab"), name = "Lucy Berry", age = 49, birthday = DateTime.Parse("11/12/1961"), gender = Gender.Female, email = "vanvo@ro.pk", children = false }, - new () { id = new Guid("203804f9-93e7-5510-8bb2-177296bafe6a"), name = "Frank Fox", age = 36, birthday = DateTime.Parse("3/19/1977"), gender = Gender.Male, email = "vav@ped.fj", children = true }, - new () { id = new Guid("a8af66fb-bad4-51eb-810c-bf3ca22337c6"), name = "Isabel Todd", age = 51, birthday = DateTime.Parse("9/16/1999"), gender = Gender.Female, email = "gu@or.bz", children = false }, - new () { id = new Guid("1a3d8a66-3e0c-50eb-99c1-a3926bce15ed"), name = $"Joseph {Environment.NewLine}Scott", age = 55, birthday = DateTime.Parse("10/26/1986"), gender = Gender.Male, email = "bup@vugeb.tt", children = false }, - new () { id = new Guid("aa7d4395-f10f-5776-9912-e3d86c4b9d3c"), name = "Gilbert Brooks", age = 56, birthday = DateTime.Parse("3/1/1956"), gender = Gender.Female, email = "epiju@ba.ly", children = true }, - new () { id = new Guid("1d25b811-4002-5744-ac40-93a50f2a442c"), name = "Louis \"Ronaldo\" Bennett", age = 25, birthday = DateTime.Parse("4/4/1967"), gender = Gender.Male, email = "ma@itrovive.tv", children = true }, - new () { id = new Guid("8e963ae5-a9ed-5572-b11c-566abc6a8a56"), name = "Norman Parker", age = 57, birthday = DateTime.Parse("4/17/1969"), gender = Gender.Male, email = "omi@hewepa.bw", children = true }, - new () { id = new Guid("4d373cfb-79e3-54ce-87ff-f2a08fde8f28"), name = "Gary Doyle", age = 20, birthday = DateTime.Parse("1/21/1958"), gender = Gender.Male, email = "orjohma@cabmofa.ps", children = true }, - new () { id = new Guid("5af00cdf-0758-5317-bcdf-c9a3337cc266"), name = "Bruce Silva", age = 39, birthday = DateTime.Parse("1/11/1968"), gender = Gender.Female, email = "ta@ovonib.ir", children = true }, - } - .Repeat(repeat); + new(10,20,30,40), + new(50,60,70,80), + new(90,100,110,120), + new(130,140,150,160), + new(870,880,890,1000), + new(890,990,1000,1010), + new(880,890,900,910), + }; var readOptions = new VariableLengthReaderOptions { - HasHeader = hasHeader, - ParallelismOptions = new() { Enabled = parallelProcessing }, + HasHeader = true, + ParallelismOptions = new() { Enabled = false }, ContainsQuotedFields = true, }; // Act - var items = streamReader.ReadRecords(readOptions, skipMismatchedColumns: true); + var items = reader.ReadRecords(readOptions, false, builder => + builder.DefaultTypeConvert(x => int.Parse(x) * 10)); // Assert - items.Should().BeEquivalentTo(expectedItems, cfg => cfg.WithStrictOrdering()); + items.Should().BeEquivalentTo(expected, cfg => cfg.WithStrictOrdering()); } } } \ No newline at end of file diff --git a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs index 41d875f..08fc0cd 100644 --- a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs +++ b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs @@ -83,8 +83,14 @@ private static IVariableLengthReader BuildParser(IEnumerable colum { try { - var expression = CreateExpression(column.Trim()); - builder.Map((dynamic)expression); + var cleaned = column + .Replace("_", string.Empty) + .Replace("'", string.Empty) + .Replace("\"", string.Empty) + .Trim(); + + dynamic expression = CreateExpression(cleaned); + builder.Map(expression); } catch when (skipMismatchedColumns) { From 9aa991d086fb32887c335d5de752a5773d0f31ee Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 13 Apr 2026 15:19:32 -0300 Subject: [PATCH 4/9] add more tests --- RecordParser.Test/FileReaderTest.cs | 133 +++++++++++++++++- RecordParser.Test/TestTypes.cs | 7 +- .../VariableLengthReaderExtensions.cs | 4 +- 3 files changed, 140 insertions(+), 4 deletions(-) diff --git a/RecordParser.Test/FileReaderTest.cs b/RecordParser.Test/FileReaderTest.cs index fb6f0f1..c3d57d0 100644 --- a/RecordParser.Test/FileReaderTest.cs +++ b/RecordParser.Test/FileReaderTest.cs @@ -658,12 +658,143 @@ public void Read_csv_file_with_header_should_automatic_bind_support_additional_c // Act - var items = reader.ReadRecords(readOptions, false, builder => + var items = reader.ReadRecords(readOptions, false, builder => builder.DefaultTypeConvert(x => int.Parse(x) * 10)); // Assert items.Should().BeEquivalentTo(expected, cfg => cfg.WithStrictOrdering()); } + + [Fact] + public void Read_csv_file_with_header_containing_nested_fields_should_automatic_bind_mapping() + { + // Arrange + + var fileContent = $""" + BirthDay ; Name ; Mother.BirthDay; Mother.Name + 2020.05.23 ; son name ; 1980.01.15 ; mother name + """; + + var reader = new StringReader(fileContent); + var expected = new Person[] + { + new Person + { + BirthDay = new DateTime(2020, 05, 23), + Name = "son name", + Mother = new Person + { + BirthDay = new DateTime(1980, 01, 15), + Name = "mother name", + } + } + }; + + var readOptions = new VariableLengthReaderOptions + { + HasHeader = true, + ParallelismOptions = new() { Enabled = false }, + ContainsQuotedFields = true, + }; + + // Act + + var items = reader.ReadRecords(readOptions, skipMismatchedColumns: false); + + // Assert + + items.Should().BeEquivalentTo(expected, cfg => cfg.WithStrictOrdering()); + } + + [Fact] + public void Read_csv_file_with_header_containing_inherited_fields_should_automatic_bind_mapping() + { + // Arrange + + var fileContent = $""" + Id ; BirthDay ; Name ; Mother.Id ; Mother.BirthDay; Mother.Name + 99 ; 2020.05.23 ; son name ; 100 ; 1980.01.15 ; mother name + """; + + var reader = new StringReader(fileContent); + var expected = new PersonDerivated[] + { + new () + { + Id = 99, + BirthDay = new DateTime(2020, 05, 23), + Name = "son name", + Mother = new () + { + // can not find 'Id' because property 'Mother' has type 'Person' and not 'PersonDerivated' + // Id = 100, + BirthDay = new DateTime(1980, 01, 15), + Name = "mother name", + } + } + }; + + var readOptions = new VariableLengthReaderOptions + { + HasHeader = true, + ParallelismOptions = new() { Enabled = false }, + ContainsQuotedFields = true, + }; + + // Act + + var items = reader.ReadRecords(readOptions, skipMismatchedColumns: true); + + // Assert + + items.Should().BeEquivalentTo(expected, cfg => cfg.WithStrictOrdering()); + } + + [Theory] + [InlineData(" ", false)] + [InlineData(" ", true)] + [InlineData("BirthDay ; Name ; Mother.BirthDay; Mother.Name", false)] + public void Read_csv_file_without_header_should_throw_when_automatic_bind_mapping(string header, bool hasHeader) + { + // Arrange + + var fileContent = $""" + {header} + 99 ; 2020.05.23 ; son name ; 100 ; 1980.01.15 ; mother name + """; + + var reader = new StringReader(fileContent); + var expected = new Person[] + { + new () + { + BirthDay = new DateTime(2020, 05, 23), + Name = "son name", + Mother = new () + { + BirthDay = new DateTime(1980, 01, 15), + Name = "mother name", + } + } + }; + + var readOptions = new VariableLengthReaderOptions + { + HasHeader = hasHeader, + ParallelismOptions = new() { Enabled = false }, + ContainsQuotedFields = true, + }; + + // Act + + var action = () => reader.ReadRecords(readOptions, skipMismatchedColumns: true); + + // Assert + + action.Should() + .Throw() + .WithMessage("Header is mandatory when using auto-binding overload."); + } } } \ No newline at end of file diff --git a/RecordParser.Test/TestTypes.cs b/RecordParser.Test/TestTypes.cs index 878a9dd..802ee25 100644 --- a/RecordParser.Test/TestTypes.cs +++ b/RecordParser.Test/TestTypes.cs @@ -27,7 +27,12 @@ public enum FlaggedEnum None = 8 } - internal class Person + public class PersonDerivated : Person + { + public int Id { get; set; } + } + + public class Person { public DateTime BirthDay { get; set; } public string Name { get; set; } diff --git a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs index 08fc0cd..b3bc4cb 100644 --- a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs +++ b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs @@ -67,7 +67,7 @@ public static IEnumerable ReadRecords(this string header; if (!options.HasHeader || string.IsNullOrEmpty(header = reader.ReadLine())) - throw new InvalidOperationException("header is mandatory"); + throw new InvalidOperationException("Header is mandatory when using auto-binding overload."); var separator = DetectDelimiter(header.AsMemory()); var columns = header.Split(separator); @@ -119,7 +119,7 @@ private static LambdaExpression CreateExpression(string propertyName) currentType.GetField(part, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); if (member == null) - throw new ArgumentException($"Can not find property or field '{part}' in type '{currentType.Name}'."); + throw new ArgumentException($"Can not bind column '{propertyName}': Type '{currentType.Name}' does not have property or field '{part}'."); body = Expression.MakeMemberAccess(body, member); } From 0bab56fbc6be4765e53ffd103a72eaa39dea7509 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 15 Apr 2026 00:02:13 -0300 Subject: [PATCH 5/9] improve docs and param rename --- RecordParser.Test/FileReaderTest.cs | 21 ++++++++-------- .../FileReader/FixedLengthReaderExtensions.cs | 4 +-- .../VariableLengthReaderExtensions.cs | 25 ++++++++++++++++--- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/RecordParser.Test/FileReaderTest.cs b/RecordParser.Test/FileReaderTest.cs index c3d57d0..a3374a9 100644 --- a/RecordParser.Test/FileReaderTest.cs +++ b/RecordParser.Test/FileReaderTest.cs @@ -574,11 +574,12 @@ public record RegularCaseRecord(int Aaa, int Bbb, int Ccc, int Ddd); [Theory] [InlineData("\"AAA\",\"BBB\",\"CCC\",\"DDD\"")] + [InlineData("Aaa,Bbb,Ccc,Ddd")] [InlineData("AAA,BBB,CCC,DDD")] [InlineData("aaa,bbb,ccc,ddd")] [InlineData("AAA , BBB , CCC , DDD")] [InlineData("A_AA,B_BB,C_CC,D_DD")] - public void Read_csv_file_with_header_in_different_case_should_automatic_bind_mapping(string header) + public void Read_csv_file_with_autobind_should_match_header_case_insensitive(string header) { // Arrange @@ -614,7 +615,7 @@ public void Read_csv_file_with_header_in_different_case_should_automatic_bind_ma // Act - var items = reader.ReadRecords(readOptions, skipMismatchedColumns: false); + var items = reader.ReadRecords(readOptions, skipUnmatchedColumns: false); // Assert @@ -622,7 +623,7 @@ public void Read_csv_file_with_header_in_different_case_should_automatic_bind_ma } [Fact] - public void Read_csv_file_with_header_should_automatic_bind_support_additional_configuration() + public void Read_csv_file_with_autobind_should_support_additional_configuration() { // Arrange @@ -658,7 +659,7 @@ public void Read_csv_file_with_header_should_automatic_bind_support_additional_c // Act - var items = reader.ReadRecords(readOptions, false, builder => + var items = reader.ReadRecords(readOptions, skipUnmatchedColumns: false, builder => builder.DefaultTypeConvert(x => int.Parse(x) * 10)); // Assert @@ -667,7 +668,7 @@ public void Read_csv_file_with_header_should_automatic_bind_support_additional_c } [Fact] - public void Read_csv_file_with_header_containing_nested_fields_should_automatic_bind_mapping() + public void Read_csv_file_with_autobind_should_support_nested_fields() { // Arrange @@ -700,7 +701,7 @@ public void Read_csv_file_with_header_containing_nested_fields_should_automatic_ // Act - var items = reader.ReadRecords(readOptions, skipMismatchedColumns: false); + var items = reader.ReadRecords(readOptions, skipUnmatchedColumns: false); // Assert @@ -708,7 +709,7 @@ public void Read_csv_file_with_header_containing_nested_fields_should_automatic_ } [Fact] - public void Read_csv_file_with_header_containing_inherited_fields_should_automatic_bind_mapping() + public void Read_csv_file_with_autobind_should_support_inherited_properties() { // Arrange @@ -744,7 +745,7 @@ public void Read_csv_file_with_header_containing_inherited_fields_should_automat // Act - var items = reader.ReadRecords(readOptions, skipMismatchedColumns: true); + var items = reader.ReadRecords(readOptions, skipUnmatchedColumns: true); // Assert @@ -755,7 +756,7 @@ public void Read_csv_file_with_header_containing_inherited_fields_should_automat [InlineData(" ", false)] [InlineData(" ", true)] [InlineData("BirthDay ; Name ; Mother.BirthDay; Mother.Name", false)] - public void Read_csv_file_without_header_should_throw_when_automatic_bind_mapping(string header, bool hasHeader) + public void Read_csv_file_with_autobind_should_throw_when_missing_header(string header, bool hasHeader) { // Arrange @@ -788,7 +789,7 @@ public void Read_csv_file_without_header_should_throw_when_automatic_bind_mappin // Act - var action = () => reader.ReadRecords(readOptions, skipMismatchedColumns: true); + var action = () => reader.ReadRecords(readOptions, skipUnmatchedColumns: true); // Assert diff --git a/RecordParser/Extensions/FileReader/FixedLengthReaderExtensions.cs b/RecordParser/Extensions/FileReader/FixedLengthReaderExtensions.cs index 0636b6a..071be59 100644 --- a/RecordParser/Extensions/FileReader/FixedLengthReaderExtensions.cs +++ b/RecordParser/Extensions/FileReader/FixedLengthReaderExtensions.cs @@ -13,7 +13,7 @@ public class FixedLengthReaderOptions /// public ParallelismOptions ParallelismOptions { get; set; } /// - /// Function which parses text into object + /// Function which parses text to object /// public FuncSpanT Parser { get; set; } } @@ -24,7 +24,7 @@ public static class FixedLengthReaderExtensions /// /// Reads the records (i.e., lines) from a fixed length file, - /// then parses the records into objects. + /// then parses the records to objects. /// /// type of objects read from file /// fixed length file diff --git a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs index b3bc4cb..5390e5e 100644 --- a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs +++ b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs @@ -35,7 +35,7 @@ public static class VariableLengthReaderExtensions { /// /// Reads the records from a variable length file, - /// then parses the records into objects. + /// then parses the records to objects. /// /// type of objects read from file /// variable length file @@ -58,10 +58,29 @@ public static IEnumerable ReadRecords(this TextReader reader, IVariableLen : ReadRecordsSequential(selector, func, options.HasHeader); } + /// + /// Reads the records from a variable length file + /// using the header of the file to auto-bind columns to properties in the process of parsing the records to objects. + /// + /// type of objects read from file + /// variable length file + /// options to configure the parsing + /// + /// If true then header columns without a matching property or field will simply be ignored; + /// otherwise, an exception will be thrown indicating the non-matching column. + /// Default value is true. + /// + /// callback to set additional bind or default type converters + /// + /// Sequence of records. + /// + /// + /// If the header is missing on or indicating that header is absent. + /// public static IEnumerable ReadRecords(this TextReader reader, VariableLengthReaderOptions options, - bool skipMismatchedColumns = false, + bool skipUnmatchedColumns = true, Action> action = null) { string header; @@ -71,7 +90,7 @@ public static IEnumerable ReadRecords(this var separator = DetectDelimiter(header.AsMemory()); var columns = header.Split(separator); - var parser = BuildParser(columns, separator, skipMismatchedColumns, action); + var parser = BuildParser(columns, separator, skipUnmatchedColumns, action); return ReadRecords(reader, parser, options with { HasHeader = false }); } From d39f8e16fd75be5bd01fcebd5042b2fd4bcd9631 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 15 Apr 2026 00:27:55 -0300 Subject: [PATCH 6/9] adds test for auto-detect separators --- RecordParser.Test/FileReaderTest.cs | 49 +++++++++++++++++++ .../VariableLengthReaderExtensions.cs | 2 +- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/RecordParser.Test/FileReaderTest.cs b/RecordParser.Test/FileReaderTest.cs index a3374a9..41563c1 100644 --- a/RecordParser.Test/FileReaderTest.cs +++ b/RecordParser.Test/FileReaderTest.cs @@ -797,5 +797,54 @@ public void Read_csv_file_with_autobind_should_throw_when_missing_header(string .Throw() .WithMessage("Header is mandatory when using auto-binding overload."); } + + [Theory] + [InlineData(",")] + [InlineData(";")] + [InlineData("\t")] + [InlineData("|")] + [InlineData(":")] + public void Read_csv_file_with_autobind_should_detect_common_separators(string separator) + { + // Arrange + + var fileContent = $""" + Aaa,Bbb,Ccc,Ddd + 1,2,3,4 + 5,6,7,8 + 9,10,11,12 + 13,14,15,16 + 87,88,89,100 + 89,99,100,101 + 88,89,90,91 + """; + + var reader = new StringReader(fileContent.Replace(",", separator)); + var expected = new RegularCaseRecord[] + { + new(1,2,3,4), + new(5,6,7,8), + new(9,10,11,12), + new(13,14,15,16), + new(87,88,89,100), + new(89,99,100,101), + new(88,89,90,91), + }; + + var readOptions = new VariableLengthReaderOptions + { + HasHeader = true, + ParallelismOptions = new() { Enabled = false }, + ContainsQuotedFields = true, + }; + + // Act + + var items = reader.ReadRecords(readOptions, skipUnmatchedColumns: false); + + // Assert + + items.Should().BeEquivalentTo(expected, cfg => cfg.WithStrictOrdering()); + } } } \ No newline at end of file diff --git a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs index 5390e5e..289f012 100644 --- a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs +++ b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs @@ -148,7 +148,7 @@ private static LambdaExpression CreateExpression(string propertyName) internal static string DetectDelimiter(ReadOnlyMemory header) { - var candidates = new string[] { ",", ";", "\t", "|", "||", "::", "@" }; + var candidates = new string[] { ",", ";", "\t", "|", ":" }; var headerSpan = header.Span; var bestCandidate = ","; var maxCount = 0; From 0c97c3999d2c78f95fd0038b685312f461ce9a43 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 15 Apr 2026 00:36:12 -0300 Subject: [PATCH 7/9] fix tests --- .../Extensions/FileReader/VariableLengthReaderExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs index 289f012..37e94af 100644 --- a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs +++ b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs @@ -85,7 +85,7 @@ public static IEnumerable ReadRecords(this { string header; - if (!options.HasHeader || string.IsNullOrEmpty(header = reader.ReadLine())) + if (!options.HasHeader || string.IsNullOrWhiteSpace(header = reader.ReadLine())) throw new InvalidOperationException("Header is mandatory when using auto-binding overload."); var separator = DetectDelimiter(header.AsMemory()); From 427e2bfbb4de53c18730d8c12929826315035e0f Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 26 Apr 2026 12:16:14 -0300 Subject: [PATCH 8/9] support explicit separator --- RecordParser.Test/FileReaderTest.cs | 101 +++++++++++++++--- .../VariableLengthReaderExtensions.cs | 31 ++++-- 2 files changed, 107 insertions(+), 25 deletions(-) diff --git a/RecordParser.Test/FileReaderTest.cs b/RecordParser.Test/FileReaderTest.cs index 41563c1..d6e78fe 100644 --- a/RecordParser.Test/FileReaderTest.cs +++ b/RecordParser.Test/FileReaderTest.cs @@ -606,16 +606,17 @@ public void Read_csv_file_with_autobind_should_match_header_case_insensitive(str new(88,89,90,91), }; - var readOptions = new VariableLengthReaderOptions + var readOptions = new VariableLengthReaderAutoBindOptions { HasHeader = true, ParallelismOptions = new() { Enabled = false }, ContainsQuotedFields = true, + SkipUnmatchedColumns = false, }; // Act - var items = reader.ReadRecords(readOptions, skipUnmatchedColumns: false); + var items = reader.ReadRecords(readOptions); // Assert @@ -650,16 +651,17 @@ public void Read_csv_file_with_autobind_should_support_additional_configuration( new(880,890,900,910), }; - var readOptions = new VariableLengthReaderOptions + var readOptions = new VariableLengthReaderAutoBindOptions { HasHeader = true, ParallelismOptions = new() { Enabled = false }, ContainsQuotedFields = true, + SkipUnmatchedColumns = false, }; // Act - var items = reader.ReadRecords(readOptions, skipUnmatchedColumns: false, builder => + var items = reader.ReadRecords(readOptions, builder => builder.DefaultTypeConvert(x => int.Parse(x) * 10)); // Assert @@ -692,16 +694,17 @@ public void Read_csv_file_with_autobind_should_support_nested_fields() } }; - var readOptions = new VariableLengthReaderOptions + var readOptions = new VariableLengthReaderAutoBindOptions { HasHeader = true, ParallelismOptions = new() { Enabled = false }, ContainsQuotedFields = true, + SkipUnmatchedColumns = false, }; // Act - var items = reader.ReadRecords(readOptions, skipUnmatchedColumns: false); + var items = reader.ReadRecords(readOptions); // Assert @@ -736,16 +739,17 @@ public void Read_csv_file_with_autobind_should_support_inherited_properties() } }; - var readOptions = new VariableLengthReaderOptions + var readOptions = new VariableLengthReaderAutoBindOptions { HasHeader = true, ParallelismOptions = new() { Enabled = false }, ContainsQuotedFields = true, + SkipUnmatchedColumns = true, }; // Act - var items = reader.ReadRecords(readOptions, skipUnmatchedColumns: true); + var items = reader.ReadRecords(readOptions); // Assert @@ -755,7 +759,7 @@ public void Read_csv_file_with_autobind_should_support_inherited_properties() [Theory] [InlineData(" ", false)] [InlineData(" ", true)] - [InlineData("BirthDay ; Name ; Mother.BirthDay; Mother.Name", false)] + [InlineData("BirthDay ; Name ; Mother.BirthDay ; Mother.Name", false)] public void Read_csv_file_with_autobind_should_throw_when_missing_header(string header, bool hasHeader) { // Arrange @@ -780,7 +784,7 @@ public void Read_csv_file_with_autobind_should_throw_when_missing_header(string } }; - var readOptions = new VariableLengthReaderOptions + var readOptions = new VariableLengthReaderAutoBindOptions { HasHeader = hasHeader, ParallelismOptions = new() { Enabled = false }, @@ -789,7 +793,7 @@ public void Read_csv_file_with_autobind_should_throw_when_missing_header(string // Act - var action = () => reader.ReadRecords(readOptions, skipUnmatchedColumns: true); + var action = () => reader.ReadRecords(readOptions); // Assert @@ -817,9 +821,10 @@ public void Read_csv_file_with_autobind_should_detect_common_separators(string s 87,88,89,100 89,99,100,101 88,89,90,91 - """; + """ + .Replace(",", separator); - var reader = new StringReader(fileContent.Replace(",", separator)); + var reader = new StringReader(fileContent); var expected = new RegularCaseRecord[] { new(1,2,3,4), @@ -831,16 +836,82 @@ public void Read_csv_file_with_autobind_should_detect_common_separators(string s new(88,89,90,91), }; - var readOptions = new VariableLengthReaderOptions + var readOptions = new VariableLengthReaderAutoBindOptions + { + HasHeader = true, + ParallelismOptions = new() { Enabled = false }, + ContainsQuotedFields = true, + SkipUnmatchedColumns = false, + }; + + // Act + + var items = reader.ReadRecords(readOptions); + + // Assert + + items.Should().BeEquivalentTo(expected, cfg => cfg.WithStrictOrdering()); + } + + [Theory] + + // inferred + [InlineData(",", null)] + [InlineData(";", null)] + [InlineData("\t", null)] + [InlineData("|", null)] + [InlineData(":", null)] + + // explicit (detectable) + [InlineData(",", ",")] + [InlineData(";", ";")] + [InlineData("\t", "\t")] + [InlineData("|", "|")] + [InlineData(":", ":")] + + // explicit (not detectable) + [InlineData("||", "||")] + [InlineData("@", "@")] + public void Read_csv_file_with_autobind_should_support_explict_separators(string fileSeparator, string informedSeparator) + { + // Arrange + + var fileContent = $""" + Aaa,Bbb,Ccc,Ddd + 1,2,3,4 + 5,6,7,8 + 9,10,11,12 + 13,14,15,16 + 87,88,89,100 + 89,99,100,101 + 88,89,90,91 + """ + .Replace(",", fileSeparator); + + var reader = new StringReader(fileContent); + var expected = new RegularCaseRecord[] + { + new(1,2,3,4), + new(5,6,7,8), + new(9,10,11,12), + new(13,14,15,16), + new(87,88,89,100), + new(89,99,100,101), + new(88,89,90,91), + }; + + var readOptions = new VariableLengthReaderAutoBindOptions { HasHeader = true, ParallelismOptions = new() { Enabled = false }, ContainsQuotedFields = true, + SkipUnmatchedColumns = false, + Separator = informedSeparator, }; // Act - var items = reader.ReadRecords(readOptions, skipUnmatchedColumns: false); + var items = reader.ReadRecords(readOptions); // Assert diff --git a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs index 37e94af..da25335 100644 --- a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs +++ b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs @@ -11,6 +11,23 @@ namespace RecordParser.Extensions { + public record class VariableLengthReaderAutoBindOptions : VariableLengthReaderOptions + { + /// + /// The text (usually a character) that delimits columns and separate values + /// If value is null then separator will be detected automatically by observing the header. + /// Default value is null. + /// + public string Separator { get; set; } = null; + + /// + /// If true then header columns without a matching property or field will simply be ignored; + /// otherwise, an exception will be thrown indicating the non-matching column. + /// Default value is true. + /// + public bool SkipUnmatchedColumns { get; set; } = true; + } + public record class VariableLengthReaderOptions { /// @@ -65,11 +82,6 @@ public static IEnumerable ReadRecords(this TextReader reader, IVariableLen /// type of objects read from file /// variable length file /// options to configure the parsing - /// - /// If true then header columns without a matching property or field will simply be ignored; - /// otherwise, an exception will be thrown indicating the non-matching column. - /// Default value is true. - /// /// callback to set additional bind or default type converters /// /// Sequence of records. @@ -79,8 +91,7 @@ public static IEnumerable ReadRecords(this TextReader reader, IVariableLen /// public static IEnumerable ReadRecords(this TextReader reader, - VariableLengthReaderOptions options, - bool skipUnmatchedColumns = true, + VariableLengthReaderAutoBindOptions options, Action> action = null) { string header; @@ -88,9 +99,9 @@ public static IEnumerable ReadRecords(this if (!options.HasHeader || string.IsNullOrWhiteSpace(header = reader.ReadLine())) throw new InvalidOperationException("Header is mandatory when using auto-binding overload."); - var separator = DetectDelimiter(header.AsMemory()); + var separator = options.Separator ?? DetectSeparator(header.AsMemory()); var columns = header.Split(separator); - var parser = BuildParser(columns, separator, skipUnmatchedColumns, action); + var parser = BuildParser(columns, separator, options.SkipUnmatchedColumns, action); return ReadRecords(reader, parser, options with { HasHeader = false }); } @@ -146,7 +157,7 @@ private static LambdaExpression CreateExpression(string propertyName) return Expression.Lambda(body, param); } - internal static string DetectDelimiter(ReadOnlyMemory header) + internal static string DetectSeparator(ReadOnlyMemory header) { var candidates = new string[] { ",", ";", "\t", "|", ":" }; var headerSpan = header.Span; From 7af0bddd82dad1374ece50e4d9bed3a9655528a7 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 26 Apr 2026 12:22:43 -0300 Subject: [PATCH 9/9] restore target frameworks --- RecordParser/RecordParser.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RecordParser/RecordParser.csproj b/RecordParser/RecordParser.csproj index 2de4a06..6dbfe18 100644 --- a/RecordParser/RecordParser.csproj +++ b/RecordParser/RecordParser.csproj @@ -2,7 +2,7 @@ RecordParser - netstandard2.1;net8.0 + netstandard2.1;net6.0;net7.0;net8.0 latest Leandro Fernandes Vieira (leandromoh)