diff --git a/RecordParser.Test/FileReaderTest.cs b/RecordParser.Test/FileReaderTest.cs index 77da323..d6e78fe 100644 --- a/RecordParser.Test/FileReaderTest.cs +++ b/RecordParser.Test/FileReaderTest.cs @@ -155,7 +155,7 @@ public void Given_record_is_too_large_for_custom_buffer_size_then_exception_shou var records = reader.ReadRecords(parser, new() { HasHeader = true, - // BufferSize = bufferSize, + // BufferSize = bufferSize, }); foreach (var item in records) @@ -176,6 +176,7 @@ public void Given_record_is_too_large_for_custom_buffer_size_then_exception_shou public static string GetFilePath(string fileName) => Path.Combine(Directory.GetCurrentDirectory(), fileName); + public const string Header = "HEADER"; public static IEnumerable Given_quoted_csv_file_should_read_quoted_properly_theory(string file) { var fileNames = new[] @@ -206,7 +207,7 @@ public static IEnumerable Given_quoted_csv_file_should_read_quoted_pro } if (hasHeader) - fileBuilder.Insert(index: 0, "Id,Date,Name,Rate,Ranking" + Environment.NewLine); + fileBuilder.Insert(index: 0, Header + Environment.NewLine); if (blankLineAtEnd) fileBuilder.AppendLine(); @@ -568,5 +569,353 @@ public void Read_plain_text_of_fixed_length_file(string fileContent, bool parall result.Should().BeEquivalentTo(expected, cfg => cfg.WithStrictOrdering()); } + + public record RegularCaseRecord(int Aaa, int Bbb, int Ccc, int Ddd); + + [Theory] + [InlineData("\"AAA\",\"BBB\",\"CCC\",\"DDD\"")] + [InlineData("Aaa,Bbb,Ccc,Ddd")] + [InlineData("AAA,BBB,CCC,DDD")] + [InlineData("aaa,bbb,ccc,ddd")] + [InlineData("AAA , BBB , CCC , DDD")] + [InlineData("A_AA,B_BB,C_CC,D_DD")] + public void Read_csv_file_with_autobind_should_match_header_case_insensitive(string header) + { + // Arrange + + var fileContent = $""" + {header} + 1,2,3,4 + 5,6,7,8 + 9,10,11,12 + 13,14,15,16 + 87,88,89,100 + 89,99,100,101 + 88,89,90,91 + """; + + var reader = new StringReader(fileContent); + var expected = new RegularCaseRecord[] + { + new(1,2,3,4), + new(5,6,7,8), + new(9,10,11,12), + new(13,14,15,16), + new(87,88,89,100), + new(89,99,100,101), + new(88,89,90,91), + }; + + var readOptions = new VariableLengthReaderAutoBindOptions + { + HasHeader = true, + ParallelismOptions = new() { Enabled = false }, + ContainsQuotedFields = true, + SkipUnmatchedColumns = false, + }; + + // Act + + var items = reader.ReadRecords(readOptions); + + // Assert + + items.Should().BeEquivalentTo(expected, cfg => cfg.WithStrictOrdering()); + } + + [Fact] + public void Read_csv_file_with_autobind_should_support_additional_configuration() + { + // Arrange + + var fileContent = $""" + AAA,BBB,CCC,DDD + 1,2,3,4 + 5,6,7,8 + 9,10,11,12 + 13,14,15,16 + 87,88,89,100 + 89,99,100,101 + 88,89,90,91 + """; + + var reader = new StringReader(fileContent); + var expected = new RegularCaseRecord[] + { + new(10,20,30,40), + new(50,60,70,80), + new(90,100,110,120), + new(130,140,150,160), + new(870,880,890,1000), + new(890,990,1000,1010), + new(880,890,900,910), + }; + + var readOptions = new VariableLengthReaderAutoBindOptions + { + HasHeader = true, + ParallelismOptions = new() { Enabled = false }, + ContainsQuotedFields = true, + SkipUnmatchedColumns = false, + }; + + // Act + + var items = reader.ReadRecords(readOptions, builder => + builder.DefaultTypeConvert(x => int.Parse(x) * 10)); + + // Assert + + items.Should().BeEquivalentTo(expected, cfg => cfg.WithStrictOrdering()); + } + + [Fact] + public void Read_csv_file_with_autobind_should_support_nested_fields() + { + // Arrange + + var fileContent = $""" + BirthDay ; Name ; Mother.BirthDay; Mother.Name + 2020.05.23 ; son name ; 1980.01.15 ; mother name + """; + + var reader = new StringReader(fileContent); + var expected = new Person[] + { + new Person + { + BirthDay = new DateTime(2020, 05, 23), + Name = "son name", + Mother = new Person + { + BirthDay = new DateTime(1980, 01, 15), + Name = "mother name", + } + } + }; + + var readOptions = new VariableLengthReaderAutoBindOptions + { + HasHeader = true, + ParallelismOptions = new() { Enabled = false }, + ContainsQuotedFields = true, + SkipUnmatchedColumns = false, + }; + + // Act + + var items = reader.ReadRecords(readOptions); + + // Assert + + items.Should().BeEquivalentTo(expected, cfg => cfg.WithStrictOrdering()); + } + + [Fact] + public void Read_csv_file_with_autobind_should_support_inherited_properties() + { + // Arrange + + var fileContent = $""" + Id ; BirthDay ; Name ; Mother.Id ; Mother.BirthDay; Mother.Name + 99 ; 2020.05.23 ; son name ; 100 ; 1980.01.15 ; mother name + """; + + var reader = new StringReader(fileContent); + var expected = new PersonDerivated[] + { + new () + { + Id = 99, + BirthDay = new DateTime(2020, 05, 23), + Name = "son name", + Mother = new () + { + // can not find 'Id' because property 'Mother' has type 'Person' and not 'PersonDerivated' + // Id = 100, + BirthDay = new DateTime(1980, 01, 15), + Name = "mother name", + } + } + }; + + var readOptions = new VariableLengthReaderAutoBindOptions + { + HasHeader = true, + ParallelismOptions = new() { Enabled = false }, + ContainsQuotedFields = true, + SkipUnmatchedColumns = true, + }; + + // Act + + var items = reader.ReadRecords(readOptions); + + // Assert + + items.Should().BeEquivalentTo(expected, cfg => cfg.WithStrictOrdering()); + } + + [Theory] + [InlineData(" ", false)] + [InlineData(" ", true)] + [InlineData("BirthDay ; Name ; Mother.BirthDay ; Mother.Name", false)] + public void Read_csv_file_with_autobind_should_throw_when_missing_header(string header, bool hasHeader) + { + // Arrange + + var fileContent = $""" + {header} + 99 ; 2020.05.23 ; son name ; 100 ; 1980.01.15 ; mother name + """; + + var reader = new StringReader(fileContent); + var expected = new Person[] + { + new () + { + BirthDay = new DateTime(2020, 05, 23), + Name = "son name", + Mother = new () + { + BirthDay = new DateTime(1980, 01, 15), + Name = "mother name", + } + } + }; + + var readOptions = new VariableLengthReaderAutoBindOptions + { + HasHeader = hasHeader, + ParallelismOptions = new() { Enabled = false }, + ContainsQuotedFields = true, + }; + + // Act + + var action = () => reader.ReadRecords(readOptions); + + // Assert + + action.Should() + .Throw() + .WithMessage("Header is mandatory when using auto-binding overload."); + } + + [Theory] + [InlineData(",")] + [InlineData(";")] + [InlineData("\t")] + [InlineData("|")] + [InlineData(":")] + public void Read_csv_file_with_autobind_should_detect_common_separators(string separator) + { + // Arrange + + var fileContent = $""" + Aaa,Bbb,Ccc,Ddd + 1,2,3,4 + 5,6,7,8 + 9,10,11,12 + 13,14,15,16 + 87,88,89,100 + 89,99,100,101 + 88,89,90,91 + """ + .Replace(",", separator); + + var reader = new StringReader(fileContent); + var expected = new RegularCaseRecord[] + { + new(1,2,3,4), + new(5,6,7,8), + new(9,10,11,12), + new(13,14,15,16), + new(87,88,89,100), + new(89,99,100,101), + new(88,89,90,91), + }; + + var readOptions = new VariableLengthReaderAutoBindOptions + { + HasHeader = true, + ParallelismOptions = new() { Enabled = false }, + ContainsQuotedFields = true, + SkipUnmatchedColumns = false, + }; + + // Act + + var items = reader.ReadRecords(readOptions); + + // Assert + + items.Should().BeEquivalentTo(expected, cfg => cfg.WithStrictOrdering()); + } + + [Theory] + + // inferred + [InlineData(",", null)] + [InlineData(";", null)] + [InlineData("\t", null)] + [InlineData("|", null)] + [InlineData(":", null)] + + // explicit (detectable) + [InlineData(",", ",")] + [InlineData(";", ";")] + [InlineData("\t", "\t")] + [InlineData("|", "|")] + [InlineData(":", ":")] + + // explicit (not detectable) + [InlineData("||", "||")] + [InlineData("@", "@")] + public void Read_csv_file_with_autobind_should_support_explict_separators(string fileSeparator, string informedSeparator) + { + // Arrange + + var fileContent = $""" + Aaa,Bbb,Ccc,Ddd + 1,2,3,4 + 5,6,7,8 + 9,10,11,12 + 13,14,15,16 + 87,88,89,100 + 89,99,100,101 + 88,89,90,91 + """ + .Replace(",", fileSeparator); + + var reader = new StringReader(fileContent); + var expected = new RegularCaseRecord[] + { + new(1,2,3,4), + new(5,6,7,8), + new(9,10,11,12), + new(13,14,15,16), + new(87,88,89,100), + new(89,99,100,101), + new(88,89,90,91), + }; + + var readOptions = new VariableLengthReaderAutoBindOptions + { + HasHeader = true, + ParallelismOptions = new() { Enabled = false }, + ContainsQuotedFields = true, + SkipUnmatchedColumns = false, + Separator = informedSeparator, + }; + + // Act + + var items = reader.ReadRecords(readOptions); + + // Assert + + items.Should().BeEquivalentTo(expected, cfg => cfg.WithStrictOrdering()); + } } } \ No newline at end of file diff --git a/RecordParser.Test/TestTypes.cs b/RecordParser.Test/TestTypes.cs index 878a9dd..802ee25 100644 --- a/RecordParser.Test/TestTypes.cs +++ b/RecordParser.Test/TestTypes.cs @@ -27,7 +27,12 @@ public enum FlaggedEnum None = 8 } - internal class Person + public class PersonDerivated : Person + { + public int Id { get; set; } + } + + public class Person { public DateTime BirthDay { get; set; } public string Name { get; set; } diff --git a/RecordParser/Extensions/FileReader/FixedLengthReaderExtensions.cs b/RecordParser/Extensions/FileReader/FixedLengthReaderExtensions.cs index 0636b6a..071be59 100644 --- a/RecordParser/Extensions/FileReader/FixedLengthReaderExtensions.cs +++ b/RecordParser/Extensions/FileReader/FixedLengthReaderExtensions.cs @@ -13,7 +13,7 @@ public class FixedLengthReaderOptions /// public ParallelismOptions ParallelismOptions { get; set; } /// - /// Function which parses text into object + /// Function which parses text to object /// public FuncSpanT Parser { get; set; } } @@ -24,7 +24,7 @@ public static class FixedLengthReaderExtensions /// /// Reads the records (i.e., lines) from a fixed length file, - /// then parses the records into objects. + /// then parses the records to objects. /// /// type of objects read from file /// fixed length file diff --git a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs index c27bbd3..da25335 100644 --- a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs +++ b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs @@ -1,13 +1,34 @@ -using RecordParser.Extensions.FileReader.RowReaders; +using RecordParser.Builders.Reader; +using RecordParser.Extensions.FileReader.RowReaders; using RecordParser.Parsers; using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Linq.Expressions; +using System.Reflection; using static RecordParser.Extensions.ReaderCommon; namespace RecordParser.Extensions { - public class VariableLengthReaderOptions + public record class VariableLengthReaderAutoBindOptions : VariableLengthReaderOptions + { + /// + /// The text (usually a character) that delimits columns and separate values + /// If value is null then separator will be detected automatically by observing the header. + /// Default value is null. + /// + public string Separator { get; set; } = null; + + /// + /// If true then header columns without a matching property or field will simply be ignored; + /// otherwise, an exception will be thrown indicating the non-matching column. + /// Default value is true. + /// + public bool SkipUnmatchedColumns { get; set; } = true; + } + + public record class VariableLengthReaderOptions { /// /// Indicates if there is a header record present in the reader's content. @@ -31,7 +52,7 @@ public static class VariableLengthReaderExtensions { /// /// Reads the records from a variable length file, - /// then parses the records into objects. + /// then parses the records to objects. /// /// type of objects read from file /// variable length file @@ -53,5 +74,123 @@ public static IEnumerable ReadRecords(this TextReader reader, IVariableLen ? ReadRecordsParallel(selector, func, options.HasHeader, parallelOptions) : ReadRecordsSequential(selector, func, options.HasHeader); } + + /// + /// Reads the records from a variable length file + /// using the header of the file to auto-bind columns to properties in the process of parsing the records to objects. + /// + /// type of objects read from file + /// variable length file + /// options to configure the parsing + /// callback to set additional bind or default type converters + /// + /// Sequence of records. + /// + /// + /// If the header is missing on or indicating that header is absent. + /// + public static IEnumerable ReadRecords(this + TextReader reader, + VariableLengthReaderAutoBindOptions options, + Action> action = null) + { + string header; + + if (!options.HasHeader || string.IsNullOrWhiteSpace(header = reader.ReadLine())) + throw new InvalidOperationException("Header is mandatory when using auto-binding overload."); + + var separator = options.Separator ?? DetectSeparator(header.AsMemory()); + var columns = header.Split(separator); + var parser = BuildParser(columns, separator, options.SkipUnmatchedColumns, action); + + return ReadRecords(reader, parser, options with { HasHeader = false }); + } + + private static IVariableLengthReader BuildParser(IEnumerable columns, string separator, bool skipMismatchedColumns, Action> action) + { + var builder = new VariableLengthReaderSequentialBuilder(); + foreach (var column in columns) + { + try + { + var cleaned = column + .Replace("_", string.Empty) + .Replace("'", string.Empty) + .Replace("\"", string.Empty) + .Trim(); + + dynamic expression = CreateExpression(cleaned); + builder.Map(expression); + } + catch when (skipMismatchedColumns) + { + builder.Skip(1); + } + } + + action?.Invoke(builder); + + return builder.Build(separator); + } + + private static LambdaExpression CreateExpression(string propertyName) + { + var param = Expression.Parameter(typeof(T), "x"); + Expression body = param; + var parts = propertyName.Split('.'); + + foreach (var part in parts) + { + // starts in T, then loop recursively + var currentType = body.Type; + + var member = (MemberInfo) + currentType.GetProperty(part, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance) ?? + currentType.GetField(part, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + + if (member == null) + throw new ArgumentException($"Can not bind column '{propertyName}': Type '{currentType.Name}' does not have property or field '{part}'."); + + body = Expression.MakeMemberAccess(body, member); + } + + return Expression.Lambda(body, param); + } + + internal static string DetectSeparator(ReadOnlyMemory header) + { + var candidates = new string[] { ",", ";", "\t", "|", ":" }; + var headerSpan = header.Span; + var bestCandidate = ","; + var maxCount = 0; + + foreach (var candidate in candidates) + { + var currentCount = 0; + var candidateSpan = candidate.AsSpan(); + var candidateLen = candidateSpan.Length; + + if (candidateLen == 0 || candidateLen > headerSpan.Length) + continue; + + for (var i = 0; i <= headerSpan.Length - candidateLen; i++) + { + if (headerSpan.Slice(i, candidateLen).SequenceEqual(candidateSpan)) + { + currentCount++; + // avoid overlap + i += candidateLen - 1; + } + } + + if (currentCount > maxCount) + { + maxCount = currentCount; + bestCandidate = candidate; + } + } + + return bestCandidate; + } } } diff --git a/RecordParser/RecordParser.csproj b/RecordParser/RecordParser.csproj index 9e00fba..6dbfe18 100644 --- a/RecordParser/RecordParser.csproj +++ b/RecordParser/RecordParser.csproj @@ -35,7 +35,13 @@ <_Parameter1>RecordParser.Test - + + + + all + + +