diff --git a/RecordParser.Benchmark/VariableLengthReaderBenchmark.cs b/RecordParser.Benchmark/VariableLengthReaderBenchmark.cs index 95065c1..17004a4 100644 --- a/RecordParser.Benchmark/VariableLengthReaderBenchmark.cs +++ b/RecordParser.Benchmark/VariableLengthReaderBenchmark.cs @@ -7,10 +7,12 @@ using FlatFiles; using FlatFiles.TypeMapping; using RecordParser.Builders.Reader; +using RecordParser.Engines.Reader; using RecordParser.Extensions.FileReader; using System; using System.Globalization; using System.IO; +using System.Linq; using System.Text; using System.Threading.Tasks; using TinyCsvParser; @@ -79,7 +81,7 @@ public async Task Read_VariableLength_RecordParser() await ProcessCSVFile(parser.Parse); } - [Benchmark] + // [Benchmark] [Arguments(false, true)] [Arguments(true, true)] public void Read_VariableLength_FullQuoted_RecordParser_Parallel(bool parallel, bool quoted) @@ -120,7 +122,7 @@ public void Read_VariableLength_FullQuoted_RecordParser_Parallel(bool parallel, throw new Exception($"read {i} records but expected {LimitRecord}"); } - [Benchmark] + // [Benchmark] [Arguments(true, true)] [Arguments(true, false)] [Arguments(false, true)] @@ -163,7 +165,7 @@ public void Read_VariableLength_RecordParser_Parallel(bool parallel, bool quoted throw new Exception($"read {i} records but expected {LimitRecord}"); } - [Benchmark] + // [Benchmark] [Arguments(true, true)] [Arguments(true, false)] [Arguments(false, true)] @@ -210,6 +212,110 @@ Person PersonFactory(Func getColumnValue) } } + [Benchmark] + [Arguments(true, true)] + [Arguments(true, false)] + [Arguments(false, true)] + [Arguments(false, false)] + public void Read_VariableLength_RecordParser_Fast(bool parallel, bool quoted) + { + using var fileStream = File.OpenRead(PathSampleDataCSV); + using var streamReader = new StreamReader(fileStream, Encoding.UTF8, true, BufferSize); + + var readOptions = new VariableLengthReaderOptions + { + HasHeader = false, + ParallelOptions = new() { Enabled = parallel }, + ContainsQuotedFields = quoted, + }; + + var caches = Enumerable.Range(0, 10).Select(_ => new InternPool()).ToArray(); + var cache2 = new InternPool(); + + var items = streamReader.GetRecordsFast(readOptions, ",", PersonFactory); + + var i = 0; + foreach (var person in items) + { + if (i++ == LimitRecord) return; + } + + if (i != LimitRecord) + throw new Exception($"read {i} records but expected {LimitRecord}"); + + Person PersonFactory(TextFindHelper finder, int index) + { + //if (parallel is false) + //{ + return Parser(finder); + //} + + //var cache = caches[index % caches.Length]; + //lock (cache) + //{ + // return Parser(finder, cache); + //} + + static Person Parser(in TextFindHelper finder) + { + return new Person() + { + id = Guid.Parse(finder.GetField(0)), + name = finder.GetField(1).Trim().ToString(), + age = int.Parse(finder.GetField(2)), + birthday = DateTime.Parse(finder.GetField(3), CultureInfo.InvariantCulture), + gender = Enum.Parse(finder.GetField(4)), + email = finder.GetField(5).Trim().ToString(), + children = bool.Parse(finder.GetField(7)) + }; + } + } + } + + [Benchmark] + [Arguments(true, true)] + [Arguments(true, false)] + [Arguments(false, true)] + [Arguments(false, false)] + public void Read_VariableLength_RecordParser_Fast_2(bool parallel, bool quoted) + { + using var fileStream = File.OpenRead(PathSampleDataCSV); + using var streamReader = new StreamReader(fileStream, Encoding.UTF8, true, BufferSize); + + var readOptions = new VariableLengthReaderRawOptions + { + HasHeader = false, + ParallelOptions = new() { Enabled = parallel }, + ContainsQuotedFields = quoted, + ColumnCount = 8 + }; + + var items = streamReader.GetRecordsFast2(readOptions, ",", PersonFactory); + + var i = 0; + foreach (var person in items) + { + if (i++ == LimitRecord) return; + } + + if (i != LimitRecord) + throw new Exception($"read {i} records but expected {LimitRecord}"); + + Person PersonFactory(Func GetField) + { + return new Person() + { + id = Guid.Parse(GetField(0)), + name = GetField(1), + age = int.Parse(GetField(2)), + birthday = DateTime.Parse(GetField(3), CultureInfo.InvariantCulture), + gender = Enum.Parse(GetField(4)), + email = GetField(5), + children = bool.Parse(GetField(7)) + }; + } + } + #if TEST_ALL [Benchmark] #endif diff --git a/RecordParser.Test/TextFindHelperTest.cs b/RecordParser.Test/TextFindHelperTest.cs new file mode 100644 index 0000000..e8a69bf --- /dev/null +++ b/RecordParser.Test/TextFindHelperTest.cs @@ -0,0 +1,32 @@ +using FluentAssertions; +using RecordParser.Engines.Reader; +using Xunit; + +namespace RecordParser.Test +{ + public class TextFindHelperTest + { + [Fact] + public void TextFindHelper_GetField_Unordered() + { + // Arrage + + var record = "foo bar baz ; 2020.05.23 ; 0123.45; LightBlue"; + var finder = new TextFindHelper(record, ";", ('"', "\"")); + + // Act + + var d = finder.GetField(3); + var c = finder.GetField(2); + var b = finder.GetField(1); + var a = finder.GetField(0); + + // Assert + + a.ToString().Should().Be("foo bar baz "); + b.ToString().Should().Be(" 2020.05.23 "); + c.ToString().Should().Be(" 0123.45"); + d.ToString().Should().Be(" LightBlue"); + } + } +} diff --git a/RecordParser/Engines/Reader/TextFindHelper.cs b/RecordParser/Engines/Reader/TextFindHelper.cs index 1927b86..0f7ca13 100644 --- a/RecordParser/Engines/Reader/TextFindHelper.cs +++ b/RecordParser/Engines/Reader/TextFindHelper.cs @@ -1,9 +1,10 @@ -using System; +using RecordParser.Extensions.FileReader; +using System; using System.Buffers; namespace RecordParser.Engines.Reader { - internal ref struct TextFindHelper + public ref struct TextFindHelper { private readonly ReadOnlySpan line; private readonly string delimiter; @@ -16,6 +17,8 @@ internal ref struct TextFindHelper private char[] buffer; + private Span<(int start, int count, bool quoted)> fields; + public TextFindHelper(ReadOnlySpan source, string delimiter, (char ch, string str) quote) { this.line = source; @@ -27,6 +30,7 @@ public TextFindHelper(ReadOnlySpan source, string delimiter, (char ch, str currentIndex = -1; currentValue = default; buffer = null; + fields = new (int start, int count, bool quoted)[50]; } public void Dispose() @@ -38,6 +42,20 @@ public void Dispose() } } + public ReadOnlySpan GetField(int index) + { + while (currentIndex < index) + { + GetValue(currentIndex + 1); + } + + var x = fields[index]; + if (x.quoted) + throw new Exception("TODO"); + else + return line.Slice(x.start, x.count); + } + public ReadOnlySpan GetValue(int index) { if (index <= currentIndex) @@ -71,7 +89,9 @@ private ReadOnlySpan ParseChunk(bool match) if (isQuotedField) { - return ParseQuotedChuck(match); + var value = ParseQuotedChuck(match); + fields[currentIndex] = (scanned, position, true); + return value; } position = unlook.IndexOf(delimiter); @@ -80,6 +100,7 @@ private ReadOnlySpan ParseChunk(bool match) position = line.Length - scanned; } + fields[currentIndex] = (scanned, position, false); return line.Slice(scanned, position); } diff --git a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs index 608b471..f8c71a2 100644 --- a/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs +++ b/RecordParser/Extensions/FileReader/VariableLengthReaderExtensions.cs @@ -1,8 +1,10 @@ -using RecordParser.Extensions.FileReader.RowReaders; +using RecordParser.Engines.Reader; +using RecordParser.Extensions.FileReader.RowReaders; using RecordParser.Parsers; using System; using System.Collections.Generic; using System.IO; +using System.Linq; using static RecordParser.Extensions.FileReader.ReaderCommon; namespace RecordParser.Extensions.FileReader @@ -17,6 +19,8 @@ public class VariableLengthReaderOptions public static class VariableLengthReaderExtensions { + // maybe to remove reader parameter and adds what we need from it + // inside options parameter public static IEnumerable GetRecords(this TextReader stream, IVariableLengthReader reader, VariableLengthReaderOptions options) { Func func = options.ContainsQuotedFields @@ -30,5 +34,89 @@ public static IEnumerable GetRecords(this TextReader stream, IVariableLeng ? GetRecordsParallel(parser, func, options.HasHeader, parallelOptions) : GetRecordsSequential(parser, func, options.HasHeader); } + + // new overload that returns record itself + public static IEnumerable> GetRecords(this TextReader stream, VariableLengthReaderOptions options, string separator) + { + Func func = options.ContainsQuotedFields + ? () => new RowByQuote(stream, Length, separator) + : () => new RowByLine(stream, Length); + + var parser = (ReadOnlyMemory memory, int i) => memory; + var parallelOptions = options.ParallelOptions ?? new(); + + return parallelOptions.Enabled + ? GetRecordsParallel(parser, func, options.HasHeader, parallelOptions) + : GetRecordsSequential(parser, func, options.HasHeader); + } + + + // new overload that gives something like 'finder.GetField' to user do whatever he whats + // what about to wrapper TextFindHelper with another struct that only wrappers 'finder.GetField'? + public static IEnumerable GetRecordsFast(this TextReader stream, VariableLengthReaderOptions options, string separator, Parse getValue) + { + Func func = options.ContainsQuotedFields + ? () => new RowByQuote(stream, Length, separator) + : () => new RowByLine(stream, Length); + + var parser = (ReadOnlyMemory memory, int i) => + { + var fi = new TextFindHelper(memory.Span, separator, ('"', "\"")); + + return getValue(fi, i); + }; + + var parallelOptions = options.ParallelOptions ?? new(); + + return parallelOptions.Enabled + ? GetRecordsParallel(parser, func, options.HasHeader, parallelOptions) + : GetRecordsSequential(parser, func, options.HasHeader); + } + + // new overload that gives something like 'finder.GetField' to user do whatever he whats + public static IEnumerable GetRecordsFast2(this TextReader stream, VariableLengthReaderRawOptions options, string separator, Func, T> getValue) + { + Func func = options.ContainsQuotedFields + ? () => new RowByQuote(stream, Length, separator) + : () => new RowByLine(stream, Length); + + var caches = Enumerable.Range(0, 10).Select(_ => + { + var buffer = new string[options.ColumnCount]; + return new + { + cache = options.StringPoolFactory?.Invoke(), + buffer = buffer, + func = new Func(i => buffer[i]) + }; + }).ToArray(); + + var parser = (ReadOnlyMemory memory, int i) => + { + // memento here? + var fi = new TextFindHelper(memory.Span, separator, ('"', "\"")); + + var cache = caches[i % caches.Length]; + lock (cache) + { + for (int j = 0; j < cache.buffer.Length; j++) + { + if (cache.cache is null) + cache.buffer[j] = fi.GetValue(j).ToString(); + else + cache.buffer[j] = cache.cache(fi.GetValue(j)); + } + return getValue(cache.func); + } + }; + + var parallelOptions = options.ParallelOptions ?? new(); + + return parallelOptions.Enabled + ? GetRecordsParallel(parser, func, options.HasHeader, parallelOptions) + : GetRecordsSequential(parser, func, options.HasHeader); } } + +public delegate T Parse(TextFindHelper index, int i); +}