diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9a639f..641b60a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: - name: Setup Zig uses: goto-bus-stop/setup-zig@v2 with: - version: 0.14.0 + version: 0.16.0 - name: Install dependencies run: bun install @@ -79,7 +79,7 @@ jobs: - name: Setup Zig uses: goto-bus-stop/setup-zig@v2 with: - version: 0.14.0 + version: 0.16.0 - name: Build native library run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dbbe497..71f6b91 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,7 +50,7 @@ jobs: - name: Setup Zig uses: goto-bus-stop/setup-zig@v2 with: - version: 0.14.0 + version: 0.16.0 - name: Build native library run: | @@ -95,7 +95,7 @@ jobs: - name: Setup Zig uses: goto-bus-stop/setup-zig@v2 with: - version: 0.14.0 + version: 0.16.0 - name: Download all artifacts uses: actions/download-artifact@v4 diff --git a/build.zig b/build.zig index 4eb8575..53a2777 100644 --- a/build.zig +++ b/build.zig @@ -17,20 +17,21 @@ pub fn build(b: *std.Build) void { // Unit tests (native only) if (!is_wasm) { + const unit_tests_module = b.createModule(.{ + .root_source_file = b.path("src/zig/parser.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + }); const unit_tests = b.addTest(.{ - .root_module = b.createModule(.{ - .root_source_file = b.path("src/zig/parser.zig"), - .target = target, - .optimize = optimize, - }), + .root_module = unit_tests_module, }); - unit_tests.linkLibC(); // Link iconv library - only needed on macOS when building natively // On Linux glibc, iconv is built into libc const is_native_macos = target.result.os.tag == .macos and target.query.isNative(); if (is_native_macos) { - unit_tests.linkSystemLibrary("iconv"); + unit_tests_module.linkSystemLibrary("iconv", .{}); } const run_unit_tests = b.addRunArtifact(unit_tests); @@ -42,24 +43,23 @@ pub fn build(b: *std.Build) void { /// Build native shared library fn buildNative(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode) void { + const lib_module = b.createModule(.{ + .root_source_file = b.path("src/zig/parser.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + }); const lib = b.addLibrary(.{ .name = "turbocsv", - .root_module = b.createModule(.{ - .root_source_file = b.path("src/zig/parser.zig"), - .target = target, - .optimize = optimize, - }), + .root_module = lib_module, .linkage = .dynamic, }); - // Link libc for basic C support - lib.linkLibC(); - // Link iconv library on macOS (required for character encoding support) // On Linux (glibc), iconv is built into libc const target_os = target.result.os.tag; if (target_os == .macos) { - lib.linkSystemLibrary("iconv"); + lib_module.linkSystemLibrary("iconv", .{}); } // Install the library diff --git a/landing/src/app.tsx b/landing/src/app.tsx index 646556a..a618505 100644 --- a/landing/src/app.tsx +++ b/landing/src/app.tsx @@ -9,7 +9,7 @@ export default function App() { ( - TurboCSV - High-performance CSV parser with SIMD acceleration + TurboCSV - Fast CSV parsing with Zig, SIMD, and Fast Mode diff --git a/landing/src/components/BenchmarkChart.tsx b/landing/src/components/BenchmarkChart.tsx index 701020f..8fd008c 100644 --- a/landing/src/components/BenchmarkChart.tsx +++ b/landing/src/components/BenchmarkChart.tsx @@ -6,7 +6,7 @@ export default function BenchmarkChart() {

Performance Comparison

- Parsing 100K rows (10 MB) on Apple M1 Pro. Higher is better. + End-to-end benchmark including file reads on Apple Silicon. Clean generated CSV data. Higher is better.

@@ -17,44 +17,46 @@ export default function BenchmarkChart() { 1K rows 10K rows 100K rows - 100K (wide) - TurboCSV - 122.6 MB/s - 165.3 MB/s - 176.1 MB/s - 269.3 MB/s + TurboCSV Fast Mode + 116.5 MB/s + 177.4 MB/s + 172.4 MB/s PapaParse - 84.0 MB/s - 109.3 MB/s - 112.0 MB/s - 224.6 MB/s + 65.8 MB/s + 98.7 MB/s + 111.5 MB/s + + + TurboCSV Native + 42.5 MB/s + 53.0 MB/s + 57.6 MB/s csv-parse - 25.2 MB/s - 34.9 MB/s - 35.3 MB/s - 40.3 MB/s + 25.1 MB/s + 35.1 MB/s + 33.3 MB/s fast-csv - 24.8 MB/s - 28.7 MB/s - 30.2 MB/s - 38.1 MB/s + 22.7 MB/s + 33.5 MB/s + 32.6 MB/s

- Run the benchmark yourself: bun run benchmark:compare + Fast Mode is optimized for simple CSV without quoted delimiters or multi-line fields. Run it yourself: + {" "}bun run benchmark:compare

diff --git a/landing/src/components/CodeExample.css b/landing/src/components/CodeExample.css index f0fe414..c805d3f 100644 --- a/landing/src/components/CodeExample.css +++ b/landing/src/components/CodeExample.css @@ -10,6 +10,9 @@ margin-bottom: 1rem; border-bottom: 1px solid var(--border-color); padding-bottom: 0.5rem; + max-width: 100%; + overflow-x: auto; + scrollbar-width: thin; } .tab { @@ -22,6 +25,7 @@ cursor: pointer; border-radius: 4px; transition: all 0.2s ease; + flex: 0 0 auto; } .tab:hover { @@ -38,6 +42,7 @@ position: relative; min-height: 350px; margin-bottom: 1.5rem; + min-width: 0; } .code-block { @@ -45,6 +50,7 @@ top: 0; left: 0; right: 0; + min-width: 0; opacity: 0; visibility: hidden; transition: opacity 0.2s ease; @@ -66,6 +72,7 @@ padding: 1.5rem; border-radius: 8px; overflow-x: auto; + max-width: 100%; font-size: 0.875rem; line-height: 1.6; background: #1d1f21 !important; diff --git a/landing/src/components/CodeExample.tsx b/landing/src/components/CodeExample.tsx index 3be27ac..bd2dd18 100644 --- a/landing/src/components/CodeExample.tsx +++ b/landing/src/components/CodeExample.tsx @@ -83,7 +83,7 @@ parser.close();`, }, { id: "cli", - label: "CLI (v0.3.0)", + label: "CLI & Benchmarks", lang: "bash", code: `# Trim whitespace and skip bad rows turbocsv head --trim --skip-errors data.csv @@ -91,6 +91,10 @@ turbocsv head --trim --skip-errors data.csv # Fast mode with dynamic typing turbocsv head --fast --dynamic-typing --format json data.csv +# Run local benchmarks +bun run benchmark +bun run benchmark:compare + # Validate with structured error reporting turbocsv validate data.csv # Output: ERROR [TooFewFields] at row 42: Expected 5 fields, got 3 diff --git a/landing/src/components/Hero.tsx b/landing/src/components/Hero.tsx index 06ad9b4..3fedbc5 100644 --- a/landing/src/components/Hero.tsx +++ b/landing/src/components/Hero.tsx @@ -29,7 +29,7 @@ export default function Hero() {

- Built with Zig for native performance. Now with Fast Mode, security hardening, and 22 CLI flags. + Built with Zig for native performance. Fast Mode now leads the benchmark suite for clean CSV data.

@@ -54,17 +54,17 @@ export default function Hero() {
- 269 MB/s - Peak throughput + 177 MB/s + Fast Mode throughput
- 5.8x - faster than fast-csv + 1.79x + faster than PapaParse
- 6.35x + 6.32x faster than csv-parse
diff --git a/landing/src/components/WhatsNew.css b/landing/src/components/WhatsNew.css index a1d7216..a76de3f 100644 --- a/landing/src/components/WhatsNew.css +++ b/landing/src/components/WhatsNew.css @@ -18,10 +18,18 @@ gap: 1.25rem; } +.current-grid { + margin-bottom: 2.5rem; +} + @media (min-width: 640px) { .whats-new-grid { grid-template-columns: repeat(2, 1fr); } + + .current-grid { + grid-template-columns: repeat(3, 1fr); + } } .whats-new-card { @@ -36,6 +44,10 @@ border-color: var(--ds-gray-500); } +.current-card { + border-color: var(--ds-gray-500); +} + .whats-new-card h3 { font-size: 1rem; margin-bottom: 0.75rem; @@ -116,6 +128,10 @@ color: var(--text-secondary); } +.release-title { + margin-top: 0; +} + @keyframes fadeIn { from { opacity: 0; diff --git a/landing/src/components/WhatsNew.tsx b/landing/src/components/WhatsNew.tsx index 22d1afe..f6c4c39 100644 --- a/landing/src/components/WhatsNew.tsx +++ b/landing/src/components/WhatsNew.tsx @@ -1,6 +1,36 @@ import { For, createSignal } from "solid-js"; import "./WhatsNew.css"; +const currentFeatures = [ + { + title: "Benchmark Clarity", + features: [ + <>bun run benchmark restored with generated sample files, + <>benchmark:compare now reports native and Fast Mode separately, + "Benchmarks now time file reads for JavaScript parsers too", + "Fast Mode leads the simple CSV benchmark suite", + ], + }, + { + title: "Zig 0.16 Ready", + features: [ + "Build script migrated to current module-level linking APIs", + <>std.Io file APIs replace deprecated filesystem calls, + <>DebugAllocator replaces the old general-purpose allocator, + <>ArrayList usage updated for allocator-explicit methods, + ], + }, + { + title: "Runtime Polish", + features: [ + "Row metadata now reuses parser header arrays instead of copying per row", + "Parallel parser synchronization updated for Zig 0.16", + "Focused unit tests can generate fixtures in clean checkouts", + "Native build and Zig test suite pass on the current toolchain", + ], + }, +]; + const v3Features = [ { title: "Security & Validation", @@ -71,9 +101,29 @@ export default function WhatsNew() { return (
-

What's New in v0.3.0

+

What's New in v0.3.4

+

+ Current toolchain support, clearer benchmarks, and Fast Mode performance visibility. +

+ +
+ + {(group) => ( +
+

{group.title}

+
    + + {(feature) =>
  • {feature}
    • } +     +
+
+ )} + +
+ +

Previously in v0.3.0

- Major feature release — 22 new CLI flags, security hardening, Fast Mode, and robust error handling. + Major feature release — 22 CLI flags, security hardening, Fast Mode, and robust error handling.

diff --git a/landing/src/entry-server.tsx b/landing/src/entry-server.tsx index 52865ef..0638bc1 100644 --- a/landing/src/entry-server.tsx +++ b/landing/src/entry-server.tsx @@ -8,20 +8,20 @@ export default createHandler(() => ( - + {/* Open Graph / Social Media */} - - + + {/* Twitter Card */} - - + + {assets} diff --git a/src/ts/parser.ts b/src/ts/parser.ts index 41f1df4..0e65034 100644 --- a/src/ts/parser.ts +++ b/src/ts/parser.ts @@ -1043,7 +1043,18 @@ export class CSVParser> while (lib.csv_next_row(this.handle)) { if (!this.cachedRows.has(rowIndex)) { const fieldCount = lib.csv_get_field_count(this.handle); - const row = new CSVRow(this.handle, fieldCount, this.headers, this.options.schema ?? null); + const row = new CSVRow( + this.handle, + fieldCount, + this.headers, + this.options.schema ?? null, + false, + null, + null, + 0, + null, + this.headerRow + ); const rowData: string[] = []; for (let i = 0; i < fieldCount; i++) { @@ -1170,7 +1181,13 @@ export class CSVParser> this.handle, fieldCount, this.headers, - this.options.schema ?? null + this.options.schema ?? null, + false, + null, + null, + 0, + null, + this.headerRow ); const data = this.headers ? row.toObject() : row.toArray(); @@ -1198,7 +1215,13 @@ export class CSVParser> this.handle, fieldCount, this.headers, - this.options.schema ?? null + this.options.schema ?? null, + false, + null, + null, + 0, + null, + this.headerRow ); chunk.push(this.headers ? row.toObject() : row.toArray()); @@ -1432,6 +1455,7 @@ export class CSVParser> trimConfig, this.dataRowIndex, this.options.cast ?? null, + this.headerRow, ); // Skip records with all empty values @@ -1532,7 +1556,18 @@ export class CSVParser> // onRecord callback: extract fields, allow modification or skipping if (this.options.onRecord) { - const rawRow = new CSVRow(this.handle, fieldCount, this.headers, this.options.schema ?? null); + const rawRow = new CSVRow( + this.handle, + fieldCount, + this.headers, + this.options.schema ?? null, + false, + null, + null, + 0, + null, + this.headerRow + ); const fields: (string | null)[] = []; for (let i = 0; i < fieldCountNum; i++) { fields.push(rawRow.get(i)); @@ -1555,6 +1590,7 @@ export class CSVParser> trimConfig, this.dataRowIndex, this.options.cast ?? null, + this.headerRow, ); if (skipEmptyValues && this.isRowEmpty(row, result.length, greedyEmpty)) { @@ -1581,6 +1617,7 @@ export class CSVParser> trimConfig, this.dataRowIndex, this.options.cast ?? null, + this.headerRow, ); // Skip records with all empty values (greedy checks after trim) @@ -1672,7 +1709,18 @@ export class CSVParser> // onRecord callback: extract fields, allow modification or skipping if (this.options.onRecord) { - const rawRow = new CSVRow(this.handle, fieldCount, this.headers, this.options.schema ?? null); + const rawRow = new CSVRow( + this.handle, + fieldCount, + this.headers, + this.options.schema ?? null, + false, + null, + null, + 0, + null, + this.headerRow + ); const fields: (string | null)[] = []; for (let i = 0; i < fieldCountNum; i++) { fields.push(rawRow.get(i)); @@ -1694,6 +1742,7 @@ export class CSVParser> trimConfig, this.dataRowIndex, this.options.cast ?? null, + this.headerRow, ); if (skipEmptyValues && this.isRowEmpty(row, result.length, greedyEmpty)) { @@ -1725,6 +1774,7 @@ export class CSVParser> trimConfig, this.dataRowIndex, this.options.cast ?? null, + this.headerRow, ); // Skip records with all empty values (greedy checks after trim) diff --git a/src/ts/row.ts b/src/ts/row.ts index 36a1775..a4c5ec5 100644 --- a/src/ts/row.ts +++ b/src/ts/row.ts @@ -101,6 +101,7 @@ export class CSVRow> { trimConfig: TrimConfig | null = null, rowIndex: number = 0, castConfig: CastConfig = null, + columns: string[] | null = headers ? Array.from(headers.keys()) : null, ) { this.handle = handle; this.fieldCount = fieldCount; @@ -111,7 +112,7 @@ export class CSVRow> { this.transform = transform; this.trimConfig = trimConfig; this.index = rowIndex; - this.columns = headers ? Array.from(headers.keys()) : null; + this.columns = columns; this.castConfig = castConfig; } @@ -127,6 +128,7 @@ export class CSVRow> { trimConfig: TrimConfig | null = null, rowIndex: number = 0, castConfig: CastConfig = null, + columns: string[] | null = headers ? Array.from(headers.keys()) : null, ): CSVRow { const row = new CSVRow( 0, // no native handle @@ -138,6 +140,7 @@ export class CSVRow> { trimConfig, rowIndex, castConfig, + columns, ); row.preloadedFields = fields; return row; diff --git a/src/zig/dataframe.zig b/src/zig/dataframe.zig index 3738c74..788e2bc 100644 --- a/src/zig/dataframe.zig +++ b/src/zig/dataframe.zig @@ -92,7 +92,7 @@ pub const DataFrame = struct { column_map: std.StringHashMap(usize), /// Row references (indices into source data) - rows: std.ArrayListUnmanaged(RowRef), + rows: std.ArrayList(RowRef), /// Reference to original parser data source_data: []const u8, @@ -108,7 +108,7 @@ pub const DataFrame = struct { .allocator = allocator, .columns = &.{}, .column_map = std.StringHashMap(usize).init(allocator), - .rows = .{}, + .rows = .empty, .source_data = source_data, .owns_source = false, }; @@ -425,14 +425,14 @@ pub const DataFrame = struct { /// Compute median pub fn median(self: *Self, col_idx: usize) f64 { // Collect all numeric values - var values = std.ArrayList(f64).init(self.allocator); - defer values.deinit(); + var values: std.ArrayList(f64) = .empty; + defer values.deinit(self.allocator); for (self.rows.items) |row| { if (row.getField(col_idx)) |field| { const str = field.getData(self.source_data); if (parseFloat(str)) |val| { - values.append(val) catch continue; + values.append(self.allocator, val) catch continue; } } } @@ -523,7 +523,7 @@ pub const GroupedDataFrame = struct { source: *DataFrame, group_col: usize, /// Map from group key (as string) to row indices - groups: std.StringHashMap(std.ArrayListUnmanaged(usize)), + groups: std.StringHashMap(std.ArrayList(usize)), const Self = @This(); @@ -533,7 +533,7 @@ pub const GroupedDataFrame = struct { .allocator = allocator, .source = source, .group_col = group_col, - .groups = std.StringHashMap(std.ArrayListUnmanaged(usize)).init(allocator), + .groups = std.StringHashMap(std.ArrayList(usize)).init(allocator), }; // Build groups @@ -547,7 +547,7 @@ pub const GroupedDataFrame = struct { // Need to duplicate the key since it points to source data const key_copy = try allocator.dupe(u8, key); result.key_ptr.* = key_copy; - result.value_ptr.* = .{}; + result.value_ptr.* = .empty; } try result.value_ptr.append(allocator, row_idx); } @@ -725,7 +725,7 @@ pub fn joinDataFrames( } // Build lookup for right side - var right_lookup = std.StringHashMap(std.ArrayListUnmanaged(usize)).init(allocator); + var right_lookup = std.StringHashMap(std.ArrayList(usize)).init(allocator); defer { var it = right_lookup.iterator(); while (it.next()) |entry| { @@ -739,7 +739,7 @@ pub fn joinDataFrames( const key = field.getData(right.source_data); const entry = try right_lookup.getOrPut(key); if (!entry.found_existing) { - entry.value_ptr.* = .{}; + entry.value_ptr.* = .empty; } try entry.value_ptr.append(allocator, idx); } @@ -905,7 +905,7 @@ fn compareFields(a: FieldRef, b: FieldRef, col_type: ColumnType, source: []const // FFI Exports // ============================================================================ -var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +var gpa = std.heap.DebugAllocator(.{}).init; const global_allocator = gpa.allocator(); /// Create DataFrame from source data diff --git a/src/zig/mmap.zig b/src/zig/mmap.zig index ce4e7ac..b31112a 100644 --- a/src/zig/mmap.zig +++ b/src/zig/mmap.zig @@ -6,8 +6,9 @@ //! //! Usage: //! ```zig -//! const file = try std.fs.cwd().openFile("data.csv", .{}); -//! const stat = try file.stat(); +//! const io = std.Io.Threaded.global_single_threaded.io(); +//! const file = try std.Io.Dir.cwd().openFile(io, "data.csv", .{}); +//! const stat = try file.stat(io); //! const mapped = try MappedFile.init(file, stat.size); //! defer mapped.deinit(); //! @@ -70,7 +71,7 @@ pub const MappedFile = struct { const Self = @This(); /// Memory map a file for reading - pub fn init(file: std.fs.File, size: usize) !Self { + pub fn init(file: std.Io.File, size: usize) !Self { if (size == 0) { return error.EmptyFile; } @@ -82,11 +83,11 @@ pub const MappedFile = struct { } } - fn initPosix(file: std.fs.File, size: usize) !Self { + fn initPosix(file: std.Io.File, size: usize) !Self { const mapped = try std.posix.mmap( null, size, - std.posix.PROT.READ, + .{ .READ = true }, .{ .TYPE = .PRIVATE }, file.handle, 0, @@ -97,7 +98,7 @@ pub const MappedFile = struct { }; } - fn initWindows(file: std.fs.File, size: usize) !Self { + fn initWindows(file: std.Io.File, size: usize) !Self { // Create file mapping const file_mapping = win32.CreateFileMappingW( file.handle, diff --git a/src/zig/parallel.zig b/src/zig/parallel.zig index d0c8c08..e43c6a3 100644 --- a/src/zig/parallel.zig +++ b/src/zig/parallel.zig @@ -40,7 +40,7 @@ pub const ParsedRow = struct { pub const ChunkResult = struct { chunk_id: usize, start_row_index: usize, - rows: std.ArrayListUnmanaged(ParsedRow), + rows: std.ArrayList(ParsedRow), err_msg: ?[]const u8, bytes_processed: usize, @@ -132,12 +132,11 @@ pub const ChunkProcessor = struct { // Thread management thread_pool: ?[]std.Thread, - results: std.ArrayListUnmanaged(ChunkResult), + results: std.ArrayList(ChunkResult), chunks: []ChunkBoundary, // Synchronization - mutex: std.Thread.Mutex, - results_ready: std.Thread.Condition, + mutex: std.Io.Mutex, completed_count: usize, // Reorder buffer @@ -156,10 +155,9 @@ pub const ChunkProcessor = struct { .config = config, .data = data, .thread_pool = null, - .results = .{}, + .results = .empty, .chunks = &.{}, - .mutex = .{}, - .results_ready = .{}, + .mutex = .init, .completed_count = 0, .reorder_buffer = ReorderBuffer.init(allocator, data, config.max_buffer_rows), .total_rows_parsed = 0, @@ -194,7 +192,7 @@ pub const ChunkProcessor = struct { const thread_count = self.getOptimalThreadCount(); const target_chunk_size = @max(self.data.len / thread_count, self.config.min_chunk_size); - var chunks_list: std.ArrayListUnmanaged(ChunkBoundary) = .{}; + var chunks_list: std.ArrayList(ChunkBoundary) = .empty; var chunk_id: usize = 0; var pos: usize = 0; var estimated_row: usize = 0; @@ -301,23 +299,21 @@ pub const ChunkProcessor = struct { const error_result = ChunkResult{ .chunk_id = chunk.id, .start_row_index = chunk.estimated_start_row, - .rows = .{}, + .rows = .empty, .err_msg = std.fmt.allocPrint(self.allocator, "Chunk {} error: {}", .{ chunk.id, err }) catch null, .bytes_processed = 0, }; - self.mutex.lock(); + std.Io.Threaded.mutexLock(&self.mutex); self.results.append(self.allocator, error_result) catch {}; self.completed_count += 1; - self.mutex.unlock(); - self.results_ready.signal(); + std.Io.Threaded.mutexUnlock(&self.mutex); return; }; - self.mutex.lock(); + std.Io.Threaded.mutexLock(&self.mutex); self.results.append(self.allocator, result) catch {}; self.completed_count += 1; - self.mutex.unlock(); - self.results_ready.signal(); + std.Io.Threaded.mutexUnlock(&self.mutex); } /// Process a single chunk and return parsed rows @@ -325,7 +321,7 @@ pub const ChunkProcessor = struct { var result = ChunkResult{ .chunk_id = chunk.id, .start_row_index = chunk.estimated_start_row, - .rows = .{}, + .rows = .empty, .err_msg = null, .bytes_processed = 0, }; @@ -336,7 +332,7 @@ pub const ChunkProcessor = struct { var in_quote = false; while (pos < chunk_data.len) { - var fields: std.ArrayListUnmanaged(FieldLoc) = .{}; + var fields: std.ArrayList(FieldLoc) = .empty; var field_start = pos; const row_start = pos; @@ -542,15 +538,14 @@ test "single chunk processing" { test "parallel processing preserves order" { // Create larger test data var buffer: [4096]u8 = undefined; - var stream = std.io.fixedBufferStream(&buffer); - const writer = stream.writer(); + var writer: std.Io.Writer = .fixed(&buffer); writer.writeAll("id,value\n") catch unreachable; for (0..50) |i| { writer.print("{d},test{d}\n", .{ i, i }) catch unreachable; } - const data = stream.getWritten(); + const data = writer.buffered(); const processor = try ChunkProcessor.init(std.testing.allocator, data, .{ .min_chunk_size = 100 }); defer processor.deinit(); diff --git a/src/zig/parser.zig b/src/zig/parser.zig index 6d68097..0a25f95 100644 --- a/src/zig/parser.zig +++ b/src/zig/parser.zig @@ -12,6 +12,10 @@ const parallel = if (is_wasm) @import("parallel_stub.zig") else @import("paralle const Allocator = std.mem.Allocator; +fn defaultIo() std.Io { + return std.Io.Threaded.global_single_threaded.io(); +} + /// Opaque handle for FFI - hides internal Parser struct from JS pub const ParserHandle = *anyopaque; @@ -71,7 +75,7 @@ pub const Parser = struct { // Memory mapped file data data: []const u8, data_len: usize, - file_handle: ?std.fs.File, + file_handle: ?std.Io.File, mapped_file: ?mmap.MappedFile, // Current parsing state @@ -80,7 +84,7 @@ pub const Parser = struct { in_quote: bool, // Field index for current row - field_offsets: std.ArrayListUnmanaged(FieldLocation), + field_offsets: std.ArrayList(FieldLocation), // SIMD scanner for accelerated parsing simd_scanner: simd.SimdScanner, @@ -102,17 +106,18 @@ pub const Parser = struct { // State flags is_paused: bool, is_closed: bool, - file_mtime: i128, + file_mtime: std.Io.Timestamp, file_size: u64, const Self = @This(); /// Initialize parser from file path pub fn initFromFile(allocator: Allocator, path: []const u8, config: ParserConfig) !*Self { - const file = try std.fs.cwd().openFile(path, .{ .mode = .read_only }); - errdefer file.close(); + const io = defaultIo(); + const file = try std.Io.Dir.cwd().openFile(io, path, .{ .mode = .read_only }); + errdefer file.close(io); - const stat = try file.stat(); + const stat = try file.stat(io); const file_size = stat.size; const mtime = stat.mtime; @@ -165,7 +170,7 @@ pub const Parser = struct { .cursor = 0, .current_row = 0, .in_quote = false, - .field_offsets = .{}, + .field_offsets = .empty, .simd_scanner = simd.SimdScanner.init(config.delimiter, config.quote_char), .string_cache = std.StringHashMap([]const u8).init(allocator), .cache_size = 0, @@ -238,7 +243,7 @@ pub const Parser = struct { .cursor = 0, .current_row = 0, .in_quote = false, - .field_offsets = .{}, + .field_offsets = .empty, .simd_scanner = simd.SimdScanner.init(config.delimiter, config.quote_char), .string_cache = std.StringHashMap([]const u8).init(allocator), .cache_size = 0, @@ -256,7 +261,7 @@ pub const Parser = struct { }, .is_paused = false, .is_closed = false, - .file_mtime = 0, + .file_mtime = .zero, .file_size = data.len, }; @@ -401,8 +406,8 @@ pub const Parser = struct { /// Check if file was modified externally pub fn checkFileModified(self: *Self) bool { if (self.file_handle) |file| { - const stat = file.stat() catch return true; - return stat.mtime != self.file_mtime or stat.size != self.file_size; + const stat = file.stat(defaultIo()) catch return true; + return stat.mtime.nanoseconds != self.file_mtime.nanoseconds or stat.size != self.file_size; } return false; } @@ -438,7 +443,7 @@ pub const Parser = struct { } } if (self.file_handle) |file| { - file.close(); + file.close(defaultIo()); } self.is_closed = true; @@ -500,7 +505,7 @@ pub const Parser = struct { // FFI Exports (C ABI) // ============================================================================ -var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +var gpa: std.heap.DebugAllocator(.{}) = .init; const global_allocator = gpa.allocator(); /// Initialize parser from file path @@ -1126,27 +1131,27 @@ export fn csv_parse_all_json(handle: ParserHandle) ?[*]const u8 { json_parse_buffer = null; } - // Use ArrayListUnmanaged for dynamic JSON building - var json: std.ArrayListUnmanaged(u8) = .{}; + // Build JSON output incrementally. + var json: std.ArrayList(u8) = .empty; defer json.deinit(global_allocator); // Start array - json.append(global_allocator,'[') catch return null; + json.append(global_allocator, '[') catch return null; var first_row = true; while (parser.nextRow()) { if (!first_row) { - json.append(global_allocator,',') catch return null; + json.append(global_allocator, ',') catch return null; } first_row = false; // Start row array - json.append(global_allocator,'[') catch return null; + json.append(global_allocator, '[') catch return null; var first_field = true; for (parser.field_offsets.items) |field| { if (!first_field) { - json.append(global_allocator,',') catch return null; + json.append(global_allocator, ',') catch return null; } first_field = false; @@ -1154,10 +1159,10 @@ export fn csv_parse_all_json(handle: ParserHandle) ?[*]const u8 { if (field_data.len == 0) { // Empty field -> null - json.appendSlice(global_allocator,"null") catch return null; + json.appendSlice(global_allocator, "null") catch return null; } else if (field.needs_unescape and field_data.len >= 2 and field_data[0] == parser.config.quote_char) { // Quoted field - unescape and write as JSON string - json.append(global_allocator,'"') catch return null; + json.append(global_allocator, '"') catch return null; const inner = field_data[1 .. field_data.len - 1]; var j: usize = 0; @@ -1165,74 +1170,74 @@ export fn csv_parse_all_json(handle: ParserHandle) ?[*]const u8 { const c = inner[j]; if (c == parser.config.quote_char and j + 1 < inner.len and inner[j + 1] == parser.config.quote_char) { // Escaped quote "" -> " - json.append(global_allocator,'"') catch return null; + json.append(global_allocator, '"') catch return null; j += 2; } else if (c == '"') { // Escape quote for JSON - json.appendSlice(global_allocator,"\\\"") catch return null; + json.appendSlice(global_allocator, "\\\"") catch return null; j += 1; } else if (c == '\\') { - json.appendSlice(global_allocator,"\\\\") catch return null; + json.appendSlice(global_allocator, "\\\\") catch return null; j += 1; } else if (c == '\n') { - json.appendSlice(global_allocator,"\\n") catch return null; + json.appendSlice(global_allocator, "\\n") catch return null; j += 1; } else if (c == '\r') { - json.appendSlice(global_allocator,"\\r") catch return null; + json.appendSlice(global_allocator, "\\r") catch return null; j += 1; } else if (c == '\t') { - json.appendSlice(global_allocator,"\\t") catch return null; + json.appendSlice(global_allocator, "\\t") catch return null; j += 1; } else if (c < 0x20) { // Control character - use unicode escape - json.appendSlice(global_allocator,"\\u00") catch return null; + json.appendSlice(global_allocator, "\\u00") catch return null; const hex = "0123456789abcdef"; - json.append(global_allocator,hex[c >> 4]) catch return null; - json.append(global_allocator,hex[c & 0xf]) catch return null; + json.append(global_allocator, hex[c >> 4]) catch return null; + json.append(global_allocator, hex[c & 0xf]) catch return null; j += 1; } else { - json.append(global_allocator,c) catch return null; + json.append(global_allocator, c) catch return null; j += 1; } } - json.append(global_allocator,'"') catch return null; + json.append(global_allocator, '"') catch return null; } else { // Unquoted field - write as JSON string with escaping - json.append(global_allocator,'"') catch return null; + json.append(global_allocator, '"') catch return null; for (field_data) |c| { if (c == '"') { - json.appendSlice(global_allocator,"\\\"") catch return null; + json.appendSlice(global_allocator, "\\\"") catch return null; } else if (c == '\\') { - json.appendSlice(global_allocator,"\\\\") catch return null; + json.appendSlice(global_allocator, "\\\\") catch return null; } else if (c == '\n') { - json.appendSlice(global_allocator,"\\n") catch return null; + json.appendSlice(global_allocator, "\\n") catch return null; } else if (c == '\r') { - json.appendSlice(global_allocator,"\\r") catch return null; + json.appendSlice(global_allocator, "\\r") catch return null; } else if (c == '\t') { - json.appendSlice(global_allocator,"\\t") catch return null; + json.appendSlice(global_allocator, "\\t") catch return null; } else if (c < 0x20) { - json.appendSlice(global_allocator,"\\u00") catch return null; + json.appendSlice(global_allocator, "\\u00") catch return null; const hex = "0123456789abcdef"; - json.append(global_allocator,hex[c >> 4]) catch return null; - json.append(global_allocator,hex[c & 0xf]) catch return null; + json.append(global_allocator, hex[c >> 4]) catch return null; + json.append(global_allocator, hex[c & 0xf]) catch return null; } else { - json.append(global_allocator,c) catch return null; + json.append(global_allocator, c) catch return null; } } - json.append(global_allocator,'"') catch return null; + json.append(global_allocator, '"') catch return null; } } // End row array - json.append(global_allocator,']') catch return null; + json.append(global_allocator, ']') catch return null; } // End array and null terminate - json.append(global_allocator,']') catch return null; - json.append(global_allocator,0) catch return null; + json.append(global_allocator, ']') catch return null; + json.append(global_allocator, 0) catch return null; // Transfer ownership to static buffer json_parse_buffer = json.toOwnedSlice(global_allocator) catch return null; @@ -1280,7 +1285,7 @@ export fn csv_parse_all_fast(handle: ParserHandle) ?[*]const u8 { } // Build output buffer - var output: std.ArrayListUnmanaged(u8) = .{}; + var output: std.ArrayList(u8) = .empty; defer output.deinit(global_allocator); var row_count: u32 = 0; @@ -1289,7 +1294,7 @@ export fn csv_parse_all_fast(handle: ParserHandle) ?[*]const u8 { while (parser.nextRow()) { if (!first_row) { // Row separator - output.append(global_allocator,0x01) catch return null; + output.append(global_allocator, 0x01) catch return null; } first_row = false; row_count += 1; @@ -1298,7 +1303,7 @@ export fn csv_parse_all_fast(handle: ParserHandle) ?[*]const u8 { for (parser.field_offsets.items) |field| { if (!first_field) { // Field separator - output.append(global_allocator,0x00) catch return null; + output.append(global_allocator, 0x00) catch return null; } first_field = false; @@ -1311,16 +1316,16 @@ export fn csv_parse_all_fast(handle: ParserHandle) ?[*]const u8 { while (j < inner.len) { const c = inner[j]; if (c == parser.config.quote_char and j + 1 < inner.len and inner[j + 1] == parser.config.quote_char) { - output.append(global_allocator,parser.config.quote_char) catch return null; + output.append(global_allocator, parser.config.quote_char) catch return null; j += 2; } else { - output.append(global_allocator,c) catch return null; + output.append(global_allocator, c) catch return null; j += 1; } } } else { // Copy directly - output.appendSlice(global_allocator,field_data) catch return null; + output.appendSlice(global_allocator, field_data) catch return null; } } } diff --git a/src/zig/simd.zig b/src/zig/simd.zig index a304e3b..fc8d3d2 100644 --- a/src/zig/simd.zig +++ b/src/zig/simd.zig @@ -308,15 +308,13 @@ pub const SimdScanner = struct { field_starts: *std.ArrayList(usize), allocator: std.mem.Allocator, ) ?usize { - _ = allocator; - const scan = self.scanRowFast(data, start); if (!scan.found_row and scan.field_count == 0) return null; // Convert field_ends to field_starts format var field_start = start; for (0..scan.field_count) |i| { - field_starts.append(field_start) catch return null; + field_starts.append(allocator, field_start) catch return null; field_start = scan.field_ends[i] + 1; } @@ -335,8 +333,8 @@ pub const SimdScanner = struct { field_counts: []usize, bytes_consumed: usize, } { - var row_ends = std.ArrayList(usize).init(allocator); - var field_counts = std.ArrayList(usize).init(allocator); + var row_ends: std.ArrayList(usize) = .empty; + var field_counts: std.ArrayList(usize) = .empty; var pos = start; var rows_found: usize = 0; @@ -345,23 +343,43 @@ pub const SimdScanner = struct { const scan = self.scanRowFast(data, pos); if (scan.found_row) { - row_ends.append(scan.row_end) catch break; - field_counts.append(scan.field_count) catch break; + row_ends.append(allocator, scan.row_end) catch break; + field_counts.append(allocator, scan.field_count) catch break; pos = scan.row_end; rows_found += 1; } else if (scan.field_count > 0) { // Last row without newline - row_ends.append(scan.row_end) catch break; - field_counts.append(scan.field_count) catch break; + row_ends.append(allocator, scan.row_end) catch break; + field_counts.append(allocator, scan.field_count) catch break; break; } else { break; } } + const owned_row_ends = row_ends.toOwnedSlice(allocator) catch { + row_ends.deinit(allocator); + field_counts.deinit(allocator); + return .{ + .row_ends = &.{}, + .field_counts = &.{}, + .bytes_consumed = pos - start, + }; + }; + + const owned_field_counts = field_counts.toOwnedSlice(allocator) catch { + allocator.free(owned_row_ends); + field_counts.deinit(allocator); + return .{ + .row_ends = &.{}, + .field_counts = &.{}, + .bytes_consumed = pos - start, + }; + }; + return .{ - .row_ends = row_ends.toOwnedSlice() catch &.{}, - .field_counts = field_counts.toOwnedSlice() catch &.{}, + .row_ends = owned_row_ends, + .field_counts = owned_field_counts, .bytes_consumed = pos - start, }; } diff --git a/src/zig/writer.zig b/src/zig/writer.zig index d85468e..93f22be 100644 --- a/src/zig/writer.zig +++ b/src/zig/writer.zig @@ -1,5 +1,9 @@ const std = @import("std"); +fn defaultIo() std.Io { + return std.Io.Threaded.global_single_threaded.io(); +} + /// CSV Writer configuration pub const WriterConfig = struct { delimiter: u8 = ',', @@ -26,7 +30,7 @@ pub const Writer = struct { config: WriterConfig, // Output destination - file: ?std.fs.File, + file: ?std.Io.File, buffer: std.ArrayList(u8), // State tracking @@ -38,14 +42,15 @@ pub const Writer = struct { /// Initialize writer to file pub fn initToFile(allocator: std.mem.Allocator, path: []const u8, config: WriterConfig) !*Self { - const file = try std.fs.cwd().createFile(path, .{ .truncate = true }); + const io = defaultIo(); + const file = try std.Io.Dir.cwd().createFile(io, path, .{ .truncate = true }); const writer = try allocator.create(Self); writer.* = Self{ .allocator = allocator, .config = config, .file = file, - .buffer = std.ArrayList(u8).init(allocator), + .buffer = .empty, .rows_written = 0, .rows_in_buffer = 0, .bytes_written = 0, @@ -61,7 +66,7 @@ pub const Writer = struct { .allocator = allocator, .config = config, .file = null, - .buffer = std.ArrayList(u8).init(allocator), + .buffer = .empty, .rows_written = 0, .rows_in_buffer = 0, .bytes_written = 0, @@ -74,15 +79,15 @@ pub const Writer = struct { pub fn writeRow(self: *Self, fields: []const []const u8) !void { for (fields, 0..) |field, i| { if (i > 0) { - try self.buffer.append(self.config.delimiter); + try self.buffer.append(self.allocator, self.config.delimiter); } try self.writeField(field); } // Write line ending switch (self.config.line_ending) { - .lf => try self.buffer.append('\n'), - .crlf => try self.buffer.appendSlice("\r\n"), + .lf => try self.buffer.append(self.allocator, '\n'), + .crlf => try self.buffer.appendSlice(self.allocator, "\r\n"), } self.rows_in_buffer += 1; @@ -98,19 +103,19 @@ pub const Writer = struct { const needs_quote = self.fieldNeedsQuoting(field); if (needs_quote or self.config.quote_style == .all) { - try self.buffer.append(self.config.quote_char); + try self.buffer.append(self.allocator, self.config.quote_char); for (field) |byte| { if (byte == self.config.quote_char) { // Escape quote by doubling - try self.buffer.append(self.config.quote_char); + try self.buffer.append(self.allocator, self.config.quote_char); } - try self.buffer.append(byte); + try self.buffer.append(self.allocator, byte); } - try self.buffer.append(self.config.quote_char); + try self.buffer.append(self.allocator, self.config.quote_char); } else { - try self.buffer.appendSlice(field); + try self.buffer.appendSlice(self.allocator, field); } } @@ -131,7 +136,7 @@ pub const Writer = struct { /// Flush buffer to file pub fn flush(self: *Self) !void { if (self.file) |file| { - try file.writeAll(self.buffer.items); + try file.writeStreamingAll(defaultIo(), self.buffer.items); self.bytes_written += self.buffer.items.len; } @@ -157,23 +162,25 @@ pub const Writer = struct { // Close file if writing to file if (self.file) |file| { - file.close(); + file.close(defaultIo()); } } pub fn deinit(self: *Self) void { self.close() catch {}; - self.buffer.deinit(); + self.buffer.deinit(self.allocator); self.allocator.destroy(self); } }; /// Modifications tracker for copy-on-write pub const ModificationLog = struct { + const CellKey = struct { row: usize, col: usize }; + allocator: std.mem.Allocator, /// Cell modifications: (row, col) -> new_value - cell_edits: std.AutoHashMap(struct { row: usize, col: usize }, []const u8), + cell_edits: std.AutoHashMap(CellKey, []const u8), /// Deleted row indices deleted_rows: std.AutoHashMap(usize, void), @@ -187,7 +194,7 @@ pub const ModificationLog = struct { const log = try allocator.create(Self); log.* = Self{ .allocator = allocator, - .cell_edits = std.AutoHashMap(struct { row: usize, col: usize }, []const u8).init(allocator), + .cell_edits = std.AutoHashMap(CellKey, []const u8).init(allocator), .deleted_rows = std.AutoHashMap(usize, void).init(allocator), .inserted_rows = std.AutoHashMap(usize, []const []const u8).init(allocator), }; @@ -264,7 +271,7 @@ pub const ModificationLog = struct { // FFI Exports // ============================================================================ -var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +var gpa: std.heap.DebugAllocator(.{}) = .init; const ffi_allocator = gpa.allocator(); /// Create writer to file diff --git a/test/benchmark-comparison.ts b/test/benchmark-comparison.ts index bab4443..c862dc4 100644 --- a/test/benchmark-comparison.ts +++ b/test/benchmark-comparison.ts @@ -112,11 +112,28 @@ async function benchmarkTurboCSV(filePath: string): Promise<{ rows: number; time return { rows, timeMs }; } +// Benchmark TurboCSV fast mode (simple CSV, no quote handling) +async function benchmarkTurboCSVFastMode(filePath: string): Promise<{ rows: number; timeMs: number }> { + const start = performance.now(); + const parser = new CSVParser(filePath, { fastMode: true }); + let rows = 0; + + for (const row of parser) { + rows++; + // Access a field to ensure parsing + row.get(0); + } + + parser.close(); + const timeMs = performance.now() - start; + + return { rows, timeMs }; +} + // Benchmark PapaParse async function benchmarkPapaParse(filePath: string): Promise<{ rows: number; timeMs: number }> { - const content = readFileSync(filePath, "utf-8"); - const start = performance.now(); + const content = readFileSync(filePath, "utf-8"); const result = Papa.parse(content, { header: true, skipEmptyLines: true, @@ -128,9 +145,8 @@ async function benchmarkPapaParse(filePath: string): Promise<{ rows: number; tim // Benchmark csv-parse (sync) async function benchmarkCsvParse(filePath: string): Promise<{ rows: number; timeMs: number }> { - const content = readFileSync(filePath, "utf-8"); - const start = performance.now(); + const content = readFileSync(filePath, "utf-8"); const records = csvParse(content, { columns: true, skip_empty_lines: true, @@ -142,10 +158,9 @@ async function benchmarkCsvParse(filePath: string): Promise<{ rows: number; time // Benchmark fast-csv async function benchmarkFastCsv(filePath: string): Promise<{ rows: number; timeMs: number }> { - const content = readFileSync(filePath, "utf-8"); - return new Promise((resolve) => { const start = performance.now(); + const content = readFileSync(filePath, "utf-8"); let rows = 0; fastCsvParse(content, { headers: true }) @@ -173,6 +188,7 @@ async function benchmarkFile(filePath: string): Promise { const libraries = [ { name: "TurboCSV", fn: benchmarkTurboCSV }, + { name: "TurboCSV fast", fn: benchmarkTurboCSVFastMode }, { name: "PapaParse", fn: benchmarkPapaParse }, { name: "csv-parse", fn: benchmarkCsvParse }, { name: "fast-csv", fn: benchmarkFastCsv }, @@ -278,32 +294,36 @@ async function main() { console.log("║ Summary ║"); console.log("╚════════════════════════════════════════════════════════════════════╝"); - // Calculate average speedup for TurboCSV vs others - const turboResults = allResults.filter(r => r.library === "TurboCSV"); + // Calculate average speedup for TurboCSV variants vs others + const turboVariants = ["TurboCSV", "TurboCSV fast"]; const otherLibraries = ["PapaParse", "csv-parse", "fast-csv"]; - console.log("\nAverage speedup (TurboCSV vs others):"); - console.log("─".repeat(40)); + for (const variant of turboVariants) { + const turboResults = allResults.filter(r => r.library === variant); - for (const lib of otherLibraries) { - const libResults = allResults.filter(r => r.library === lib); - let totalSpeedup = 0; - let count = 0; + console.log(`\nAverage speedup (${variant} vs others):`); + console.log("─".repeat(40)); - for (const turbo of turboResults) { - const other = libResults.find(r => r.file === turbo.file); - if (other) { - totalSpeedup += turbo.throughputMBs / other.throughputMBs; - count++; + for (const lib of otherLibraries) { + const libResults = allResults.filter(r => r.library === lib); + let totalSpeedup = 0; + let count = 0; + + for (const turbo of turboResults) { + const other = libResults.find(r => r.file === turbo.file); + if (other) { + totalSpeedup += turbo.throughputMBs / other.throughputMBs; + count++; + } } - } - const avgSpeedup = count > 0 ? totalSpeedup / count : 0; - const speedupStr = avgSpeedup >= 1 - ? `\x1b[32m${avgSpeedup.toFixed(2)}x faster\x1b[0m` - : `\x1b[31m${(1/avgSpeedup).toFixed(2)}x slower\x1b[0m`; + const avgSpeedup = count > 0 ? totalSpeedup / count : 0; + const speedupStr = avgSpeedup >= 1 + ? `\x1b[32m${avgSpeedup.toFixed(2)}x faster\x1b[0m` + : `\x1b[31m${(1 / avgSpeedup).toFixed(2)}x slower\x1b[0m`; - console.log(` vs ${lib.padEnd(12)}: ${speedupStr}`); + console.log(` vs ${lib.padEnd(12)}: ${speedupStr}`); + } } // Winner summary @@ -321,7 +341,7 @@ async function main() { for (const [file, results] of fileGroups) { results.sort((a, b) => b.throughputMBs - a.throughputMBs); const winner = results[0]; - const winnerColor = winner.library === "TurboCSV" ? "\x1b[32m" : "\x1b[33m"; + const winnerColor = winner.library.startsWith("TurboCSV") ? "\x1b[32m" : "\x1b[33m"; console.log(` ${file.padEnd(25)}: ${winnerColor}${winner.library}\x1b[0m (${winner.throughputMBs.toFixed(1)} MB/s)`); } diff --git a/test/benchmark.ts b/test/benchmark.ts new file mode 100644 index 0000000..86d7719 --- /dev/null +++ b/test/benchmark.ts @@ -0,0 +1,211 @@ +#!/usr/bin/env bun +/** + * TurboCSV benchmark runner. + * + * This is the target of `bun run benchmark`. It benchmarks TurboCSV itself + * against generated sample files by default, or against explicit file paths + * passed on the command line. + */ + +import { existsSync, mkdirSync, statSync, writeFileSync } from "fs"; +import { basename, join } from "path"; +import { CSVParser } from "../src/ts/parser"; +import { generateCSV } from "../src/ts/testing"; + +const SAMPLES_DIR = join(import.meta.dir, "..", "samples"); +const DEFAULT_ITERATIONS = 5; + +interface Options { + iterations: number; + files: string[]; +} + +interface RunResult { + rows: number; + timeMs: number; + throughputMBps: number; +} + +function parseArgs(argv: string[]): Options { + const files: string[] = []; + let iterations = DEFAULT_ITERATIONS; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--iterations" || arg === "-i") { + const value = argv[i + 1]; + if (!value) { + throw new Error(`${arg} requires a value`); + } + iterations = Number.parseInt(value, 10); + i++; + } else if (arg?.startsWith("--iterations=")) { + iterations = Number.parseInt(arg.slice("--iterations=".length), 10); + } else if (arg === "--help" || arg === "-h") { + printUsage(); + process.exit(0); + } else if (arg) { + files.push(arg); + } + } + + if (!Number.isFinite(iterations) || iterations < 1) { + throw new Error("--iterations must be a positive integer"); + } + + return { iterations, files }; +} + +function printUsage(): void { + console.log(`TurboCSV benchmark + +Usage: + bun run benchmark + bun run benchmark -- --iterations 10 + bun run benchmark -- samples/custom.csv + +Options: + -i, --iterations Number of measured runs per file (default: ${DEFAULT_ITERATIONS}) +`); +} + +function ensureBenchmarkFile(name: string, rows: number): string { + mkdirSync(SAMPLES_DIR, { recursive: true }); + + const filePath = join(SAMPLES_DIR, name); + if (existsSync(filePath)) { + return filePath; + } + + const csv = generateCSV({ + rows, + seed: rows, + columns: [ + "id:integer", + "name:name", + "email:email", + "city:city", + "signup_date:date", + "active:boolean", + "score:float", + "department:string", + ], + }); + + writeFileSync(filePath, csv); + return filePath; +} + +function defaultFiles(): string[] { + return [ + ensureBenchmarkFile("benchmark-1k.csv", 1_000), + ensureBenchmarkFile("benchmark-10k.csv", 10_000), + ensureBenchmarkFile("benchmark-100k.csv", 100_000), + ]; +} + +function formatSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / 1024 / 1024).toFixed(2)} MB`; +} + +function average(values: number[]): number { + return values.reduce((sum, value) => sum + value, 0) / values.length; +} + +function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 0) { + return ((sorted[mid - 1] ?? 0) + (sorted[mid] ?? 0)) / 2; + } + return sorted[mid] ?? 0; +} + +function parseFile(filePath: string): number { + const parser = new CSVParser(filePath); + let rows = 0; + + for (const row of parser) { + rows++; + row.get(0); + } + + parser.close(); + return rows; +} + +function benchmarkFile(filePath: string, iterations: number): RunResult[] { + const fileSize = statSync(filePath).size; + const fileSizeMB = fileSize / 1024 / 1024; + + // Warm up native library loading and parser setup. + parseFile(filePath); + + const results: RunResult[] = []; + for (let i = 0; i < iterations; i++) { + const start = performance.now(); + const rows = parseFile(filePath); + const timeMs = performance.now() - start; + + results.push({ + rows, + timeMs, + throughputMBps: fileSizeMB / (timeMs / 1000), + }); + } + + return results; +} + +async function main(): Promise { + const options = parseArgs(Bun.argv.slice(2)); + const files = options.files.length > 0 ? options.files : defaultFiles(); + + console.log("TurboCSV Benchmark"); + console.log("=".repeat(70)); + console.log(`Iterations: ${options.iterations}`); + console.log(`SIMD width: ${CSVParser.getSIMDWidth()} bytes`); + console.log(""); + + for (const filePath of files) { + if (!existsSync(filePath)) { + throw new Error(`File not found: ${filePath}`); + } + + const fileSize = statSync(filePath).size; + const results = benchmarkFile(filePath, options.iterations); + const times = results.map((result) => result.timeMs); + const throughputs = results.map((result) => result.throughputMBps); + const rows = results[0]?.rows ?? 0; + + console.log(`${basename(filePath)} (${formatSize(fileSize)})`); + console.log("-".repeat(70)); + + for (const [index, result] of results.entries()) { + console.log( + ` Run ${String(index + 1).padStart(2)}: ` + + `${result.timeMs.toFixed(1).padStart(8)} ms ` + + `${result.throughputMBps.toFixed(1).padStart(8)} MB/s ` + + `${result.rows.toLocaleString()} rows` + ); + } + + console.log( + ` Avg: ${average(times).toFixed(1).padStart(8)} ms ` + + `${average(throughputs).toFixed(1).padStart(8)} MB/s` + ); + console.log( + ` Median: ${median(times).toFixed(1).padStart(8)} ms ` + + `${median(throughputs).toFixed(1).padStart(8)} MB/s` + ); + console.log(` Rows: ${rows.toLocaleString()}`); + console.log(""); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +}); diff --git a/test/fixtures/.gitkeep b/test/fixtures/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/test/fixtures/.gitkeep @@ -0,0 +1 @@ +