diff --git a/include/internal/ARCHITECTURE.md b/include/internal/ARCHITECTURE.md index ebd509b5..b745c58d 100644 --- a/include/internal/ARCHITECTURE.md +++ b/include/internal/ARCHITECTURE.md @@ -245,14 +245,37 @@ edit strategies symmetric. Current structural edit policy: -- Row insert/erase: mutate `rows`, `keys_`, and sparse row-overlay slots - directly because rows are the native storage unit. -- Column insert: write the current visible table, including sparse overlay - edits, through the CSV writer and reparse it into fresh row storage. This - bakes visible edits into the rebuilt rows and clears sparse overlays. +- Clean row/cell access: `data_frame::DataFrameStorage` centrally selects a + concrete `CleanDataFrame` or `DirtyDataFrame` through an atomic tagged pointer. + Calls remain direct and inlineable, with no vtable or C++17 variant dependency. + Clean handling owns physical rows/names; dirty handling owns visible names, + sparse overlays, and logical-to-physical mapping. Facade features never inspect + the active state themselves. +- First-edit promotion: construct all dirty state before release publication and + retain the clean backing object for readers that captured the previous backend. + Mutable cells bind to the stable selector. Structural changes and moves are + exclusive; read operations do not acquire a transition lock. +- Read-only column views: `data_frame::RowViewAccessor` reuses `CSVRow`'s decoded, + trimmed field extraction without constructing a scalar-aware `CSVField`. + Both backends use it for parsed values; dirty overlays retain their own lookup. + The measurements and rejected physical-column caching alternative are retained + in [data_frame/PERFORMANCE.md](data_frame/PERFORMANCE.md). +- Row insert/erase: mutate `rows` and `keys_` directly and update the handler's + row bindings and overlay slots because rows are the native storage unit. +- Column insert: delegate validation, schema/key-column remapping, and rebuilding + to `DirtyDataFrame`, including for a clean input frame. Construct fresh shared + backing chunks from visible values, + including sparse edits, without CSV serialization/reparsing. This preserves + empty fields and zero-column rows and returns the frame to clean handling. + Stored keys remain stable row identities even after key-column cell edits. + View keys are retargeted into private owned key-byte storage before their + parsed backing is released; owning-key types require no additional work. - Column erase: soft-delete the visible column by removing it from the visible column-name list and logical-to-physical column map. Underlying `CSVRow` storage is intentionally left unchanged. +- Selection: share immutable parsed rows and independently snapshot editing + state and visible-column mapping. Row proxies resolve current overlays when + creating cells; raw underlying rows intentionally exclude logical edits. This asymmetry is intentional. The consistent principle is to use the cheapest reliable operation that preserves visible semantics and keeps ordinary row diff --git a/include/internal/col_names.cpp b/include/internal/col_names.cpp index 69843854..0bdf7172 100644 --- a/include/internal/col_names.cpp +++ b/include/internal/col_names.cpp @@ -48,10 +48,6 @@ namespace csv { return this->_policy; } - CSV_INLINE size_t ColNames::size() const noexcept { - return this->col_names.size(); - } - CSV_INLINE const std::string& ColNames::operator[](size_t i) const { if (i >= this->col_names.size()) throw_column_index_out_of_bounds(); diff --git a/include/internal/col_names.hpp b/include/internal/col_names.hpp index 24b93f09..2a71a5a7 100644 --- a/include/internal/col_names.hpp +++ b/include/internal/col_names.hpp @@ -39,7 +39,7 @@ namespace csv { csv::ColumnNamePolicy get_policy() const noexcept; bool empty() const noexcept { return this->col_names.empty(); } - size_t size() const noexcept; + inline size_t size() const noexcept { return this->col_names.size(); } /** Retrieve column name by index. Throws if index is out of bounds. */ const std::string& operator[](size_t i) const; diff --git a/include/internal/csv_row.hpp b/include/internal/csv_row.hpp index 90f6e352..ce8bc84e 100644 --- a/include/internal/csv_row.hpp +++ b/include/internal/csv_row.hpp @@ -49,6 +49,8 @@ namespace csv { struct CSVRowFragment; } + namespace data_frame { struct RowViewAccessor; } + static const std::string ERROR_NAN = "Not a number."; static const std::string ERROR_OVERFLOW = "Overflow error."; static const std::string ERROR_FLOAT_TO_INT = @@ -546,6 +548,7 @@ namespace csv { template friend class internals::CSVParserCore; friend struct internals::CSVRowRowPolicy; + friend struct internals::data_frame::RowViewAccessor; friend internals::parser::CSVParserDriverBase; friend struct internals::speculative::CSVRowFragment; diff --git a/include/internal/data_frame/AGENTS.md b/include/internal/data_frame/AGENTS.md index 6f7f3e7c..67460c1a 100644 --- a/include/internal/data_frame/AGENTS.md +++ b/include/internal/data_frame/AGENTS.md @@ -5,8 +5,14 @@ namespace `csv`; the folder split is for maintainability, not a namespace move. ## Storage Model -`DataFrame` is row-backed. Its primary storage is `std::vector` plus -per-row sparse edit overlays. Do not turn normal row/cell access into a +`DataFrame` delegates storage operations through private +`internals::data_frame::DataFrameStorage`. Its C++11 tagged pointer selects +concrete `CleanDataFrame` or `DirtyDataFrame` implementations without virtual +dispatch. State checks belong only in this selector, never in facade features. +`CleanDataFrame` owns the native `std::vector` and physical column names. +`DirtyDataFrame` owns visible names, sparse overlays, and column mapping. +Clean reads access parsed rows directly, without overlay slots or mapping. +Do not turn normal row/cell access into a columnar abstraction just to make structural edit implementations symmetric. The guiding rule is: @@ -16,11 +22,17 @@ The guiding rule is: Current structural edit strategy: -- Row insert/erase mutates row storage, keyed metadata, and sparse overlay slots - directly. -- Column insert materializes the current visible table through `CSVWriter`, - reparses it into fresh row storage, and clears sparse overlays because visible - edits are baked into the rebuilt rows. +- Cell assignment activates dirty handling lazily. Taking mutable row/cell + proxies does not allocate overlays or activate the flag. +- Row insert/erase delegates to the active concrete backend. Dirty handling + shifts overlay slots alongside native rows; row and slot capacity is prepared + before committing keyed insertion metadata. +- Column insert builds fresh owned row storage from visible values, without CSV + serialization or reparsing. Shared backing chunks avoid allocation per row + and preserve empty values and zero-column row cardinality. Successful + materialization returns the selector to clean handling. `DirtyDataFrame` + owns schema validation, key-column remapping, and row rebuilding, including + structural insertion into an otherwise clean frame. - Column erase is a soft delete: visible column names and the logical-to-physical column map change, while underlying `CSVRow` storage stays intact. @@ -41,13 +53,32 @@ compaction/materialization API over adding hot-path indirection for all access. - Sparse overlays are keyed by physical column index. Any feature that changes physical row storage or logical-to-physical mapping must account for existing overlays. +- Stored keys identify rows independently of key-column cell edits. Structural + materialization must preserve those keys, including custom-function keys. +- `csv::string_view` keys must be copied into private owned key storage before + replacing parsed rows, and the cached key index must be invalidated when views + are retargeted. Owning-key types use a compile-time no-op storage policy. +- Row proxies resolve current editing state when constructing cells; never + capture overlay allocation history as the authority for later row access. +- Selection shares parsed rows but snapshots overlays independently and keeps + the visible column mapping. Never reconstruct a logical frame solely from + `get_underlying_row()`: it omits edits and includes hidden columns. +- First-edit promotion fully constructs the dirty backend and slot array before + release publication. Retain the clean backing object so readers that loaded + the old backend can finish without per-read locking. Bind mutable cells to the + stable selector, not to whichever concrete backend was active at capture. +- Structural operations and moves require exclusive access. Successful column + materialization may retire dirty state; ordinary cell edits retain row locks. - DataFrame iterators should follow the library's cached-proxy convention: store the current proxy inside the iterator and expose `operator*` / `operator->` reference-like access, as `CSVReader` and `CSVRow` do. ## Test Expectations -Put DataFrame behavior tests in `tests/test_data_frame.cpp`. For writer +Put general DataFrame behavior tests in `tests/test_data_frame.cpp`; put +clean/dirty transitions and edit-preservation regressions in +`tests/test_data_frame_dirty.cpp`. Both belong to the `data_frame_test` target. +For writer compatibility, also check `tests/test_write_csv.cpp` when row-like output or `to_sv_range()` behavior changes. diff --git a/include/internal/data_frame/PERFORMANCE.md b/include/internal/data_frame/PERFORMANCE.md new file mode 100644 index 00000000..d2171c72 --- /dev/null +++ b/include/internal/data_frame/PERFORMANCE.md @@ -0,0 +1,115 @@ +# DataFrameColumn performance investigation — 2026-10-10 + +The direct string-view extraction investigated here is retained in the DataFrame implementation. Physical-column caching was investigated separately and rejected. Both experiments used baseline commit `030bbed158dcf0b8226dfdbe294877f3feb8fe18`. The results below record the isolated prototypes, before integration. + +## Change + +Add an internal `csv::internals::data_frame::RowViewAccessor`, forward-declared and friended by CSVRow. Its inline accessor calls CSVRow's existing `get_field_impl(column, row.data)`. CleanDataFrame::view and DirtyDataFrame::view's unedited fallback call it instead of constructing CSVField through CSVRow::operator[]. Sparse overlay retrieval and its locking remain untouched. There are no new public methods, objects, caches, member fields, virtuals, or state checks. CSVRow layout unchanged; DataFrameColumn remains 48 bytes. + +This reuses the canonical decoding, realized quote storage, trimming, and column bounds checks. The backend still validates visible column bounds and row bounds. It removes wrapper construction and scalar metadata work and exposes the existing inline field extraction to the optimizer. It does not change the actual DataFrameColumn cell-proxy iterator path. + +## Method + +Windows Intel Core i5-12400 (6 cores/12 logical), MSVC `/O2 /MD /EHsc /std:c++20`. Identical retained source [benchmark source](../../../tools/benchmarks/data_frame/benchmark.cpp) for all builds. Multiheader variants link identical existing release csv.lib; header-only variants generated from copied baseline/candidate include trees through `generate_single_header` target using canonical single_header.json and generator. Neither uses the in-repo shim. + +Data: 500001 distinct rows, four columns, more than 10 MB, parsed once for isolated scans. Name indexed scans do 12 repetitions (~6 million cells), actual cell-proxy iterator scans 4 repetitions. Dirty frames edit every 499th row; mapped dirty frame additionally hides the id column. Quoted indexed scan exercises realized arena storage. chunk_parallel_apply includes fresh stream parsing, chunk creation and executor(2) scanning all columns with chunk size 50000. One untimed warmup per executable followed by 7 paired samples, order alternated each pair. Agent CPU workloads were serialized. + +All checksums agree across all 14 measured runs for each metric. Raw samples: [compiled-library runs](performance/direct-library-samples.json) and [generated-header runs](performance/direct-header-samples.json). Summaries contain medians, min/max, IQR and checksums. No sampling instrumentation or forced inlining. + +## Median results (milliseconds; negative percentage means less time) + +| Metric | Multiheader baseline | Multiheader candidate | Change | Header-only baseline | Header-only candidate | Change | +|---|---:|---:|---:|---:|---:|---:| +| clean_indexed | 81.48 | 49.40 | -39.4% | 82.34 | 44.05 | -46.5% | +| clean_iterator | 61.25 | 61.76 | +0.8% | 62.90 | 62.47 | -0.7% | +| clean_quoted_indexed | 90.60 | 78.09 | -13.8% | 128.26 | 74.34 | -42.0% | +| dirty_indexed | 86.10 | 53.66 | -37.7% | 87.80 | 49.53 | -43.6% | +| dirty_iterator | 60.28 | 62.68 | +4.0% | 63.00 | 62.37 | -1.0% | +| mapped_indexed | 86.80 | 53.58 | -38.3% | 87.03 | 52.65 | -39.5% | +| mapped_iterator | 61.84 | 61.84 | -0.0% | 62.69 | 62.87 | +0.3% | +| chunk_parallel | 175.45 | 130.28 | -25.7% | 159.33 | 120.23 | -24.5% | + +The indexed improvements were consistently larger than their within-build dispersion; e.g. multiheader clean IQR 3.06 ms baseline/5.40 ms candidate versus 32.08 ms median reduction. Header clean IQR 9.86 ms/4.24 ms versus 38.30 ms reduction. chunk_parallel is noisier (multiheader IQR 24.67 ms/15.07 ms; header 17.70 ms/6.20 ms), but all 7 candidate times are below their paired baseline times in both build styles. A few shared-runtime outliers remain; full samples retain them. Iterator differences fall within dispersion and the implementation path is unchanged. + +## Correctness and size + +- Full DataFrame and ETL copied-header regressions: 109016 assertions across 69 cases pass. +- Supplemental view-equivalence test (subsequently integrated into `tests/test_data_frame_dirty.cpp`): 54 assertions on mmap and stream paths, 500001 rows, quote realization, configured trimming, eager scalar classification, empty fields, first/sparse edit visibility, hidden-column mapping, lossless materialization, row bounds, and invalidated proxies pass. +- Same supplemental regressions against generated candidate header with MSVC C++14: 54 assertions pass. The change uses only C++11 constructs; true C++11 compiler execution is not available locally. +- Multiheader executable 228864 → 229376 bytes (+512 B); header-only executable 207360 bytes unchanged. Executable file size is a coarse size check, not exact text-section size. + +## Recommendation and limits + +Retain direct extraction and leave out physical-column caching. It is a small private extraction reusing existing behavior and produced substantial indexed-scan improvement on this machine; both distribution styles benefit in these measurements. It keeps schema/dirty dispatch decisions in concrete backends/selector. + +The experiment does not improve DataFrameColumn iterator traversal; that still constructs DataFrameCell proxies. These are microbenchmark and one representative chunked pipeline measurements on one compiler/processor, not universal expected percentages. No GCC/Clang performance, eager-scalar performance, or concurrent edit stress benchmark was run. Shared library benchmarks use identical old csv.lib for both variants; the added friendship changes no ABI and all changed functions are inline, and generated-header results independently avoid that mixing. + +## Rejected alternative: physical-column caching + +Recommendation: do not integrate this prototype. It shows a small clean-scan gain but no broad improvement, slows dirty indexed access, increases DFC size, and adds constructor work. + +### Design + +An eight-byte cached physical column index is resolved once at DFC construction. Cached views still resolve the current clean/dirty backend through DataFrameStorage every read; only schema validation and physical mapping are skipped. No clean-backend pointer or overlay snapshot is cached. Invalid manually constructed DFCs retain deferred validation through a SIZE_MAX sentinel fallback. Structural operations are exclusive and invalidate DFC generations; copying/moving a DFC copies/moves the cache with its existing generation token. + +Changed four copied headers: DataFrameColumn, DataFrameStorage, CleanDataFrame, DirtyDataFrame. The experimental [patch](performance/cached-column.patch) is retained for reproduction against the baseline commit; it is not applied to the implementation. No public API or C++11 language feature changes; MSVC C++14 syntax check passed. Actual C++11 compiler unavailable locally. + +### Measurements + +MSVC /O2 /MD /std:c++20, the same retained `tools/benchmarks/data_frame/benchmark.cpp`, cached canonical csv.lib. Intel i5-12400, six cores/twelve logical. Warmup followed by seven paired baseline/candidate runs, alternating order. Other agents held all tests and compilation during timed runs. Shared dataset: 500001 distinct rows, clean, sparse-dirty, dirty with erased-column mapping; actual DFC iterator and chunk_parallel_apply with two workers. Indexed scans twelve repetitions, iterators four. All checksums matched. + +Times below are milliseconds; change is candidate/baseline - 1, so negative is faster. MAD is median absolute deviation. + +| Measurement | Baseline median | Candidate median | Change | Baseline MAD | Candidate MAD | +|---|---:|---:|---:|---:|---:| +| clean_indexed | 86.855 | 82.426 | -5.1% | 2.519 | 4.792 | +| clean_iterator | 63.233 | 60.031 | -5.1% | 2.738 | 1.556 | +| clean_quoted_indexed | 92.231 | 89.415 | -3.1% | 1.162 | 6.184 | +| dirty_indexed | 86.581 | 92.883 | +7.3% | 1.787 | 4.489 | +| dirty_iterator | 62.633 | 60.824 | -2.9% | 1.764 | 0.474 | +| mapped_indexed | 84.686 | 89.064 | +5.2% | 1.236 | 6.461 | +| mapped_iterator | 63.395 | 63.787 | +0.6% | 2.859 | 2.656 | +| chunk_parallel | 166.242 | 167.541 | +0.8% | 10.237 | 1.661 | + +DFC layout: 48 bytes baseline, 56 bytes candidate. DataFrameCell layout unchanged. Each callback-by-value DFC copy carries an extra size_t; no allocation is added. + +Full samples, checksums, median/min/max/MAD: [cached-column-results.json](performance/cached-column-results.json). There is meaningful noise: candidate clean_indexed ranges 71.903–134.306 ms despite its 82.426 ms median. Clean gains are modest and should not be overinterpreted. Dirty indexed median regressions are a reason to reject this as a general improvement. + +A plausible explanation is that replacing schema/mapping checks with an extra sentinel branch changes code layout while much of dirty cost remains in selector/slot/overlay access. That is a hypothesis, not established causality. No assembly analysis or counter measurements were performed. + +### Supplemental constructor/copy cost + +Separate [construction.cpp](../../../tools/benchmarks/data_frame/construction.cpp); same compiler/link setup, warmup + seven alternating pairs. Six million emplacements/copies across twelve batches of 500000 DFCs, vector capacity pre-reserved; measurements include clear/destruction between batches and a final 500000-view get_sv checksum scan. This is a batch-cost estimate, not a precise isolated per-constructor cost. + +| Batch | Baseline median | Candidate median | Change | +|---|---:|---:|---:| +| construct | 84.925 | 102.388 | +20.6% | +| copy | 81.674 | 85.393 | +4.6% | + +Construction now reads schema validity and resolves physical mapping through the selector once, in addition to existing factory/name lookup validation. The measured construction batch is +20.6%; copying is +4.6%. Supplement data: [construction results](performance/cached-column-construction-results.json). + +### Correctness + +Copied-header full DataFrame + dirty + ETL test suite: 69 cases / 109016 assertions passed. Separate probe.cpp uses both mmap and stream with 500001 rows. Baseline/candidate output matches exactly for deferred invalid public constructors, null/default views, invalid row/column exception ordering, ragged bounds, copied/moved DFCs, captured clean views observing first edit, dirty physical mapping, and structural invalidation. Valid mapped values and subsequent edits also matched. C++14 /c probe compiled successfully. C++14 linkage against cached C++20 csv.lib is intentionally not used because their string_view ABI differs. + +## Reproduction + +The retained [benchmark source](../../../tools/benchmarks/data_frame/benchmark.cpp) and [paired runner](../../../tools/benchmarks/data_frame/run.py) reproduce the timed workload. Use two isolated checkouts: the baseline commit above and the implementation commit containing direct extraction. For the rejected caching experiment, apply `performance/cached-column.patch` to the baseline instead. Keep the retained harness identical across checkouts. + +In an x64 MSVC developer command prompt, configure/build each checkout and compile the harness against its library: + +```text +cmake -S . -B build/dfc-perf -DCSV_CXX_STANDARD=20 -DCSV_BUILD_TESTS=OFF +cmake --build build/dfc-perf --config Release --target csv +cl /nologo /EHsc /O2 /MD /std:c++20 /I include build/dfc-perf/include/internal/Release/csv.lib /Fe: /Fo: +``` + +For generated-header mode, first build `generate_single_header` in each checkout, then compile with `/I build/dfc-perf/single_include_generated` and omit the library argument. This requires the canonical `single_header` generator available to CMake. Never compile against the in-repository `single_include/csv.hpp` shim. + +Run the executable pair serially on an otherwise idle machine: + +```text +python tools/benchmarks/data_frame/run.py --output build/dfc-perf-results +``` + +The runner performs warmups, alternates pair order, verifies checksums, and retains raw samples and median/range/IQR summaries. Use the same build mode/compiler flags for each pair; compare variants within a pair rather than absolute times across build styles. Standalone benchmark sources live under `tools/benchmarks/data_frame/`, outside the amalgamation input tree. They are manual investigation tools and are not part of the library or test targets. diff --git a/include/internal/data_frame/clean_data_frame.hpp b/include/internal/data_frame/clean_data_frame.hpp new file mode 100644 index 00000000..38683a95 --- /dev/null +++ b/include/internal/data_frame/clean_data_frame.hpp @@ -0,0 +1,66 @@ +#pragma once + +#include "row_storage.hpp" +#include "row_view.hpp" + +namespace csv { + namespace internals { + namespace data_frame { + /** Direct parsed-row backend and stable physical backing for editing. + * First-edit promotion never moves or destroys this storage. + */ + class CleanDataFrame : public RowBackend { + public: + CleanDataFrame() = default; + CleanDataFrame(const CleanDataFrame&) = delete; + CleanDataFrame& operator=(const CleanDataFrame&) = delete; + CleanDataFrame(CleanDataFrame&&) noexcept = default; + CleanDataFrame& operator=(CleanDataFrame&&) noexcept = default; + + inline const ConstColNamesPtr& names() const noexcept { return names_; } + inline void set_names(ConstColNamesPtr names) { names_ = std::move(names); } + inline std::vector& rows() noexcept { return rows_; } + inline const std::vector& rows() const noexcept { return rows_; } + inline size_t n_cols() const noexcept { return names_->size(); } + inline const RowOverlay* find_edits(size_t) const noexcept { return nullptr; } + + inline CellBinding bind(size_t, size_t column) const { + validate_column(column, n_cols()); + return CellBinding{nullptr, column}; + } + + inline csv::string_view view(size_t row, size_t column) const { + validate_column(column, n_cols()); + return RowViewAccessor::view(rows_.at(row), column); + } + + inline CSVRow make_inserted_row(const std::vector& row) const { + validate_row(row, n_cols()); + return RowStorageBuilder::make_row(row, names_); + } + + inline void reserve_insert() { reserve_next(rows_); } + inline void insert_row(size_t index, CSVRow&& row) { + rows_.insert(rows_.begin() + index, std::move(row)); + } + inline void erase_row(size_t index) { rows_.erase(rows_.begin() + index); } + + inline CleanDataFrame selected_rows(const std::vector& mask) const { + CleanDataFrame selected; + selected.names_ = names_; + selected.rows_ = select_parsed_rows(rows_, mask); + return selected; + } + + inline void replace_rows(std::vector&& rows, ConstColNamesPtr names) noexcept { + rows_ = std::move(rows); + names_ = std::move(names); + } + + private: + ConstColNamesPtr names_ = std::make_shared(); + std::vector rows_; + }; + } + } +} diff --git a/include/internal/data_frame/data_frame.hpp b/include/internal/data_frame/data_frame.hpp index fe467337..84dee49f 100644 --- a/include/internal/data_frame/data_frame.hpp +++ b/include/internal/data_frame/data_frame.hpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -22,7 +21,9 @@ #include "data_frame_executor.hpp" #include "data_frame_options.hpp" #include "data_frame_row.hpp" +#include "data_frame_storage.hpp" #include "fwd.hpp" +#include "key_storage.hpp" #include "row_overlay.hpp" namespace csv { @@ -135,10 +136,10 @@ namespace csv { CSVReader& reader, KeyFunc key_func, DuplicateKeyPolicy policy = DuplicateKeyPolicy::OVERWRITE - ) : col_names_(reader.col_names_ptr()) { - this->is_keyed = true; + ) { + this->storage_.set_names(reader.col_names_ptr()); + this->is_keyed_ = true; this->key_column_index_ = CSV_NOT_FOUND; - this->physical_col_names_ = this->col_names_; this->build_from_key_function(reader, key_func, policy); } @@ -155,19 +156,19 @@ namespace csv { /** Get the number of rows in the DataFrame. */ size_t size() const noexcept { - return rows.size(); + return storage_.rows().size(); } /** Check if the DataFrame is empty (has no rows). */ bool empty() const noexcept { - return rows.empty(); + return storage_.rows().empty(); } /** Get the number of rows in the DataFrame. Alias for size(). */ - size_t n_rows() const noexcept { return rows.size(); } + size_t n_rows() const noexcept { return storage_.rows().size(); } /** Get the number of visible columns in the DataFrame. */ - size_t n_cols() const noexcept { return col_names_->size(); } + size_t n_cols() const noexcept { return storage_.n_cols(); } /** Check if a column exists in the DataFrame. */ bool has_column(const std::string& name) const { @@ -176,73 +177,53 @@ namespace csv { /** Get the index of a column by name. */ int index_of(const std::string& name) const { - return this->col_names_->index_of(name); + return this->storage_.names()->index_of(name); } /** Get the column names in order. */ - const std::vector& columns() const noexcept { return this->col_names_->get_col_names(); } + const std::vector& columns() const noexcept { return this->storage_.names()->get_col_names(); } /** Insert a new row at the specified index. Structural mutation invalidates outstanding row and cell proxies. */ void insert_row(size_t index, const std::vector& row) { - if (index > this->rows.size()) { - throw std::out_of_range("DataFrame insert_row index out of range"); - } - - this->validate_inserted_row(row); - std::vector physical_row = this->expand_visible_row(row); - CSVRow inserted_row = this->make_inserted_csv_row(physical_row, this->physical_col_names_); - - if (this->is_keyed) { + if (index > this->size()) { throw std::out_of_range("DataFrame insert_row index out of range"); } + CSVRow inserted_row = this->storage_.make_inserted_row(row); + if (this->is_keyed_) { if (this->key_column_index_ == CSV_NOT_FOUND) { throw std::runtime_error("insert_row requires a column-keyed DataFrame"); } - KeyType key = inserted_row[static_cast(this->key_column_index_)].template get(); this->ensure_key_index(); - if (this->key_index->find(key) != this->key_index->end()) { + if (this->key_index_->find(key) != this->key_index_->end()) { throw std::runtime_error(internals::ERROR_DUPLICATE_KEY); } - + this->storage_.reserve_insert(); + reserve_for_append(this->keys_, 1); this->keys_.insert(this->keys_.begin() + index, std::move(key)); } - - this->rows.insert(this->rows.begin() + index, std::move(inserted_row)); - this->edits.insert(this->edits.begin() + index, RowOverlaySlot()); + else { this->storage_.reserve_insert(); } + this->storage_.insert_row(index, std::move(inserted_row)); this->invalidate_key_index(); } - /** Insert a new column with empty values. Structural mutation materializes visible rows into fresh row storage. */ + /** Insert a new column with empty values. + * Materializes visible rows into fresh row storage and invalidates outstanding proxies. + */ void insert_column(size_t index, const std::string& name) { this->insert_column(index, name, std::string()); } - /** Insert a new column with a default value. Structural mutation materializes visible rows into fresh row storage. */ + /** Insert a new column with a default value. + * Materializes visible rows into fresh row storage and invalidates outstanding proxies. + * Established row keys remain unchanged even when key-column values have been edited. + */ void insert_column(size_t index, const std::string& name, const std::string& default_value) { - if (index > this->n_cols()) { - throw std::out_of_range("DataFrame insert_column index out of range"); - } - - this->validate_inserted_column_name(name); - - std::vector new_columns = this->columns(); - new_columns.insert(new_columns.begin() + index, name); - - std::stringstream serialized; - auto writer = make_csv_writer(serialized); - writer << new_columns; - - const DataFrame& self = *this; - for (size_t row_index = 0; row_index < this->rows.size(); ++row_index) { - std::vector row_values = self.at(row_index); - row_values.insert(row_values.begin() + index, default_value); - writer << row_values; - } - - this->replace_with_reparsed_column_insert( - serialized.str(), - this->col_names_->get_policy(), - index - ); + this->storage_.insert_column(index, name, default_value, this->key_column_index_, [this]() { + if (this->is_keyed_) { + this->key_storage_.preserve(this->keys_, [this]() { this->invalidate_key_index(); }); + } + }); + this->invalidate_column_proxies(); + this->invalidate_json_converter(); } /** Append a new column with empty values. */ @@ -262,28 +243,8 @@ namespace csv { * Existing cell edits are copied; subsequent edits to either frame are independent. */ DataFrame selected_rows(const std::vector& include_rows) const { - if (include_rows.size() != this->rows.size()) { - throw std::invalid_argument("selected row mask size must match DataFrame row count"); - } - DataFrame result; - result.rows.reserve(this->rows.size()); - for (size_t row_index = 0; row_index < this->rows.size(); ++row_index) { - if (include_rows[row_index]) { - result.rows.push_back(this->rows[row_index]); - result.edits.emplace_back(); - if (const RowOverlay* overlay = this->find_row_edits(row_index)) { - RowOverlay snapshot = overlay->snapshot(); - if (!snapshot.empty()) { - *result.edits.back().ensure() = std::move(snapshot); - } - } - } - } - - result.col_names_ = this->col_names_; - result.physical_col_names_ = this->physical_col_names_; - result.column_indices_ = this->column_indices_; + result.storage_ = this->storage_.selected_rows(include_rows); return result; } @@ -348,36 +309,30 @@ namespace csv { * @throws std::out_of_range if index is out of bounds */ DataFrameRow at(size_t i) { - const auto& row = rows.at(i); - auto* row_edits = this->ensure_row_edits(i); - return DataFrameRow(&row, this, i, row_edits, this->key_ptr_at(i)); + return this->make_row_proxy(i); } /** Access a row by position with bounds checking (const version). */ DataFrameRow at(size_t i) const { - const auto& row = rows.at(i); - const RowOverlay* row_edits = this->find_row_edits(i); - return DataFrameRow(&row, this, i, row_edits, this->key_ptr_at(i)); + return this->make_const_row_proxy(i); } /** * Access a row by its key. + * Stored keys identify rows independently of subsequent key-column cell edits. * * @throws std::runtime_error if the DataFrame was not created with a key column * @throws std::out_of_range if the key is not found */ DataFrameRow operator[](const KeyType& key) { this->require_keyed_frame(); - auto position = this->position_of(key); - return DataFrameRow(&rows.at(position), this, position, this->ensure_row_edits(position), this->key_ptr_at(position)); + return this->make_row_proxy(this->position_of(key)); } /** Access a row by its key (const version). */ DataFrameRow operator[](const KeyType& key) const { this->require_keyed_frame(); - auto position = this->position_of(key); - const RowOverlay* row_edits = this->find_row_edits(position); - return DataFrameRow(&rows.at(position), this, position, row_edits, this->key_ptr_at(position)); + return this->make_const_row_proxy(this->position_of(key)); } /** @@ -388,7 +343,7 @@ namespace csv { bool contains(const KeyType& key) const { this->require_keyed_frame(); this->ensure_key_index(); - return key_index->find(key) != key_index->end(); + return key_index_->find(key) != key_index_->end(); } /** @@ -403,8 +358,8 @@ namespace csv { const size_t col_idx = this->find_column(name); std::vector values; - values.reserve(rows.size()); - for (size_t row_index = 0; row_index < rows.size(); ++row_index) { + values.reserve(storage_.rows().size()); + for (size_t row_index = 0; row_index < storage_.rows().size(); ++row_index) { values.push_back(this->at(row_index)[col_idx].template get()); } @@ -519,7 +474,7 @@ namespace csv { std::unordered_map> group_by(GroupFunc group_func) const { std::unordered_map> grouped; - for (size_t i = 0; i < rows.size(); i++) { + for (size_t i = 0; i < storage_.rows().size(); i++) { GroupKey group_key = group_func(this->at(i)); grouped[group_key].push_back(i); } @@ -536,7 +491,7 @@ namespace csv { const size_t col_idx = this->find_column(name); std::unordered_map> grouped; - for (size_t i = 0; i < rows.size(); i++) { + for (size_t i = 0; i < storage_.rows().size(); i++) { grouped[this->at(i)[col_idx].template get()].push_back(i); } @@ -563,46 +518,27 @@ namespace csv { private: /** Whether this DataFrame was created with a key. */ - bool is_keyed = false; + bool is_keyed_ = false; - /** Visible column names in order. */ - internals::ConstColNamesPtr col_names_ = std::make_shared(); - - /** Physical column names used by the underlying CSVRow storage. */ - internals::ConstColNamesPtr physical_col_names_ = col_names_; - - /** Maps visible column positions to physical CSVRow field positions. */ - std::vector column_indices_; - - /** Internal storage for row data. */ - std::vector rows; + internals::data_frame::DataFrameStorage storage_; /** Stored keys for keyed DataFrames only. Empty for unkeyed frames. */ std::vector keys_; + internals::data_frame::KeyStorage key_storage_; /** Column index used for keyed frames constructed from a named column. */ int key_column_index_ = CSV_NOT_FOUND; /** Lazily-built index mapping keys to row positions (mutable for const methods). */ - mutable std::unique_ptr> key_index; + mutable std::unique_ptr> key_index_; mutable internals::lazy_shared_ptr json_converter_; std::shared_ptr column_generation_{ new size_t(0) }; - /** - * Sparse per-row edit overlays. Slots stay cheap at load time; the - * heavier synchronized overlay map is allocated only for rows that are - * actually edited. - */ - std::vector edits; - std::shared_ptr edits_creation_lock_{ new std::mutex() }; - /** Initialize an unkeyed DataFrame from a CSV reader. */ void init_unkeyed_from_reader(CSVReader& reader) { this->assert_fresh_storage(false); - this->is_keyed = false; - this->col_names_ = reader.col_names_ptr(); - this->physical_col_names_ = this->col_names_; - this->reset_visible_column_indices(); + this->is_keyed_ = false; + this->storage_.set_names(reader.col_names_ptr()); std::vector batch; while (reader.read_chunk(batch, dataframe_read_chunk_rows())) { @@ -613,28 +549,24 @@ namespace csv { /** Initialize an unkeyed DataFrame from an existing row batch. */ void init_unkeyed_from_rows(std::vector& source_rows) { this->assert_fresh_storage(false); - this->is_keyed = false; - this->col_names_ = source_rows.empty() + this->is_keyed_ = false; + this->storage_.set_names(source_rows.empty() ? internals::ConstColNamesPtr(std::make_shared()) - : source_rows.front().col_names_ptr(); - this->physical_col_names_ = this->col_names_; - this->reset_visible_column_indices(); - this->rows = std::move(source_rows); - this->edits.resize(this->rows.size()); + : source_rows.front().col_names_ptr()); + this->storage_.rows() = std::move(source_rows); } /** Initialize a keyed DataFrame from a CSV reader using column-based key extraction. */ void init_from_reader(CSVReader& reader, const DataFrameOptions& options) { this->assert_fresh_storage(false); - this->is_keyed = true; - this->col_names_ = reader.col_names_ptr(); - this->physical_col_names_ = this->col_names_; + this->is_keyed_ = true; + this->storage_.set_names(reader.col_names_ptr()); const std::string key_column = options.get_key_column(); if (key_column.empty()) throw std::runtime_error(internals::ERROR_KEY_COLUMN_EMPTY); - const int key_column_index = this->col_names_->index_of(key_column); + const int key_column_index = this->storage_.names()->index_of(key_column); if (key_column_index == CSV_NOT_FOUND) throw std::runtime_error(std::string(internals::ERROR_KEY_COLUMN_NOT_FOUND) + key_column); this->key_column_index_ = key_column_index; @@ -668,7 +600,6 @@ namespace csv { ) { std::unordered_map key_to_pos; this->assert_fresh_storage(true); - this->reset_visible_column_indices(); std::vector batch; while (reader.read_chunk(batch, dataframe_read_chunk_rows())) { @@ -704,31 +635,13 @@ namespace csv { } void append_unkeyed_batch(std::vector& batch) { - reserve_for_append(rows, batch.size()); - reserve_for_append(edits, batch.size()); + reserve_for_append(storage_.rows(), batch.size()); for (auto& row : batch) { - rows.push_back(std::move(row)); - edits.emplace_back(); - } - } - - void reset_visible_column_indices() { - this->column_indices_.clear(); - this->column_indices_.reserve(this->col_names_->size()); - for (size_t i = 0; i < this->col_names_->size(); ++i) { - this->column_indices_.push_back(i); + storage_.rows().push_back(std::move(row)); } } - size_t physical_n_cols() const noexcept { - return this->physical_col_names_ ? this->physical_col_names_->size() : 0; - } - - size_t physical_column_index(size_t logical_col_index) const { - return this->column_indices_.at(logical_col_index); - } - template void append_keyed_batch( std::vector& batch, @@ -736,9 +649,8 @@ namespace csv { DuplicateKeyPolicy policy, std::unordered_map& key_to_pos ) { - reserve_for_append(rows, batch.size()); + reserve_for_append(storage_.rows(), batch.size()); reserve_for_append(keys_, batch.size()); - reserve_for_append(edits, batch.size()); for (auto& row : batch) { KeyType key = key_func(row); @@ -749,15 +661,14 @@ namespace csv { throw std::runtime_error(internals::ERROR_DUPLICATE_KEY); if (policy == DuplicateKeyPolicy::OVERWRITE) - rows[existing->second] = std::move(row); + storage_.rows()[existing->second] = std::move(row); continue; } - rows.push_back(std::move(row)); + storage_.rows().push_back(std::move(row)); keys_.push_back(key); - edits.emplace_back(); - key_to_pos[key] = rows.size() - 1; + key_to_pos[key] = storage_.rows().size() - 1; } } @@ -767,218 +678,45 @@ namespace csv { throw std::out_of_range(std::string(internals::ERROR_COLUMN_NOT_FOUND) + name); } - /** Return the overlay for a specific row index. */ - const RowOverlay* find_row_edits(size_t row_index) const { - return edits.at(row_index).get(); + DataFrameCell make_cell(size_t row_index, size_t column_index, bool mutable_access) const { + return this->make_cell(&this->storage_.rows().at(row_index), row_index, column_index, mutable_access); } - /** Return the row overlay, allocating it only when mutable access needs it. */ - RowOverlay* ensure_row_edits(size_t row_index) { - RowOverlaySlot& slot = edits.at(row_index); - RowOverlay* overlay = slot.get(); - if (overlay) { - return overlay; - } + DataFrameCell make_cell(const CSVRow* row, size_t row_index, size_t column_index, bool mutable_access) const { + const auto binding = this->storage_.bind(row_index, column_index); + return mutable_access + ? DataFrameCell(row, binding.overlay, binding.physical_column, &this->storage_, true) + : DataFrameCell(row, binding.overlay, binding.physical_column); + } - std::lock_guard lock(*edits_creation_lock_); - return slot.ensure(); + csv::string_view cell_view(size_t row_index, size_t column_index) const { + return this->storage_.view(row_index, column_index); } DataFrameRow make_row_proxy(size_t row_index) { - const auto& row = rows.at(row_index); - return DataFrameRow(&row, this, row_index, this->ensure_row_edits(row_index), this->key_ptr_at(row_index)); + const auto& row = storage_.rows().at(row_index); + return DataFrameRow(&row, this, row_index, static_cast(nullptr), this->key_ptr_at(row_index)); } DataFrameRow make_const_row_proxy(size_t row_index) const { - const auto& row = rows.at(row_index); - return DataFrameRow(&row, this, row_index, this->find_row_edits(row_index), this->key_ptr_at(row_index)); - } - - void erase_row_edits(size_t row_index) { - if (row_index < edits.size()) { - edits.erase(edits.begin() + row_index); - } - } - - static CSVRow make_inserted_csv_row( - const std::vector& row, - internals::ConstColNamesPtr col_names - ) { - internals::RawCSVDataPtr data = std::make_shared(); - data->col_names = std::const_pointer_cast(col_names); - - size_t total_bytes = 0; - for (const auto& field : row) { - total_bytes += field.size(); - } - data->quote_arena.reserve_for_source_size(total_bytes); - data->fields.reserve_for_source_size(row.size()); - - for (const auto& field : row) { - const size_t offset = data->quote_arena.append(csv::string_view(field)); - data->fields.emplace_back(offset, field.size(), true); - } - - return CSVRow(data, 0, 0, row.size()); - } - - void validate_inserted_row(const std::vector& row) const { - if (this->n_cols() == 0 && !row.empty()) { - throw std::invalid_argument("cannot insert a non-empty row into a DataFrame without columns"); - } - - if (this->col_names_ && !this->col_names_->empty() && row.size() != this->n_cols()) { - throw std::invalid_argument("inserted row field count must match DataFrame column count"); - } - } - - std::vector expand_visible_row(const std::vector& row) const { - if (this->column_indices_.empty() || this->physical_n_cols() == this->n_cols()) { - return row; - } - - std::vector physical_row(this->physical_n_cols()); - for (size_t i = 0; i < row.size(); ++i) { - physical_row[this->physical_column_index(i)] = row[i]; - } - - return physical_row; - } - - void validate_inserted_column_name(const std::string& name) const { - if (name.empty()) { - throw std::invalid_argument("inserted column name must not be empty"); - } - - if (this->has_column(name)) { - throw std::invalid_argument("inserted column name must not duplicate an existing column"); - } - } - - std::vector build_keys_from_rows( - const std::vector& source_rows, - size_t key_column_index - ) const { - std::vector rebuilt_keys; - std::unordered_map seen_keys; - rebuilt_keys.reserve(source_rows.size()); - - for (size_t row_index = 0; row_index < source_rows.size(); ++row_index) { - KeyType key = source_rows[row_index][key_column_index].template get(); - auto inserted = seen_keys.emplace(key, row_index); - if (!inserted.second) { - throw std::runtime_error(internals::ERROR_DUPLICATE_KEY); - } - - rebuilt_keys.push_back(std::move(key)); - } - - return rebuilt_keys; - } - - void replace_with_reparsed_column_insert( - const std::string& serialized, - ColumnNamePolicy column_name_policy, - size_t inserted_column_index - ) { - const bool was_keyed = this->is_keyed; - const int old_key_column_index = this->key_column_index_; - int new_key_column_index = CSV_NOT_FOUND; - if (old_key_column_index != CSV_NOT_FOUND) { - const auto key_position = std::find( - this->column_indices_.begin(), - this->column_indices_.end(), - static_cast(old_key_column_index) - ); - if (key_position == this->column_indices_.end()) { - throw std::runtime_error("key column is not visible"); - } - - const size_t key_column_index = static_cast( - std::distance(this->column_indices_.begin(), key_position) - ); - new_key_column_index = static_cast( - inserted_column_index <= key_column_index - ? key_column_index + 1 - : key_column_index - ); - } - - CSVFormat format; - format.delimiter(',') - .header_row(0) - .column_names_policy(column_name_policy); - - std::istringstream input(serialized); - CSVReader reader(input, format); - DataFrame rebuilt(reader); - - std::vector rebuilt_keys; - if (was_keyed && new_key_column_index != CSV_NOT_FOUND) { - rebuilt_keys = this->build_keys_from_rows( - rebuilt.rows, - static_cast(new_key_column_index) - ); - } - - this->col_names_ = rebuilt.col_names_; - this->physical_col_names_ = rebuilt.col_names_; - this->reset_visible_column_indices(); - this->rows = std::move(rebuilt.rows); - this->edits.clear(); - this->edits.resize(this->rows.size()); - - if (was_keyed) { - this->is_keyed = true; - this->key_column_index_ = new_key_column_index; - if (new_key_column_index != CSV_NOT_FOUND) { - this->keys_ = std::move(rebuilt_keys); - } - } - else { - this->is_keyed = false; - this->key_column_index_ = CSV_NOT_FOUND; - this->keys_.clear(); - } - - this->invalidate_key_index(); - this->invalidate_column_proxies(); - this->invalidate_json_converter(); + const auto& row = storage_.rows().at(row_index); + return DataFrameRow(&row, this, row_index, static_cast(nullptr), this->key_ptr_at(row_index)); } bool erase_column_at_index(size_t column_index) { - if (column_index >= this->n_cols()) { - return false; - } - - const size_t physical_index = this->physical_column_index(column_index); - if (this->is_keyed && this->key_column_index_ != CSV_NOT_FOUND && - physical_index == static_cast(this->key_column_index_)) { - throw std::runtime_error("cannot erase key column from DataFrame"); - } - - std::vector names = this->columns(); - names.erase(names.begin() + column_index); - - internals::ColNamesPtr visible_names(new internals::ColNames()); - visible_names->set_policy(this->col_names_->get_policy()); - visible_names->set_col_names(names); - - this->col_names_ = visible_names; - this->column_indices_.erase(this->column_indices_.begin() + column_index); + if (!this->storage_.erase_column(column_index, this->key_column_index_)) { return false; } this->invalidate_column_proxies(); this->invalidate_json_converter(); return true; } bool erase_at_index(size_t row_index) { - if (row_index >= rows.size()) { + if (row_index >= storage_.rows().size()) { return false; } - this->erase_row_edits(row_index); - rows.erase(rows.begin() + row_index); - if (this->is_keyed) { + this->storage_.erase_row(row_index); + if (this->is_keyed_) { keys_.erase(keys_.begin() + row_index); } this->invalidate_key_index(); @@ -995,13 +733,13 @@ namespace csv { /** Validate that this DataFrame was created with a key column. */ void require_keyed_frame() const { - if (!is_keyed) + if (!is_keyed_) throw std::runtime_error(internals::ERROR_UNKEYED_DATA_FRAME); } /** Invalidate the lazy key index after structural changes. */ void invalidate_key_index() { - key_index.reset(); + key_index_.reset(); } /** Invalidate cached JSON column mapping after structural column changes. */ @@ -1016,38 +754,37 @@ namespace csv { /** Debug-only check that constructor helpers are starting from a pristine batch state. */ void assert_fresh_storage(bool expected_is_keyed) const { - CSV_DEBUG_ASSERT(this->rows.empty()); + CSV_DEBUG_ASSERT(this->storage_.rows().empty()); CSV_DEBUG_ASSERT(this->keys_.empty()); - CSV_DEBUG_ASSERT(this->edits.empty()); - CSV_DEBUG_ASSERT(this->column_indices_.empty()); - CSV_DEBUG_ASSERT(this->key_index.get() == nullptr); + CSV_DEBUG_ASSERT(this->storage_.pristine()); + CSV_DEBUG_ASSERT(this->key_index_.get() == nullptr); CSV_DEBUG_ASSERT(this->json_converter_.get() == nullptr); - CSV_DEBUG_ASSERT(this->is_keyed == expected_is_keyed); + CSV_DEBUG_ASSERT(this->is_keyed_ == expected_is_keyed); } /** Build the key index if it doesn't exist (lazy initialization). */ void ensure_key_index() const { - if (key_index) return; + if (key_index_) return; - key_index = std::unique_ptr>( + key_index_ = std::unique_ptr>( new std::unordered_map() ); - for (size_t i = 0; i < rows.size(); i++) { - (*key_index)[keys_[i]] = i; + for (size_t i = 0; i < storage_.rows().size(); i++) { + (*key_index_)[keys_[i]] = i; } } /** Find the position of a key in the rows vector. */ size_t position_of(const KeyType& key) const { this->ensure_key_index(); - auto it = key_index->find(key); - return it == key_index->end() ? throw std::out_of_range(internals::ERROR_KEY_NOT_FOUND) + auto it = key_index_->find(key); + return it == key_index_->end() ? throw std::out_of_range(internals::ERROR_KEY_NOT_FOUND) : it->second; } const KeyType* key_ptr_at(size_t row_index) const { - return this->is_keyed ? &keys_.at(row_index) : nullptr; + return this->is_keyed_ ? &keys_.at(row_index) : nullptr; } }; diff --git a/include/internal/data_frame/data_frame_cell.hpp b/include/internal/data_frame/data_frame_cell.hpp index 6c7e7a6f..75d3cfd7 100644 --- a/include/internal/data_frame/data_frame_cell.hpp +++ b/include/internal/data_frame/data_frame_cell.hpp @@ -6,6 +6,8 @@ #include "../csv_exceptions.hpp" #include "../csv_row.hpp" +#include "data_frame_storage.hpp" +#include "fwd.hpp" #include "row_overlay.hpp" namespace csv { @@ -21,24 +23,26 @@ namespace csv { using CSVField::try_get; using CSVField::type; - DataFrameCell() : CSVField(csv::string_view()), row(nullptr), row_overlay(nullptr), col_index(0), can_mutate(false) {} + DataFrameCell() : CSVField(csv::string_view()), row_(nullptr), edit_context_(nullptr), col_index_(0), can_mutate_(false) {} DataFrameCell(const DataFrameCell& other) : CSVField(csv::string_view()), - row(other.row), - row_overlay(other.row_overlay), - col_index(other.col_index), - can_mutate(other.can_mutate), + row_(other.row_), + edit_context_(other.edit_context_), + col_index_(other.col_index_), + can_mutate_(other.can_mutate_), + frame_bound_(other.frame_bound_), owned_value_(other.owned_value_) { this->refresh_value(); } DataFrameCell(DataFrameCell&& other) noexcept : CSVField(csv::string_view()), - row(other.row), - row_overlay(other.row_overlay), - col_index(other.col_index), - can_mutate(other.can_mutate), + row_(other.row_), + edit_context_(other.edit_context_), + col_index_(other.col_index_), + can_mutate_(other.can_mutate_), + frame_bound_(other.frame_bound_), owned_value_(std::move(other.owned_value_)) { this->refresh_value(); } @@ -48,11 +52,11 @@ namespace csv { RowOverlay* _row_overlay, size_t _col_index ) : CSVField(csv::string_view()), - row(_row), - row_overlay(_row_overlay), - col_index(_col_index), - can_mutate(true) { - this->refresh_value(); + row_(_row), + edit_context_(_row_overlay), + col_index_(_col_index), + can_mutate_(true) { + this->refresh_value(_row_overlay); } DataFrameCell( @@ -60,20 +64,21 @@ namespace csv { const RowOverlay* _row_overlay, size_t _col_index ) : CSVField(csv::string_view()), - row(_row), - row_overlay(_row_overlay), - col_index(_col_index), - can_mutate(false) { - this->refresh_value(); + row_(_row), + edit_context_(_row_overlay), + col_index_(_col_index), + can_mutate_(false) { + this->refresh_value(_row_overlay); } DataFrameCell& operator=(const DataFrameCell& other) { if (this != &other) { - row = other.row; - row_overlay = other.row_overlay; - col_index = other.col_index; - can_mutate = other.can_mutate; + row_ = other.row_; + edit_context_ = other.edit_context_; + col_index_ = other.col_index_; + can_mutate_ = other.can_mutate_; owned_value_ = other.owned_value_; + frame_bound_ = other.frame_bound_; this->refresh_value(); } @@ -82,11 +87,12 @@ namespace csv { DataFrameCell& operator=(DataFrameCell&& other) noexcept { if (this != &other) { - row = other.row; - row_overlay = other.row_overlay; - col_index = other.col_index; - can_mutate = other.can_mutate; + row_ = other.row_; + edit_context_ = other.edit_context_; + col_index_ = other.col_index_; + can_mutate_ = other.can_mutate_; owned_value_ = std::move(other.owned_value_); + frame_bound_ = other.frame_bound_; this->refresh_value(); } @@ -133,36 +139,73 @@ namespace csv { } private: + template friend class DataFrame; + + // Frame-bound cells can defer overlay allocation until assignment. Cell + // reads retain the existing snapshot semantics; row proxies resolve afresh. + DataFrameCell( + const CSVRow* source, const RowOverlay* overlay, size_t column, + const internals::data_frame::DataFrameStorage* storage, + bool mutable_access + ) : CSVField(csv::string_view()), row_(source), + edit_context_(mutable_access ? static_cast(storage) : static_cast(overlay)), + col_index_(column), can_mutate_(mutable_access), frame_bound_(mutable_access) { + this->refresh_value(overlay); + } + + const internals::data_frame::DataFrameStorage* storage() const { + return static_cast(edit_context_); + } + + const RowOverlay* find_overlay() const { + if (frame_bound_) { + return this->storage()->find_edits(row_); + } + return static_cast(edit_context_); + } + void refresh_value() { - if (!row) { + this->refresh_value(this->find_overlay()); + } + + void refresh_value(const RowOverlay* overlay) { + if (!row_) { CSVField::operator=(CSVField(csv::string_view())); return; } - if (row_overlay && row_overlay->try_get_copy(col_index, owned_value_)) { + if (overlay && overlay->try_get_copy(col_index_, owned_value_)) { CSVField::operator=(CSVField(csv::string_view(owned_value_))); return; } owned_value_.clear(); - CSVField::operator=(CSVField((*row)[col_index].template get())); + CSVField::operator=(CSVField((*row_)[col_index_].template get())); } DataFrameCell& assign(std::string stored) { - if (!can_mutate || !row_overlay) { + if (!can_mutate_ || !edit_context_) { throw std::runtime_error(internals::ERROR_CANNOT_EDIT_CONST_DF_CELL); } owned_value_ = stored; - const_cast(row_overlay)->set(col_index, std::move(stored)); + auto* state = frame_bound_ + ? const_cast(this->storage()) : nullptr; + RowOverlay* overlay = state ? state->ensure_edits(row_) + : const_cast(static_cast(edit_context_)); + overlay->set(col_index_, std::move(stored)); CSVField::operator=(CSVField(csv::string_view(owned_value_))); return *this; } - const CSVRow* row; - const RowOverlay* row_overlay; - size_t col_index; - bool can_mutate; + const CSVRow* row_; + // Tagged context: standalone/read-only cells hold an overlay; mutable + // frame-bound cells hold the lazy handler. This uses existing padding + // for the tag instead of adding an owner pointer and row index per cell. + const void* edit_context_; + size_t col_index_; + bool can_mutate_; + bool frame_bound_ = false; std::string owned_value_; }; } diff --git a/include/internal/data_frame/data_frame_column.hpp b/include/internal/data_frame/data_frame_column.hpp index 29840a42..8b62c0bc 100644 --- a/include/internal/data_frame/data_frame_column.hpp +++ b/include/internal/data_frame/data_frame_column.hpp @@ -41,7 +41,7 @@ namespace csv { /** Column name. */ const std::string& name() const { this->require_valid(); - return (*frame_->col_names_)[col_index_]; + return frame_->columns().at(col_index_); } /** Zero-based column position. */ @@ -66,9 +66,7 @@ namespace csv { /** Access a visible cell value by row index. */ DataFrameCell operator[](size_t row_index) const { this->require_valid(); - const auto& row = frame_->rows.at(row_index); - const auto* row_edits = frame_->find_row_edits(row_index); - return DataFrameCell(&row, row_edits, frame_->physical_column_index(col_index_)); + return frame_->make_cell(row_index, col_index_, false); } /** Access a visible cell value as a string_view without materializing a DataFrameCell. @@ -79,15 +77,7 @@ namespace csv { */ csv::string_view get_sv(size_t row_index) const { this->require_valid(); - const auto& row = frame_->rows.at(row_index); - const auto* row_edits = frame_->find_row_edits(row_index); - const size_t physical_index = frame_->physical_column_index(col_index_); - csv::string_view edited_value; - if (row_edits && row_edits->try_get_view(physical_index, edited_value)) { - return edited_value; - } - - return row[physical_index].template get(); + return frame_->cell_view(row_index, col_index_); } /** Materialize this column as a vector of converted values. */ diff --git a/include/internal/data_frame/data_frame_row.hpp b/include/internal/data_frame/data_frame_row.hpp index 192b9c5e..5583ed6c 100644 --- a/include/internal/data_frame/data_frame_row.hpp +++ b/include/internal/data_frame/data_frame_row.hpp @@ -64,7 +64,9 @@ namespace csv { /** Get column names. */ const std::vector& get_col_names() const { return frame ? frame->columns() : row->get_col_names(); } - /** Get the underlying CSVRow for compatibility. */ + /** Get the underlying CSVRow without applying cell edits or column visibility. + * Use row conversion to obtain the current visible values. + */ const CSVRow& get_underlying_row() const { return *row; } /** Get the key for this row (only valid for keyed DataFrames). */ @@ -148,7 +150,7 @@ namespace csv { DataFrameCell make_cell(size_t col_index) { if (frame) { - col_index = frame->physical_column_index(col_index); + return frame->make_cell(row, row_index, col_index, can_mutate); } return can_mutate @@ -158,7 +160,7 @@ namespace csv { DataFrameCell make_cell(size_t col_index) const { if (frame) { - col_index = frame->physical_column_index(col_index); + return frame->make_cell(row, row_index, col_index, false); } return DataFrameCell(row, row_overlay, col_index); diff --git a/include/internal/data_frame/data_frame_storage.hpp b/include/internal/data_frame/data_frame_storage.hpp new file mode 100644 index 00000000..7b66b7f2 --- /dev/null +++ b/include/internal/data_frame/data_frame_storage.hpp @@ -0,0 +1,172 @@ +#pragma once + +#include +#include "dirty_data_frame.hpp" + +namespace csv { + namespace internals { + namespace data_frame { + /** C++11 tagged choice of concrete backends, without a vtable. + * + * The clean backing remains alive after first-edit publication. A reader + * that loaded the old pointer can safely finish without a per-read lock. + * Only structural operations (including moves) retire a dirty backend; + * those require exclusive access and invalidate proxies. + */ + class DataFrameStorage { + public: + inline DataFrameStorage() : active_(&clean_) {} + DataFrameStorage(const DataFrameStorage&) = delete; + DataFrameStorage& operator=(const DataFrameStorage&) = delete; + + inline DataFrameStorage(DataFrameStorage&& other) noexcept + : clean_(std::move(other.clean_)), dirty_(std::move(other.dirty_)), active_(&clean_) { + restore_choice(); + other.active_.store(&other.clean_, std::memory_order_release); + } + inline DataFrameStorage& operator=(DataFrameStorage&& other) noexcept { + if (this != &other) { + clean_ = std::move(other.clean_); + dirty_ = std::move(other.dirty_); + restore_choice(); + other.active_.store(&other.clean_, std::memory_order_release); + } + return *this; + } + + // Both concrete backends use the same native row vector. + inline std::vector& rows() noexcept { return clean_.rows(); } + inline const std::vector& rows() const noexcept { return clean_.rows(); } + inline void set_names(ConstColNamesPtr names) { clean_.set_names(std::move(names)); } + inline const ConstColNamesPtr& names() const noexcept { return dispatch(Names()); } + inline size_t n_cols() const noexcept { return dispatch(Count()); } + inline CellBinding bind(size_t row, size_t column) const { return dispatch(Bind{row, column}); } + inline csv::string_view view(size_t row, size_t column) const { return dispatch(View{row, column}); } + inline const RowOverlay* find_edits(const CSVRow* row) const { + return dispatch(Edits{row_index(row)}); + } + inline RowOverlay* ensure_edits(const CSVRow* row) { + return promote().ensure_edits(row_index(row)); + } + inline CSVRow make_inserted_row(const std::vector& row) const { + return dispatch(MakeRow{row}); + } + inline void reserve_insert() { dispatch_mutable(Reserve()); } + inline void insert_row(size_t index, CSVRow&& row) { dispatch_mutable(Insert{index, row}); } + inline void erase_row(size_t index) { dispatch_mutable(Erase{index}); } + + inline bool erase_column(size_t index, int key_column) { + if (index >= n_cols()) { return false; } + return promote().erase_column(index, key_column); + } + + template + inline void insert_column( + size_t index, const std::string& name, const std::string& value, + int& key_column, const BeforeReplace& before_replace + ) { + RowBackend* choice = active_.load(std::memory_order_acquire); + if (choice == &clean_) { + // Structural mutation is exclusive: this transient dirty backend + // needs no overlay slots and never becomes visible to cell readers. + DirtyDataFrame mutation(clean_, false); + mutation.insert_column(index, name, value, key_column, before_replace); + } + else { + static_cast(choice)->insert_column(index, name, value, key_column, before_replace); + active_.store(&clean_, std::memory_order_release); + dirty_.reset(); + } + } + + inline DataFrameStorage selected_rows(const std::vector& mask) const { + DataFrameStorage selected; + selected.clean_ = clean_.selected_rows(mask); + const RowBackend* choice = active_.load(std::memory_order_acquire); + if (choice != &clean_) { + selected.dirty_ = static_cast(choice)->selected_rows(selected.clean_, mask); + selected.active_.store(selected.dirty_.get(), std::memory_order_release); + } + return selected; + } + + inline bool pristine() const noexcept { + return active_.load(std::memory_order_acquire) == &clean_ && rows().empty(); + } + + private: + template + inline auto dispatch(const Operation& operation) const + -> decltype(operation(std::declval())) { + const RowBackend* choice = active_.load(std::memory_order_acquire); + if (choice == &clean_) { return operation(clean_); } + return operation(*static_cast(choice)); + } + template + inline auto dispatch_mutable(const Operation& operation) + -> decltype(operation(std::declval())) { + RowBackend* choice = active_.load(std::memory_order_acquire); + if (choice == &clean_) { return operation(clean_); } + return operation(*static_cast(choice)); + } + + inline DirtyDataFrame& promote() { + RowBackend* choice = active_.load(std::memory_order_acquire); + if (choice != &clean_) { return *static_cast(choice); } + std::lock_guard lock(transition_lock_); + choice = active_.load(std::memory_order_acquire); + if (choice == &clean_) { + // Construct the schema and complete slot array before release. + dirty_.reset(new DirtyDataFrame(clean_)); + choice = dirty_.get(); + active_.store(choice, std::memory_order_release); + } + return *static_cast(choice); + } + inline size_t row_index(const CSVRow* row) const noexcept { + return static_cast(row - rows().data()); + } + inline void restore_choice() noexcept { + if (dirty_) { + dirty_->rebind(clean_); + active_.store(dirty_.get(), std::memory_order_release); + } + else { active_.store(&clean_, std::memory_order_release); } + } + + struct Names { template const ConstColNamesPtr& operator()(const B& b) const { return b.names(); } }; + struct Count { template size_t operator()(const B& b) const { return b.n_cols(); } }; + struct Bind { + size_t row, column; + template CellBinding operator()(const B& b) const { return b.bind(row, column); } + }; + struct View { + size_t row, column; + template csv::string_view operator()(const B& b) const { return b.view(row, column); } + }; + struct Edits { + size_t row; + template const RowOverlay* operator()(const B& b) const { return b.find_edits(row); } + }; + struct MakeRow { + const std::vector& row; + template CSVRow operator()(const B& b) const { return b.make_inserted_row(row); } + }; + struct Reserve { template void operator()(B& b) const { b.reserve_insert(); } }; + struct Insert { + size_t index; CSVRow& row; + template void operator()(B& b) const { b.insert_row(index, std::move(row)); } + }; + struct Erase { + size_t index; + template void operator()(B& b) const { b.erase_row(index); } + }; + + CleanDataFrame clean_; + std::unique_ptr dirty_; + std::mutex transition_lock_; + std::atomic active_; + }; + } + } +} diff --git a/include/internal/data_frame/dirty_data_frame.hpp b/include/internal/data_frame/dirty_data_frame.hpp new file mode 100644 index 00000000..5c7ddb16 --- /dev/null +++ b/include/internal/data_frame/dirty_data_frame.hpp @@ -0,0 +1,161 @@ +#pragma once + +#include +#include "clean_data_frame.hpp" + +namespace csv { + namespace internals { + namespace data_frame { + /** Overlay-aware backend. Owns visible schema and all column mutation logic. + * Physical rows stay in the stable clean backing; promotion leaves readers + * and proxies bound to the same objects. Structural operations are exclusive. + */ + class DirtyDataFrame : public RowBackend { + public: + explicit inline DirtyDataFrame(CleanDataFrame& backing, bool allocate_edits = true) + : backing_(&backing), names_(backing.names()) { + if (allocate_edits) { edits_.resize(backing.rows().size()); } + } + DirtyDataFrame(const DirtyDataFrame&) = delete; + DirtyDataFrame& operator=(const DirtyDataFrame&) = delete; + + inline void rebind(CleanDataFrame& backing) noexcept { backing_ = &backing; } + inline const ConstColNamesPtr& names() const noexcept { return names_; } + inline size_t n_cols() const noexcept { return names_->size(); } + inline const RowOverlay* find_edits(size_t row) const { + // Empty slots exist only in an unpublished, exclusive mutation + // backend. Published backends initialize every slot before release. + return edits_.empty() ? nullptr : edits_.at(row).get(); + } + inline RowOverlay* ensure_edits(size_t row) { + if (auto* overlay = edits_.at(row).get()) { return overlay; } + std::lock_guard lock(creation_lock_); + return edits_.at(row).ensure(); + } + inline size_t physical_column(size_t column) const { + return column_indices_.empty() ? column : column_indices_.at(column); + } + inline size_t logical_column(size_t physical) const { + if (column_indices_.empty()) { return physical; } + const auto position = std::find(column_indices_.begin(), column_indices_.end(), physical); + if (position == column_indices_.end()) { throw std::runtime_error("key column is not visible"); } + return static_cast(std::distance(column_indices_.begin(), position)); + } + inline CellBinding bind(size_t row, size_t column) const { + validate_column(column, n_cols()); + return CellBinding{find_edits(row), physical_column(column)}; + } + inline csv::string_view view(size_t row, size_t column) const { + const CellBinding cell = bind(row, column); + csv::string_view value; + if (cell.overlay && cell.overlay->try_get_view(cell.physical_column, value)) { return value; } + return RowViewAccessor::view(backing_->rows().at(row), cell.physical_column); + } + inline CSVRow make_inserted_row(const std::vector& row) const { + validate_row(row, n_cols()); + if (column_indices_.empty() && n_cols() == backing_->n_cols()) { return RowStorageBuilder::make_row(row, backing_->names()); } + std::vector expanded(backing_->n_cols()); + for (size_t i = 0; i < row.size(); ++i) { expanded[physical_column(i)] = row[i]; } + return RowStorageBuilder::make_row(expanded, backing_->names()); + } + inline void reserve_insert() { + backing_->reserve_insert(); + reserve_next(edits_); + } + inline void insert_row(size_t index, CSVRow&& row) { + // Call reserve_insert before mutating keyed metadata. Slot moves + // and CSVRow moves cannot allocate once capacity is available. + backing_->insert_row(index, std::move(row)); + edits_.insert(edits_.begin() + index, RowOverlaySlot()); + } + inline void erase_row(size_t index) { + edits_.erase(edits_.begin() + index); + backing_->erase_row(index); + } + inline bool erase_column(size_t index, int key_column) { + if (index >= n_cols()) { return false; } + const size_t physical = physical_column(index); + if (key_column != CSV_NOT_FOUND && physical == static_cast(key_column)) { + throw std::runtime_error("cannot erase key column from DataFrame"); + } + std::vector columns = names_->get_col_names(); + columns.erase(columns.begin() + index); + ColNamesPtr names = std::make_shared(); + names->set_policy(names_->get_policy()); + names->set_col_names(columns); + std::vector mapping = column_indices_; + if (mapping.empty()) { + mapping.reserve(backing_->n_cols()); + for (size_t i = 0; i < backing_->n_cols(); ++i) { mapping.push_back(i); } + } + mapping.erase(mapping.begin() + index); + names_ = std::move(names); + column_indices_ = std::move(mapping); + return true; + } + inline std::unique_ptr selected_rows( + CleanDataFrame& selected, const std::vector& mask + ) const { + std::unique_ptr result(new DirtyDataFrame(selected)); + result->names_ = names_; + result->column_indices_ = column_indices_; + size_t target = 0; + for (size_t i = 0; i < mask.size(); ++i) { + if (mask[i]) { + if (const auto* overlay = find_edits(i)) { + *result->edits_[target].ensure() = overlay->snapshot(); + } + ++target; + } + } + return result; + } + + template + inline void insert_column( + size_t index, const std::string& name, const std::string& default_value, + int& key_column, const BeforeReplace& before_replace + ) { + if (index > n_cols()) { throw std::out_of_range("DataFrame insert_column index out of range"); } + if (name.empty()) { throw std::invalid_argument("inserted column name must not be empty"); } + if (names_->index_of(name) != CSV_NOT_FOUND) { + throw std::invalid_argument("inserted column name must not duplicate an existing column"); + } + std::vector columns = names_->get_col_names(); + columns.insert(columns.begin() + index, name); + ColNamesPtr names = std::make_shared(); + names->set_policy(names_->get_policy()); + names->set_col_names(columns); + int new_key_column = key_column; + if (key_column != CSV_NOT_FOUND) { + const size_t logical = logical_column(static_cast(key_column)); + new_key_column = static_cast(logical + (index <= logical ? 1 : 0)); + } + // Select a concrete materializer once. An unpublished clean-input + // mutation backend has neither slot allocation nor per-field lookups. + std::vector rebuilt = edits_.empty() && column_indices_.empty() + ? RowStorageBuilder::insert_column(backing_->rows().size(), names, index, default_value, + [this](size_t row, size_t column) { return backing_->rows()[row][column].get(); }) + : RowStorageBuilder::insert_column(backing_->rows().size(), names, index, default_value, + [this](size_t row, size_t column) -> std::string { + const CellBinding cell = bind(row, column); + std::string value; + if (cell.overlay && cell.overlay->try_get_copy(cell.physical_column, value)) { return value; } + return backing_->rows()[row][cell.physical_column].get(); + }); + // Prepare everything before releasing parsed backing or view-key bytes. + before_replace(); + backing_->replace_rows(std::move(rebuilt), std::move(names)); + key_column = new_key_column; + } + + private: + CleanDataFrame* backing_; + ConstColNamesPtr names_; + std::vector edits_; + std::vector column_indices_; + std::mutex creation_lock_; + }; + } + } +} diff --git a/include/internal/data_frame/key_storage.hpp b/include/internal/data_frame/key_storage.hpp new file mode 100644 index 00000000..abf5f4b9 --- /dev/null +++ b/include/internal/data_frame/key_storage.hpp @@ -0,0 +1,71 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "../common.hpp" + +namespace csv { + namespace internals { + namespace data_frame { + // Owning key types need no additional lifetime handling or runtime work. + template + class KeyStorage { + public: + template + inline void preserve(std::vector&, const InvalidateIndex&) const noexcept {} + }; + + /** Keep established view keys alive when their parsed rows are replaced. + * Only key bytes are retained, rather than entire obsolete CSV chunks. + */ + template<> + class KeyStorage { + public: + template + inline void preserve(std::vector& keys, const InvalidateIndex& invalidate_index) { + if (keys.empty()) { + invalidate_index(); + this->storage_.reset(); + return; + } + + std::unique_ptr storage(new std::string()); + size_t bytes = 0; + for (const auto& key : keys) { + if (key.size() > storage->max_size() - bytes) { + throw std::length_error("DataFrame key storage is too large"); + } + bytes += key.size(); + } + storage->reserve(bytes); + for (const auto& key : keys) { + if (!key.empty()) { + storage->append(key.data(), key.size()); + } + } + + // Form views only after the last append. Heap-own the string object + // so even its small-string buffer stays stable across frame moves. + std::vector preserved; + preserved.reserve(keys.size()); + size_t offset = 0; + for (const auto& key : keys) { + preserved.emplace_back(storage->data() + offset, key.size()); + offset += key.size(); + } + // Drop index views while both old backing stores are still alive. + invalidate_index(); + keys.swap(preserved); + this->storage_ = std::move(storage); + } + + private: + std::unique_ptr storage_; + }; + } + } +} diff --git a/include/internal/data_frame/performance/cached-column-construction-results.json b/include/internal/data_frame/performance/cached-column-construction-results.json new file mode 100644 index 00000000..37e12d98 --- /dev/null +++ b/include/internal/data_frame/performance/cached-column-construction-results.json @@ -0,0 +1,104 @@ +{ + "construct": { + "baseline": { + "samples": [ + 84.9254, + 80.0115, + 93.8199, + 92.3752, + 87.6782, + 80.9792, + 82.2074 + ], + "checksums": [ + 3000000, + 3000000, + 3000000, + 3000000, + 3000000, + 3000000, + 3000000 + ], + "median": 84.9254, + "min": 80.0115, + "max": 93.8199, + "mad": 3.9461999999999904 + }, + "candidate": { + "samples": [ + 103.728, + 130.808, + 93.5402, + 100.544, + 104.319, + 99.4639, + 102.388 + ], + "checksums": [ + 3000000, + 3000000, + 3000000, + 3000000, + 3000000, + 3000000, + 3000000 + ], + "median": 102.388, + "min": 93.5402, + "max": 130.808, + "mad": 1.9309999999999974 + }, + "candidate_change_pct": 20.562281720192075 + }, + "copy": { + "baseline": { + "samples": [ + 79.739, + 83.2895, + 84.6133, + 86.7707, + 81.6743, + 77.807, + 79.9222 + ], + "checksums": [ + 3000000, + 3000000, + 3000000, + 3000000, + 3000000, + 3000000, + 3000000 + ], + "median": 81.6743, + "min": 77.807, + "max": 86.7707, + "mad": 1.935299999999998 + }, + "candidate": { + "samples": [ + 97.8638, + 85.0175, + 82.3975, + 86.0181, + 82.9437, + 92.4242, + 85.3929 + ], + "checksums": [ + 3000000, + 3000000, + 3000000, + 3000000, + 3000000, + 3000000, + 3000000 + ], + "median": 85.3929, + "min": 82.3975, + "max": 97.8638, + "mad": 2.4491999999999905 + }, + "candidate_change_pct": 4.552962192513421 + } +} \ No newline at end of file diff --git a/include/internal/data_frame/performance/cached-column-results.json b/include/internal/data_frame/performance/cached-column-results.json new file mode 100644 index 00000000..b161180c --- /dev/null +++ b/include/internal/data_frame/performance/cached-column-results.json @@ -0,0 +1,414 @@ +{ + "layout_dfc": { + "baseline": 48, + "candidate": 56 + }, + "clean_indexed": { + "baseline": { + "samples": [ + 85.1142, + 86.8551, + 99.731, + 89.3737, + 82.1837, + 84.6808, + 92.8627 + ], + "checksums": [ + 76666836, + 76666836, + 76666836, + 76666836, + 76666836, + 76666836, + 76666836 + ], + "median": 86.8551, + "min": 82.1837, + "max": 99.731, + "mad": 2.5186000000000064 + }, + "candidate": { + "samples": [ + 77.0366, + 87.2176, + 134.3057, + 85.6111, + 71.903, + 81.6378, + 82.426 + ], + "checksums": [ + 76666836, + 76666836, + 76666836, + 76666836, + 76666836, + 76666836, + 76666836 + ], + "median": 82.426, + "min": 71.903, + "max": 134.3057, + "mad": 4.7916000000000025 + }, + "candidate_change_pct": -5.099412700002636 + }, + "clean_iterator": { + "baseline": { + "samples": [ + 60.222, + 63.8219, + 63.2333, + 61.1363, + 60.4957, + 70.5118, + 93.2978 + ], + "checksums": [ + 25555612, + 25555612, + 25555612, + 25555612, + 25555612, + 25555612, + 25555612 + ], + "median": 63.2333, + "min": 60.222, + "max": 93.2978, + "mad": 2.7376000000000005 + }, + "candidate": { + "samples": [ + 58.0797, + 60.0305, + 57.4663, + 59.1636, + 60.326, + 61.8406, + 61.5868 + ], + "checksums": [ + 25555612, + 25555612, + 25555612, + 25555612, + 25555612, + 25555612, + 25555612 + ], + "median": 60.0305, + "min": 57.4663, + "max": 61.8406, + "mad": 1.5562999999999931 + }, + "candidate_change_pct": -5.065052749105292 + }, + "clean_quoted_indexed": { + "baseline": { + "samples": [ + 94.1876, + 86.5565, + 92.2312, + 92.3973, + 91.6157, + 90.3518, + 93.3929 + ], + "checksums": [ + 100666884, + 100666884, + 100666884, + 100666884, + 100666884, + 100666884, + 100666884 + ], + "median": 92.2312, + "min": 86.5565, + "max": 94.1876, + "mad": 1.1616999999999962 + }, + "candidate": { + "samples": [ + 84.1158, + 95.5986, + 82.6454, + 89.4145, + 86.246, + 95.8819, + 99.4147 + ], + "checksums": [ + 100666884, + 100666884, + 100666884, + 100666884, + 100666884, + 100666884, + 100666884 + ], + "median": 89.4145, + "min": 82.6454, + "max": 99.4147, + "mad": 6.184100000000001 + }, + "candidate_change_pct": -3.0539557112994276 + }, + "dirty_indexed": { + "baseline": { + "samples": [ + 88.3677, + 87.2941, + 84.5807, + 81.7566, + 81.2683, + 86.5807, + 88.3462 + ], + "checksums": [ + 76669560, + 76669560, + 76669560, + 76669560, + 76669560, + 76669560, + 76669560 + ], + "median": 86.5807, + "min": 81.2683, + "max": 88.3677, + "mad": 1.7870000000000061 + }, + "candidate": { + "samples": [ + 98.5898, + 92.8826, + 86.3935, + 97.0365, + 91.1092, + 88.3935, + 103.2631 + ], + "checksums": [ + 76669560, + 76669560, + 76669560, + 76669560, + 76669560, + 76669560, + 76669560 + ], + "median": 92.8826, + "min": 86.3935, + "max": 103.2631, + "mad": 4.489099999999993 + }, + "candidate_change_pct": 7.278642930814838 + }, + "dirty_iterator": { + "baseline": { + "samples": [ + 60.8691, + 62.6327, + 60.6711, + 63.3954, + 64.7791, + 64.9701, + 61.9638 + ], + "checksums": [ + 25556520, + 25556520, + 25556520, + 25556520, + 25556520, + 25556520, + 25556520 + ], + "median": 62.6327, + "min": 60.6711, + "max": 64.9701, + "mad": 1.7635999999999967 + }, + "candidate": { + "samples": [ + 77.954, + 60.8924, + 61.2981, + 60.4558, + 60.8236, + 57.7917, + 58.7522 + ], + "checksums": [ + 25556520, + 25556520, + 25556520, + 25556520, + 25556520, + 25556520, + 25556520 + ], + "median": 60.8236, + "min": 57.7917, + "max": 77.954, + "mad": 0.47449999999999903 + }, + "candidate_change_pct": -2.888427291175377 + }, + "mapped_indexed": { + "baseline": { + "samples": [ + 94.0223, + 84.5073, + 90.7443, + 84.2819, + 84.6862, + 83.45, + 90.5745 + ], + "checksums": [ + 76669560, + 76669560, + 76669560, + 76669560, + 76669560, + 76669560, + 76669560 + ], + "median": 84.6862, + "min": 83.45, + "max": 94.0223, + "mad": 1.2361999999999966 + }, + "candidate": { + "samples": [ + 109.3394, + 100.9515, + 82.3192, + 89.0643, + 85.2335, + 82.6034, + 90.3202 + ], + "checksums": [ + 76669560, + 76669560, + 76669560, + 76669560, + 76669560, + 76669560, + 76669560 + ], + "median": 89.0643, + "min": 82.3192, + "max": 109.3394, + "mad": 6.460900000000009 + }, + "candidate_change_pct": 5.1697915362833635 + }, + "mapped_iterator": { + "baseline": { + "samples": [ + 62.7091, + 63.3954, + 60.5365, + 67.2625, + 65.3821, + 59.7352, + 69.4581 + ], + "checksums": [ + 25556520, + 25556520, + 25556520, + 25556520, + 25556520, + 25556520, + 25556520 + ], + "median": 63.3954, + "min": 59.7352, + "max": 69.4581, + "mad": 2.8589000000000055 + }, + "candidate": { + "samples": [ + 66.4432, + 59.3271, + 69.8818, + 61.636, + 63.7874, + 59.989, + 63.7992 + ], + "checksums": [ + 25556520, + 25556520, + 25556520, + 25556520, + 25556520, + 25556520, + 25556520 + ], + "median": 63.7874, + "min": 59.3271, + "max": 69.8818, + "mad": 2.6558000000000064 + }, + "candidate_change_pct": 0.6183413938550597 + }, + "chunk_parallel": { + "baseline": { + "samples": [ + 166.2424, + 150.9798, + 160.7823, + 170.1355, + 191.7566, + 180.3124, + 156.0052 + ], + "checksums": [ + 21007980, + 21007980, + 21007980, + 21007980, + 21007980, + 21007980, + 21007980 + ], + "median": 166.2424, + "min": 150.9798, + "max": 191.7566, + "mad": 10.237200000000001 + }, + "candidate": { + "samples": [ + 165.8798, + 197.6012, + 173.2929, + 174.4019, + 167.3851, + 167.5406, + 166.902 + ], + "checksums": [ + 21007980, + 21007980, + 21007980, + 21007980, + 21007980, + 21007980, + 21007980 + ], + "median": 167.5406, + "min": 165.8798, + "max": 197.6012, + "mad": 1.6608000000000231 + }, + "candidate_change_pct": 0.780907879097037 + } +} \ No newline at end of file diff --git a/include/internal/data_frame/performance/cached-column.patch b/include/internal/data_frame/performance/cached-column.patch new file mode 100644 index 00000000..2d033b79 --- /dev/null +++ b/include/internal/data_frame/performance/cached-column.patch @@ -0,0 +1,153 @@ +--- a/include/internal/data_frame/clean_data_frame.hpp ++++ b/include/internal/data_frame/clean_data_frame.hpp +@@ -33,6 +33,17 @@ + return rows_.at(row)[column].get(); + } + ++ inline size_t resolve_column(size_t column) const { ++ validate_column(column, n_cols()); ++ return column; ++ } ++ inline CellBinding bind_physical(size_t, size_t column) const { ++ return CellBinding{nullptr, column}; ++ } ++ inline csv::string_view view_physical(size_t row, size_t column) const { ++ return rows_.at(row)[column].get(); ++ } ++ + inline CSVRow make_inserted_row(const std::vector& row) const { + validate_row(row, n_cols()); + return RowStorageBuilder::make_row(row, names_); +--- a/include/internal/data_frame/dirty_data_frame.hpp ++++ b/include/internal/data_frame/dirty_data_frame.hpp +@@ -46,11 +46,22 @@ + return CellBinding{find_edits(row), physical_column(column)}; + } + inline csv::string_view view(size_t row, size_t column) const { +- const CellBinding cell = bind(row, column); ++ return view_physical(row, resolve_column(column)); ++ } ++ inline size_t resolve_column(size_t column) const { ++ validate_column(column, n_cols()); ++ return physical_column(column); ++ } ++ inline CellBinding bind_physical(size_t row, size_t column) const { ++ return CellBinding{find_edits(row), column}; ++ } ++ inline csv::string_view view_physical(size_t row, size_t column) const { ++ const RowOverlay* overlay = find_edits(row); + csv::string_view value; +- if (cell.overlay && cell.overlay->try_get_view(cell.physical_column, value)) { return value; } +- return backing_->rows().at(row)[cell.physical_column].get(); ++ if (overlay && overlay->try_get_view(column, value)) { return value; } ++ return backing_->rows().at(row)[column].get(); + } ++ + inline CSVRow make_inserted_row(const std::vector& row) const { + validate_row(row, n_cols()); + if (column_indices_.empty() && n_cols() == backing_->n_cols()) { return RowStorageBuilder::make_row(row, backing_->names()); } +--- a/include/internal/data_frame/data_frame_storage.hpp ++++ b/include/internal/data_frame/data_frame_storage.hpp +@@ -42,6 +42,9 @@ + inline size_t n_cols() const noexcept { return dispatch(Count()); } + inline CellBinding bind(size_t row, size_t column) const { return dispatch(Bind{row, column}); } + inline csv::string_view view(size_t row, size_t column) const { return dispatch(View{row, column}); } ++ inline size_t resolve_column(size_t column) const { return dispatch(ResolveColumn{column}); } ++ inline CellBinding bind_physical(size_t row, size_t column) const { return dispatch(BindPhysical{row, column}); } ++ inline csv::string_view view_physical(size_t row, size_t column) const { return dispatch(ViewPhysical{row, column}); } + inline const RowOverlay* find_edits(const CSVRow* row) const { + return dispatch(Edits{row_index(row)}); + } +@@ -144,6 +147,18 @@ + size_t row, column; + template csv::string_view operator()(const B& b) const { return b.view(row, column); } + }; ++ struct ResolveColumn { ++ size_t column; ++ template size_t operator()(const B& b) const { return b.resolve_column(column); } ++ }; ++ struct BindPhysical { ++ size_t row, column; ++ template CellBinding operator()(const B& b) const { return b.bind_physical(row, column); } ++ }; ++ struct ViewPhysical { ++ size_t row, column; ++ template csv::string_view operator()(const B& b) const { return b.view_physical(row, column); } ++ }; + struct Edits { + size_t row; + template const RowOverlay* operator()(const B& b) const { return b.find_edits(row); } +--- a/include/internal/data_frame/data_frame_column.hpp ++++ b/include/internal/data_frame/data_frame_column.hpp +@@ -1,6 +1,7 @@ + #pragma once + + #include ++#include + #include + #include + #include +@@ -22,21 +23,25 @@ + using iterator = internals::indexed_proxy_iterator, DataFrameCell, cell_accessor>; + using const_iterator = iterator; + +- DataFrameColumn() : frame_(nullptr), mutable_frame_(nullptr), col_index_(0), generation_value_(0) {} ++ DataFrameColumn() : frame_(nullptr), mutable_frame_(nullptr), col_index_(0), generation_value_(0), physical_column_(std::numeric_limits::max()) {} + + DataFrameColumn(const DataFrame* frame, size_t col_index) + : frame_(frame), + mutable_frame_(nullptr), + col_index_(col_index), + generation_(frame ? frame->column_generation_ : std::shared_ptr()), +- generation_value_(generation_ ? *generation_ : 0) {} ++ generation_value_(generation_ ? *generation_ : 0), ++ physical_column_(frame && col_index < frame->n_cols() ++ ? frame->storage_.resolve_column(col_index) : std::numeric_limits::max()) {} + + DataFrameColumn(DataFrame* frame, size_t col_index) + : frame_(frame), + mutable_frame_(frame), + col_index_(col_index), + generation_(frame ? frame->column_generation_ : std::shared_ptr()), +- generation_value_(generation_ ? *generation_ : 0) {} ++ generation_value_(generation_ ? *generation_ : 0), ++ physical_column_(frame && col_index < frame->n_cols() ++ ? frame->storage_.resolve_column(col_index) : std::numeric_limits::max()) {} + + /** Column name. */ + const std::string& name() const { +@@ -66,7 +71,12 @@ + /** Access a visible cell value by row index. */ + DataFrameCell operator[](size_t row_index) const { + this->require_valid(); +- return frame_->make_cell(row_index, col_index_, false); ++ if (physical_column_ == std::numeric_limits::max()) { ++ return frame_->make_cell(row_index, col_index_, false); ++ } ++ const auto* row = &frame_->storage_.rows().at(row_index); ++ const auto binding = frame_->storage_.bind_physical(row_index, physical_column_); ++ return DataFrameCell(row, binding.overlay, binding.physical_column); + } + + /** Access a visible cell value as a string_view without materializing a DataFrameCell. +@@ -77,7 +87,10 @@ + */ + csv::string_view get_sv(size_t row_index) const { + this->require_valid(); +- return frame_->cell_view(row_index, col_index_); ++ if (physical_column_ == std::numeric_limits::max()) { ++ return frame_->cell_view(row_index, col_index_); ++ } ++ return frame_->storage_.view_physical(row_index, physical_column_); + } + + /** Materialize this column as a vector of converted values. */ +@@ -147,5 +160,8 @@ + size_t col_index_; + std::shared_ptr generation_; + size_t generation_value_; ++ // Exclusive structural mutation invalidates views when the mapping changes. ++ // Invalid manually constructed views retain the original deferred validation. ++ size_t physical_column_; + }; + } diff --git a/include/internal/data_frame/performance/direct-header-samples.json b/include/internal/data_frame/performance/direct-header-samples.json new file mode 100644 index 00000000..aaddd717 --- /dev/null +++ b/include/internal/data_frame/performance/direct-header-samples.json @@ -0,0 +1,884 @@ +[ + { + "pair": 0, + "side": "baseline", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 0, + "side": "baseline", + "metric": "clean_indexed", + "ms": 84.8333, + "checksum": 76666836 + }, + { + "pair": 0, + "side": "baseline", + "metric": "clean_iterator", + "ms": 67.3415, + "checksum": 25555612 + }, + { + "pair": 0, + "side": "baseline", + "metric": "clean_quoted_indexed", + "ms": 129.8762, + "checksum": 100666884 + }, + { + "pair": 0, + "side": "baseline", + "metric": "dirty_indexed", + "ms": 87.6133, + "checksum": 76669560 + }, + { + "pair": 0, + "side": "baseline", + "metric": "dirty_iterator", + "ms": 65.2783, + "checksum": 25556520 + }, + { + "pair": 0, + "side": "baseline", + "metric": "mapped_indexed", + "ms": 86.8675, + "checksum": 76669560 + }, + { + "pair": 0, + "side": "baseline", + "metric": "mapped_iterator", + "ms": 64.5963, + "checksum": 25556520 + }, + { + "pair": 0, + "side": "baseline", + "metric": "chunk_parallel", + "ms": 144.3011, + "checksum": 21007980 + }, + { + "pair": 0, + "side": "candidate", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 0, + "side": "candidate", + "metric": "clean_indexed", + "ms": 42.2933, + "checksum": 76666836 + }, + { + "pair": 0, + "side": "candidate", + "metric": "clean_iterator", + "ms": 62.3278, + "checksum": 25555612 + }, + { + "pair": 0, + "side": "candidate", + "metric": "clean_quoted_indexed", + "ms": 77.2193, + "checksum": 100666884 + }, + { + "pair": 0, + "side": "candidate", + "metric": "dirty_indexed", + "ms": 49.535, + "checksum": 76669560 + }, + { + "pair": 0, + "side": "candidate", + "metric": "dirty_iterator", + "ms": 60.263, + "checksum": 25556520 + }, + { + "pair": 0, + "side": "candidate", + "metric": "mapped_indexed", + "ms": 51.5204, + "checksum": 76669560 + }, + { + "pair": 0, + "side": "candidate", + "metric": "mapped_iterator", + "ms": 62.8704, + "checksum": 25556520 + }, + { + "pair": 0, + "side": "candidate", + "metric": "chunk_parallel", + "ms": 121.1497, + "checksum": 21007980 + }, + { + "pair": 1, + "side": "candidate", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 1, + "side": "candidate", + "metric": "clean_indexed", + "ms": 48.0208, + "checksum": 76666836 + }, + { + "pair": 1, + "side": "candidate", + "metric": "clean_iterator", + "ms": 64.5614, + "checksum": 25555612 + }, + { + "pair": 1, + "side": "candidate", + "metric": "clean_quoted_indexed", + "ms": 76.7822, + "checksum": 100666884 + }, + { + "pair": 1, + "side": "candidate", + "metric": "dirty_indexed", + "ms": 51.0939, + "checksum": 76669560 + }, + { + "pair": 1, + "side": "candidate", + "metric": "dirty_iterator", + "ms": 63.8719, + "checksum": 25556520 + }, + { + "pair": 1, + "side": "candidate", + "metric": "mapped_indexed", + "ms": 53.0207, + "checksum": 76669560 + }, + { + "pair": 1, + "side": "candidate", + "metric": "mapped_iterator", + "ms": 63.1135, + "checksum": 25556520 + }, + { + "pair": 1, + "side": "candidate", + "metric": "chunk_parallel", + "ms": 136.0833, + "checksum": 21007980 + }, + { + "pair": 1, + "side": "baseline", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 1, + "side": "baseline", + "metric": "clean_indexed", + "ms": 89.9999, + "checksum": 76666836 + }, + { + "pair": 1, + "side": "baseline", + "metric": "clean_iterator", + "ms": 64.4432, + "checksum": 25555612 + }, + { + "pair": 1, + "side": "baseline", + "metric": "clean_quoted_indexed", + "ms": 128.6186, + "checksum": 100666884 + }, + { + "pair": 1, + "side": "baseline", + "metric": "dirty_indexed", + "ms": 91.8604, + "checksum": 76669560 + }, + { + "pair": 1, + "side": "baseline", + "metric": "dirty_iterator", + "ms": 61.8485, + "checksum": 25556520 + }, + { + "pair": 1, + "side": "baseline", + "metric": "mapped_indexed", + "ms": 90.4979, + "checksum": 76669560 + }, + { + "pair": 1, + "side": "baseline", + "metric": "mapped_iterator", + "ms": 69.4281, + "checksum": 25556520 + }, + { + "pair": 1, + "side": "baseline", + "metric": "chunk_parallel", + "ms": 161.6356, + "checksum": 21007980 + }, + { + "pair": 2, + "side": "baseline", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 2, + "side": "baseline", + "metric": "clean_indexed", + "ms": 80.1391, + "checksum": 76666836 + }, + { + "pair": 2, + "side": "baseline", + "metric": "clean_iterator", + "ms": 60.3223, + "checksum": 25555612 + }, + { + "pair": 2, + "side": "baseline", + "metric": "clean_quoted_indexed", + "ms": 126.8111, + "checksum": 100666884 + }, + { + "pair": 2, + "side": "baseline", + "metric": "dirty_indexed", + "ms": 87.9141, + "checksum": 76669560 + }, + { + "pair": 2, + "side": "baseline", + "metric": "dirty_iterator", + "ms": 62.6347, + "checksum": 25556520 + }, + { + "pair": 2, + "side": "baseline", + "metric": "mapped_indexed", + "ms": 87.0762, + "checksum": 76669560 + }, + { + "pair": 2, + "side": "baseline", + "metric": "mapped_iterator", + "ms": 61.2209, + "checksum": 25556520 + }, + { + "pair": 2, + "side": "baseline", + "metric": "chunk_parallel", + "ms": 159.3291, + "checksum": 21007980 + }, + { + "pair": 2, + "side": "candidate", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 2, + "side": "candidate", + "metric": "clean_indexed", + "ms": 43.0649, + "checksum": 76666836 + }, + { + "pair": 2, + "side": "candidate", + "metric": "clean_iterator", + "ms": 65.7639, + "checksum": 25555612 + }, + { + "pair": 2, + "side": "candidate", + "metric": "clean_quoted_indexed", + "ms": 75.9153, + "checksum": 100666884 + }, + { + "pair": 2, + "side": "candidate", + "metric": "dirty_indexed", + "ms": 48.7432, + "checksum": 76669560 + }, + { + "pair": 2, + "side": "candidate", + "metric": "dirty_iterator", + "ms": 62.3675, + "checksum": 25556520 + }, + { + "pair": 2, + "side": "candidate", + "metric": "mapped_indexed", + "ms": 50.4738, + "checksum": 76669560 + }, + { + "pair": 2, + "side": "candidate", + "metric": "mapped_iterator", + "ms": 63.7299, + "checksum": 25556520 + }, + { + "pair": 2, + "side": "candidate", + "metric": "chunk_parallel", + "ms": 120.2332, + "checksum": 21007980 + }, + { + "pair": 3, + "side": "candidate", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 3, + "side": "candidate", + "metric": "clean_indexed", + "ms": 46.5295, + "checksum": 76666836 + }, + { + "pair": 3, + "side": "candidate", + "metric": "clean_iterator", + "ms": 62.4749, + "checksum": 25555612 + }, + { + "pair": 3, + "side": "candidate", + "metric": "clean_quoted_indexed", + "ms": 72.7694, + "checksum": 100666884 + }, + { + "pair": 3, + "side": "candidate", + "metric": "dirty_indexed", + "ms": 44.9264, + "checksum": 76669560 + }, + { + "pair": 3, + "side": "candidate", + "metric": "dirty_iterator", + "ms": 61.8533, + "checksum": 25556520 + }, + { + "pair": 3, + "side": "candidate", + "metric": "mapped_indexed", + "ms": 51.3345, + "checksum": 76669560 + }, + { + "pair": 3, + "side": "candidate", + "metric": "mapped_iterator", + "ms": 59.5951, + "checksum": 25556520 + }, + { + "pair": 3, + "side": "candidate", + "metric": "chunk_parallel", + "ms": 115.4544, + "checksum": 21007980 + }, + { + "pair": 3, + "side": "baseline", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 3, + "side": "baseline", + "metric": "clean_indexed", + "ms": 82.3419, + "checksum": 76666836 + }, + { + "pair": 3, + "side": "baseline", + "metric": "clean_iterator", + "ms": 61.8598, + "checksum": 25555612 + }, + { + "pair": 3, + "side": "baseline", + "metric": "clean_quoted_indexed", + "ms": 126.6269, + "checksum": 100666884 + }, + { + "pair": 3, + "side": "baseline", + "metric": "dirty_indexed", + "ms": 87.8002, + "checksum": 76669560 + }, + { + "pair": 3, + "side": "baseline", + "metric": "dirty_iterator", + "ms": 63.8521, + "checksum": 25556520 + }, + { + "pair": 3, + "side": "baseline", + "metric": "mapped_indexed", + "ms": 86.7878, + "checksum": 76669560 + }, + { + "pair": 3, + "side": "baseline", + "metric": "mapped_iterator", + "ms": 62.6873, + "checksum": 25556520 + }, + { + "pair": 3, + "side": "baseline", + "metric": "chunk_parallel", + "ms": 162.0011, + "checksum": 21007980 + }, + { + "pair": 4, + "side": "baseline", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 4, + "side": "baseline", + "metric": "clean_indexed", + "ms": 81.4786, + "checksum": 76666836 + }, + { + "pair": 4, + "side": "baseline", + "metric": "clean_iterator", + "ms": 62.3033, + "checksum": 25555612 + }, + { + "pair": 4, + "side": "baseline", + "metric": "clean_quoted_indexed", + "ms": 128.2576, + "checksum": 100666884 + }, + { + "pair": 4, + "side": "baseline", + "metric": "dirty_indexed", + "ms": 86.5854, + "checksum": 76669560 + }, + { + "pair": 4, + "side": "baseline", + "metric": "dirty_iterator", + "ms": 62.5954, + "checksum": 25556520 + }, + { + "pair": 4, + "side": "baseline", + "metric": "mapped_indexed", + "ms": 87.0274, + "checksum": 76669560 + }, + { + "pair": 4, + "side": "baseline", + "metric": "mapped_iterator", + "ms": 60.9933, + "checksum": 25556520 + }, + { + "pair": 4, + "side": "baseline", + "metric": "chunk_parallel", + "ms": 169.5046, + "checksum": 21007980 + }, + { + "pair": 4, + "side": "candidate", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 4, + "side": "candidate", + "metric": "clean_indexed", + "ms": 41.4912, + "checksum": 76666836 + }, + { + "pair": 4, + "side": "candidate", + "metric": "clean_iterator", + "ms": 58.2093, + "checksum": 25555612 + }, + { + "pair": 4, + "side": "candidate", + "metric": "clean_quoted_indexed", + "ms": 73.5064, + "checksum": 100666884 + }, + { + "pair": 4, + "side": "candidate", + "metric": "dirty_indexed", + "ms": 48.6395, + "checksum": 76669560 + }, + { + "pair": 4, + "side": "candidate", + "metric": "dirty_iterator", + "ms": 60.9346, + "checksum": 25556520 + }, + { + "pair": 4, + "side": "candidate", + "metric": "mapped_indexed", + "ms": 58.1039, + "checksum": 76669560 + }, + { + "pair": 4, + "side": "candidate", + "metric": "mapped_iterator", + "ms": 61.9691, + "checksum": 25556520 + }, + { + "pair": 4, + "side": "candidate", + "metric": "chunk_parallel", + "ms": 118.1028, + "checksum": 21007980 + }, + { + "pair": 5, + "side": "candidate", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 5, + "side": "candidate", + "metric": "clean_indexed", + "ms": 44.7658, + "checksum": 76666836 + }, + { + "pair": 5, + "side": "candidate", + "metric": "clean_iterator", + "ms": 62.7599, + "checksum": 25555612 + }, + { + "pair": 5, + "side": "candidate", + "metric": "clean_quoted_indexed", + "ms": 74.3427, + "checksum": 100666884 + }, + { + "pair": 5, + "side": "candidate", + "metric": "dirty_indexed", + "ms": 71.2611, + "checksum": 76669560 + }, + { + "pair": 5, + "side": "candidate", + "metric": "dirty_iterator", + "ms": 74.8841, + "checksum": 25556520 + }, + { + "pair": 5, + "side": "candidate", + "metric": "mapped_indexed", + "ms": 52.9142, + "checksum": 76669560 + }, + { + "pair": 5, + "side": "candidate", + "metric": "mapped_iterator", + "ms": 61.6841, + "checksum": 25556520 + }, + { + "pair": 5, + "side": "candidate", + "metric": "chunk_parallel", + "ms": 121.6521, + "checksum": 21007980 + }, + { + "pair": 5, + "side": "baseline", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 5, + "side": "baseline", + "metric": "clean_indexed", + "ms": 79.4975, + "checksum": 76666836 + }, + { + "pair": 5, + "side": "baseline", + "metric": "clean_iterator", + "ms": 62.8974, + "checksum": 25555612 + }, + { + "pair": 5, + "side": "baseline", + "metric": "clean_quoted_indexed", + "ms": 130.4737, + "checksum": 100666884 + }, + { + "pair": 5, + "side": "baseline", + "metric": "dirty_indexed", + "ms": 85.1837, + "checksum": 76669560 + }, + { + "pair": 5, + "side": "baseline", + "metric": "dirty_iterator", + "ms": 63.0006, + "checksum": 25556520 + }, + { + "pair": 5, + "side": "baseline", + "metric": "mapped_indexed", + "ms": 85.3658, + "checksum": 76669560 + }, + { + "pair": 5, + "side": "baseline", + "metric": "mapped_iterator", + "ms": 62.1051, + "checksum": 25556520 + }, + { + "pair": 5, + "side": "baseline", + "metric": "chunk_parallel", + "ms": 152.4847, + "checksum": 21007980 + }, + { + "pair": 6, + "side": "baseline", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 6, + "side": "baseline", + "metric": "clean_indexed", + "ms": 122.9126, + "checksum": 76666836 + }, + { + "pair": 6, + "side": "baseline", + "metric": "clean_iterator", + "ms": 63.5963, + "checksum": 25555612 + }, + { + "pair": 6, + "side": "baseline", + "metric": "clean_quoted_indexed", + "ms": 123.7773, + "checksum": 100666884 + }, + { + "pair": 6, + "side": "baseline", + "metric": "dirty_indexed", + "ms": 132.7802, + "checksum": 76669560 + }, + { + "pair": 6, + "side": "baseline", + "metric": "dirty_iterator", + "ms": 66.6538, + "checksum": 25556520 + }, + { + "pair": 6, + "side": "baseline", + "metric": "mapped_indexed", + "ms": 127.8014, + "checksum": 76669560 + }, + { + "pair": 6, + "side": "baseline", + "metric": "mapped_iterator", + "ms": 65.3799, + "checksum": 25556520 + }, + { + "pair": 6, + "side": "baseline", + "metric": "chunk_parallel", + "ms": 142.5849, + "checksum": 21007980 + }, + { + "pair": 6, + "side": "candidate", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 6, + "side": "candidate", + "metric": "clean_indexed", + "ms": 44.0467, + "checksum": 76666836 + }, + { + "pair": 6, + "side": "candidate", + "metric": "clean_iterator", + "ms": 62.0897, + "checksum": 25555612 + }, + { + "pair": 6, + "side": "candidate", + "metric": "clean_quoted_indexed", + "ms": 74.2583, + "checksum": 100666884 + }, + { + "pair": 6, + "side": "candidate", + "metric": "dirty_indexed", + "ms": 49.8976, + "checksum": 76669560 + }, + { + "pair": 6, + "side": "candidate", + "metric": "dirty_iterator", + "ms": 64.0686, + "checksum": 25556520 + }, + { + "pair": 6, + "side": "candidate", + "metric": "mapped_indexed", + "ms": 52.646, + "checksum": 76669560 + }, + { + "pair": 6, + "side": "candidate", + "metric": "mapped_iterator", + "ms": 64.2999, + "checksum": 25556520 + }, + { + "pair": 6, + "side": "candidate", + "metric": "chunk_parallel", + "ms": 111.1839, + "checksum": 21007980 + } +] \ No newline at end of file diff --git a/include/internal/data_frame/performance/direct-header-summary.csv b/include/internal/data_frame/performance/direct-header-summary.csv new file mode 100644 index 00000000..900ef2e9 --- /dev/null +++ b/include/internal/data_frame/performance/direct-header-summary.csv @@ -0,0 +1,10 @@ +metric,baseline_median,candidate_median,change_percent,baseline_min,baseline_max,candidate_min,candidate_max,baseline_iqr,candidate_iqr,checksum +layout_dfc,48.0,48.0,0.0,48.0,48.0,48.0,48.0,0.0,0.0,0 +clean_indexed,82.3419,44.0467,-46.5075496193311,79.4975,122.9126,41.4912,48.0208,9.860799999999998,4.236199999999997,76666836 +clean_iterator,62.8974,62.4749,-0.671728879095157,60.3223,67.3415,58.2093,65.7639,2.5834000000000046,2.4717000000000056,25555612 +clean_quoted_indexed,128.2576,74.3427,-42.036417335113086,123.7773,130.4737,72.7694,77.2193,3.249300000000005,3.275800000000004,100666884 +dirty_indexed,87.8002,49.535,-43.582133070311926,85.1837,132.7802,44.9264,71.2611,5.2749999999999915,2.4543999999999997,76669560 +dirty_iterator,63.0006,62.3675,-1.0049110643390646,61.8485,66.6538,60.263,74.8841,2.6829000000000036,3.1340000000000003,25556520 +mapped_indexed,87.0274,52.646,-39.50640832657301,85.3658,127.8014,50.4738,58.1039,3.710099999999997,1.6861999999999995,76669560 +mapped_iterator,62.6873,62.8704,0.29208468062909,60.9933,69.4281,59.5951,64.2999,4.159000000000006,2.0458,25556520 +chunk_parallel,159.3291,120.2332,-24.537827678685197,142.5849,169.5046,111.1839,136.0833,17.700000000000017,6.1976999999999975,21007980 diff --git a/include/internal/data_frame/performance/direct-library-samples.json b/include/internal/data_frame/performance/direct-library-samples.json new file mode 100644 index 00000000..035e27e4 --- /dev/null +++ b/include/internal/data_frame/performance/direct-library-samples.json @@ -0,0 +1,884 @@ +[ + { + "pair": 0, + "side": "baseline", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 0, + "side": "baseline", + "metric": "clean_indexed", + "ms": 81.4832, + "checksum": 76666836 + }, + { + "pair": 0, + "side": "baseline", + "metric": "clean_iterator", + "ms": 60.4561, + "checksum": 25555612 + }, + { + "pair": 0, + "side": "baseline", + "metric": "clean_quoted_indexed", + "ms": 95.274, + "checksum": 100666884 + }, + { + "pair": 0, + "side": "baseline", + "metric": "dirty_indexed", + "ms": 89.7099, + "checksum": 76669560 + }, + { + "pair": 0, + "side": "baseline", + "metric": "dirty_iterator", + "ms": 62.5834, + "checksum": 25556520 + }, + { + "pair": 0, + "side": "baseline", + "metric": "mapped_indexed", + "ms": 86.6604, + "checksum": 76669560 + }, + { + "pair": 0, + "side": "baseline", + "metric": "mapped_iterator", + "ms": 62.2849, + "checksum": 25556520 + }, + { + "pair": 0, + "side": "baseline", + "metric": "chunk_parallel", + "ms": 180.3432, + "checksum": 21007980 + }, + { + "pair": 0, + "side": "candidate", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 0, + "side": "candidate", + "metric": "clean_indexed", + "ms": 51.4364, + "checksum": 76666836 + }, + { + "pair": 0, + "side": "candidate", + "metric": "clean_iterator", + "ms": 62.5339, + "checksum": 25555612 + }, + { + "pair": 0, + "side": "candidate", + "metric": "clean_quoted_indexed", + "ms": 77.4908, + "checksum": 100666884 + }, + { + "pair": 0, + "side": "candidate", + "metric": "dirty_indexed", + "ms": 53.6648, + "checksum": 76669560 + }, + { + "pair": 0, + "side": "candidate", + "metric": "dirty_iterator", + "ms": 68.276, + "checksum": 25556520 + }, + { + "pair": 0, + "side": "candidate", + "metric": "mapped_indexed", + "ms": 54.0978, + "checksum": 76669560 + }, + { + "pair": 0, + "side": "candidate", + "metric": "mapped_iterator", + "ms": 61.8363, + "checksum": 25556520 + }, + { + "pair": 0, + "side": "candidate", + "metric": "chunk_parallel", + "ms": 145.4327, + "checksum": 21007980 + }, + { + "pair": 1, + "side": "candidate", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 1, + "side": "candidate", + "metric": "clean_indexed", + "ms": 49.4033, + "checksum": 76666836 + }, + { + "pair": 1, + "side": "candidate", + "metric": "clean_iterator", + "ms": 64.2089, + "checksum": 25555612 + }, + { + "pair": 1, + "side": "candidate", + "metric": "clean_quoted_indexed", + "ms": 75.8269, + "checksum": 100666884 + }, + { + "pair": 1, + "side": "candidate", + "metric": "dirty_indexed", + "ms": 57.4458, + "checksum": 76669560 + }, + { + "pair": 1, + "side": "candidate", + "metric": "dirty_iterator", + "ms": 61.7057, + "checksum": 25556520 + }, + { + "pair": 1, + "side": "candidate", + "metric": "mapped_indexed", + "ms": 51.8408, + "checksum": 76669560 + }, + { + "pair": 1, + "side": "candidate", + "metric": "mapped_iterator", + "ms": 62.2153, + "checksum": 25556520 + }, + { + "pair": 1, + "side": "candidate", + "metric": "chunk_parallel", + "ms": 129.7577, + "checksum": 21007980 + }, + { + "pair": 1, + "side": "baseline", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 1, + "side": "baseline", + "metric": "clean_indexed", + "ms": 77.1536, + "checksum": 76666836 + }, + { + "pair": 1, + "side": "baseline", + "metric": "clean_iterator", + "ms": 57.3018, + "checksum": 25555612 + }, + { + "pair": 1, + "side": "baseline", + "metric": "clean_quoted_indexed", + "ms": 93.7211, + "checksum": 100666884 + }, + { + "pair": 1, + "side": "baseline", + "metric": "dirty_indexed", + "ms": 84.733, + "checksum": 76669560 + }, + { + "pair": 1, + "side": "baseline", + "metric": "dirty_iterator", + "ms": 65.5025, + "checksum": 25556520 + }, + { + "pair": 1, + "side": "baseline", + "metric": "mapped_indexed", + "ms": 88.6847, + "checksum": 76669560 + }, + { + "pair": 1, + "side": "baseline", + "metric": "mapped_iterator", + "ms": 66.2181, + "checksum": 25556520 + }, + { + "pair": 1, + "side": "baseline", + "metric": "chunk_parallel", + "ms": 155.0051, + "checksum": 21007980 + }, + { + "pair": 2, + "side": "baseline", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 2, + "side": "baseline", + "metric": "clean_indexed", + "ms": 95.6338, + "checksum": 76666836 + }, + { + "pair": 2, + "side": "baseline", + "metric": "clean_iterator", + "ms": 64.7396, + "checksum": 25555612 + }, + { + "pair": 2, + "side": "baseline", + "metric": "clean_quoted_indexed", + "ms": 90.148, + "checksum": 100666884 + }, + { + "pair": 2, + "side": "baseline", + "metric": "dirty_indexed", + "ms": 87.4327, + "checksum": 76669560 + }, + { + "pair": 2, + "side": "baseline", + "metric": "dirty_iterator", + "ms": 66.6726, + "checksum": 25556520 + }, + { + "pair": 2, + "side": "baseline", + "metric": "mapped_indexed", + "ms": 91.7942, + "checksum": 76669560 + }, + { + "pair": 2, + "side": "baseline", + "metric": "mapped_iterator", + "ms": 61.8414, + "checksum": 25556520 + }, + { + "pair": 2, + "side": "baseline", + "metric": "chunk_parallel", + "ms": 160.4757, + "checksum": 21007980 + }, + { + "pair": 2, + "side": "candidate", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 2, + "side": "candidate", + "metric": "clean_indexed", + "ms": 46.0401, + "checksum": 76666836 + }, + { + "pair": 2, + "side": "candidate", + "metric": "clean_iterator", + "ms": 61.198, + "checksum": 25555612 + }, + { + "pair": 2, + "side": "candidate", + "metric": "clean_quoted_indexed", + "ms": 80.386, + "checksum": 100666884 + }, + { + "pair": 2, + "side": "candidate", + "metric": "dirty_indexed", + "ms": 49.9958, + "checksum": 76669560 + }, + { + "pair": 2, + "side": "candidate", + "metric": "dirty_iterator", + "ms": 62.2209, + "checksum": 25556520 + }, + { + "pair": 2, + "side": "candidate", + "metric": "mapped_indexed", + "ms": 57.2916, + "checksum": 76669560 + }, + { + "pair": 2, + "side": "candidate", + "metric": "mapped_iterator", + "ms": 59.8991, + "checksum": 25556520 + }, + { + "pair": 2, + "side": "candidate", + "metric": "chunk_parallel", + "ms": 117.0433, + "checksum": 21007980 + }, + { + "pair": 3, + "side": "candidate", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 3, + "side": "candidate", + "metric": "clean_indexed", + "ms": 51.7975, + "checksum": 76666836 + }, + { + "pair": 3, + "side": "candidate", + "metric": "clean_iterator", + "ms": 74.8575, + "checksum": 25555612 + }, + { + "pair": 3, + "side": "candidate", + "metric": "clean_quoted_indexed", + "ms": 77.4408, + "checksum": 100666884 + }, + { + "pair": 3, + "side": "candidate", + "metric": "dirty_indexed", + "ms": 64.375, + "checksum": 76669560 + }, + { + "pair": 3, + "side": "candidate", + "metric": "dirty_iterator", + "ms": 62.677, + "checksum": 25556520 + }, + { + "pair": 3, + "side": "candidate", + "metric": "mapped_indexed", + "ms": 53.584, + "checksum": 76669560 + }, + { + "pair": 3, + "side": "candidate", + "metric": "mapped_iterator", + "ms": 61.6691, + "checksum": 25556520 + }, + { + "pair": 3, + "side": "candidate", + "metric": "chunk_parallel", + "ms": 130.2794, + "checksum": 21007980 + }, + { + "pair": 3, + "side": "baseline", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 3, + "side": "baseline", + "metric": "clean_indexed", + "ms": 80.4935, + "checksum": 76666836 + }, + { + "pair": 3, + "side": "baseline", + "metric": "clean_iterator", + "ms": 61.2479, + "checksum": 25555612 + }, + { + "pair": 3, + "side": "baseline", + "metric": "clean_quoted_indexed", + "ms": 90.6001, + "checksum": 100666884 + }, + { + "pair": 3, + "side": "baseline", + "metric": "dirty_indexed", + "ms": 82.4714, + "checksum": 76669560 + }, + { + "pair": 3, + "side": "baseline", + "metric": "dirty_iterator", + "ms": 57.3193, + "checksum": 25556520 + }, + { + "pair": 3, + "side": "baseline", + "metric": "mapped_indexed", + "ms": 86.7973, + "checksum": 76669560 + }, + { + "pair": 3, + "side": "baseline", + "metric": "mapped_iterator", + "ms": 58.6402, + "checksum": 25556520 + }, + { + "pair": 3, + "side": "baseline", + "metric": "chunk_parallel", + "ms": 164.4455, + "checksum": 21007980 + }, + { + "pair": 4, + "side": "baseline", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 4, + "side": "baseline", + "metric": "clean_indexed", + "ms": 82.1476, + "checksum": 76666836 + }, + { + "pair": 4, + "side": "baseline", + "metric": "clean_iterator", + "ms": 57.9454, + "checksum": 25555612 + }, + { + "pair": 4, + "side": "baseline", + "metric": "clean_quoted_indexed", + "ms": 88.3762, + "checksum": 100666884 + }, + { + "pair": 4, + "side": "baseline", + "metric": "dirty_indexed", + "ms": 84.4099, + "checksum": 76669560 + }, + { + "pair": 4, + "side": "baseline", + "metric": "dirty_iterator", + "ms": 58.9203, + "checksum": 25556520 + }, + { + "pair": 4, + "side": "baseline", + "metric": "mapped_indexed", + "ms": 83.3691, + "checksum": 76669560 + }, + { + "pair": 4, + "side": "baseline", + "metric": "mapped_iterator", + "ms": 61.5866, + "checksum": 25556520 + }, + { + "pair": 4, + "side": "baseline", + "metric": "chunk_parallel", + "ms": 175.4507, + "checksum": 21007980 + }, + { + "pair": 4, + "side": "candidate", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 4, + "side": "candidate", + "metric": "clean_indexed", + "ms": 45.3533, + "checksum": 76666836 + }, + { + "pair": 4, + "side": "candidate", + "metric": "clean_iterator", + "ms": 59.8365, + "checksum": 25555612 + }, + { + "pair": 4, + "side": "candidate", + "metric": "clean_quoted_indexed", + "ms": 78.0938, + "checksum": 100666884 + }, + { + "pair": 4, + "side": "candidate", + "metric": "dirty_indexed", + "ms": 49.6732, + "checksum": 76669560 + }, + { + "pair": 4, + "side": "candidate", + "metric": "dirty_iterator", + "ms": 61.254, + "checksum": 25556520 + }, + { + "pair": 4, + "side": "candidate", + "metric": "mapped_indexed", + "ms": 52.809, + "checksum": 76669560 + }, + { + "pair": 4, + "side": "candidate", + "metric": "mapped_iterator", + "ms": 68.0847, + "checksum": 25556520 + }, + { + "pair": 4, + "side": "candidate", + "metric": "chunk_parallel", + "ms": 124.0867, + "checksum": 21007980 + }, + { + "pair": 5, + "side": "candidate", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 5, + "side": "candidate", + "metric": "clean_indexed", + "ms": 49.047, + "checksum": 76666836 + }, + { + "pair": 5, + "side": "candidate", + "metric": "clean_iterator", + "ms": 57.9701, + "checksum": 25555612 + }, + { + "pair": 5, + "side": "candidate", + "metric": "clean_quoted_indexed", + "ms": 79.1299, + "checksum": 100666884 + }, + { + "pair": 5, + "side": "candidate", + "metric": "dirty_indexed", + "ms": 53.1781, + "checksum": 76669560 + }, + { + "pair": 5, + "side": "candidate", + "metric": "dirty_iterator", + "ms": 80.5874, + "checksum": 25556520 + }, + { + "pair": 5, + "side": "candidate", + "metric": "mapped_indexed", + "ms": 77.3931, + "checksum": 76669560 + }, + { + "pair": 5, + "side": "candidate", + "metric": "mapped_iterator", + "ms": 62.4743, + "checksum": 25556520 + }, + { + "pair": 5, + "side": "candidate", + "metric": "chunk_parallel", + "ms": 139.1549, + "checksum": 21007980 + }, + { + "pair": 5, + "side": "baseline", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 5, + "side": "baseline", + "metric": "clean_indexed", + "ms": 81.5087, + "checksum": 76666836 + }, + { + "pair": 5, + "side": "baseline", + "metric": "clean_iterator", + "ms": 62.4591, + "checksum": 25555612 + }, + { + "pair": 5, + "side": "baseline", + "metric": "clean_quoted_indexed", + "ms": 94.7737, + "checksum": 100666884 + }, + { + "pair": 5, + "side": "baseline", + "metric": "dirty_indexed", + "ms": 96.3079, + "checksum": 76669560 + }, + { + "pair": 5, + "side": "baseline", + "metric": "dirty_iterator", + "ms": 57.9784, + "checksum": 25556520 + }, + { + "pair": 5, + "side": "baseline", + "metric": "mapped_indexed", + "ms": 90.5802, + "checksum": 76669560 + }, + { + "pair": 5, + "side": "baseline", + "metric": "mapped_iterator", + "ms": 60.9193, + "checksum": 25556520 + }, + { + "pair": 5, + "side": "baseline", + "metric": "chunk_parallel", + "ms": 195.7814, + "checksum": 21007980 + }, + { + "pair": 6, + "side": "baseline", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 6, + "side": "baseline", + "metric": "clean_indexed", + "ms": 79.0901, + "checksum": 76666836 + }, + { + "pair": 6, + "side": "baseline", + "metric": "clean_iterator", + "ms": 61.8569, + "checksum": 25555612 + }, + { + "pair": 6, + "side": "baseline", + "metric": "clean_quoted_indexed", + "ms": 89.1303, + "checksum": 100666884 + }, + { + "pair": 6, + "side": "baseline", + "metric": "dirty_indexed", + "ms": 86.0969, + "checksum": 76669560 + }, + { + "pair": 6, + "side": "baseline", + "metric": "dirty_iterator", + "ms": 60.2804, + "checksum": 25556520 + }, + { + "pair": 6, + "side": "baseline", + "metric": "mapped_indexed", + "ms": 85.6166, + "checksum": 76669560 + }, + { + "pair": 6, + "side": "baseline", + "metric": "mapped_iterator", + "ms": 68.7473, + "checksum": 25556520 + }, + { + "pair": 6, + "side": "baseline", + "metric": "chunk_parallel", + "ms": 185.1495, + "checksum": 21007980 + }, + { + "pair": 6, + "side": "candidate", + "metric": "layout_dfc", + "ms": 48.0, + "checksum": 0 + }, + { + "pair": 6, + "side": "candidate", + "metric": "clean_indexed", + "ms": 50.9871, + "checksum": 76666836 + }, + { + "pair": 6, + "side": "candidate", + "metric": "clean_iterator", + "ms": 61.7581, + "checksum": 25555612 + }, + { + "pair": 6, + "side": "candidate", + "metric": "clean_quoted_indexed", + "ms": 83.2574, + "checksum": 100666884 + }, + { + "pair": 6, + "side": "candidate", + "metric": "dirty_indexed", + "ms": 58.2018, + "checksum": 76669560 + }, + { + "pair": 6, + "side": "candidate", + "metric": "dirty_iterator", + "ms": 66.4012, + "checksum": 25556520 + }, + { + "pair": 6, + "side": "candidate", + "metric": "mapped_indexed", + "ms": 53.2425, + "checksum": 76669560 + }, + { + "pair": 6, + "side": "candidate", + "metric": "mapped_iterator", + "ms": 59.5207, + "checksum": 25556520 + }, + { + "pair": 6, + "side": "candidate", + "metric": "chunk_parallel", + "ms": 135.7137, + "checksum": 21007980 + } +] \ No newline at end of file diff --git a/include/internal/data_frame/performance/direct-library-summary.csv b/include/internal/data_frame/performance/direct-library-summary.csv new file mode 100644 index 00000000..14ffee67 --- /dev/null +++ b/include/internal/data_frame/performance/direct-library-summary.csv @@ -0,0 +1,10 @@ +metric,baseline_median,candidate_median,change_percent,baseline_min,baseline_max,candidate_min,candidate_max,baseline_iqr,candidate_iqr,checksum +layout_dfc,48.0,48.0,0.0,48.0,48.0,48.0,48.0,0.0,0.0,0 +clean_indexed,81.4832,49.4033,-39.36995601547313,77.1536,95.6338,45.3533,51.7975,3.0574999999999903,5.3962999999999965,76666836 +clean_iterator,61.2479,61.7581,0.8330081521162214,57.3018,64.7396,57.9701,74.8575,4.5137,4.372399999999999,25555612 +clean_quoted_indexed,90.6001,78.0938,-13.803847898622623,88.3762,95.274,75.8269,83.2574,5.6434,2.9452,100666884 +dirty_indexed,86.0969,53.6648,-37.66930052069238,82.4714,96.3079,49.6732,64.375,5.300000000000011,8.205999999999996,76669560 +dirty_iterator,60.2804,62.677,3.975753312851271,57.3193,66.6726,61.254,80.5874,7.524099999999997,6.570299999999996,25556520 +mapped_indexed,86.7973,53.584,-38.26536078887247,83.3691,91.7942,51.8408,77.3931,4.9636,4.482600000000005,76669560 +mapped_iterator,61.8414,61.8363,-0.008246902560415936,58.6402,68.7473,59.5207,68.0847,5.298800000000007,2.5752000000000024,25556520 +chunk_parallel,175.4507,130.2794,-25.74586479278794,155.0051,195.7814,117.0433,145.4327,24.6738,15.068200000000004,21007980 diff --git a/include/internal/data_frame/row_storage.hpp b/include/internal/data_frame/row_storage.hpp new file mode 100644 index 00000000..4ecdb7c1 --- /dev/null +++ b/include/internal/data_frame/row_storage.hpp @@ -0,0 +1,122 @@ +#pragma once + +#include +#include +#include +#include "../csv_exceptions.hpp" +#include "../csv_row.hpp" +#include "../raw_csv_data.hpp" +#include "row_overlay.hpp" + +namespace csv { + namespace internals { + namespace data_frame { + // Concrete backend identity only: deliberately no virtual functions. + struct RowBackend {}; + + struct CellBinding { + const RowOverlay* overlay; + size_t physical_column; + }; + + template + inline void reserve_next(Container& values) { + const size_t required = values.size() + 1; + if (required > values.capacity()) { + values.reserve(std::max(required, values.capacity() * 2)); + } + } + + inline void validate_column(size_t index, size_t count) { + if (index >= count) { throw_column_index_out_of_range(); } + } + + inline void validate_row(const std::vector& row, size_t columns) { + if (columns == 0 && !row.empty()) { + throw std::invalid_argument("cannot insert a non-empty row into a DataFrame without columns"); + } + if (row.size() != columns) { + throw std::invalid_argument("inserted row field count must match DataFrame column count"); + } + } + + inline std::vector select_parsed_rows( + const std::vector& rows, const std::vector& mask + ) { + if (mask.size() != rows.size()) { + throw std::invalid_argument("selected row mask size must match DataFrame row count"); + } + std::vector selected; + selected.reserve(static_cast(std::count_if(mask.begin(), mask.end(), + [](std::uint8_t value) { return value != 0; }))); + for (size_t i = 0; i < rows.size(); ++i) { + if (mask[i]) { selected.push_back(rows[i]); } + } + return selected; + } + + /** Shared lossless owned-row construction for insertion and materialization. */ + class RowStorageBuilder { + public: + explicit inline RowStorageBuilder(ConstColNamesPtr names) : names_(std::move(names)) {} + + inline void begin_row() { + // Offsets are chunk-local. Rotate between rows instead of accumulating + // arbitrarily large offsets when rebuilding a large table. + if (!data_ || bytes_ >= CSV_CHUNK_SIZE_DEFAULT || data_->fields.size() >= CSV_CHUNK_SIZE_DEFAULT) { + data_ = std::make_shared(); + data_->col_names = std::const_pointer_cast(names_); + bytes_ = 0; + } + field_start_ = data_->fields.size(); + data_->fields.reserve_for_source_size(field_start_ + names_->size()); + } + + inline void append_field(csv::string_view value) { + const size_t offset = data_->quote_arena.append(value); + data_->fields.emplace_back(offset, value.size(), true); + bytes_ += value.size(); + } + + inline CSVRow finish_row() const { + return CSVRow(data_, 0, field_start_, data_->fields.size() - field_start_); + } + + static inline CSVRow make_row(const std::vector& values, ConstColNamesPtr names) { + RowStorageBuilder storage(std::move(names)); + storage.begin_row(); + for (const auto& value : values) { storage.append_field(csv::string_view(value)); } + return storage.finish_row(); + } + + template + static inline std::vector insert_column( + size_t count, ConstColNamesPtr names, size_t index, + const std::string& default_value, const FieldAt& field_at + ) { + std::vector rebuilt; + rebuilt.reserve(count); + RowStorageBuilder storage(names); + for (size_t row = 0; row < count; ++row) { + storage.begin_row(); + for (size_t column = 0; column < names->size(); ++column) { + if (column == index) { storage.append_field(csv::string_view(default_value)); } + else { + const auto value = field_at(row, column < index ? column : column - 1); + storage.append_field(csv::string_view(value)); + } + } + rebuilt.push_back(storage.finish_row()); + } + return rebuilt; + } + + private: + ConstColNamesPtr names_; + RawCSVDataPtr data_; + size_t bytes_ = 0; + size_t field_start_ = 0; + }; + } + } +} diff --git a/include/internal/data_frame/row_view.hpp b/include/internal/data_frame/row_view.hpp new file mode 100644 index 00000000..dee2cbf9 --- /dev/null +++ b/include/internal/data_frame/row_view.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include "../csv_row.hpp" + +namespace csv { + namespace internals { + namespace data_frame { + // Read the same decoded, trimmed field bytes without scalar metadata. + struct RowViewAccessor { + static inline csv::string_view view(const CSVRow& row, size_t column) { + return row.get_field_impl(column, row.data); + } + }; + } + } +} diff --git a/tests/AGENTS.md b/tests/AGENTS.md index 88f0595c..6b66cabd 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -263,6 +263,13 @@ TEST_CASE("Multithreaded parsing", "[threading]") { - **test_csv_row_json.cpp**: JSON export functionality +- **test_data_frame.cpp**: General DataFrame API and structural operations + +- **test_data_frame_dirty.cpp**: Clean/dirty transitions, edit preservation, + proxy visibility, and structural composition regressions + +- **test_data_frame_etl.cpp**: DataFrame batch processing and parallel editing + ### Key Patterns 1. **Validation Lambdas**: Write once, test both paths diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d8e44c92..8434de8d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -105,5 +105,6 @@ add_csv_parser_test_target(csv_write_test add_csv_parser_test_target(data_frame_test test_data_frame.cpp + test_data_frame_dirty.cpp test_data_frame_etl.cpp ) diff --git a/tests/test_data_frame.cpp b/tests/test_data_frame.cpp index 6db54e0e..6e5fdd28 100644 --- a/tests/test_data_frame.cpp +++ b/tests/test_data_frame.cpp @@ -947,107 +947,6 @@ TEST_CASE("DataFrame: const row erase is rejected", "[data_frame]") { REQUIRE_THROWS_AS(row["name"] = "Blocked", std::runtime_error); } -TEST_CASE("DataFrame: selected rows preserve independent sparse edits", "[data_frame][selected_rows]") { - std::istringstream input( - "id,name,value\n" - "1,Alice,10\n" - "2,Bob,20\n" - "3,Carol,30\n" - "4,David,40\n" - ); - CSVReader reader(input); - DataFrame<> frame; - SECTION("Unkeyed source") { frame = DataFrame<>(reader); } - SECTION("Keyed source") { frame = DataFrame<>(reader, "id"); } - - frame.at(0)["name"] = "Excluded"; - frame.at(1)["name"] = "Robert"; - frame.at(1)["value"] = "95"; - frame.at(3)["value"] = "45"; - - const auto& source = frame; - auto selected = source.selected_rows({0, 1, 0, 2}); - REQUIRE(selected.size() == 2); - REQUIRE(selected.columns() == frame.columns()); - REQUIRE_THROWS_AS(selected["2"], std::runtime_error); - REQUIRE(std::vector(selected.at(0)) == std::vector{"2", "Robert", "95"}); - REQUIRE(std::vector(selected.at(1)) == std::vector{"4", "David", "45"}); - REQUIRE(selected.column("value") == std::vector{"95", "45"}); - REQUIRE(selected.column_view(1)[0].get() == "Robert"); - REQUIRE(selected.at(0).to_json() == "{\"id\":2,\"name\":\"Robert\",\"value\":95}"); - REQUIRE(selected.at(0).to_json_array() == "[2,\"Robert\",95]"); - - // Selection should still share parsed bytes, not materialize edited rows. - REQUIRE(selected.at(0).get_underlying_row()["name"].get_sv().data() - == source.at(1).get_underlying_row()["name"].get_sv().data()); - - std::stringstream output; - auto writer = make_csv_writer(output); - writer << selected.columns(); - for (const auto& row : selected) { -#ifdef CSV_HAS_CXX20 - writer << row; -#else - writer << std::vector(row); -#endif - } - REQUIRE(output.str() == "id,name,value\n2,Robert,95\n4,David,45\n"); - - const auto& const_selected = selected; - REQUIRE_THROWS_AS(const_selected.at(0)["name"] = "Blocked", std::runtime_error); - REQUIRE_THROWS_AS(const_selected.column_view("name")[0] = "Blocked", std::runtime_error); - - // Edits on either frame must not leak into the other frame's overlay. - frame.at(1)["name"] = "Source only"; - REQUIRE(selected.at(0)["name"].get() == "Robert"); - selected.at(0)["value"] = "100"; - REQUIRE(frame.at(1)["value"].get() == "95"); - - auto repeated = selected.selected_rows({1, 0}); - REQUIRE(std::vector(repeated.at(0)) == std::vector{"2", "Robert", "100"}); - selected.at(0)["name"] = "Selected only"; - REQUIRE(repeated.at(0)["name"].get() == "Robert"); -} - -TEST_CASE("DataFrame: selected rows preserve physical edit indices after structural changes", "[data_frame][selected_rows]") { - std::istringstream input("id,name,value\n1,Alice,10\n2,Bob,20\n3,Carol,30\n"); - CSVFormat format; - format.column_names_policy(ColumnNamePolicy::CASE_INSENSITIVE); - CSVReader reader(input, format); - DataFrame<> frame(reader, "id"); - frame.at(0)["name"] = "Hidden"; - frame.at(1)["value"] = "95"; - frame.at(2)["value"] = ""; - REQUIRE(frame.column_view("name").erase()); - REQUIRE(frame.at(0).erase()); - frame.insert_row(0, {"4", "40"}); - - auto selected = frame.selected_rows({0, 1, 1}); - REQUIRE(selected.columns() == std::vector{"id", "value"}); - REQUIRE(selected.index_of("VALUE") == 1); - REQUIRE_FALSE(selected.has_column("name")); - REQUIRE(std::vector(selected.at(0)) == std::vector{"2", "95"}); - REQUIRE(std::vector(selected.at(1)) == std::vector{"3", ""}); - REQUIRE(selected.column_view(1)[0].get() == "95"); - REQUIRE(selected.at(0).to_json() == "{\"id\":2,\"value\":95}"); - REQUIRE(selected.at(1)["VALUE"].get().empty()); - - std::stringstream output; - auto writer = make_csv_writer(output); - writer << selected.columns(); - for (const auto& row : selected) { -#ifdef CSV_HAS_CXX20 - writer << row; -#else - writer << std::vector(row); -#endif - } - REQUIRE(output.str() == "id,value\n2,95\n3,\n"); - - auto repeated = selected.selected_rows({0, 1}); - REQUIRE(std::vector(repeated.at(0)) == std::vector{"3", ""}); -} - TEST_CASE("DataFrame: selected row masks keep boundary behavior", "[data_frame][selected_rows]") { auto input = make_people_stream(); CSVReader reader(input); diff --git a/tests/test_data_frame_dirty.cpp b/tests/test_data_frame_dirty.cpp new file mode 100644 index 00000000..2cabc7cd --- /dev/null +++ b/tests/test_data_frame_dirty.cpp @@ -0,0 +1,607 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include "csv.hpp" +#include "shared/generated_file.hpp" +#include "shared/timeout_helper.hpp" + +using namespace csv; + +namespace { + const std::string& issue_333_file() { + static csv_test::GeneratedFile file("csv_parser_issue_333.csv"); + return file.path([](std::ofstream& out) { + out << "id,name,value\n"; + for (size_t i = 1; i <= 500001; ++i) { + out << i << ",person_" << i << ',' << i * 7 << '\n'; + } + }); + } +} + +TEST_CASE("DataFrame: const row proxies observe first and subsequent edits - issue 333", + "[data_frame][issue_333]") { + const auto& filename = issue_333_file(); + auto validate = [](CSVReader& reader) { + DataFrame<> frame(reader); + REQUIRE(frame.size() == 500001); + const auto& source = frame; + auto first = source.at(0); + const auto last = source.at(500000); + auto names = source.column_view("name"); + auto pending = frame.at(100)["name"]; + auto copied = pending; + DataFrameCell assigned; + assigned = pending; + DataFrameCell moved(std::move(pending)); + + // These proxies predate both the dirty transition and their row overlays. + frame.at(0)["name"] = "first edit"; + frame.at(500000)["name"] = "last edit"; + CHECK(first["name"].get() == "first edit"); + CHECK(last["name"].get() == "last edit"); + CHECK(names[0].get() == "first edit"); + CHECK(names.get_sv(500000) == "last edit"); + copied = "copied cell"; + assigned = "assigned cell"; + moved = "moved cell"; + CHECK(source.at(100)["name"].get() == "moved cell"); + + auto after = source.at(0); + frame.at(0)["value"] = "changed value"; + CHECK(first["value"].get() == "changed value"); + CHECK(after["value"].get() == "changed value"); + CHECK(std::vector(first) == + std::vector{"1", "first edit", "changed value"}); + CHECK(first.to_json() == "{\"id\":1,\"name\":\"first edit\",\"value\":\"changed value\"}"); + CHECK(first.get_underlying_row()["name"].get() == "person_1"); + CHECK_THROWS_AS(first["name"] = "blocked", std::runtime_error); + DataFrame<> moved_frame(std::move(frame)); + moved_frame.at(500000)["value"] = "after frame move"; + CHECK(moved_frame.at(500000)["value"].get() == "after frame move"); + DataFrame<> assigned_frame; + assigned_frame = std::move(moved_frame); + assigned_frame.at(0)["value"] = "after frame assignment"; + CHECK(assigned_frame.at(0)["value"].get() == "after frame assignment"); + }; + SECTION("mmap") { + CSVReader reader(filename, CSVFormat()); + validate(reader); + } + SECTION("stream") { + std::ifstream input(filename, std::ios::binary); + CSVReader reader(input, CSVFormat()); + validate(reader); + } +} + +TEST_CASE("DataFrame: column rebuild preserves established typed keys - issue 333", + "[data_frame][issue_333]") { + const auto& filename = issue_333_file(); + const bool custom_keys = GENERATE(false, true); + auto validate = [custom_keys](CSVReader& reader) { + DataFrame frame = custom_keys + ? DataFrame(reader, [](const CSVRow& row) { return row["id"].get(); }) + : DataFrame(reader, "id"); + REQUIRE(frame.size() == 500001); + // Duplicate and unconvertible edited values must not redefine row identity. + frame.at(0)["id"] = "2"; + frame.at(1)["id"] = "not an integer"; + frame.at(500000)["name"] = "line\rbreak\n\"quoted\",tail"; + REQUIRE(frame.contains(1)); + REQUIRE(frame.contains(2)); + REQUIRE_NOTHROW(frame.insert_column(0, "extra", "constant")); + REQUIRE(frame.size() == 500001); + CHECK(frame.at(0).key() == 1); + CHECK(frame.at(1).key() == 2); + CHECK(frame[1]["id"].get() == "2"); + CHECK(frame[2]["id"].get() == "not an integer"); + CHECK(frame[500001]["name"].get() == "line\rbreak\n\"quoted\",tail"); + CHECK_THROWS_AS(frame.insert_row(0, {"constant", "1", "duplicate", "0"}), std::runtime_error); + REQUIRE(frame.column_view("name").erase()); + REQUIRE_NOTHROW(frame.append_column("tail")); + CHECK(frame[1]["id"].get() == "2"); + CHECK(frame[2]["value"].get() == 14); + CHECK(frame[500001]["value"].get() == 3500007); + if (!custom_keys) { + CHECK_THROWS_AS(frame.column_view("id").erase(), std::runtime_error); + } + }; + SECTION("mmap") { + CSVReader reader(filename, CSVFormat()); + validate(reader); + } + SECTION("stream") { + std::ifstream input(filename, std::ios::binary); + CSVReader reader(input, CSVFormat()); + validate(reader); + } +} + +TEST_CASE("DataFrame: empty column insertion retains zero-column rows - issue 333", + "[data_frame][issue_333]") { + DataFrame<> frame; + frame.insert_row(0, {}); + frame.insert_row(1, {}); + REQUIRE(frame.size() == 2); + std::string default_value; + SECTION("empty default") {} + SECTION("nonempty default") { default_value = "present"; } + frame.append_column("empty", default_value); + REQUIRE(frame.size() == 2); + REQUIRE(frame.n_cols() == 1); + CHECK(std::vector(frame.at(0)) == std::vector{default_value}); + CHECK(std::vector(frame.at(1)) == std::vector{default_value}); + REQUIRE(frame.column_view(0).erase()); + REQUIRE(frame.n_cols() == 0); + frame.append_column("again"); + REQUIRE(frame.size() == 2); + CHECK(frame.at(1)[0].get().empty()); +} + +TEST_CASE("DataFrame: selection snapshots edits and hidden columns independently", + "[data_frame][issue_333]") { + const auto& filename = issue_333_file(); + auto validate = [](CSVReader& reader) { + DataFrame<> frame(reader); + frame.at(500000)["name"] = "snapshot"; + REQUIRE(frame.column_view("value").erase()); + std::vector mask(frame.size(), 0); + mask[500000] = 1; + auto selected = frame.selected_rows(mask); + REQUIRE(selected.size() == 1); + CHECK(selected.columns() == std::vector{"id", "name"}); + CHECK(selected.at(0)["name"].get() == "snapshot"); + frame.at(500000)["name"] = "source changed"; + CHECK(selected.at(0)["name"].get() == "snapshot"); + selected.at(0)["name"] = "selection changed"; + CHECK(frame.at(500000)["name"].get() == "source changed"); + selected.append_column("extra", "x"); + CHECK(selected.at(0)["name"].get() == "selection changed"); + CHECK(selected.n_cols() == 3); + CHECK_FALSE(selected.has_column("value")); + auto empty = frame.selected_rows(std::vector(frame.size(), 0)); + CHECK(empty.empty()); + CHECK(empty.columns() == frame.columns()); + }; + SECTION("mmap") { + CSVReader reader(filename, CSVFormat()); + validate(reader); + } + SECTION("stream") { + std::ifstream input(filename, std::ios::binary); + CSVReader reader(input, CSVFormat()); + validate(reader); + } +} + +TEST_CASE("DataFrame: selected rows preserve independent sparse edits", "[data_frame][selected_rows]") { + std::istringstream input( + "id,name,value\n" + "1,Alice,10\n" + "2,Bob,20\n" + "3,Carol,30\n" + "4,David,40\n" + ); + CSVReader reader(input); + DataFrame<> frame; + SECTION("Unkeyed source") { frame = DataFrame<>(reader); } + SECTION("Keyed source") { frame = DataFrame<>(reader, "id"); } + + frame.at(0)["name"] = "Excluded"; + frame.at(1)["name"] = "Robert"; + frame.at(1)["value"] = "95"; + frame.at(3)["value"] = "45"; + + const auto& source = frame; + auto selected = source.selected_rows({0, 1, 0, 2}); + REQUIRE(selected.size() == 2); + REQUIRE(selected.columns() == frame.columns()); + REQUIRE_THROWS_AS(selected["2"], std::runtime_error); + REQUIRE(std::vector(selected.at(0)) == std::vector{"2", "Robert", "95"}); + REQUIRE(std::vector(selected.at(1)) == std::vector{"4", "David", "45"}); + REQUIRE(selected.column("value") == std::vector{"95", "45"}); + REQUIRE(selected.column_view(1)[0].get() == "Robert"); + REQUIRE(selected.at(0).to_json() == "{\"id\":2,\"name\":\"Robert\",\"value\":95}"); + REQUIRE(selected.at(0).to_json_array() == "[2,\"Robert\",95]"); + + // Selection should still share parsed bytes, not materialize edited rows. + REQUIRE(selected.at(0).get_underlying_row()["name"].get_sv().data() + == source.at(1).get_underlying_row()["name"].get_sv().data()); + + std::stringstream output; + auto writer = make_csv_writer(output); + writer << selected.columns(); + for (const auto& row : selected) { +#ifdef CSV_HAS_CXX20 + writer << row; +#else + writer << std::vector(row); +#endif + } + REQUIRE(output.str() == "id,name,value\n2,Robert,95\n4,David,45\n"); + + const auto& const_selected = selected; + REQUIRE_THROWS_AS(const_selected.at(0)["name"] = "Blocked", std::runtime_error); + REQUIRE_THROWS_AS(const_selected.column_view("name")[0] = "Blocked", std::runtime_error); + + // Edits on either frame must not leak into the other frame's overlay. + frame.at(1)["name"] = "Source only"; + REQUIRE(selected.at(0)["name"].get() == "Robert"); + selected.at(0)["value"] = "100"; + REQUIRE(frame.at(1)["value"].get() == "95"); + + auto repeated = selected.selected_rows({1, 0}); + REQUIRE(std::vector(repeated.at(0)) == std::vector{"2", "Robert", "100"}); + selected.at(0)["name"] = "Selected only"; + REQUIRE(repeated.at(0)["name"].get() == "Robert"); +} + +TEST_CASE("DataFrame: selected rows preserve physical edit indices after structural changes", "[data_frame][selected_rows]") { + std::istringstream input("id,name,value\n1,Alice,10\n2,Bob,20\n3,Carol,30\n"); + CSVFormat format; + format.column_names_policy(ColumnNamePolicy::CASE_INSENSITIVE); + CSVReader reader(input, format); + DataFrame<> frame(reader, "id"); + frame.at(0)["name"] = "Hidden"; + frame.at(1)["value"] = "95"; + frame.at(2)["value"] = ""; + REQUIRE(frame.column_view("name").erase()); + REQUIRE(frame.at(0).erase()); + frame.insert_row(0, {"4", "40"}); + + auto selected = frame.selected_rows({0, 1, 1}); + REQUIRE(selected.columns() == std::vector{"id", "value"}); + REQUIRE(selected.index_of("VALUE") == 1); + REQUIRE_FALSE(selected.has_column("name")); + REQUIRE(std::vector(selected.at(0)) == std::vector{"2", "95"}); + REQUIRE(std::vector(selected.at(1)) == std::vector{"3", ""}); + REQUIRE(selected.column_view(1)[0].get() == "95"); + REQUIRE(selected.at(0).to_json() == "{\"id\":2,\"value\":95}"); + REQUIRE(selected.at(1)["VALUE"].get().empty()); + + std::stringstream output; + auto writer = make_csv_writer(output); + writer << selected.columns(); + for (const auto& row : selected) { +#ifdef CSV_HAS_CXX20 + writer << row; +#else + writer << std::vector(row); +#endif + } + REQUIRE(output.str() == "id,value\n2,95\n3,\n"); + + auto repeated = selected.selected_rows({0, 1}); + REQUIRE(std::vector(repeated.at(0)) == std::vector{"3", ""}); +} + +TEST_CASE("DataFrame: materialization owns preserved view keys after reader destruction", + "[data_frame][view_keys]") { + const auto& filename = issue_333_file(); + const bool custom_keys = GENERATE(false, true); + DataFrame frame; + auto load = [custom_keys, &frame](CSVReader& reader) { + frame = custom_keys + ? DataFrame(reader, [](const CSVRow& row) { return row["id"].get(); }) + : DataFrame(reader, "id"); + }; + SECTION("mmap") { + CSVReader reader(filename, CSVFormat()); + load(reader); + } + SECTION("stream") { + std::ifstream input(filename, std::ios::binary); + CSVReader reader(input, CSVFormat()); + load(reader); + } + + // Neither parser nor input remains alive. Build the cached index before the + // original backing rows are released; it also contains borrowed key views. + REQUIRE(frame.size() == 500001); + REQUIRE(frame.contains("1")); + frame.at(0)["id"] = "edited key value"; + frame.append_column("extra", "x"); + REQUIRE(frame.at(0).key() == csv::string_view("1")); + REQUIRE(frame.contains("1")); + REQUIRE(frame["1"]["id"].get() == "edited key value"); + REQUIRE(frame.contains("500001")); + + if (!custom_keys) { + frame.insert_row(1, {"inserted", "new row", "99", "x"}); + } + frame.append_column("again"); + REQUIRE(frame.contains("1")); + REQUIRE(frame.at(0).key() == csv::string_view("1")); + if (!custom_keys) { + REQUIRE(frame["inserted"]["value"].get() == 99); + } + DataFrame moved(std::move(frame)); + moved.append_column("after_move"); + REQUIRE(moved.contains("500001")); + REQUIRE(moved["1"]["id"].get() == "edited key value"); + + // Compact the preserved keys to fewer bytes than a string's inline buffer. + // Moving their owner must also preserve these small-string-backed views. + while (moved.size() > 2) { + moved.at(moved.size() - 1).erase(); + } + moved.append_column("small_keys"); + DataFrame small(std::move(moved)); + REQUIRE(small.contains("1")); + REQUIRE(small.at(0).key() == csv::string_view("1")); + small.at(1).erase(); + small.at(0).erase(); + small.append_column("no_keys"); + REQUIRE(small.empty()); + REQUIRE_FALSE(small.contains("1")); +} + +TEST_CASE("DataFrame: standalone mutable proxies retain edits and access permissions", + "[data_frame][coverage]") { + // A row built in memory isolates proxy binding from the parser paths. + DataFrame<> source; + source.append_column("name"); + source.append_column("value"); + source.insert_row(0, {"original", "22"}); + const auto source_row = source.at(0); + const auto& raw = source_row.get_underlying_row(); + RowOverlay overlay; + DataFrameRow detached(&raw, static_cast*>(nullptr), 0, &overlay, nullptr); + + auto cell = detached["name"]; + cell = "standalone edited value"; + REQUIRE(detached["name"].get() == "standalone edited value"); + REQUIRE(raw["name"].get() == "original"); + auto copied = cell; + DataFrameCell moved(std::move(cell)); + overlay.set(0, "later overlay edit"); + REQUIRE(copied.get() == "standalone edited value"); + REQUIRE(moved.get() == "standalone edited value"); + REQUIRE(detached["name"].get() == "later overlay edit"); + + const DataFrameCell read_only(&raw, static_cast(&overlay), 0); + moved = read_only; + REQUIRE(moved.get() == "later overlay edit"); + REQUIRE_THROWS_AS(moved = "blocked", std::runtime_error); + moved = detached["value"]; + REQUIRE(moved.get() == 22); + moved = "33"; + REQUIRE(detached["value"].get() == 33); + + DataFrameCell empty; + DataFrameCell empty_copy(empty); + DataFrameCell empty_move(std::move(empty_copy)); + REQUIRE(empty_move.is_null()); + copied = empty; + REQUIRE(copied.get_sv().empty()); + moved = std::move(empty_move); + REQUIRE(moved.is_null()); + REQUIRE_THROWS_AS(moved = "unbound", std::runtime_error); + + DataFrameCell no_overlay(&raw, static_cast(nullptr), 0); + REQUIRE(no_overlay.get() == "original"); + REQUIRE_THROWS_AS(no_overlay = "unbound", std::runtime_error); +} + +TEST_CASE("DataFrame: cell position bounds remain enforced after edits", + "[data_frame][coverage]") { + DataFrame<> frame; + frame.append_column("name"); + frame.insert_row(0, {"original"}); + SECTION("clean") {} + SECTION("dirty") { frame.at(0)["name"] = "edited"; } + REQUIRE_THROWS_AS(frame.at(0)[frame.n_cols()], std::out_of_range); + const auto& source = frame; + REQUIRE_THROWS_AS(source.at(0)[source.n_cols()], std::out_of_range); + REQUIRE_THROWS_AS(source.column_view("name").get_sv(source.size()), std::out_of_range); +} + +TEST_CASE("DataFrame: row insertion with identity mapping shifts sparse edits", + "[data_frame][coverage]") { + const auto& filename = issue_333_file(); + const bool keyed = GENERATE(false, true); + auto validate = [keyed](CSVReader& reader) { + DataFrame<> frame = keyed ? DataFrame<>(reader, "id") : DataFrame<>(reader); + REQUIRE(frame.size() == 500001); + if (keyed) { REQUIRE(frame.contains("1")); } + frame.at(0)["name"] = "edited first"; + frame.at(500000)["name"] = "edited last"; + // No column has been deleted: dirty insertion uses the identity mapping. + frame.insert_row(0, {"new-row", "inserted", "17"}); + REQUIRE(frame.size() == 500002); + REQUIRE(frame.at(1)["name"].get() == "edited first"); + REQUIRE(frame.at(500001)["name"].get() == "edited last"); + frame.at(0)["name"] = "edited inserted"; + REQUIRE(frame.at(0).to_json() == "{\"id\":\"new-row\",\"name\":\"edited inserted\",\"value\":17}"); + frame.append_column("extra", "default"); + REQUIRE(frame.at(0)["name"].get() == "edited inserted"); + REQUIRE(frame.at(1)["name"].get() == "edited first"); + REQUIRE(frame.at(500001)["value"].get() == 3500007); + REQUIRE(frame.at(500001)["extra"].get() == "default"); + if (keyed) { + REQUIRE(frame["new-row"]["name"].get() == "edited inserted"); + REQUIRE(frame["1"]["name"].get() == "edited first"); + REQUIRE(frame["500001"]["name"].get() == "edited last"); + } + }; + SECTION("mmap") { + CSVReader reader(filename, CSVFormat()); + validate(reader); + } + SECTION("stream") { + std::ifstream input(filename, std::ios::binary); + CSVReader reader(input, CSVFormat()); + validate(reader); + } +} + +TEST_CASE("DataFrame: materialization preserves an empty borrowed key", + "[data_frame][coverage][view_keys]") { + const auto& filename = issue_333_file(); + DataFrame frame; + auto load = [&frame](CSVReader& reader) { + frame = DataFrame(reader, [](const CSVRow& row) -> csv::string_view { + const auto key = row["id"].get(); + return key == csv::string_view("1") ? csv::string_view() : key; + }); + }; + SECTION("mmap") { + CSVReader reader(filename, CSVFormat()); + load(reader); + } + SECTION("stream") { + std::ifstream input(filename, std::ios::binary); + CSVReader reader(input, CSVFormat()); + load(reader); + } + REQUIRE(frame.size() == 500001); + REQUIRE(frame.contains(csv::string_view())); + frame.at(0)["name"] = "empty key row"; + frame.append_column("extra"); + REQUIRE(frame.at(0).key().empty()); + REQUIRE(frame.contains(csv::string_view())); + REQUIRE(frame[csv::string_view()]["name"].get() == "empty key row"); + REQUIRE(frame.contains("500001")); + frame.insert_column(0, "prefix", "x"); + REQUIRE(frame[csv::string_view()]["prefix"].get() == "x"); + REQUIRE(frame[csv::string_view()]["name"].get() == "empty key row"); +} + +TEST_CASE("DataFrame: repeated backend transitions preserve schema and proxy reads", + "[data_frame][backend]") { + DataFrame<> frame; + frame.append_column("id"); + frame.append_column("value"); + frame.insert_row(0, {"1", "original"}); + for (size_t i = 0; i < 20; ++i) { + const auto& source = frame; + const auto row = source.at(0); + auto column = source.column_view("value"); + auto pending = frame.at(0)["value"]; + const std::string edited = "edit_" + std::to_string(i); + pending = edited; + REQUIRE(row["value"].get() == edited); + REQUIRE(column[0].get() == edited); + frame.insert_column(0, "extra", "default"); + REQUIRE_THROWS_AS(column.name(), std::runtime_error); + REQUIRE(frame.at(0)["value"].get() == edited); + REQUIRE(frame.columns() == std::vector{"extra", "id", "value"}); + REQUIRE(frame.column_view("extra").erase()); + auto selected = frame.selected_rows({1}); + REQUIRE(selected.columns() == std::vector{"id", "value"}); + REQUIRE(selected.at(0)["value"].get() == edited); + frame.append_column("materialize"); + REQUIRE(frame.column_view("materialize").erase()); + frame = frame.selected_rows({1}); + } +} + +#if CSV_ENABLE_THREADS +TEST_CASE("DataFrame: concurrent first edits retain clean readers and captured cells", + "[data_frame][backend][threading]") { + const auto& filename = issue_333_file(); + auto validate = [](CSVReader& reader) { + auto frame = std::make_shared>(reader); + REQUIRE(frame->size() == 500001); + auto errors = std::make_shared(); + test_with_timeout([frame, errors]() { + const auto& source = *frame; + const auto row = source.at(0); + auto pending_name = frame->at(0)["name"]; + auto pending_value = frame->at(0)["value"]; + const auto* names = &source.columns(); + std::atomic start(false), valid(true); + auto wait = [&start]() { while (!start.load(std::memory_order_acquire)) { std::this_thread::yield(); } }; + std::thread name_writer([&]() { wait(); pending_name = "edited name"; }); + std::thread value_writer([&]() { wait(); pending_value = "99"; }); + std::thread observer([&]() { + wait(); + for (size_t i = 0; i < 2000; ++i) { + const auto name = row["name"].get(); + const auto value = row["value"].get(); + if ((name != "person_1" && name != "edited name") || + (value != "7" && value != "99") || &source.columns() != names) { + valid.store(false, std::memory_order_relaxed); + } + } + }); + start.store(true, std::memory_order_release); + name_writer.join(); + value_writer.join(); + observer.join(); + if (!valid.load(std::memory_order_relaxed)) { errors->add_error("first-edit reader observed corrupt state"); } + if (row["name"].get() != "edited name") { errors->add_error("captured name cell lost its write"); } + if (row["value"].get() != 99) { errors->add_error("captured value cell lost its write"); } + }); + errors->check_and_fail_if_errors(); + }; + SECTION("mmap") { + CSVReader reader(filename, CSVFormat()); + validate(reader); + } + SECTION("stream") { + std::ifstream input(filename, std::ios::binary); + CSVReader reader(input, CSVFormat()); + validate(reader); + } +} +#endif + +TEST_CASE("DataFrame: direct column views preserve parsed and edited field bytes", + "[data_frame][column_view]") { + static csv_test::GeneratedFile file("csv_parser_direct_column_view.csv"); + const auto& filename = file.path([](std::ofstream& out) { + out << "id,name,value,quoted,empty\n"; + for (size_t i = 0; i < 500001; ++i) { + out << i << ", name_" << i << " ," << i * 7 + << ",\" quoted_" << i << "_\"\"x\"\" \",\n"; + } + }); + CSVFormat format; + format.trim({' '}).eager_field_classification(true); + auto validate = [](CSVReader& reader) { + DataFrame<> frame(reader); + REQUIRE(frame.size() == 500001); + const auto& source = frame; + // Direct extraction must retain CSVRow decoding/trimming even when the + // parser has cached scalar metadata that this read path does not need. + for (const size_t row : {size_t(0), size_t(250000), size_t(500000)}) { + for (size_t column = 0; column < 5; ++column) { + CHECK(source.column_view(column).get_sv(row) == + source.at(row).get_underlying_row()[column].get_sv()); + } + } + CHECK(source.column_view("quoted").get_sv(500000) == "quoted_500000_\"x\""); + auto names = source.column_view("name"); + frame.at(0)[1] = "edited,\n\"name\""; + CHECK(names.get_sv(0) == "edited,\n\"name\""); + CHECK(names.get_sv(500000) == "name_500000"); + CHECK_THROWS_AS(names.get_sv(frame.size()), std::out_of_range); + REQUIRE(frame.column_view("value").erase()); + CHECK_THROWS_AS(names.get_sv(0), std::runtime_error); + CHECK(source.column_view("quoted").get_sv(500000) == "quoted_500000_\"x\""); + CHECK(source.column_view("empty").get_sv(500000).empty()); + // Materialized fields contain literal bytes, with no parser trimming. + frame.insert_column(1, "new", " materialized\"value "); + CHECK(source.column_view("new").get_sv(500000) == " materialized\"value "); + CHECK(source.column_view("name").get_sv(0) == "edited,\n\"name\""); + CHECK(source.column_view("quoted").get_sv(500000) == "quoted_500000_\"x\""); + CHECK(source.column_view("empty").get_sv(500000).empty()); + }; + SECTION("mmap") { + CSVReader reader(filename, format); + validate(reader); + } + SECTION("stream") { + std::ifstream input(filename, std::ios::binary); + CSVReader reader(input, format); + validate(reader); + } +} diff --git a/tools/benchmarks/data_frame/benchmark.cpp b/tools/benchmarks/data_frame/benchmark.cpp new file mode 100644 index 00000000..7365dd78 --- /dev/null +++ b/tools/benchmarks/data_frame/benchmark.cpp @@ -0,0 +1,84 @@ +#include +#include +#include +#include +#include +#include +#include +#include "csv.hpp" + +using Clock = std::chrono::steady_clock; +static volatile std::uint64_t sink = 0; + +template +void measure(const char* name, Fn fn) { + const auto start = Clock::now(); + const std::uint64_t sum = fn(); + const double ms = std::chrono::duration(Clock::now() - start).count(); + sink = sum; + std::cout << name << ',' << std::fixed << std::setprecision(6) << ms << ',' << sum << '\n'; +} + +void scan(const char* indexed_name, const char* iterator_name, const csv::DataFrame<>& frame) { + const auto column = frame.column_view("name"); + measure(indexed_name, [&]() { + std::uint64_t sum = 0; + for (size_t repeat = 0; repeat < 12; ++repeat) + for (size_t i = 0; i < column.size(); ++i) + sum += column.get_sv(i).size(); + return sum; + }); + measure(iterator_name, [&]() { + std::uint64_t sum = 0; + for (size_t repeat = 0; repeat < 4; ++repeat) + for (auto it = column.begin(), end = column.end(); it != end; ++it) + sum += it->get_sv().size(); + return sum; + }); +} + +int main() { + std::ostringstream generated; + generated << "id,name,value,quoted\n"; + for (size_t i = 0; i < 500001; ++i) + generated << i << ",person_" << i << ',' << i * 7 << ",\"quoted_" << i << "_\"\"x\"\"\"\n"; + const std::string text = generated.str(); + std::istringstream input(text); + csv::CSVReader reader(input); + std::vector rows(reader.begin(), reader.end()); + csv::DataFrame<> clean(rows); + csv::DataFrame<> dirty(rows); + for (size_t i = 0; i < dirty.size(); i += 499) + dirty.at(i)[1] = csv::string_view("edited_person"); + csv::DataFrame<> mapped(rows); + mapped.column_view("id").erase(); + for (size_t i = 0; i < mapped.size(); i += 499) + mapped.at(i)[0] = csv::string_view("edited_person"); + std::cout << "layout_dfc," << sizeof(csv::DataFrameColumn) << ",0\n"; + scan("clean_indexed", "clean_iterator", clean); + const auto quoted_column = static_cast&>(clean).column_view("quoted"); + measure("clean_quoted_indexed", [&]() { + std::uint64_t sum = 0; + for (size_t repeat = 0; repeat < 12; ++repeat) + for (size_t i = 0; i < quoted_column.size(); ++i) + sum += quoted_column.get_sv(i).size(); + return sum; + }); + scan("dirty_indexed", "dirty_iterator", dirty); + scan("mapped_indexed", "mapped_iterator", mapped); + csv::DataFrameExecutor executor(2); + measure("chunk_parallel", [&]() { + std::istringstream chunk_input(text); + csv::CSVReader chunk_reader(chunk_input); + std::vector states(4, 0); + csv::chunk_parallel_apply(chunk_reader, executor, states, + [](csv::DataFrame<>::column_type column, std::uint64_t& state) { + for (size_t i = 0; i < column.size(); ++i) + state += column.get_sv(i).size(); + }, 50000); + std::uint64_t sum = 0; + for (const auto value : states) sum += value; + return sum; + }); + return sink == 0 ? 1 : 0; +} diff --git a/tools/benchmarks/data_frame/construction.cpp b/tools/benchmarks/data_frame/construction.cpp new file mode 100644 index 00000000..14090fd2 --- /dev/null +++ b/tools/benchmarks/data_frame/construction.cpp @@ -0,0 +1,12 @@ +#include "csv.hpp" +#include +#include +#include +#include +#include +static volatile std::uint64_t sink; +using Column=csv::DataFrameColumn; +template void measure(const char* name,Fn fn){auto start=std::chrono::steady_clock::now();auto sum=fn(); sink=sum;std::cout<(std::chrono::steady_clock::now()-start).count()<<','< frame(reader);const size_t N=500000;std::vector columns; columns.reserve(N); std::vector copies;copies.reserve(N); +measure("construct",[&](){for(size_t rep=0;rep<12;++rep){columns.clear();for(size_t i=0;i