diff --git a/haystack/components/converters/csv.py b/haystack/components/converters/csv.py index 9a4a102da5..785bdec00f 100644 --- a/haystack/components/converters/csv.py +++ b/haystack/components/converters/csv.py @@ -15,6 +15,7 @@ logger = logging.getLogger(__name__) _ROW_MODE_SIZE_WARN_BYTES = 5 * 1024 * 1024 # ~5MB; warn when parsing rows might be memory-heavy +_RAGGED_ROW_RESTKEY = object() # Unique sentinel for DictReader restkey to avoid colliding with CSV headers @component @@ -167,8 +168,9 @@ def run( # ``restkey`` ensures surplus fields on ragged rows (rows with more values than the # header, e.g. an unquoted comma inside a value) land under an explicit string key # instead of the default ``None`` key, which would break ``Document`` id generation. + # A private sentinel is used so an actual CSV column named "extra_columns" is not overwritten. reader = csv.DictReader( - io.StringIO(data), delimiter=self.delimiter, quotechar=self.quotechar, restkey="extra_columns" + io.StringIO(data), delimiter=self.delimiter, quotechar=self.quotechar, restkey=_RAGGED_ROW_RESTKEY ) except Exception as e: raise RuntimeError(f"CSVToDocument(row): could not parse CSV rows for {source}: {e}") from e @@ -224,7 +226,7 @@ def _build_document_from_row( for k, v in row.items(): if k == content_column: continue - key_to_use = k + key_to_use = "extra_columns" if k is _RAGGED_ROW_RESTKEY else k if key_to_use in row_meta: # Avoid clobbering existing meta like file_path/encoding; prefix and de-dupe base_key = f"csv_{key_to_use}" diff --git a/releasenotes/notes/csv-row-ragged-extra-columns-75c6c07dc1cdc4ff.yaml b/releasenotes/notes/csv-row-ragged-extra-columns-75c6c07dc1cdc4ff.yaml new file mode 100644 index 0000000000..be71a82255 --- /dev/null +++ b/releasenotes/notes/csv-row-ragged-extra-columns-75c6c07dc1cdc4ff.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixed an issue in ``CSVToDocument`` (row mode) where a ragged row with surplus + fields overwrote an actual CSV column named ``extra_columns``. A private sentinel + is now used for ``DictReader.restkey``, preserving both the original column data + and storing surplus fields under collision-safe metadata keys. diff --git a/test/components/converters/test_csv_to_document.py b/test/components/converters/test_csv_to_document.py index dfb6afcdd6..76f600b4a3 100644 --- a/test/components/converters/test_csv_to_document.py +++ b/test/components/converters/test_csv_to_document.py @@ -243,6 +243,30 @@ def test_row_mode_ragged_row_does_not_crash(self): assert None not in ragged_doc.meta assert "state" in ragged_doc.meta["extra_columns"] + def test_row_mode_ragged_row_with_real_extra_columns_column(self): + # When a CSV header contains an 'extra_columns' column, ragged rows with + # surplus fields must preserve the real column value and place surplus fields + # under a collision-prefixed key. + source = ByteStream( + data=b"text,extra_columns\r\nhello,real value,overflow\r\nregular,normal value\r\n", + meta={"file_path": "ragged_extra.csv"}, + ) + conv = CSVToDocument(conversion_mode="row") + docs = conv.run(sources=[source], content_column="text")["documents"] + + assert len(docs) == 2 + # Ragged row preserves both the original extra_columns value and the surplus fields under a collision-safe key + ragged_doc = docs[0] + assert ragged_doc.content == "hello" + assert ragged_doc.meta["extra_columns"] == "real value" + assert "overflow" in ragged_doc.meta["csv_extra_columns"] + + # Regular row preserves the extra_columns value without adding surplus keys + regular_doc = docs[1] + assert regular_doc.content == "regular" + assert regular_doc.meta["extra_columns"] == "normal value" + assert "csv_extra_columns" not in regular_doc.meta + def test_run_utf8_with_bom(self, tmp_path): """ A CSV saved as UTF-8 with a byte order mark must not leak the BOM into the content.