Speed up SAV import and export (0.9.1)

CHANGELOG.md

@@ -2,6 +2,24 @@
 
 All notable changes to this reference implementation are documented here.
 
+## 0.9.1 - 2026-11-09
+
+This patch release speeds up SAV import and export. Behavior, error codes and
+the catalog contract are unchanged.
+
+### Changed
+
+- Prove strict row limits column by column before falling back to the exact
+  row-by-row checks, which still run (and report the same first failing row)
+  whenever a column is not provably within limits.
+- Convert imported frames and exported rows column-wise instead of cell by cell.
+- Insert imported rows through the driver after preflight instead of
+  SQLAlchemy's per-row parameter pass.
+- Declare the direct `numpy>=1.26` dependency (already required by pandas).
+
+Local SQLite timings on a 50k x 200 SAV: import 29.2 s to 8.6 s, export
+22.4 s to 17.4 s.
+
 ## 0.9.0 - 2026-09-10
 
 This minor release adds the explicitly selected optional official SPSS Frontend

README.md

@@ -4,7 +4,7 @@ The reference Python implementation of the OpenStatSpec specification.
 
 This package implements the specification; it does not define or extend it.
 The normative model lives in the `OpenStatSpec/specification` repository.
-Python 0.9.0 pins released specification `v0.5.0` at
+Python 0.9.1 pins released specification `v0.5.0` at
 `864e84479f554b8ee250ffed44c4dfb963750d4a` and selects SAV/ZSAV 1.0 with
 `database_io_policy=openstatspec-database-io-v1`. It implements the explicitly
 selected optional official SPSS Frontend 0.3 over unchanged Plan 0.1/0.2, not

pyproject.toml

@@ -4,13 +4,13 @@ build-backend = "hatchling.build"
 
 [project]
 name = "openstatspec"
-version = "0.9.0"
+version = "0.9.1"
 description = "Reference adapter for the OpenStatSpec relational contract"
 readme = "README.md"
 requires-python = ">=3.11"
 license = {text = "Apache-2.0"}
 authors = [{name = "OpenStatSpec"}]
-dependencies = ["SQLAlchemy>=2.0", "pandas>=2.2", "openstatspec-pyspssio==0.5.1.post2", "sqlglot>=30.14", "rfc8785>=0.1.4"]
+dependencies = ["SQLAlchemy>=2.0", "pandas>=2.2", "numpy>=1.26", "openstatspec-pyspssio==0.5.1.post2", "sqlglot>=30.14", "rfc8785>=0.1.4"]
 
 [project.optional-dependencies]
 dev = ["pytest>=8"]

src/openstatspec/spss/sav.py

@@ -10,11 +10,11 @@
 import os
 import json
 import errno
-import math
 import ctypes
 from contextlib import ExitStack, contextmanager
 from datetime import UTC, datetime
 from functools import wraps
+from operator import itemgetter
 from pathlib import Path
 from tempfile import mkstemp, TemporaryDirectory
 from typing import Any, Callable
@@ -24,6 +24,7 @@
 except ImportError:  # pragma: no cover - the supported CI/runtime is POSIX.
     fcntl = None
 
+import numpy as np
 import pandas as pd
 import pyspssio
 
@@ -206,18 +207,16 @@ def _pyspssio_missing_rules(encoded: str) -> dict[str, Any] | None:
 
 
 def _rows(frame: pd.DataFrame, variables: list[dict[str, Any]]) -> list[dict[str, Any]]:
-    records: list[dict[str, Any]] = []
-    for input_row in frame.to_dict(orient="records"):
-        output: dict[str, Any] = {}
-        for item in variables:
-            value = input_row[item["source_name"]]
-            if item["storage_kind"] == "numeric" and pd.isna(value):
-                value = None
-            elif item["storage_kind"] == "string" and (value is None or pd.isna(value)):
-                value = ""
-            output[item["physical_name"]] = value
-        records.append(output)
-    return records
+    columns = []
+    for item in variables:
+        series = frame[item["source_name"]]
+        empty = None if item["storage_kind"] == "numeric" else ""
+        columns.append([
+            empty if missing else value
+            for value, missing in zip(series.tolist(), series.isna().tolist())
+        ])
+    names = [item["physical_name"] for item in variables]
+    return [dict(zip(names, values)) for values in zip(*columns)]
 
 
 def inspect_sav(source: str | Path) -> dict[str, Any]:
@@ -586,15 +585,14 @@ def export_sav_dataset(
         raise UnsupportedOperationError("Export requires explicit allow_loss for: " + ", ".join(rejected))
 
     frame = pd.DataFrame(
-        [
-            {
-                variable["source_name"]: (
-                    math.nan if row[variable["physical_name"]] is None else float(row[variable["physical_name"]])
-                ) if variable["storage_kind"] == "numeric" else row[variable["physical_name"]]
-                for variable in variables
-            }
-            for row in rows
-        ],
+        {
+            variable["source_name"]: (
+                # Preflight already proved numeric cells are binary64 or None.
+                np.array(column, dtype=float) if variable["storage_kind"] == "numeric" else column
+            )
+            for variable in variables
+            for column in [list(map(itemgetter(variable["physical_name"]), rows))]
+        },
         columns=[variable["source_name"] for variable in variables],
     )
     publication_state: dict[str, Any] = {"published": False}

src/openstatspec/sql/profiles.py

@@ -8,9 +8,12 @@
 from dataclasses import dataclass
 import math
 from numbers import Real
+from operator import itemgetter
 from typing import Any
 from urllib.parse import urlparse
 
+import numpy as np
+
 from ..core import UnsupportedOperationError
 
 
@@ -125,8 +128,12 @@ def preflight(
     *,
     rows: Iterable[Mapping[str, Any]] | None = None,
     require_canonical_mapping: bool = True,
-) -> None:
-    """Validate strict target capabilities before any source dataset is created."""
+) -> bool:
+    """Validate strict target capabilities before any source dataset is created.
+
+    Returns True only when every cell was proven to be float/None (numeric) or
+    str, so callers may bypass per-value bind processing.
+    """
     variables = None if isinstance(variables_or_count, int) else list(variables_or_count)
     variable_count = variables_or_count if isinstance(variables_or_count, int) else len(variables)
     if variable_count > profile.max_source_variables:
@@ -140,7 +147,7 @@ def preflight(
             max_physical=profile.max_source_variables + 1,
         )
     if variables is None:
-        return
+        return False
 
     used = {"__case_ordinal"}
     source_names: set[str] = set()
@@ -211,7 +218,11 @@ def preflight(
             f"{profile.name} permits {profile.max_row_bytes}.",
             declared_row_bytes=declared_row_bytes, maximum=profile.max_row_bytes,
         )
-    for row_ordinal, row in enumerate(rows or (), start=1):
+    rows = list(rows or ())
+    if not rows or _rows_within_limits(profile, variables, rows):
+        return True
+    # Slow path only reports the first failing row in row-major order.
+    for row_ordinal, row in enumerate(rows, start=1):
         row_bytes = 8
         for variable in variables:
             if variable.get("storage_kind") == "numeric":
@@ -310,6 +321,54 @@ def preflight(
                 row_ordinal=row_ordinal, statement_bytes=statement_bytes,
                 maximum=profile.max_statement_bytes,
             )
+    return False
+
+
+def _rows_within_limits(
+    profile: SqlProfile, variables: list[Mapping[str, Any]],
+    rows: list[Mapping[str, Any]],
+) -> bool:
+    """Column-wise proof that the row-by-row checks in preflight cannot fail.
+
+    Per-column maxima bound every row's row and statement size from above, so
+    True means no row exceeds a limit; False defers to the exact slow path.
+    """
+    row_bytes, statement_bytes = 8, 32
+    for variable in variables:
+        name = str(variable["physical_name"])
+        try:
+            column = list(map(itemgetter(name), rows))
+        except KeyError:
+            return False
+        if variable.get("storage_kind") == "numeric":
+            if not set(map(type, column)) <= _PLAIN_NUMERIC_TYPES:
+                return False
+            values = np.array(column, dtype=float)  # None becomes NaN
+            if np.isinf(values).any() or np.isnan(values).sum() != column.count(None):
+                return False
+            row_bytes += 8
+            statement_bytes += 24
+            continue
+        if set(map(type, column)) != {str}:
+            return False
+        try:
+            longest = max(map(len, map(str.encode, column)))
+        except UnicodeEncodeError:
+            return False
+        if longest > profile.max_text_value_bytes:
+            return False
+        row_bytes += 20 if profile.name in MYSQL_WIRE_PROFILES else longest
+        statement_bytes += longest + 8
+    return (
+        (profile.max_row_bytes is None or row_bytes <= profile.max_row_bytes)
+        and (
+            profile.max_statement_bytes is None
+            or statement_bytes <= profile.max_statement_bytes
+        )
+    )
+
+
+_PLAIN_NUMERIC_TYPES = {float, type(None)}
 
 
 def _row_storage_bytes(profile: SqlProfile, variable: Mapping[str, Any]) -> int:

src/openstatspec/sql/wide.py

@@ -760,10 +760,24 @@ def _bounded_batches(
     maximum_statement_bytes: int | None,
 ) -> Iterable[list[Mapping[str, Any]]]:
     """Partition rows without exceeding the profile's statement payload budget."""
+    if maximum_statement_bytes is None:
+        rows = list(rows)
+        if rows:
+            yield rows
+        return
+    variables = list(variables)
+    strings = [
+        str(variable["physical_name"]) for variable in variables
+        if variable.get("storage_kind") != "numeric"
+    ]
+    # Same arithmetic as statement_payload_bytes, with numeric widths hoisted.
+    fixed_size = 32 + 24 * (len(variables) - len(strings)) + 8 * len(strings)
     batch: list[Mapping[str, Any]] = []
     size = 0
     for row in rows:
-        row_size = statement_payload_bytes(row, variables)
+        row_size = fixed_size + sum(
+            len(str(row.get(name, "")).encode("utf-8")) for name in strings
+        )
         if (
             batch and maximum_statement_bytes is not None
             and size + row_size > maximum_statement_bytes
@@ -789,13 +803,19 @@ def _canonicalize_database_numeric_rows(
     normalized = []
     for source in rows:
         row = dict(source)
+        if set(map(type, row.values())) <= _PLAIN_ROW_VALUE_TYPES:
+            normalized.append(row)
+            continue
         for name in numeric_names:
             if isinstance(row.get(name), Decimal) and row[name].is_finite():
                 row[name] = float(row[name])
         normalized.append(row)
     return normalized
 
 
+_PLAIN_ROW_VALUE_TYPES = {float, int, str, bool, bytes, type(None)}
+
+
 def data_table_name(dataset_id: str) -> str:
     stem = _IDENTIFIER.sub("_", dataset_id).strip("_").lower() or "dataset"
     return f"data_{stem[:48]}"
@@ -851,7 +871,7 @@ def create_wide_dataset(
     fidelity_events = tuple(fidelity_events)
     source_rows = _canonicalize_database_numeric_rows(rows, variables)
     try:
-        preflight(profile, variables, rows=source_rows)
+        plain_rows = preflight(profile, variables, rows=source_rows)
         validate_spss_catalog(
             variables,
             case_weight_variable=case_weight_variable,
@@ -962,10 +982,21 @@ def create_wide_dataset(
                 ),
             )
             if source_rows:
+                # Plain rows hold only float/None and str, so the only bind
+                # processor (SQLite to_float) is a no-op: hand them straight to
+                # the driver. Anything else (e.g. exact ints) keeps SQLAlchemy.
+                compiled = insert(data_table).compile(dialect=connection.dialect)
+                keys = compiled.positiontup
                 for batch in _bounded_batches(
                     source_rows, variables, profile.max_statement_bytes,
                 ):
-                    connection.execute(insert(data_table), batch)
+                    if not plain_rows:
+                        connection.execute(insert(data_table), batch)
+                        continue
+                    connection.exec_driver_sql(compiled.string, (
+                        [tuple(map(row.__getitem__, keys)) for row in batch]
+                        if keys else batch
+                    ))
             finish_normative_operation(
                 connection, normative, operation_id=operation_id,
                 status="succeeded",

tests/test_cli.py

@@ -42,7 +42,7 @@ def test_cli_import_inspect_validate_and_export_emit_json(tmp_path, capsys) -> N
 
 def test_capability_matrix_is_public_and_cli_matches_engine_boundary(capsys) -> None:
     matrix = openstatspec.capability_matrix()
-    assert matrix["adapter_version"] == "0.9.0"
+    assert matrix["adapter_version"] == "0.9.1"
     assert matrix["database_io_policy"] == "openstatspec-database-io-v1"
     assert matrix["specification_status"] == "released"
     assert matrix["specification_release"] == "v0.5.0"

tests/test_sql_profiles.py

@@ -752,3 +752,46 @@ def test_generated_data_table_identifier_uses_effective_limit() -> None:
     assert error.value.details["reason"] == "identifier_limit"
     assert error.value.details["role"] == "physical data-table identifier"
     assert error.value.details["maximum"] == 5
+
+
+def test_column_preflight_fast_path_agrees_with_row_checks():
+    from openstatspec.sql.profiles import _rows_within_limits
+
+    variables = [
+        {"source_name": "q", "physical_name": "q", "ordinal": 1, "storage_kind": "numeric"},
+        {"source_name": "s", "physical_name": "s", "ordinal": 2, "storage_kind": "string", "string_width": 8},
+    ]
+    plain = [{"q": 1.0, "s": "Pärnu"}, {"q": None, "s": ""}]
+    assert _rows_within_limits(MYSQL, variables, plain)
+    preflight(MYSQL, variables, rows=plain)
+    # Anything not provably safe falls back to the exact row-major checks.
+    for bad in ({"q": float("inf"), "s": ""}, {"q": True, "s": ""}, {"q": 1.0, "s": None}, {"s": ""}):
+        assert not _rows_within_limits(MYSQL, variables, [*plain, bad])
+        with pytest.raises(UnsupportedOperationError):
+            preflight(MYSQL, variables, rows=[*plain, bad])
+    too_long = [{"q": 1.0, "s": "x" * (MYSQL.max_text_value_bytes + 1)}]
+    assert not _rows_within_limits(MYSQL, variables, too_long)
+    assert not _rows_within_limits(MYSQL, variables, [{"q": 1.0, "s": "\ud800"}])
+    # Exact ints are valid but not plain, so callers keep bind processing.
+    assert preflight(MYSQL, variables, rows=[*plain, {"q": 2**63, "s": ""}]) is False
+    with pytest.raises(UnsupportedOperationError, match="binary64"):
+        preflight(MYSQL, variables, rows=[*plain, {"q": 2**53 + 1, "s": ""}])
+
+
+def test_import_stores_exact_integer_cells_as_binary64(tmp_path):
+    from openstatspec.sql.wide import create_wide_dataset
+    import sqlite3
+
+    database = f"sqlite:///{tmp_path / 'ints.sqlite'}"
+    variables = [{
+        "source_name": "q", "physical_name": "q", "ordinal": 1,
+        "storage_kind": "numeric", "format": "F20.0",
+    }]
+    create_wide_dataset(
+        database_url=database, dataset_id="ints", source_name="ints.sav",
+        source_format="SAV", rows=[{"q": 2**63}, {"q": None}], variables=variables,
+    )
+    stored = sqlite3.connect(tmp_path / "ints.sqlite").execute(
+        "select q from data_ints order by __case_ordinal"
+    ).fetchall()
+    assert stored == [(float(2**63),), (None,)]