From 78a93e0ce93bb6c541934291e69f564ca4ddb31f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 9 Nov 2026 13:10:14 +0200 Subject: [PATCH 1/2] Speed up SAV import and export preflight (0.9.1) Prove strict row limits column by column and fall back to the unchanged row-major checks only when a column is not provably safe. Convert frames column-wise and insert rows through the driver after preflight. SQLite 50k x 200 SAV: import 29.2 s -> 8.6 s, export 22.4 s -> 17.4 s. --- CHANGELOG.md | 18 +++++++++++ README.md | 2 +- pyproject.toml | 4 +-- src/openstatspec/spss/sav.py | 42 ++++++++++++------------- src/openstatspec/sql/profiles.py | 53 +++++++++++++++++++++++++++++++- src/openstatspec/sql/wide.py | 32 +++++++++++++++++-- tests/test_cli.py | 2 +- tests/test_sql_profiles.py | 19 ++++++++++++ 8 files changed, 143 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0cec82..0593c9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ All notable changes to this reference implementation are documented here. +## 0.9.1 - 2026-11-09 + +This patch release speeds up SAV import and export. Behavior, error codes and +the catalog contract are unchanged. + +### Changed + +- Prove strict row limits column by column before falling back to the exact + row-by-row checks, which still run (and report the same first failing row) + whenever a column is not provably within limits. +- Convert imported frames and exported rows column-wise instead of cell by cell. +- Insert imported rows through the driver after preflight instead of + SQLAlchemy's per-row parameter pass. +- Declare the direct `numpy>=1.26` dependency (already required by pandas). + +Local SQLite timings on a 50k x 200 SAV: import 29.2 s to 8.6 s, export +22.4 s to 17.4 s. + ## 0.9.0 - 2026-09-10 This minor release adds the explicitly selected optional official SPSS Frontend diff --git a/README.md b/README.md index 60da07a..fbe9bf4 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ The reference Python implementation of the OpenStatSpec specification. This package implements the specification; it does not define or extend it. The normative model lives in the `OpenStatSpec/specification` repository. -Python 0.9.0 pins released specification `v0.5.0` at +Python 0.9.1 pins released specification `v0.5.0` at `864e84479f554b8ee250ffed44c4dfb963750d4a` and selects SAV/ZSAV 1.0 with `database_io_policy=openstatspec-database-io-v1`. It implements the explicitly selected optional official SPSS Frontend 0.3 over unchanged Plan 0.1/0.2, not diff --git a/pyproject.toml b/pyproject.toml index d679991..8aa8d8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,13 +4,13 @@ build-backend = "hatchling.build" [project] name = "openstatspec" -version = "0.9.0" +version = "0.9.1" description = "Reference adapter for the OpenStatSpec relational contract" readme = "README.md" requires-python = ">=3.11" license = {text = "Apache-2.0"} authors = [{name = "OpenStatSpec"}] -dependencies = ["SQLAlchemy>=2.0", "pandas>=2.2", "openstatspec-pyspssio==0.5.1.post2", "sqlglot>=30.14", "rfc8785>=0.1.4"] +dependencies = ["SQLAlchemy>=2.0", "pandas>=2.2", "numpy>=1.26", "openstatspec-pyspssio==0.5.1.post2", "sqlglot>=30.14", "rfc8785>=0.1.4"] [project.optional-dependencies] dev = ["pytest>=8"] diff --git a/src/openstatspec/spss/sav.py b/src/openstatspec/spss/sav.py index 5a9af41..12cc204 100644 --- a/src/openstatspec/spss/sav.py +++ b/src/openstatspec/spss/sav.py @@ -10,11 +10,11 @@ import os import json import errno -import math import ctypes from contextlib import ExitStack, contextmanager from datetime import UTC, datetime from functools import wraps +from operator import itemgetter from pathlib import Path from tempfile import mkstemp, TemporaryDirectory from typing import Any, Callable @@ -24,6 +24,7 @@ except ImportError: # pragma: no cover - the supported CI/runtime is POSIX. fcntl = None +import numpy as np import pandas as pd import pyspssio @@ -206,18 +207,16 @@ def _pyspssio_missing_rules(encoded: str) -> dict[str, Any] | None: def _rows(frame: pd.DataFrame, variables: list[dict[str, Any]]) -> list[dict[str, Any]]: - records: list[dict[str, Any]] = [] - for input_row in frame.to_dict(orient="records"): - output: dict[str, Any] = {} - for item in variables: - value = input_row[item["source_name"]] - if item["storage_kind"] == "numeric" and pd.isna(value): - value = None - elif item["storage_kind"] == "string" and (value is None or pd.isna(value)): - value = "" - output[item["physical_name"]] = value - records.append(output) - return records + columns = [] + for item in variables: + series = frame[item["source_name"]] + empty = None if item["storage_kind"] == "numeric" else "" + columns.append([ + empty if missing else value + for value, missing in zip(series.tolist(), series.isna().tolist()) + ]) + names = [item["physical_name"] for item in variables] + return [dict(zip(names, values)) for values in zip(*columns)] def inspect_sav(source: str | Path) -> dict[str, Any]: @@ -586,15 +585,14 @@ def export_sav_dataset( raise UnsupportedOperationError("Export requires explicit allow_loss for: " + ", ".join(rejected)) frame = pd.DataFrame( - [ - { - variable["source_name"]: ( - math.nan if row[variable["physical_name"]] is None else float(row[variable["physical_name"]]) - ) if variable["storage_kind"] == "numeric" else row[variable["physical_name"]] - for variable in variables - } - for row in rows - ], + { + variable["source_name"]: ( + # Preflight already proved numeric cells are binary64 or None. + np.array(column, dtype=float) if variable["storage_kind"] == "numeric" else column + ) + for variable in variables + for column in [list(map(itemgetter(variable["physical_name"]), rows))] + }, columns=[variable["source_name"] for variable in variables], ) publication_state: dict[str, Any] = {"published": False} diff --git a/src/openstatspec/sql/profiles.py b/src/openstatspec/sql/profiles.py index 3c3c8e8..97cf14b 100644 --- a/src/openstatspec/sql/profiles.py +++ b/src/openstatspec/sql/profiles.py @@ -8,9 +8,12 @@ from dataclasses import dataclass import math from numbers import Real +from operator import itemgetter from typing import Any from urllib.parse import urlparse +import numpy as np + from ..core import UnsupportedOperationError @@ -211,7 +214,11 @@ def preflight( f"{profile.name} permits {profile.max_row_bytes}.", declared_row_bytes=declared_row_bytes, maximum=profile.max_row_bytes, ) - for row_ordinal, row in enumerate(rows or (), start=1): + rows = list(rows or ()) + if not rows or _rows_within_limits(profile, variables, rows): + return + # Slow path only reports the first failing row in row-major order. + for row_ordinal, row in enumerate(rows, start=1): row_bytes = 8 for variable in variables: if variable.get("storage_kind") == "numeric": @@ -312,6 +319,50 @@ def preflight( ) +def _rows_within_limits( + profile: SqlProfile, variables: list[Mapping[str, Any]], + rows: list[Mapping[str, Any]], +) -> bool: + """Column-wise proof that the row-by-row checks in preflight cannot fail. + + Per-column maxima bound every row's row and statement size from above, so + True means no row exceeds a limit; False defers to the exact slow path. + """ + row_bytes, statement_bytes = 8, 32 + for variable in variables: + name = str(variable["physical_name"]) + try: + column = list(map(itemgetter(name), rows)) + except KeyError: + return False + if variable.get("storage_kind") == "numeric": + if not set(map(type, column)) <= _PLAIN_NUMERIC_TYPES: + return False + values = np.array(column, dtype=float) # None becomes NaN + if np.isinf(values).any() or np.isnan(values).sum() != column.count(None): + return False + row_bytes += 8 + statement_bytes += 24 + continue + if set(map(type, column)) != {str}: + return False + longest = max(map(len, map(str.encode, column))) + if longest > profile.max_text_value_bytes: + return False + row_bytes += 20 if profile.name in MYSQL_WIRE_PROFILES else longest + statement_bytes += longest + 8 + return ( + (profile.max_row_bytes is None or row_bytes <= profile.max_row_bytes) + and ( + profile.max_statement_bytes is None + or statement_bytes <= profile.max_statement_bytes + ) + ) + + +_PLAIN_NUMERIC_TYPES = {float, type(None)} + + def _row_storage_bytes(profile: SqlProfile, variable: Mapping[str, Any]) -> int: if variable.get("storage_kind") == "numeric": return 8 diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index 1f0a72a..bae46c9 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -760,10 +760,24 @@ def _bounded_batches( maximum_statement_bytes: int | None, ) -> Iterable[list[Mapping[str, Any]]]: """Partition rows without exceeding the profile's statement payload budget.""" + if maximum_statement_bytes is None: + rows = list(rows) + if rows: + yield rows + return + variables = list(variables) + strings = [ + str(variable["physical_name"]) for variable in variables + if variable.get("storage_kind") != "numeric" + ] + # Same arithmetic as statement_payload_bytes, with numeric widths hoisted. + fixed_size = 32 + 24 * (len(variables) - len(strings)) + 8 * len(strings) batch: list[Mapping[str, Any]] = [] size = 0 for row in rows: - row_size = statement_payload_bytes(row, variables) + row_size = fixed_size + sum( + len(str(row.get(name, "")).encode("utf-8")) for name in strings + ) if ( batch and maximum_statement_bytes is not None and size + row_size > maximum_statement_bytes @@ -789,6 +803,9 @@ def _canonicalize_database_numeric_rows( normalized = [] for source in rows: row = dict(source) + if set(map(type, row.values())) <= _PLAIN_ROW_VALUE_TYPES: + normalized.append(row) + continue for name in numeric_names: if isinstance(row.get(name), Decimal) and row[name].is_finite(): row[name] = float(row[name]) @@ -796,6 +813,9 @@ def _canonicalize_database_numeric_rows( return normalized +_PLAIN_ROW_VALUE_TYPES = {float, int, str, bool, bytes, type(None)} + + def data_table_name(dataset_id: str) -> str: stem = _IDENTIFIER.sub("_", dataset_id).strip("_").lower() or "dataset" return f"data_{stem[:48]}" @@ -962,10 +982,18 @@ def create_wide_dataset( ), ) if source_rows: + # Preflight proved cells are binary64/None or str, so the only + # bind processor (SQLite to_float) is a no-op; skip the per-row + # SQLAlchemy parameter pass and hand rows to the driver. + compiled = insert(data_table).compile(dialect=connection.dialect) + keys = compiled.positiontup for batch in _bounded_batches( source_rows, variables, profile.max_statement_bytes, ): - connection.execute(insert(data_table), batch) + connection.exec_driver_sql(compiled.string, ( + [tuple(map(row.__getitem__, keys)) for row in batch] + if keys else batch + )) finish_normative_operation( connection, normative, operation_id=operation_id, status="succeeded", diff --git a/tests/test_cli.py b/tests/test_cli.py index 829a3a8..e408845 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -42,7 +42,7 @@ def test_cli_import_inspect_validate_and_export_emit_json(tmp_path, capsys) -> N def test_capability_matrix_is_public_and_cli_matches_engine_boundary(capsys) -> None: matrix = openstatspec.capability_matrix() - assert matrix["adapter_version"] == "0.9.0" + assert matrix["adapter_version"] == "0.9.1" assert matrix["database_io_policy"] == "openstatspec-database-io-v1" assert matrix["specification_status"] == "released" assert matrix["specification_release"] == "v0.5.0" diff --git a/tests/test_sql_profiles.py b/tests/test_sql_profiles.py index 3b1ca73..1963897 100755 --- a/tests/test_sql_profiles.py +++ b/tests/test_sql_profiles.py @@ -752,3 +752,22 @@ def test_generated_data_table_identifier_uses_effective_limit() -> None: assert error.value.details["reason"] == "identifier_limit" assert error.value.details["role"] == "physical data-table identifier" assert error.value.details["maximum"] == 5 + + +def test_column_preflight_fast_path_agrees_with_row_checks(): + from openstatspec.sql.profiles import _rows_within_limits + + variables = [ + {"source_name": "q", "physical_name": "q", "ordinal": 1, "storage_kind": "numeric"}, + {"source_name": "s", "physical_name": "s", "ordinal": 2, "storage_kind": "string", "string_width": 8}, + ] + plain = [{"q": 1.0, "s": "Pärnu"}, {"q": None, "s": ""}] + assert _rows_within_limits(MYSQL, variables, plain) + preflight(MYSQL, variables, rows=plain) + # Anything not provably safe falls back to the exact row-major checks. + for bad in ({"q": float("inf"), "s": ""}, {"q": True, "s": ""}, {"q": 1.0, "s": None}, {"s": ""}): + assert not _rows_within_limits(MYSQL, variables, [*plain, bad]) + with pytest.raises(UnsupportedOperationError): + preflight(MYSQL, variables, rows=[*plain, bad]) + too_long = [{"q": 1.0, "s": "x" * (MYSQL.max_text_value_bytes + 1)}] + assert not _rows_within_limits(MYSQL, variables, too_long) From 2ab215e52caedeeb0a48bc5e17cf0338bf405410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 9 Nov 2026 13:21:19 +0200 Subject: [PATCH 2/2] Keep bind processing for non-plain import cells Exact integer cells pass preflight but SQLite cannot bind integers beyond int64, so only rows proven float/None/str bypass SQLAlchemy. Strings that cannot be UTF-8 encoded fall back to the row-major checks. --- src/openstatspec/sql/profiles.py | 18 +++++++++++++----- src/openstatspec/sql/wide.py | 11 +++++++---- tests/test_sql_profiles.py | 24 ++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/src/openstatspec/sql/profiles.py b/src/openstatspec/sql/profiles.py index 97cf14b..01b914a 100644 --- a/src/openstatspec/sql/profiles.py +++ b/src/openstatspec/sql/profiles.py @@ -128,8 +128,12 @@ def preflight( *, rows: Iterable[Mapping[str, Any]] | None = None, require_canonical_mapping: bool = True, -) -> None: - """Validate strict target capabilities before any source dataset is created.""" +) -> bool: + """Validate strict target capabilities before any source dataset is created. + + Returns True only when every cell was proven to be float/None (numeric) or + str, so callers may bypass per-value bind processing. + """ variables = None if isinstance(variables_or_count, int) else list(variables_or_count) variable_count = variables_or_count if isinstance(variables_or_count, int) else len(variables) if variable_count > profile.max_source_variables: @@ -143,7 +147,7 @@ def preflight( max_physical=profile.max_source_variables + 1, ) if variables is None: - return + return False used = {"__case_ordinal"} source_names: set[str] = set() @@ -216,7 +220,7 @@ def preflight( ) rows = list(rows or ()) if not rows or _rows_within_limits(profile, variables, rows): - return + return True # Slow path only reports the first failing row in row-major order. for row_ordinal, row in enumerate(rows, start=1): row_bytes = 8 @@ -317,6 +321,7 @@ def preflight( row_ordinal=row_ordinal, statement_bytes=statement_bytes, maximum=profile.max_statement_bytes, ) + return False def _rows_within_limits( @@ -346,7 +351,10 @@ def _rows_within_limits( continue if set(map(type, column)) != {str}: return False - longest = max(map(len, map(str.encode, column))) + try: + longest = max(map(len, map(str.encode, column))) + except UnicodeEncodeError: + return False if longest > profile.max_text_value_bytes: return False row_bytes += 20 if profile.name in MYSQL_WIRE_PROFILES else longest diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index bae46c9..d31b783 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -871,7 +871,7 @@ def create_wide_dataset( fidelity_events = tuple(fidelity_events) source_rows = _canonicalize_database_numeric_rows(rows, variables) try: - preflight(profile, variables, rows=source_rows) + plain_rows = preflight(profile, variables, rows=source_rows) validate_spss_catalog( variables, case_weight_variable=case_weight_variable, @@ -982,14 +982,17 @@ def create_wide_dataset( ), ) if source_rows: - # Preflight proved cells are binary64/None or str, so the only - # bind processor (SQLite to_float) is a no-op; skip the per-row - # SQLAlchemy parameter pass and hand rows to the driver. + # Plain rows hold only float/None and str, so the only bind + # processor (SQLite to_float) is a no-op: hand them straight to + # the driver. Anything else (e.g. exact ints) keeps SQLAlchemy. compiled = insert(data_table).compile(dialect=connection.dialect) keys = compiled.positiontup for batch in _bounded_batches( source_rows, variables, profile.max_statement_bytes, ): + if not plain_rows: + connection.execute(insert(data_table), batch) + continue connection.exec_driver_sql(compiled.string, ( [tuple(map(row.__getitem__, keys)) for row in batch] if keys else batch diff --git a/tests/test_sql_profiles.py b/tests/test_sql_profiles.py index 1963897..4ff870d 100755 --- a/tests/test_sql_profiles.py +++ b/tests/test_sql_profiles.py @@ -771,3 +771,27 @@ def test_column_preflight_fast_path_agrees_with_row_checks(): preflight(MYSQL, variables, rows=[*plain, bad]) too_long = [{"q": 1.0, "s": "x" * (MYSQL.max_text_value_bytes + 1)}] assert not _rows_within_limits(MYSQL, variables, too_long) + assert not _rows_within_limits(MYSQL, variables, [{"q": 1.0, "s": "\ud800"}]) + # Exact ints are valid but not plain, so callers keep bind processing. + assert preflight(MYSQL, variables, rows=[*plain, {"q": 2**63, "s": ""}]) is False + with pytest.raises(UnsupportedOperationError, match="binary64"): + preflight(MYSQL, variables, rows=[*plain, {"q": 2**53 + 1, "s": ""}]) + + +def test_import_stores_exact_integer_cells_as_binary64(tmp_path): + from openstatspec.sql.wide import create_wide_dataset + import sqlite3 + + database = f"sqlite:///{tmp_path / 'ints.sqlite'}" + variables = [{ + "source_name": "q", "physical_name": "q", "ordinal": 1, + "storage_kind": "numeric", "format": "F20.0", + }] + create_wide_dataset( + database_url=database, dataset_id="ints", source_name="ints.sav", + source_format="SAV", rows=[{"q": 2**63}, {"q": None}], variables=variables, + ) + stored = sqlite3.connect(tmp_path / "ints.sqlite").execute( + "select q from data_ints order by __case_ordinal" + ).fetchall() + assert stored == [(float(2**63),), (None,)]