Skip to content

Commit 28a60e6

Browse files
Merge pull request #34 from OpenStatSpec/perf/column-preflight
Speed up SAV import and export (0.9.1)
2 parents a41849a + 2ab215e commit 28a60e6

8 files changed

Lines changed: 182 additions & 33 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,24 @@
22

33
All notable changes to this reference implementation are documented here.
44

5+
## 0.9.1 - 2026-11-09
6+
7+
This patch release speeds up SAV import and export. Behavior, error codes and
8+
the catalog contract are unchanged.
9+
10+
### Changed
11+
12+
- Prove strict row limits column by column before falling back to the exact
13+
row-by-row checks, which still run (and report the same first failing row)
14+
whenever a column is not provably within limits.
15+
- Convert imported frames and exported rows column-wise instead of cell by cell.
16+
- Insert imported rows through the driver after preflight instead of
17+
SQLAlchemy's per-row parameter pass.
18+
- Declare the direct `numpy>=1.26` dependency (already required by pandas).
19+
20+
Local SQLite timings on a 50k x 200 SAV: import 29.2 s to 8.6 s, export
21+
22.4 s to 17.4 s.
22+
523
## 0.9.0 - 2026-09-10
624

725
This minor release adds the explicitly selected optional official SPSS Frontend

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ The reference Python implementation of the OpenStatSpec specification.
44

55
This package implements the specification; it does not define or extend it.
66
The normative model lives in the `OpenStatSpec/specification` repository.
7-
Python 0.9.0 pins released specification `v0.5.0` at
7+
Python 0.9.1 pins released specification `v0.5.0` at
88
`864e84479f554b8ee250ffed44c4dfb963750d4a` and selects SAV/ZSAV 1.0 with
99
`database_io_policy=openstatspec-database-io-v1`. It implements the explicitly
1010
selected optional official SPSS Frontend 0.3 over unchanged Plan 0.1/0.2, not

‎pyproject.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@ build-backend = "hatchling.build"
44

55
[project]
66
name = "openstatspec"
7-
version = "0.9.0"
7+
version = "0.9.1"
88
description = "Reference adapter for the OpenStatSpec relational contract"
99
readme = "README.md"
1010
requires-python = ">=3.11"
1111
license = {text = "Apache-2.0"}
1212
authors = [{name = "OpenStatSpec"}]
13-
dependencies = ["SQLAlchemy>=2.0", "pandas>=2.2", "openstatspec-pyspssio==0.5.1.post2", "sqlglot>=30.14", "rfc8785>=0.1.4"]
13+
dependencies = ["SQLAlchemy>=2.0", "pandas>=2.2", "numpy>=1.26", "openstatspec-pyspssio==0.5.1.post2", "sqlglot>=30.14", "rfc8785>=0.1.4"]
1414

1515
[project.optional-dependencies]
1616
dev = ["pytest>=8"]

‎src/openstatspec/spss/sav.py‎

Lines changed: 20 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,11 @@
1010
import os
1111
import json
1212
import errno
13-
import math
1413
import ctypes
1514
from contextlib import ExitStack, contextmanager
1615
from datetime import UTC, datetime
1716
from functools import wraps
17+
from operator import itemgetter
1818
from pathlib import Path
1919
from tempfile import mkstemp, TemporaryDirectory
2020
from typing import Any, Callable
@@ -24,6 +24,7 @@
2424
except ImportError: # pragma: no cover - the supported CI/runtime is POSIX.
2525
fcntl = None
2626

27+
import numpy as np
2728
import pandas as pd
2829
import pyspssio
2930

@@ -206,18 +207,16 @@ def _pyspssio_missing_rules(encoded: str) -> dict[str, Any] | None:
206207

207208

208209
def _rows(frame: pd.DataFrame, variables: list[dict[str, Any]]) -> list[dict[str, Any]]:
209-
records: list[dict[str, Any]] = []
210-
for input_row in frame.to_dict(orient="records"):
211-
output: dict[str, Any] = {}
212-
for item in variables:
213-
value = input_row[item["source_name"]]
214-
if item["storage_kind"] == "numeric" and pd.isna(value):
215-
value = None
216-
elif item["storage_kind"] == "string" and (value is None or pd.isna(value)):
217-
value = ""
218-
output[item["physical_name"]] = value
219-
records.append(output)
220-
return records
210+
columns = []
211+
for item in variables:
212+
series = frame[item["source_name"]]
213+
empty = None if item["storage_kind"] == "numeric" else ""
214+
columns.append([
215+
empty if missing else value
216+
for value, missing in zip(series.tolist(), series.isna().tolist())
217+
])
218+
names = [item["physical_name"] for item in variables]
219+
return [dict(zip(names, values)) for values in zip(*columns)]
221220

222221

223222
def inspect_sav(source: str | Path) -> dict[str, Any]:
@@ -586,15 +585,14 @@ def export_sav_dataset(
586585
raise UnsupportedOperationError("Export requires explicit allow_loss for: " + ", ".join(rejected))
587586

588587
frame = pd.DataFrame(
589-
[
590-
{
591-
variable["source_name"]: (
592-
math.nan if row[variable["physical_name"]] is None else float(row[variable["physical_name"]])
593-
) if variable["storage_kind"] == "numeric" else row[variable["physical_name"]]
594-
for variable in variables
595-
}
596-
for row in rows
597-
],
588+
{
589+
variable["source_name"]: (
590+
# Preflight already proved numeric cells are binary64 or None.
591+
np.array(column, dtype=float) if variable["storage_kind"] == "numeric" else column
592+
)
593+
for variable in variables
594+
for column in [list(map(itemgetter(variable["physical_name"]), rows))]
595+
},
598596
columns=[variable["source_name"] for variable in variables],
599597
)
600598
publication_state: dict[str, Any] = {"published": False}

‎src/openstatspec/sql/profiles.py‎

Lines changed: 63 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,12 @@
88
from dataclasses import dataclass
99
import math
1010
from numbers import Real
11+
from operator import itemgetter
1112
from typing import Any
1213
from urllib.parse import urlparse
1314

15+
import numpy as np
16+
1417
from ..core import UnsupportedOperationError
1518

1619

@@ -125,8 +128,12 @@ def preflight(
125128
*,
126129
rows: Iterable[Mapping[str, Any]] | None = None,
127130
require_canonical_mapping: bool = True,
128-
) -> None:
129-
"""Validate strict target capabilities before any source dataset is created."""
131+
) -> bool:
132+
"""Validate strict target capabilities before any source dataset is created.
133+
134+
Returns True only when every cell was proven to be float/None (numeric) or
135+
str, so callers may bypass per-value bind processing.
136+
"""
130137
variables = None if isinstance(variables_or_count, int) else list(variables_or_count)
131138
variable_count = variables_or_count if isinstance(variables_or_count, int) else len(variables)
132139
if variable_count > profile.max_source_variables:
@@ -140,7 +147,7 @@ def preflight(
140147
max_physical=profile.max_source_variables + 1,
141148
)
142149
if variables is None:
143-
return
150+
return False
144151

145152
used = {"__case_ordinal"}
146153
source_names: set[str] = set()
@@ -211,7 +218,11 @@ def preflight(
211218
f"{profile.name} permits {profile.max_row_bytes}.",
212219
declared_row_bytes=declared_row_bytes, maximum=profile.max_row_bytes,
213220
)
214-
for row_ordinal, row in enumerate(rows or (), start=1):
221+
rows = list(rows or ())
222+
if not rows or _rows_within_limits(profile, variables, rows):
223+
return True
224+
# Slow path only reports the first failing row in row-major order.
225+
for row_ordinal, row in enumerate(rows, start=1):
215226
row_bytes = 8
216227
for variable in variables:
217228
if variable.get("storage_kind") == "numeric":
@@ -310,6 +321,54 @@ def preflight(
310321
row_ordinal=row_ordinal, statement_bytes=statement_bytes,
311322
maximum=profile.max_statement_bytes,
312323
)
324+
return False
325+
326+
327+
def _rows_within_limits(
328+
profile: SqlProfile, variables: list[Mapping[str, Any]],
329+
rows: list[Mapping[str, Any]],
330+
) -> bool:
331+
"""Column-wise proof that the row-by-row checks in preflight cannot fail.
332+
333+
Per-column maxima bound every row's row and statement size from above, so
334+
True means no row exceeds a limit; False defers to the exact slow path.
335+
"""
336+
row_bytes, statement_bytes = 8, 32
337+
for variable in variables:
338+
name = str(variable["physical_name"])
339+
try:
340+
column = list(map(itemgetter(name), rows))
341+
except KeyError:
342+
return False
343+
if variable.get("storage_kind") == "numeric":
344+
if not set(map(type, column)) <= _PLAIN_NUMERIC_TYPES:
345+
return False
346+
values = np.array(column, dtype=float) # None becomes NaN
347+
if np.isinf(values).any() or np.isnan(values).sum() != column.count(None):
348+
return False
349+
row_bytes += 8
350+
statement_bytes += 24
351+
continue
352+
if set(map(type, column)) != {str}:
353+
return False
354+
try:
355+
longest = max(map(len, map(str.encode, column)))
356+
except UnicodeEncodeError:
357+
return False
358+
if longest > profile.max_text_value_bytes:
359+
return False
360+
row_bytes += 20 if profile.name in MYSQL_WIRE_PROFILES else longest
361+
statement_bytes += longest + 8
362+
return (
363+
(profile.max_row_bytes is None or row_bytes <= profile.max_row_bytes)
364+
and (
365+
profile.max_statement_bytes is None
366+
or statement_bytes <= profile.max_statement_bytes
367+
)
368+
)
369+
370+
371+
_PLAIN_NUMERIC_TYPES = {float, type(None)}
313372

314373

315374
def _row_storage_bytes(profile: SqlProfile, variable: Mapping[str, Any]) -> int:

‎src/openstatspec/sql/wide.py‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -760,10 +760,24 @@ def _bounded_batches(
760760
maximum_statement_bytes: int | None,
761761
) -> Iterable[list[Mapping[str, Any]]]:
762762
"""Partition rows without exceeding the profile's statement payload budget."""
763+
if maximum_statement_bytes is None:
764+
rows = list(rows)
765+
if rows:
766+
yield rows
767+
return
768+
variables = list(variables)
769+
strings = [
770+
str(variable["physical_name"]) for variable in variables
771+
if variable.get("storage_kind") != "numeric"
772+
]
773+
# Same arithmetic as statement_payload_bytes, with numeric widths hoisted.
774+
fixed_size = 32 + 24 * (len(variables) - len(strings)) + 8 * len(strings)
763775
batch: list[Mapping[str, Any]] = []
764776
size = 0
765777
for row in rows:
766-
row_size = statement_payload_bytes(row, variables)
778+
row_size = fixed_size + sum(
779+
len(str(row.get(name, "")).encode("utf-8")) for name in strings
780+
)
767781
if (
768782
batch and maximum_statement_bytes is not None
769783
and size + row_size > maximum_statement_bytes
@@ -789,13 +803,19 @@ def _canonicalize_database_numeric_rows(
789803
normalized = []
790804
for source in rows:
791805
row = dict(source)
806+
if set(map(type, row.values())) <= _PLAIN_ROW_VALUE_TYPES:
807+
normalized.append(row)
808+
continue
792809
for name in numeric_names:
793810
if isinstance(row.get(name), Decimal) and row[name].is_finite():
794811
row[name] = float(row[name])
795812
normalized.append(row)
796813
return normalized
797814

798815

816+
_PLAIN_ROW_VALUE_TYPES = {float, int, str, bool, bytes, type(None)}
817+
818+
799819
def data_table_name(dataset_id: str) -> str:
800820
stem = _IDENTIFIER.sub("_", dataset_id).strip("_").lower() or "dataset"
801821
return f"data_{stem[:48]}"
@@ -851,7 +871,7 @@ def create_wide_dataset(
851871
fidelity_events = tuple(fidelity_events)
852872
source_rows = _canonicalize_database_numeric_rows(rows, variables)
853873
try:
854-
preflight(profile, variables, rows=source_rows)
874+
plain_rows = preflight(profile, variables, rows=source_rows)
855875
validate_spss_catalog(
856876
variables,
857877
case_weight_variable=case_weight_variable,
@@ -962,10 +982,21 @@ def create_wide_dataset(
962982
),
963983
)
964984
if source_rows:
985+
# Plain rows hold only float/None and str, so the only bind
986+
# processor (SQLite to_float) is a no-op: hand them straight to
987+
# the driver. Anything else (e.g. exact ints) keeps SQLAlchemy.
988+
compiled = insert(data_table).compile(dialect=connection.dialect)
989+
keys = compiled.positiontup
965990
for batch in _bounded_batches(
966991
source_rows, variables, profile.max_statement_bytes,
967992
):
968-
connection.execute(insert(data_table), batch)
993+
if not plain_rows:
994+
connection.execute(insert(data_table), batch)
995+
continue
996+
connection.exec_driver_sql(compiled.string, (
997+
[tuple(map(row.__getitem__, keys)) for row in batch]
998+
if keys else batch
999+
))
9691000
finish_normative_operation(
9701001
connection, normative, operation_id=operation_id,
9711002
status="succeeded",

‎tests/test_cli.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ def test_cli_import_inspect_validate_and_export_emit_json(tmp_path, capsys) -> N
4242

4343
def test_capability_matrix_is_public_and_cli_matches_engine_boundary(capsys) -> None:
4444
matrix = openstatspec.capability_matrix()
45-
assert matrix["adapter_version"] == "0.9.0"
45+
assert matrix["adapter_version"] == "0.9.1"
4646
assert matrix["database_io_policy"] == "openstatspec-database-io-v1"
4747
assert matrix["specification_status"] == "released"
4848
assert matrix["specification_release"] == "v0.5.0"

‎tests/test_sql_profiles.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -752,3 +752,46 @@ def test_generated_data_table_identifier_uses_effective_limit() -> None:
752752
assert error.value.details["reason"] == "identifier_limit"
753753
assert error.value.details["role"] == "physical data-table identifier"
754754
assert error.value.details["maximum"] == 5
755+
756+
757+
def test_column_preflight_fast_path_agrees_with_row_checks():
758+
from openstatspec.sql.profiles import _rows_within_limits
759+
760+
variables = [
761+
{"source_name": "q", "physical_name": "q", "ordinal": 1, "storage_kind": "numeric"},
762+
{"source_name": "s", "physical_name": "s", "ordinal": 2, "storage_kind": "string", "string_width": 8},
763+
]
764+
plain = [{"q": 1.0, "s": "Pärnu"}, {"q": None, "s": ""}]
765+
assert _rows_within_limits(MYSQL, variables, plain)
766+
preflight(MYSQL, variables, rows=plain)
767+
# Anything not provably safe falls back to the exact row-major checks.
768+
for bad in ({"q": float("inf"), "s": ""}, {"q": True, "s": ""}, {"q": 1.0, "s": None}, {"s": ""}):
769+
assert not _rows_within_limits(MYSQL, variables, [*plain, bad])
770+
with pytest.raises(UnsupportedOperationError):
771+
preflight(MYSQL, variables, rows=[*plain, bad])
772+
too_long = [{"q": 1.0, "s": "x" * (MYSQL.max_text_value_bytes + 1)}]
773+
assert not _rows_within_limits(MYSQL, variables, too_long)
774+
assert not _rows_within_limits(MYSQL, variables, [{"q": 1.0, "s": "\ud800"}])
775+
# Exact ints are valid but not plain, so callers keep bind processing.
776+
assert preflight(MYSQL, variables, rows=[*plain, {"q": 2**63, "s": ""}]) is False
777+
with pytest.raises(UnsupportedOperationError, match="binary64"):
778+
preflight(MYSQL, variables, rows=[*plain, {"q": 2**53 + 1, "s": ""}])
779+
780+
781+
def test_import_stores_exact_integer_cells_as_binary64(tmp_path):
782+
from openstatspec.sql.wide import create_wide_dataset
783+
import sqlite3
784+
785+
database = f"sqlite:///{tmp_path / 'ints.sqlite'}"
786+
variables = [{
787+
"source_name": "q", "physical_name": "q", "ordinal": 1,
788+
"storage_kind": "numeric", "format": "F20.0",
789+
}]
790+
create_wide_dataset(
791+
database_url=database, dataset_id="ints", source_name="ints.sav",
792+
source_format="SAV", rows=[{"q": 2**63}, {"q": None}], variables=variables,
793+
)
794+
stored = sqlite3.connect(tmp_path / "ints.sqlite").execute(
795+
"select q from data_ints order by __case_ordinal"
796+
).fetchall()
797+
assert stored == [(float(2**63),), (None,)]

0 commit comments

Comments
 (0)