Skip to content

Commit 37658e4

Browse files
Merge pull request #24 from OpenStatSpec/perf/memory-bounds
perf: remove duplicate case-row materialization
2 parents 06b346a + efeda69 commit 37658e4

2 files changed

Lines changed: 77 additions & 5 deletions

File tree

‎src/openstatspec/sql/wide.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1104,10 +1104,9 @@ def _read_wide_dataset(
11041104
for row, variable in zip(source_variables, variables, strict=True)
11051105
}
11061106
variable_ids = tuple(variables_by_id)
1107-
rows = [dict(row) for row in connection.execute(
1107+
rows = _canonicalize_database_numeric_rows(connection.execute(
11081108
select(data_table).order_by(data_table.c.__case_ordinal)
1109-
).mappings()]
1110-
rows = _canonicalize_database_numeric_rows(rows, variables)
1109+
).mappings(), variables)
11111110
documents = connection.execute(
11121111
select(normative.document)
11131112
.where(normative.document.c.dataset_id == core_id)

‎tests/test_read_only_export.py‎

Lines changed: 75 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
python -m pytest tests/test_read_only_export.py
66
The fixture creates and removes its own database and SELECT-only user.
77
"""
8+
from collections.abc import Mapping
89
import json
910
import os
1011
import sqlite3
@@ -30,8 +31,14 @@ def seeded_catalog(request, tmp_path):
3031
if request.param == "dolt" and not admin_url:
3132
pytest.skip("Live Dolt requires OPENSTATSPEC_TEST_DOLT_ADMIN_URL")
3233
source = tmp_path / "source.sav"
34+
frame = pd.DataFrame({"answer": [1.0, 2.0], "name": ["one", "two"]})
35+
if request.param in {"sqlite-mixed", "sqlite-empty"}:
36+
frame = pd.DataFrame({
37+
"answer": [1.0000000000000002, None, -2.5, 0.0],
38+
"name": ["Õun", "", "東京", "last"],
39+
})
3340
pyspssio.write_sav(
34-
str(source), pd.DataFrame({"answer": [1.0, 2.0], "name": ["one", "two"]}),
41+
str(source), frame,
3542
var_labels={"answer": "Answer"},
3643
var_value_labels={"answer": {1.0: "Yes", 2.0: "No"}},
3744
var_missing_values={"answer": {"values": [2.0]}},
@@ -43,7 +50,13 @@ def seeded_catalog(request, tmp_path):
4350
logical = catalog(MetaData())
4451
with source_engine.connect() as connection:
4552
dataset = connection.execute(select(logical.dataset)).mappings().one()
46-
if request.param == "sqlite":
53+
if request.param == "sqlite-empty":
54+
_, variables, _ = wide.read_wide_dataset(database_url=url, dataset_id="sample")
55+
wide.create_wide_dataset(
56+
database_url=url, dataset_id="empty", source_name="source.sav",
57+
source_format="SAV", variables=variables, rows=[], source_encoding="UTF-8",
58+
)
59+
if request.param.startswith("sqlite"):
4760
with sqlite3.connect(path) as connection:
4861
connection.execute("PRAGMA journal_mode = WAL")
4962

@@ -156,6 +169,66 @@ def test_export_and_reads_leave_no_database_trace(read_catalog, tmp_path, suffix
156169
assert not list(tmp_path.glob(".*.staging.*"))
157170

158171

172+
@pytest.mark.parametrize("seeded_catalog,dataset_id", [
173+
("sqlite-mixed", "sample"), ("sqlite-empty", "empty"),
174+
], indirect=["seeded_catalog"])
175+
@pytest.mark.parametrize("operation", ["descriptor", "validate", "export"])
176+
def test_public_reads_copy_each_case_once(read_catalog, tmp_path, monkeypatch, dataset_id, operation):
177+
url, snapshot, source = read_catalog
178+
expected_rows = [] if dataset_id == "empty" else [
179+
{"__case_ordinal": 1, "answer": 1.0000000000000002, "name": "Õun"},
180+
{"__case_ordinal": 2, "answer": None, "name": ""},
181+
{"__case_ordinal": 3, "answer": -2.5, "name": "東京"},
182+
{"__case_ordinal": 4, "answer": 0.0, "name": "last"},
183+
]
184+
before = snapshot()
185+
output = tmp_path / "copies.sav"
186+
copies = 0
187+
188+
def counted_dict(*args, **kwargs):
189+
nonlocal copies
190+
if args and isinstance(args[0], Mapping) and "__case_ordinal" in args[0]:
191+
copies += 1
192+
return dict(*args, **kwargs)
193+
194+
with monkeypatch.context() as patch:
195+
patch.setattr(wide, "dict", counted_dict, raising=False)
196+
if operation == "descriptor":
197+
dataset, variables, rows = wide.read_wide_dataset(database_url=url, dataset_id=dataset_id)
198+
elif operation == "validate":
199+
result = openstatspec.validate(database_url=url, dataset_id=dataset_id)
200+
else:
201+
result = openstatspec.export_sav(database_url=url, dataset_id=dataset_id, destination=output)
202+
203+
if operation == "descriptor":
204+
assert isinstance(rows, list) and all(type(row) is dict for row in rows)
205+
assert rows == expected_rows
206+
assert dataset["case_count"] == len(expected_rows)
207+
assert [(v["source_name"], v["physical_name"]) for v in variables] == [
208+
("answer", "answer"), ("name", "name"),
209+
]
210+
assert variables[0]["label"] == "Answer"
211+
assert json.loads(variables[0]["value_labels"]) == {"1.0": "Yes", "2.0": "No"}
212+
else:
213+
assert not result.diagnostics
214+
if operation == "validate":
215+
assert result["valid"]
216+
assert result["case_count"] == len(expected_rows)
217+
assert result["variable_count"] == 2
218+
else:
219+
assert "operation_id" not in result
220+
expected, expected_meta = pyspssio.read_sav(str(source), include_user_missing=True)
221+
actual, actual_meta = pyspssio.read_sav(str(output), include_user_missing=True)
222+
if not expected_rows:
223+
# pyspssio decodes an empty string column as object, not StringDtype.
224+
expected = expected.iloc[:0].astype({"name": object})
225+
pd.testing.assert_frame_equal(actual, expected, check_exact=True)
226+
for key in ("var_labels", "var_value_labels", "var_missing_values", "var_formats", "var_measure_levels"):
227+
assert actual_meta[key] == expected_meta[key]
228+
assert snapshot() == before
229+
assert copies == len(expected_rows), "one case dictionary per fetched row, not two"
230+
231+
159232
@pytest.mark.parametrize("failure", ["read", "loss", "writer", "publish", "restore", "staging", "backup"])
160233
def test_failed_export_never_writes_database(read_catalog, tmp_path, monkeypatch, failure):
161234
url, snapshot, _source = read_catalog

0 commit comments

Comments
 (0)