From f7fd1633901b1f745e2fc2ed76d73fc9fb49a8b6 Mon Sep 17 00:00:00 2001 From: breken-ai <312387581+breken-ai@users.noreply.github.com> Date: Fri, 25 Sep 2026 18:22:54 -0700 Subject: [PATCH 1/4] Key the zone-map sidecar by the column's offset in its file Two same-length memmapped columns from one file (rows of a 2-D table memmap, or np.memmap(offset=...) columns) shared one `.xyzones` sidecar, so a rebuild loaded one column's min/max for the other and the axis range pointed away from the data. Name the sidecar by the column's byte offset too, and bump the sidecar magic so a v1 file that may hold another column's maps is refolded once. --- news/+zone-cache-views.bugfix.md | 4 +++ python/xy/_ooc.py | 24 +++++++++++++++ python/xy/columns.py | 18 +++++++++-- spec/process/perf-audit-2026-07-22.md | 5 +++- tests/test_ooc.py | 43 +++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 news/+zone-cache-views.bugfix.md diff --git a/news/+zone-cache-views.bugfix.md b/news/+zone-cache-views.bugfix.md new file mode 100644 index 000000000..5d2277da9 --- /dev/null +++ b/news/+zone-cache-views.bugfix.md @@ -0,0 +1,4 @@ +Charts over several disk-backed columns from one file (the rows of a 2-D +`np.memmap` table, or `np.memmap(offset=...)` columns packed into one file) no +longer reuse one column's cached statistics for another, which gave a column +the other column's axis range when the chart was built again. diff --git a/python/xy/_ooc.py b/python/xy/_ooc.py index 68c89721a..8b36fe355 100644 --- a/python/xy/_ooc.py +++ b/python/xy/_ooc.py @@ -133,6 +133,30 @@ def backing_path(arr: Any) -> str | None: return None +def backing_offset(arr: Any) -> int | None: + """Byte offset of ``arr``'s first element within its backing file, or + ``None`` when ``arr`` is not disk-backed. + + Views of one file share its path, so the path alone does not name a + column: the rows of a 2-D table memmap, or two ``np.memmap(offset=...)`` + columns packed into one file, all map the same file. A view's + ``.offset`` is inherited from the memmap it was sliced from, so the + offset is measured from the root memmap (the one mapping the file + directly, whose ``.offset`` is its file position) to ``arr``'s data. + """ + root = None + seen = arr + while isinstance(seen, np.ndarray): + if isinstance(seen, np.memmap): + root = seen + seen = seen.base + if root is None: + return None + start = int(root.__array_interface__["data"][0]) + here = int(np.asarray(arr).__array_interface__["data"][0]) + return int(root.offset) + (here - start) + + def open_f64(path: str | os.PathLike[str]) -> npt.NDArray[np.float64]: """Reopen an existing canonical f64 file as a read-only memmap column. diff --git a/python/xy/columns.py b/python/xy/columns.py index 7d37a745c..daf9f515e 100644 --- a/python/xy/columns.py +++ b/python/xy/columns.py @@ -98,7 +98,10 @@ def null_count(self) -> int: # restart) loads a ~10 MB sidecar in well under a second. The cache self- # invalidates if the source file's size or mtime changed, or the chunk size / # row count no longer matches, so a stale/edited column is never trusted. -_ZONE_CACHE_MAGIC = b"XYZONEC1" +# Version 2 keys the sidecar by the column's byte offset in its file (see +# `_zone_cache_path`); a version-1 sidecar may hold another view's maps, so it +# is rejected and refolded once. +_ZONE_CACHE_MAGIC = b"XYZONEC2" _ZONE_CACHE_SUFFIX = ".xyzones" # 6 f64 planes + 2 u64 planes per chunk (mirrors ZoneMaps' fields, in order). _ZONE_F64_FIELDS = ("mins", "maxs", "sums", "sum_sqs", "positive_mins", "positive_maxs") @@ -106,8 +109,19 @@ def null_count(self) -> int: def _zone_cache_path(arr: Any) -> str | None: + """Sidecar path for one memmapped column. + + Several columns can map the same file (rows of a 2-D table memmap, or + `np.memmap(offset=...)` columns packed into one file), so the sidecar is + named by the column's byte offset as well as the file: sharing one path + let a same-length column reload another column's maps (its min/max, hence + its axis range). + """ bp = _ooc.backing_path(arr) - return (bp + _ZONE_CACHE_SUFFIX) if bp is not None else None + offset = _ooc.backing_offset(arr) + if bp is None or offset is None: + return None + return f"{bp}.{offset}{_ZONE_CACHE_SUFFIX}" if offset else bp + _ZONE_CACHE_SUFFIX def _zone_source_stamp(source: str) -> tuple[int, int]: diff --git a/spec/process/perf-audit-2026-07-22.md b/spec/process/perf-audit-2026-07-22.md index 78f099419..f7e4bdb21 100644 --- a/spec/process/perf-audit-2026-07-22.md +++ b/spec/process/perf-audit-2026-07-22.md @@ -38,7 +38,10 @@ Fix (`columns.py`, Python side — no kernel/ABI change): persist the fold to a `.xyzones` sidecar and reload it on the next build instead of rescanning. The sidecar carries the source file's size + mtime and the `(ZONE_CHUNK, row-count)` it was folded at; any mismatch rejects it, so an -edited or truncated column is never trusted. First build pays the scan and +edited or truncated column is never trusted. A column that starts partway into +its file (a row of a 2-D table memmap, or an `np.memmap(offset=...)` column) +gets its own `..xyzones` sidecar, since columns sharing a +file and a row count would otherwise reload each other's maps. First build pays the scan and writes the sidecar (21 MB for both planet columns); every later build loads it bit-identically. Measured on the real 10.74 B-row planet columns: **52.8 s → 0.01 s** figure build, identical domain. The fused `zone_maps_pair` still does diff --git a/tests/test_ooc.py b/tests/test_ooc.py index 59fa22c97..0030f398d 100644 --- a/tests/test_ooc.py +++ b/tests/test_ooc.py @@ -168,3 +168,46 @@ def test_zone_map_cache_roundtrip_and_staleness(tmp_path): os.utime(tmp_path / "z.f64", ns=(0, 0)) reopened = open_f64(tmp_path / "z.f64") assert _load_zone_cache(reopened) is None + + +def test_zone_map_cache_keeps_views_of_one_file_apart(tmp_path): + """Two same-length columns mapped from one file (rows of a 2-D table memmap) + each keep their own zone-map sidecar, so a warm rebuild does not hand one + column the other's min/max.""" + n = 1000 + path = tmp_path / "table.f64" + table = np.memmap(path, dtype=np.float64, mode="w+", shape=(2, n)) + table[0] = np.linspace(0.0, 1.0, n) + table[1] = np.linspace(1000.0, 2000.0, n) + table.flush() + del table + + def build_ranges(): + mapped = np.memmap(path, dtype=np.float64, mode="r", shape=(2, n)) + fig = xy.chart(xy.scatter(x=mapped[0], y=mapped[1])).figure() + trace = fig.traces[0] + return (trace.x.min, trace.x.max), (trace.y.min, trace.y.max) + + cold = build_ranges() # folds both columns and writes the sidecars + warm = build_ranges() # reloads them + assert cold == ((0.0, 1.0), (1000.0, 2000.0)) + assert warm == cold + + +def test_zone_map_cache_keys_on_file_offset(tmp_path): + """`np.memmap(..., offset=)` columns of one file cache separately too.""" + from xy import columns + + n = 500 + path = tmp_path / "packed.f64" + np.concatenate([np.full(n, -5.0), np.full(n, 7.0)]).tofile(path) + + def view(offset): + return np.memmap(path, dtype=np.float64, mode="r", offset=offset, shape=(n,)) + + assert columns._zone_maps_for(view(0)).max == -5.0 + assert columns._zone_maps_for(view(n * 8)).min == 7.0 + # Warm reloads read each column's own sidecar. + assert columns._zone_maps_for(view(0)).max == -5.0 + assert columns._zone_maps_for(view(n * 8)).min == 7.0 + assert columns._zone_cache_path(view(0)) != columns._zone_cache_path(view(n * 8)) From 060ed72bb47c463d81f02c5ce5b170314627ca25 Mon Sep 17 00:00:00 2001 From: breken-ai <312387581+breken-ai@users.noreply.github.com> Date: Fri, 25 Sep 2026 18:23:11 -0700 Subject: [PATCH 2/4] Name the news fragment after the pull request --- news/{+zone-cache-views.bugfix.md => 524.bugfix.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename news/{+zone-cache-views.bugfix.md => 524.bugfix.md} (100%) diff --git a/news/+zone-cache-views.bugfix.md b/news/524.bugfix.md similarity index 100% rename from news/+zone-cache-views.bugfix.md rename to news/524.bugfix.md From c28e5c6e75462376885f87bbdc817b86defb492c Mon Sep 17 00:00:00 2001 From: breken-ai <312387581+breken-ai@users.noreply.github.com> Date: Fri, 25 Sep 2026 18:29:35 -0700 Subject: [PATCH 3/4] Always write the offset in the sidecar name `table` at offset 8 and a file named `table.8` at offset 0 both mapped to `table.8.xyzones`; writing `.0` too makes every file/offset pair distinct. A memmap-typed view with no mapping behind it now counts as not disk-backed instead of raising, the tests assert each column persisted a loadable sidecar, and the spec paragraph is rewrapped. --- python/xy/_ooc.py | 5 +++-- python/xy/columns.py | 19 +++++++++++-------- spec/process/perf-audit-2026-07-22.md | 12 ++++++------ tests/test_ooc.py | 27 +++++++++++++++++++++++++-- 4 files changed, 45 insertions(+), 18 deletions(-) diff --git a/python/xy/_ooc.py b/python/xy/_ooc.py index 8b36fe355..a97f3e621 100644 --- a/python/xy/_ooc.py +++ b/python/xy/_ooc.py @@ -150,11 +150,12 @@ def backing_offset(arr: Any) -> int | None: if isinstance(seen, np.memmap): root = seen seen = seen.base - if root is None: + root_offset = getattr(root, "offset", None) + if root is None or root_offset is None: return None start = int(root.__array_interface__["data"][0]) here = int(np.asarray(arr).__array_interface__["data"][0]) - return int(root.offset) + (here - start) + return int(root_offset) + (here - start) def open_f64(path: str | os.PathLike[str]) -> npt.NDArray[np.float64]: diff --git a/python/xy/columns.py b/python/xy/columns.py index daf9f515e..dfd6b01c1 100644 --- a/python/xy/columns.py +++ b/python/xy/columns.py @@ -98,9 +98,9 @@ def null_count(self) -> int: # restart) loads a ~10 MB sidecar in well under a second. The cache self- # invalidates if the source file's size or mtime changed, or the chunk size / # row count no longer matches, so a stale/edited column is never trusted. -# Version 2 keys the sidecar by the column's byte offset in its file (see -# `_zone_cache_path`); a version-1 sidecar may hold another view's maps, so it -# is rejected and refolded once. +# Version 2 names the sidecar by the column's byte offset in its file (see +# `_zone_cache_path`). A version-1 sidecar may hold another column's maps, so +# the magic changed with the name and old files are never read. _ZONE_CACHE_MAGIC = b"XYZONEC2" _ZONE_CACHE_SUFFIX = ".xyzones" # 6 f64 planes + 2 u64 planes per chunk (mirrors ZoneMaps' fields, in order). @@ -113,15 +113,18 @@ def _zone_cache_path(arr: Any) -> str | None: Several columns can map the same file (rows of a 2-D table memmap, or `np.memmap(offset=...)` columns packed into one file), so the sidecar is - named by the column's byte offset as well as the file: sharing one path - let a same-length column reload another column's maps (its min/max, hence - its axis range). + named `..xyzones`: sharing one path let a same-length + column reload another column's maps (its min/max, hence its axis range). + The offset is always written, even when it is 0, so `table` at offset 8 + and a file named `table.8` at offset 0 cannot share a name either. """ bp = _ooc.backing_path(arr) + if bp is None: + return None offset = _ooc.backing_offset(arr) - if bp is None or offset is None: + if offset is None: return None - return f"{bp}.{offset}{_ZONE_CACHE_SUFFIX}" if offset else bp + _ZONE_CACHE_SUFFIX + return f"{bp}.{offset}{_ZONE_CACHE_SUFFIX}" def _zone_source_stamp(source: str) -> tuple[int, int]: diff --git a/spec/process/perf-audit-2026-07-22.md b/spec/process/perf-audit-2026-07-22.md index f7e4bdb21..a7b758d7a 100644 --- a/spec/process/perf-audit-2026-07-22.md +++ b/spec/process/perf-audit-2026-07-22.md @@ -35,14 +35,14 @@ columns) that is a ~53 s NVMe-bound scan paid on **every** process start an immutable file. Fix (`columns.py`, Python side — no kernel/ABI change): persist the fold to a -`.xyzones` sidecar and reload it on the next build instead of +sidecar file and reload it on the next build instead of rescanning. The sidecar carries the source file's size + mtime and the `(ZONE_CHUNK, row-count)` it was folded at; any mismatch rejects it, so an -edited or truncated column is never trusted. A column that starts partway into -its file (a row of a 2-D table memmap, or an `np.memmap(offset=...)` column) -gets its own `..xyzones` sidecar, since columns sharing a -file and a row count would otherwise reload each other's maps. First build pays the scan and -writes the sidecar (21 MB for both planet columns); every later build loads it +edited or truncated column is never trusted. The sidecar is named +`..xyzones` because several columns can map one file (the +rows of a 2-D table memmap, or `np.memmap(offset=...)` columns); sharing a +name let a same-length column reload another column's maps. First build pays +the scan and writes the sidecar (21 MB for both planet columns); every later build loads it bit-identically. Measured on the real 10.74 B-row planet columns: **52.8 s → 0.01 s** figure build, identical domain. The fused `zone_maps_pair` still does both columns in one pass on a cold miss; only genuinely disk-backed columns are diff --git a/tests/test_ooc.py b/tests/test_ooc.py index 0030f398d..0060978b6 100644 --- a/tests/test_ooc.py +++ b/tests/test_ooc.py @@ -188,7 +188,12 @@ def build_ranges(): trace = fig.traces[0] return (trace.x.min, trace.x.max), (trace.y.min, trace.y.max) + from xy.columns import _load_zone_cache + cold = build_ranges() # folds both columns and writes the sidecars + mapped = np.memmap(path, dtype=np.float64, mode="r", shape=(2, n)) + assert _load_zone_cache(mapped[0]).max == 1.0 # each row persisted its own + assert _load_zone_cache(mapped[1]).min == 1000.0 warm = build_ranges() # reloads them assert cold == ((0.0, 1.0), (1000.0, 2000.0)) assert warm == cold @@ -208,6 +213,24 @@ def view(offset): assert columns._zone_maps_for(view(0)).max == -5.0 assert columns._zone_maps_for(view(n * 8)).min == 7.0 # Warm reloads read each column's own sidecar. - assert columns._zone_maps_for(view(0)).max == -5.0 - assert columns._zone_maps_for(view(n * 8)).min == 7.0 + assert columns._load_zone_cache(view(0)).max == -5.0 + assert columns._load_zone_cache(view(n * 8)).min == 7.0 assert columns._zone_cache_path(view(0)) != columns._zone_cache_path(view(n * 8)) + + # A file whose name ends in `.` gets a distinct sidecar from the + # same-named column at that offset of the shorter-named file. + suffixed = tmp_path / f"packed.f64.{n * 8}" + np.full(n, 3.0).tofile(suffixed) + at_zero = np.memmap(suffixed, dtype=np.float64, mode="r", shape=(n,)) + assert columns._zone_cache_path(at_zero) != columns._zone_cache_path(view(n * 8)) + assert columns._zone_maps_for(at_zero).max == 3.0 + assert columns._load_zone_cache(view(n * 8)).min == 7.0 + + +def test_backing_offset_without_file_metadata_is_not_disk_backed(): + """A memmap-typed view with no mapping behind it has no file offset.""" + from xy._ooc import backing_offset + + orphan = np.arange(4, dtype=np.float64).view(np.memmap) + assert backing_offset(orphan) is None + assert backing_offset(np.arange(4, dtype=np.float64)) is None From 3f1e7b353359a2901e8d5f252ce577d1c79a10e6 Mon Sep 17 00:00:00 2001 From: breken Date: Fri, 25 Sep 2026 18:34:53 -0700 Subject: [PATCH 4/4] docs: fix zone-sidecar prose and wrapping --- spec/process/perf-audit-2026-07-22.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/spec/process/perf-audit-2026-07-22.md b/spec/process/perf-audit-2026-07-22.md index a7b758d7a..2da12a438 100644 --- a/spec/process/perf-audit-2026-07-22.md +++ b/spec/process/perf-audit-2026-07-22.md @@ -41,9 +41,10 @@ rescanning. The sidecar carries the source file's size + mtime and the edited or truncated column is never trusted. The sidecar is named `..xyzones` because several columns can map one file (the rows of a 2-D table memmap, or `np.memmap(offset=...)` columns); sharing a -name let a same-length column reload another column's maps. First build pays -the scan and writes the sidecar (21 MB for both planet columns); every later build loads it -bit-identically. Measured on the real 10.74 B-row planet columns: **52.8 s → +name lets a same-length column reload another column's maps. First build pays +the scan and writes the sidecar (21 MB for both planet columns); every later +build loads it bit-identically. Measured on the real 10.74 B-row planet +columns: **52.8 s → 0.01 s** figure build, identical domain. The fused `zone_maps_pair` still does both columns in one pass on a cold miss; only genuinely disk-backed columns are cached (in-RAM ingest is byte-for-byte unchanged). Full range-pruning is