From 54ac05764679e7e404cddb9db010ab2a131423fc Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Mon, 5 Oct 2026 12:22:28 +0530 Subject: [PATCH 01/18] add: diskmanager exception handling --- src/seriousdb/exceptions.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/seriousdb/exceptions.py b/src/seriousdb/exceptions.py index d0cba68..4ca6b63 100644 --- a/src/seriousdb/exceptions.py +++ b/src/seriousdb/exceptions.py @@ -41,3 +41,8 @@ class ServiceUnavailableError(ApplicationError): """A dependency the application needs is currently not usable.""" default_detail = "The service is temporarily unavailable" + +class DiskManagerError(ApplicationError): + """A raw page read, write or allocation on the database file failed.""" + + default_detail = "The database file could not be accessed as fixed-size pages" From 6a2da9a45128211186f1b25dad6a855edfec35f9 Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Mon, 5 Oct 2026 22:50:46 +0530 Subject: [PATCH 02/18] add: disk_manager.py, implemented pagecount, readpage, writepage,appendpage --- src/seriousdb/disk_manager.py | 192 ++++++++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 src/seriousdb/disk_manager.py diff --git a/src/seriousdb/disk_manager.py b/src/seriousdb/disk_manager.py new file mode 100644 index 0000000..f6e342b --- /dev/null +++ b/src/seriousdb/disk_manager.py @@ -0,0 +1,192 @@ + +from __future__ import annotations + +import operator +import os +from pathlib import Path +from threading import RLock +from typing import Self, SupportsIndex + +from .exceptions import DiskManagerError +from .storage_format import PAGE_SIZE +from .types import UInt32 + +_O_BINARY = getattr(os, "O_BINARY", 0) # Windows only; stops newline translation +_HAS_POSITIONAL_IO = hasattr(os, "pread") and hasattr(os, "pwrite") + +__all__ = ["DiskManager"] + + +class DiskManager: + """Page ``n`` is stored at byte offset ``n * PAGE_SIZE``. This class does raw +page I/O only: it knows nothing about page contents, and locking is left to +the concurrency manager. + + Parameters + ---------- + path : str or Path + Path of the database file. It is created if it does not exist. + repair_torn_page : bool, optional + If the file ends in a partial page (for example after a crash while + growing it), truncate it to the last whole page instead of raising. + Only ever removes the incomplete tail. Defaults to ``False``. + + Raises + ------ + DiskManagerError + If the file size is not a multiple of ``PAGE_SIZE`` and + `repair_torn_page` is false. + OSError + If the file cannot be opened. + + + + A new file has no pages, so the first :meth:`append_page` returns 0. + The layer that owns the metadata must append and write page 0 first. + Free-list reuse is also that layer's job; this class only appends. + """ + + def __init__(self, path: str | Path, *, repair_torn_page: bool = False): + self._path = Path(path) + self._lock = RLock() + self._fd: int | None = None + fd = os.open(self._path, os.O_RDWR | os.O_CREAT | _O_BINARY, 0o644) + try: + size = os.fstat(fd).st_size + tail = size % PAGE_SIZE + if tail: + if not repair_torn_page: + raise DiskManagerError( + f"{self._path} is {size} bytes, not a multiple of " + f"{PAGE_SIZE}; the last page is incomplete" + ) + os.ftruncate(fd, size - tail) + os.fsync(fd) + except BaseException: + os.close(fd) + raise + self._fd = fd + + @property + def path(self): + #Path of the database file. + return self._path + + @property + def page_count(self) : + """Number of whole pages currently in the file. + + Read from the file size on every call, so pages appended by another + process are visible. + """ + return os.fstat(self._require_open()).st_size // PAGE_SIZE + + def read_page(self, page_id: SupportsIndex): + """Return the ``PAGE_SIZE`` bytes of page `page_id`. + + Raises + ------ + DiskManagerError + If `page_id` is out of range, the file is closed, or the file + ends before a full page could be read. + """ + fd = self._require_open() + offset = self._offset(fd, page_id) + buf = bytearray() + while len(buf) < PAGE_SIZE: + chunk = self._read_at(fd, PAGE_SIZE - len(buf), offset + len(buf)) + if not chunk: + raise DiskManagerError(f"unexpected end of file reading page {page_id}") + buf += chunk + return bytes(buf) + + def write_page(self, page_id: SupportsIndex, data: bytes): + """Overwrite existing page `page_id` with `data`. + Raises + ------ + DiskManagerError + If `data` is not exactly ``PAGE_SIZE`` bytes, `page_id` has not + been allocated, or the file is closed. + """ + fd = self._require_open() + if len(data) != PAGE_SIZE: + raise DiskManagerError( + f"page data must be {PAGE_SIZE} bytes, got {len(data)}" + ) + self._write_all(fd, data, self._offset(fd, page_id)) + + def append_page(self): + """Grow the file by one zero-filled page and return its ID. + + Not fsynced; call :meth:`sync` before relying on the new size + after a crash. + + Raises + ------ + DiskManagerError + If the file is closed or page IDs would exceed the ``UInt32`` range. + """ + fd = self._require_open() + with self._lock: + count = os.fstat(fd).st_size // PAGE_SIZE + if count > UInt32.MAX: + raise DiskManagerError("page ID would exceed the UInt32 range") + self._write_all(fd, bytes(PAGE_SIZE), count * PAGE_SIZE) + return UInt32(count) + + def sync(self): + #Flush file contents and size to stable storage (``fsync``). + os.fsync(self._require_open()) + + def close(self): + #Close the file. Safe to call more than once. + with self._lock: + if self._fd is not None: + os.close(self._fd) + self._fd = None + + def __enter__(self): + #Return the manager for use in a ``with`` block. + return self + + def __exit__(self, *exc_info: object): + #Close the file when leaving the ``with`` block + self.close() + + # -- internals --------------------------------------------------------- + + def _require_open(self): + if self._fd is None: + raise DiskManagerError("the database file is closed") + return self._fd + + def _offset(self, fd: int, page_id: SupportsIndex): + try: + pid = operator.index(page_id) + except TypeError as e: + raise DiskManagerError(f"invalid page ID: {page_id!r}") from e + count = os.fstat(fd).st_size // PAGE_SIZE + if not 0 <= pid < count: + raise DiskManagerError(f"page {pid} is out of range (0..{count - 1})") + return pid * PAGE_SIZE + + def _read_at(self, fd: int, size: int, offset: int): + if _HAS_POSITIONAL_IO: + return os.pread(fd, size, offset) + with self._lock: # no pread on Windows: seek + read must be atomic + os.lseek(fd, offset, os.SEEK_SET) + return os.read(fd, size) + + def _write_all(self, fd: int, data: bytes, offset: int): + view = memoryview(data) + written = 0 + while written < len(view): + if _HAS_POSITIONAL_IO: + n = os.pwrite(fd, view[written:], offset + written) + else: + with self._lock: + os.lseek(fd, offset + written, os.SEEK_SET) + n = os.write(fd, view[written:]) + if n <= 0: + raise DiskManagerError("write made no progress") + written += n From 9bce2d4330c224ec07d5befd3f283e17bf590a5f Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Mon, 5 Oct 2026 22:52:22 +0530 Subject: [PATCH 03/18] add: test_disk_manager.py --- tests/test_disk_manager.py | 168 +++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 tests/test_disk_manager.py diff --git a/tests/test_disk_manager.py b/tests/test_disk_manager.py new file mode 100644 index 0000000..8c0feb0 --- /dev/null +++ b/tests/test_disk_manager.py @@ -0,0 +1,168 @@ +import os +from threading import Barrier, Thread + +import pytest + +from seriousdb import disk_manager +from seriousdb.disk_manager import DiskManager +from seriousdb.exceptions import DiskManagerError +from seriousdb.storage_format import PAGE_SIZE + + +@pytest.fixture +def path(tmp_path): + return tmp_path / "test.sdb" + + +def page(byte: int) -> bytes: + return bytes([byte]) * PAGE_SIZE + + +def test_new_file_is_empty_and_first_append_is_page_zero(path): + with DiskManager(path) as dm: + assert dm.page_count == 0 + assert dm.append_page() == 0 + assert dm.append_page() == 1 + assert dm.page_count == 2 + assert path.stat().st_size == 2 * PAGE_SIZE + + +def test_round_trip_and_reopen(path): + with DiskManager(path) as dm: + for i in range(3): + dm.write_page(dm.append_page(), page(i + 1)) + dm.sync() + with DiskManager(path) as dm: + assert dm.page_count == 3 + assert [dm.read_page(i) for i in range(3)] == [page(1), page(2), page(3)] + + +def test_fresh_page_is_zeroed(path): + with DiskManager(path) as dm: + pid = dm.append_page() + assert dm.read_page(pid) == bytes(PAGE_SIZE) + + +def test_misaligned_file_rejected(path): + path.write_bytes(bytes(PAGE_SIZE + 10)) + with pytest.raises(DiskManagerError): + DiskManager(path) + assert path.stat().st_size == PAGE_SIZE + 10 # untouched + + +def test_repair_truncates_only_the_torn_tail(path): + path.write_bytes(page(7) + page(8) + b"x" * 100) + with DiskManager(path, repair_torn_page=True) as dm: + assert dm.page_count == 2 + assert dm.read_page(0) == page(7) + assert dm.read_page(1) == page(8) + assert path.stat().st_size == 2 * PAGE_SIZE + + +@pytest.mark.parametrize("size", [PAGE_SIZE - 1, PAGE_SIZE + 1, 0]) +def test_wrong_size_write_rejected(path, size): + with DiskManager(path) as dm: + dm.append_page() + with pytest.raises(DiskManagerError): + dm.write_page(0, bytes(size)) + + +@pytest.mark.parametrize("bad", [-1, 1, 99, "0", 1.5]) +def test_bad_page_id_rejected(path, bad): + with DiskManager(path) as dm: + dm.append_page() + with pytest.raises(DiskManagerError): + dm.read_page(bad) + with pytest.raises(DiskManagerError): + dm.write_page(bad, bytes(PAGE_SIZE)) + + +def test_write_never_grows_file(path): + with DiskManager(path) as dm: + with pytest.raises(DiskManagerError): + dm.write_page(0, bytes(PAGE_SIZE)) + assert dm.page_count == 0 + + +def test_use_after_close_raises(path): + dm = DiskManager(path) + dm.close() + dm.close() # idempotent + for call in (dm.append_page, dm.sync, lambda: dm.read_page(0)): + with pytest.raises(DiskManagerError): + call() + + +def test_page_count_sees_other_handles_appends(path): + with DiskManager(path) as a, DiskManager(path) as b: + a.append_page() + assert b.page_count == 1 + assert b.read_page(0) == bytes(PAGE_SIZE) + + +@pytest.mark.skipif(not hasattr(os, "pread"), reason="needs os.pread") +def test_short_reads_are_retried(path, monkeypatch): + real = os.pread + monkeypatch.setattr( + disk_manager.os, "pread", lambda fd, n, off: real(fd, min(n, 1000), off) + ) + with DiskManager(path) as dm: + dm.write_page(dm.append_page(), page(5)) + assert dm.read_page(0) == page(5) + + +@pytest.mark.skipif(not hasattr(os, "pread"), reason="needs os.pread") +def test_eof_mid_page_raises(path, monkeypatch): + with DiskManager(path) as dm: + dm.append_page() + monkeypatch.setattr(disk_manager.os, "pread", lambda fd, n, off: b"") + with pytest.raises(DiskManagerError): + dm.read_page(0) + + +def test_concurrent_readers_and_writer_never_see_mixed_pages(path): + n_pages, rounds = 8, 200 + with DiskManager(path) as dm: + for _ in range(n_pages): + dm.write_page(dm.append_page(), page(0)) + start = Barrier(5) + errors: list[str] = [] + + def writer(): + start.wait() + for r in range(rounds): + for p in range(n_pages): + dm.write_page(p, page(r % 250 + 1)) + + def reader(): + start.wait() + for _ in range(rounds): + for p in range(n_pages): + data = dm.read_page(p) + if data != data[:1] * PAGE_SIZE: + errors.append(f"torn read on page {p}") + + threads = [Thread(target=writer)] + [Thread(target=reader) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + assert not errors + + +def test_concurrent_appends_get_unique_ids(path): + with DiskManager(path) as dm: + ids: list[int] = [] + start = Barrier(8) + + def work(): + start.wait() + ids.extend(int(dm.append_page()) for _ in range(25)) + + threads = [Thread(target=work) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + assert sorted(ids) == list(range(200)) + assert dm.page_count == 200 From dbad9b2cdedde37ac29c24236fa9e63ac575b3e4 Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Mon, 5 Oct 2026 23:23:26 +0530 Subject: [PATCH 04/18] fix: os.pread and pwrite, since they dont exist in windows windows issue, also fixed formatting and docstring issues --- .idea/.gitignore | 5 + .idea/inspectionProfiles/Project_Default.xml | 12 +++ .../inspectionProfiles/profiles_settings.xml | 6 ++ .idea/misc.xml | 7 ++ .idea/modules.xml | 8 ++ .idea/seriousdb.iml | 18 ++++ .idea/vcs.xml | 6 ++ flake.lock | 44 ++++---- src/seriousdb/disk_manager.py | 102 +++++++++--------- src/seriousdb/exceptions.py | 1 + tests/test_disk_manager.py | 79 ++++++-------- 11 files changed, 166 insertions(+), 122 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/inspectionProfiles/Project_Default.xml create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/seriousdb.iml create mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..b58b603 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,5 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..e235b72 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,12 @@ + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..62f6c46 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..fe04a66 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/seriousdb.iml b/.idea/seriousdb.iml new file mode 100644 index 0000000..7d7ee5f --- /dev/null +++ b/.idea/seriousdb.iml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/flake.lock b/flake.lock index fc28473..4864c7e 100644 --- a/flake.lock +++ b/flake.lock @@ -1,27 +1,23 @@ { - "nodes": { - "nixpkgs": { - "locked": { - "lastModified": 1789006805, - "narHash": "sha256-xB8mKMOx1IA9vTDNLmJZ6n4wCMq/cuWBBOzGCRnqxrU=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "8ce4ef6cb6f871616146b9fe26d2a5ae594e94fe", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" - } + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1789006805, + "narHash": "sha256-xB8mKMOx1IA9vTDNLmJZ6n4wCMq/cuWBBOzGCRnqxrU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "8ce4ef6cb6f871616146b9fe26d2a5ae594e94fe", + "type": "github", + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github", + }, + }, + "root": {"inputs": {"nixpkgs": "nixpkgs"}}, }, - "root": { - "inputs": { - "nixpkgs": "nixpkgs" - } - } - }, - "root": "root", - "version": 7 + "root": "root", + "version": 7, } diff --git a/src/seriousdb/disk_manager.py b/src/seriousdb/disk_manager.py index f6e342b..de41b33 100644 --- a/src/seriousdb/disk_manager.py +++ b/src/seriousdb/disk_manager.py @@ -1,3 +1,4 @@ +"""Raw fixed-size page I/O on the ``.sdb`` database file.""" from __future__ import annotations @@ -11,25 +12,29 @@ from .storage_format import PAGE_SIZE from .types import UInt32 -_O_BINARY = getattr(os, "O_BINARY", 0) # Windows only; stops newline translation -_HAS_POSITIONAL_IO = hasattr(os, "pread") and hasattr(os, "pwrite") - __all__ = ["DiskManager"] +# os.pread/os.pwrite do not exist on Windows. Looked up with getattr so that +# type checkers do not depend on the platform they run on. +_pread = getattr(os, "pread", None) +_pwrite = getattr(os, "pwrite", None) +# Windows only: stops newline translation. +_O_BINARY = getattr(os, "O_BINARY", 0) + class DiskManager: - """Page ``n`` is stored at byte offset ``n * PAGE_SIZE``. This class does raw -page I/O only: it knows nothing about page contents, and locking is left to -the concurrency manager. + """Read and write fixed-size pages of the database file by page ID. + + Page ``n`` is stored at byte offset ``n * PAGE_SIZE``. This class does raw + page I/O only: it knows nothing about page contents, and locking is left + to the concurrency manager. Parameters ---------- path : str or Path - Path of the database file. It is created if it does not exist. + Database file, created if missing. repair_torn_page : bool, optional - If the file ends in a partial page (for example after a crash while - growing it), truncate it to the last whole page instead of raising. - Only ever removes the incomplete tail. Defaults to ``False``. + Truncate a partial last page instead of raising. Default ``False``. Raises ------ @@ -39,14 +44,13 @@ class DiskManager: OSError If the file cannot be opened. - - - A new file has no pages, so the first :meth:`append_page` returns 0. - The layer that owns the metadata must append and write page 0 first. - Free-list reuse is also that layer's job; this class only appends. + Notes + ----- + A new file is empty, so the first :meth:`allocate_page` returns 0 and the + caller must write the metadata page there. """ - def __init__(self, path: str | Path, *, repair_torn_page: bool = False): + def __init__(self, path: str | Path, *, repair_torn_page: bool = False) -> None: self._path = Path(path) self._lock = RLock() self._fd: int | None = None @@ -68,20 +72,16 @@ def __init__(self, path: str | Path, *, repair_torn_page: bool = False): self._fd = fd @property - def path(self): - #Path of the database file. + def path(self) -> Path: + """Path of the database file.""" return self._path @property - def page_count(self) : - """Number of whole pages currently in the file. - - Read from the file size on every call, so pages appended by another - process are visible. - """ + def page_count(self) -> int: + """Number of whole pages in the file, read from the file size.""" return os.fstat(self._require_open()).st_size // PAGE_SIZE - def read_page(self, page_id: SupportsIndex): + def read_page(self, page_id: SupportsIndex) -> bytes: """Return the ``PAGE_SIZE`` bytes of page `page_id`. Raises @@ -100,8 +100,11 @@ def read_page(self, page_id: SupportsIndex): buf += chunk return bytes(buf) - def write_page(self, page_id: SupportsIndex, data: bytes): - """Overwrite existing page `page_id` with `data`. + def write_page(self, page_id: SupportsIndex, data: bytes) -> None: + """Overwrite the existing page `page_id` with `data`. + + Not fsynced; call :meth:`sync` when the write must be durable. + Raises ------ DiskManagerError @@ -115,11 +118,12 @@ def write_page(self, page_id: SupportsIndex, data: bytes): ) self._write_all(fd, data, self._offset(fd, page_id)) - def append_page(self): - """Grow the file by one zero-filled page and return its ID. + def allocate_page(self) -> UInt32: + """Allocate a new page at the end of the file and return its ID. - Not fsynced; call :meth:`sync` before relying on the new size - after a crash. + The page is zero-filled. Reusing freed pages is the caller's job. + Not fsynced; call :meth:`sync` before relying on the new size after + a crash. Raises ------ @@ -134,33 +138,31 @@ def append_page(self): self._write_all(fd, bytes(PAGE_SIZE), count * PAGE_SIZE) return UInt32(count) - def sync(self): - #Flush file contents and size to stable storage (``fsync``). + def sync(self) -> None: + """Flush file contents and size to stable storage (``fsync``).""" os.fsync(self._require_open()) - def close(self): - #Close the file. Safe to call more than once. + def close(self) -> None: + """Close the file. Safe to call more than once.""" with self._lock: if self._fd is not None: os.close(self._fd) self._fd = None - def __enter__(self): - #Return the manager for use in a ``with`` block. + def __enter__(self) -> Self: + """Return the manager for use in a ``with`` block.""" return self - def __exit__(self, *exc_info: object): - #Close the file when leaving the ``with`` block + def __exit__(self, *exc_info: object) -> None: + """Close the file when leaving the ``with`` block.""" self.close() - # -- internals --------------------------------------------------------- - - def _require_open(self): + def _require_open(self) -> int: if self._fd is None: raise DiskManagerError("the database file is closed") return self._fd - def _offset(self, fd: int, page_id: SupportsIndex): + def _offset(self, fd: int, page_id: SupportsIndex) -> int: try: pid = operator.index(page_id) except TypeError as e: @@ -170,19 +172,19 @@ def _offset(self, fd: int, page_id: SupportsIndex): raise DiskManagerError(f"page {pid} is out of range (0..{count - 1})") return pid * PAGE_SIZE - def _read_at(self, fd: int, size: int, offset: int): - if _HAS_POSITIONAL_IO: - return os.pread(fd, size, offset) - with self._lock: # no pread on Windows: seek + read must be atomic + def _read_at(self, fd: int, size: int, offset: int) -> bytes: + if _pread is not None: + return _pread(fd, size, offset) + with self._lock: # seek + read must be atomic without pread os.lseek(fd, offset, os.SEEK_SET) return os.read(fd, size) - def _write_all(self, fd: int, data: bytes, offset: int): + def _write_all(self, fd: int, data: bytes, offset: int) -> None: view = memoryview(data) written = 0 while written < len(view): - if _HAS_POSITIONAL_IO: - n = os.pwrite(fd, view[written:], offset + written) + if _pwrite is not None: + n = _pwrite(fd, view[written:], offset + written) else: with self._lock: os.lseek(fd, offset + written, os.SEEK_SET) diff --git a/src/seriousdb/exceptions.py b/src/seriousdb/exceptions.py index 4ca6b63..2be7743 100644 --- a/src/seriousdb/exceptions.py +++ b/src/seriousdb/exceptions.py @@ -42,6 +42,7 @@ class ServiceUnavailableError(ApplicationError): default_detail = "The service is temporarily unavailable" + class DiskManagerError(ApplicationError): """A raw page read, write or allocation on the database file failed.""" diff --git a/tests/test_disk_manager.py b/tests/test_disk_manager.py index 8c0feb0..c23636b 100644 --- a/tests/test_disk_manager.py +++ b/tests/test_disk_manager.py @@ -1,4 +1,3 @@ -import os from threading import Barrier, Thread import pytest @@ -21,8 +20,8 @@ def page(byte: int) -> bytes: def test_new_file_is_empty_and_first_append_is_page_zero(path): with DiskManager(path) as dm: assert dm.page_count == 0 - assert dm.append_page() == 0 - assert dm.append_page() == 1 + assert dm.allocate_page() == 0 + assert dm.allocate_page() == 1 assert dm.page_count == 2 assert path.stat().st_size == 2 * PAGE_SIZE @@ -30,7 +29,7 @@ def test_new_file_is_empty_and_first_append_is_page_zero(path): def test_round_trip_and_reopen(path): with DiskManager(path) as dm: for i in range(3): - dm.write_page(dm.append_page(), page(i + 1)) + dm.write_page(dm.allocate_page(), page(i + 1)) dm.sync() with DiskManager(path) as dm: assert dm.page_count == 3 @@ -39,7 +38,7 @@ def test_round_trip_and_reopen(path): def test_fresh_page_is_zeroed(path): with DiskManager(path) as dm: - pid = dm.append_page() + pid = dm.allocate_page() assert dm.read_page(pid) == bytes(PAGE_SIZE) @@ -62,7 +61,7 @@ def test_repair_truncates_only_the_torn_tail(path): @pytest.mark.parametrize("size", [PAGE_SIZE - 1, PAGE_SIZE + 1, 0]) def test_wrong_size_write_rejected(path, size): with DiskManager(path) as dm: - dm.append_page() + dm.allocate_page() with pytest.raises(DiskManagerError): dm.write_page(0, bytes(size)) @@ -70,7 +69,7 @@ def test_wrong_size_write_rejected(path, size): @pytest.mark.parametrize("bad", [-1, 1, 99, "0", 1.5]) def test_bad_page_id_rejected(path, bad): with DiskManager(path) as dm: - dm.append_page() + dm.allocate_page() with pytest.raises(DiskManagerError): dm.read_page(bad) with pytest.raises(DiskManagerError): @@ -88,76 +87,60 @@ def test_use_after_close_raises(path): dm = DiskManager(path) dm.close() dm.close() # idempotent - for call in (dm.append_page, dm.sync, lambda: dm.read_page(0)): + for call in (dm.allocate_page, dm.sync, lambda: dm.read_page(0)): with pytest.raises(DiskManagerError): call() def test_page_count_sees_other_handles_appends(path): with DiskManager(path) as a, DiskManager(path) as b: - a.append_page() + a.allocate_page() assert b.page_count == 1 assert b.read_page(0) == bytes(PAGE_SIZE) -@pytest.mark.skipif(not hasattr(os, "pread"), reason="needs os.pread") +@pytest.mark.skipif(disk_manager._pread is None, reason="needs os.pread") def test_short_reads_are_retried(path, monkeypatch): - real = os.pread + real = disk_manager._pread + assert real is not None monkeypatch.setattr( - disk_manager.os, "pread", lambda fd, n, off: real(fd, min(n, 1000), off) + disk_manager, "_pread", lambda fd, n, off: real(fd, min(n, 1000), off) ) with DiskManager(path) as dm: - dm.write_page(dm.append_page(), page(5)) + dm.write_page(dm.allocate_page(), page(5)) assert dm.read_page(0) == page(5) -@pytest.mark.skipif(not hasattr(os, "pread"), reason="needs os.pread") -def test_eof_mid_page_raises(path, monkeypatch): +@pytest.mark.skipif(disk_manager._pwrite is None, reason="needs os.pwrite") +def test_short_writes_are_retried(path, monkeypatch): + real = disk_manager._pwrite + assert real is not None + monkeypatch.setattr( + disk_manager, "_pwrite", lambda fd, data, off: real(fd, bytes(data[:1000]), off) + ) with DiskManager(path) as dm: - dm.append_page() - monkeypatch.setattr(disk_manager.os, "pread", lambda fd, n, off: b"") - with pytest.raises(DiskManagerError): - dm.read_page(0) + dm.write_page(dm.allocate_page(), page(9)) + assert dm.read_page(0) == page(9) + assert path.read_bytes() == page(9) -def test_concurrent_readers_and_writer_never_see_mixed_pages(path): - n_pages, rounds = 8, 200 +@pytest.mark.skipif(disk_manager._pread is None, reason="needs os.pread") +def test_eof_mid_page_raises(path, monkeypatch): with DiskManager(path) as dm: - for _ in range(n_pages): - dm.write_page(dm.append_page(), page(0)) - start = Barrier(5) - errors: list[str] = [] - - def writer(): - start.wait() - for r in range(rounds): - for p in range(n_pages): - dm.write_page(p, page(r % 250 + 1)) - - def reader(): - start.wait() - for _ in range(rounds): - for p in range(n_pages): - data = dm.read_page(p) - if data != data[:1] * PAGE_SIZE: - errors.append(f"torn read on page {p}") - - threads = [Thread(target=writer)] + [Thread(target=reader) for _ in range(4)] - for t in threads: - t.start() - for t in threads: - t.join() - assert not errors + dm.allocate_page() + monkeypatch.setattr(disk_manager, "_pread", lambda fd, n, off: b"") + with pytest.raises(DiskManagerError): + dm.read_page(0) -def test_concurrent_appends_get_unique_ids(path): +def test_concurrent_allocations_get_unique_ids(path): with DiskManager(path) as dm: ids: list[int] = [] start = Barrier(8) def work(): start.wait() - ids.extend(int(dm.append_page()) for _ in range(25)) + ids.extend(int(dm.allocate_page()) for _ in range(25)) threads = [Thread(target=work) for _ in range(8)] for t in threads: From eb9544c2212de433cc5981ac5063b74ac349a37a Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Mon, 5 Oct 2026 23:30:24 +0530 Subject: [PATCH 05/18] removed ide config from tracking --- .idea/.gitignore | 5 ----- .idea/inspectionProfiles/Project_Default.xml | 12 ------------ .idea/inspectionProfiles/profiles_settings.xml | 6 ------ .idea/misc.xml | 7 ------- .idea/modules.xml | 8 -------- .idea/seriousdb.iml | 18 ------------------ .idea/vcs.xml | 6 ------ 7 files changed, 62 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/inspectionProfiles/Project_Default.xml delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/seriousdb.iml delete mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index b58b603..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index e235b72..0000000 --- a/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2d..0000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 62f6c46..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index fe04a66..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/seriousdb.iml b/.idea/seriousdb.iml deleted file mode 100644 index 7d7ee5f..0000000 --- a/.idea/seriousdb.iml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1dd..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From 617f10d968b5d3590c269abcfed63d73824274ea Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Tue, 6 Oct 2026 00:22:18 +0530 Subject: [PATCH 06/18] formatting --- .gitignore | Bin 4838 -> 5102 bytes src/seriousdb/disk_manager.py | 26 +++++++++++++------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 4a001109052ef04d2bc57a8ad250ff42f91c1f12..9b10f1c77d8b1b0cb48806edabbc71f69dae242b 100644 GIT binary patch delta 2201 zcmZuy&2QX96yK;HBreWJ5mKN~EZVBDu-*d*iBgc-G@(JXO_R0=A#lg`?Cxaknd!{z zCPqTNaN`2_h}An^}yLr9#s!G#0ji1-uuy|H%_AtY<&eb1YDzxVvtrN1wKdHGrx zB#EvvmW_Cl^uwSV>*}LS<)3~G`l7DltYFrLK^R6Akbm{?+~Y8a zcPckhG7Qp*D00Nf`IHw*_xNyVi51q-rVeDGSWco=6SsRIQV6lW@j77Z zcQE3-@Zu3Oys!P4y@RcpxJ$u$LJky+VWrC6i*nt@Ah4JZl{0=24-Bcm*wV4^-?9VMt8Q3+{N&jfCyx z!tl%~Q#ERO!~0UW$RTXhk1{poh7EaSxK_At7>rz5K>C={cj}Nf*=#dgNM>Z*C59>& z22xp^YMqS_%OVWsCZ6jyE_4wpD4hz=QKgj|aVzLSp&<9fhwD>8Q!cYn$;_D4-UmC~ z;GSX97R=31P%0l-;r^t)vCrgCMb=FQgDa%L>!7YGV?}}-osdI|(}0CEwvP(o_-n`M z-oBa`AsH?~(YAoT@jRHXsLVJee85uxlRMjMpMcEwGQ)H=w~wvon^1dtB;;X~t87wo z>HG!4`tgZv)cmItH&+Oqne8Vng6ro`uHzjj>P~f?Q0-q$S6owdlPy> z6Wvyhr%H{HqR^F06VxXgYb7KVm@Kzvy`qg8h=M0MAFxS5f#5-Hlw?W{#4ui_cOn;E zaXAg^A5Pv!vP-9KMkJ(~!MKNVQcJd$LVMHl)I*s5;@95kPe6Zh`qDGxwj(s1in?S- zk|g&Zgmn>Y*1;3!VE4izS%0>8ah-IN=`lmCxHAylg^ArQNq%<&kGzq6^3^)_K`$uG=!RIpclAp`eKA@{DUj)V~D` zq2D_55a-64tb(xq?#u_MD@{GkvN77v`oTx88|Yor!Uv%DT*{g2Srjp!6lk;~Ap+*j zg?2fn6hYnODQ!#*d`ryrEKP;GbvF!3iGE}Kt6^ZgtbBx`&&XcR9Ba{ZEpuYZ$%T2U zd^FeA-#kgc#P~(-zLO~RO*s`tNkTSKi3tOKHJDrR)U#L+ooH;Ew;x~y9j9#Eid&Mx z?db}5kATp9Ot5IyxQJy*vu;{VBbYN`{Z2^9#YSmBt+nx9=oS-WI9G=I&jamd2xSubt@0Td&>0}t$ zs!UaZx5}E9^U~IEr`o^|1}J@T`NLD>8P$KRUax;#ehP59x^}aEZiT4+O7b+$5ILWh zEw#=tw8F|2=zm|igSET*@&#BiuWAo?Anb${(Vm;+LPZ-Y=lJieQXl-fDzIK%OR#>u zwjI*x@zmmvj9b+ItiAds<_<44`Rq=l@4nBHOi{n_>Rn_xG9Ukqgw6yR)H c0b6`P!#wWvJ8?&J@(%B?P8iVlf3h(67r4`hkpKVy delta 1883 zcmYjSPj3`A6j%6@fCC8usz`{EkpQ7a>`Wyp1i3*7O)4b~3524m%z9?LyUxzoj_uuK zDpmUds!B+{Rq7|`p?AJO5B&zM)MF1-%elw??AcxAu-^CQ=V$x9-+S}n-d}g$-5ZfK zRdr7D9!t}Ntj4PTI+w+_-;mWis*dxL8biqKXgy*lmI=e9FhCjUjnBBTnr8zGYm`Ka z=6%}bqD$i4+V-R%*?^ZtLJBryrBp4}?HXvru@T70zi*jH^CIi;Ip-NX&(?v@BT>=bbL@3>Vpcva|JkWB;YU z{cgP_%xr*Yw>k5u|2VF`OK{u;vSt@ z8Ihi?%3Ka{w3R1VMOa~Bnsyn9CsMo$*=L*xX_&=PCO)Z3GWC09wSnW`m%+y%aj#a= z_L$*#Kg@n-8UjJhMoi?rit0X`0iXDrztmKilKKkyx>eE#lr~iFkczI1j2(13YcS(Q zq?9==Q(VfTj0R_rB8$yY$t`@EloiHaTM)kNjNxDoRla_%|h_zck_3ZL_ zQFVYb(8E&QWuO-TUkrSTbhINa%cSgs5vmqhip1xACAqMH*;`|-DlI#_yvx#pb?Bf( z48T2$(_D%U@A_mQs;Ze7y36NZx%~X*O&EM3fNG&azB>YjD-ZhZeB0fhZMz?4-@1Qh z?|$Sj%@qz|$aUWL$X5q$b*}BcpL^^6p1X4$mcel9RFYX;1F+%x7gi=9w|FtV4PfJK zz*EgiP{%uqD(w5?;!XG8#kPAx_8vCo>4u^LO@WSqZ8dLbFUz*s;z^rpE;*c~CM&ZOfnt#h_%TY=~6&Eqe^z z4m#>0!%Afb>%>K~jN9!Oq!MTXnh+ff)sdG*-d7#93Toj&peMQILk2Vr!fDq&`f;1u z9!8cntcZL~!dQcST;~ljv~X+9E4Efp5obNnHY!*)=ticlGFcjuH;fO)fZDCsF=#9I z=fWHJXz`6ZcXiYKzIe@jSWK@WNX!d1&US`x0d?7xHTTn%FWtXaK3O3#4aN}YfSUm= zqkTIlxQy0i!QA7e_Dtk{T;hnDLMpCq&HI=`gD((E-21Ed9}vqb<2NF8ZhYsTsyNDpfZJ4a3_i?8{3R(yx#f8>)k~uid<7f#n78{@Gg@Kn gp int: return os.fstat(self._require_open()).st_size // PAGE_SIZE def read_page(self, page_id: SupportsIndex) -> bytes: - """Return the ``PAGE_SIZE`` bytes of page `page_id`. + """Return the PAGE_SIZE bytes of page page_id. Raises ------ DiskManagerError - If `page_id` is out of range, the file is closed, or the file + If page_id is out of range, the file is closed, or the file ends before a full page could be read. """ fd = self._require_open() @@ -101,14 +101,14 @@ def read_page(self, page_id: SupportsIndex) -> bytes: return bytes(buf) def write_page(self, page_id: SupportsIndex, data: bytes) -> None: - """Overwrite the existing page `page_id` with `data`. + """Overwrite the existing page page_id with data. - Not fsynced; call :meth:`sync` when the write must be durable. + Not fsynced; call :meth: sync when the write must be durable. Raises ------ DiskManagerError - If `data` is not exactly ``PAGE_SIZE`` bytes, `page_id` has not + If data is not exactly PAGE_SIZE bytes, page_id has not been allocated, or the file is closed. """ fd = self._require_open() @@ -128,7 +128,7 @@ def allocate_page(self) -> UInt32: Raises ------ DiskManagerError - If the file is closed or page IDs would exceed the ``UInt32`` range. + If the file is closed or page IDs would exceed the UInt32 range. """ fd = self._require_open() with self._lock: @@ -139,7 +139,7 @@ def allocate_page(self) -> UInt32: return UInt32(count) def sync(self) -> None: - """Flush file contents and size to stable storage (``fsync``).""" + """Flush file contents and size to stable storage (fsync).""" os.fsync(self._require_open()) def close(self) -> None: @@ -154,7 +154,7 @@ def __enter__(self) -> Self: return self def __exit__(self, *exc_info: object) -> None: - """Close the file when leaving the ``with`` block.""" + """Close the file when leaving the with block.""" self.close() def _require_open(self) -> int: From c3ae9e14dbd670e7ca6e1bc2dea37b53fc9a446d Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Tue, 6 Oct 2026 21:46:42 +0530 Subject: [PATCH 07/18] Resolved int type issue, chaged int32 to UInt --- src/seriousdb/disk_manager.py | 46 ++++++++++++++++------------------- tests/test_disk_manager.py | 29 +++++++++++++--------- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/seriousdb/disk_manager.py b/src/seriousdb/disk_manager.py index 058a541..668d209 100644 --- a/src/seriousdb/disk_manager.py +++ b/src/seriousdb/disk_manager.py @@ -40,7 +40,7 @@ class DiskManager: ------ DiskManagerError If the file size is not a multiple of PAGE_SIZE and - `repair_torn_page` is false. + repair_torn_page is false. OSError If the file cannot be opened. @@ -50,7 +50,7 @@ class DiskManager: caller must write the metadata page there. """ - def __init__(self, path: str | Path, *, repair_torn_page: bool = False) -> None: + def __init__(self, path: str | Path, *, repair_torn_page: bool = False): self._path = Path(path) self._lock = RLock() self._fd: int | None = None @@ -72,16 +72,16 @@ def __init__(self, path: str | Path, *, repair_torn_page: bool = False) -> None: self._fd = fd @property - def path(self) -> Path: + def path(self): """Path of the database file.""" return self._path @property - def page_count(self) -> int: + def page_count(self): """Number of whole pages in the file, read from the file size.""" return os.fstat(self._require_open()).st_size // PAGE_SIZE - def read_page(self, page_id: SupportsIndex) -> bytes: + def read_page(self, page_id: UInt32): """Return the PAGE_SIZE bytes of page page_id. Raises @@ -100,10 +100,10 @@ def read_page(self, page_id: SupportsIndex) -> bytes: buf += chunk return bytes(buf) - def write_page(self, page_id: SupportsIndex, data: bytes) -> None: + def write_page(self, page_id: UInt32, data: bytes): """Overwrite the existing page page_id with data. - Not fsynced; call :meth: sync when the write must be durable. + Not fsynced; call :meth:sync when the write must be durable. Raises ------ @@ -118,11 +118,11 @@ def write_page(self, page_id: SupportsIndex, data: bytes) -> None: ) self._write_all(fd, data, self._offset(fd, page_id)) - def allocate_page(self) -> UInt32: + def allocate_page(self): """Allocate a new page at the end of the file and return its ID. The page is zero-filled. Reusing freed pages is the caller's job. - Not fsynced; call :meth:`sync` before relying on the new size after + Not fsynced; call :meth:sync before relying on the new size after a crash. Raises @@ -138,48 +138,44 @@ def allocate_page(self) -> UInt32: self._write_all(fd, bytes(PAGE_SIZE), count * PAGE_SIZE) return UInt32(count) - def sync(self) -> None: + def sync(self): """Flush file contents and size to stable storage (fsync).""" os.fsync(self._require_open()) - def close(self) -> None: + def close(self): """Close the file. Safe to call more than once.""" with self._lock: if self._fd is not None: os.close(self._fd) self._fd = None - def __enter__(self) -> Self: + def __enter__(self): """Return the manager for use in a ``with`` block.""" return self - def __exit__(self, *exc_info: object) -> None: - """Close the file when leaving the with block.""" + def __exit__(self, *exc_info: object): + """Close the file when leaving the ``with`` block.""" self.close() - def _require_open(self) -> int: + def _require_open(self): if self._fd is None: raise DiskManagerError("the database file is closed") return self._fd - def _offset(self, fd: int, page_id: SupportsIndex) -> int: - try: - pid = operator.index(page_id) - except TypeError as e: - raise DiskManagerError(f"invalid page ID: {page_id!r}") from e + def _offset(self, fd: int, page_id: UInt32): count = os.fstat(fd).st_size // PAGE_SIZE - if not 0 <= pid < count: - raise DiskManagerError(f"page {pid} is out of range (0..{count - 1})") - return pid * PAGE_SIZE + if not 0 <= page_id < count: + raise DiskManagerError(f"page {page_id} is out of range (0..{count - 1})") + return page_id * PAGE_SIZE - def _read_at(self, fd: int, size: int, offset: int) -> bytes: + def _read_at(self, fd: int, size: int, offset: int): if _pread is not None: return _pread(fd, size, offset) with self._lock: # seek + read must be atomic without pread os.lseek(fd, offset, os.SEEK_SET) return os.read(fd, size) - def _write_all(self, fd: int, data: bytes, offset: int) -> None: + def _write_all(self, fd: int, data: bytes, offset: int): view = memoryview(data) written = 0 while written < len(view): diff --git a/tests/test_disk_manager.py b/tests/test_disk_manager.py index c23636b..b2fa63c 100644 --- a/tests/test_disk_manager.py +++ b/tests/test_disk_manager.py @@ -6,6 +6,7 @@ from seriousdb.disk_manager import DiskManager from seriousdb.exceptions import DiskManagerError from seriousdb.storage_format import PAGE_SIZE +from seriousdb.types import UInt32 @pytest.fixture @@ -33,7 +34,11 @@ def test_round_trip_and_reopen(path): dm.sync() with DiskManager(path) as dm: assert dm.page_count == 3 - assert [dm.read_page(i) for i in range(3)] == [page(1), page(2), page(3)] + assert [dm.read_page(UInt32(i)) for i in range(3)] == [ + page(1), + page(2), + page(3), + ] def test_fresh_page_is_zeroed(path): @@ -53,8 +58,8 @@ def test_repair_truncates_only_the_torn_tail(path): path.write_bytes(page(7) + page(8) + b"x" * 100) with DiskManager(path, repair_torn_page=True) as dm: assert dm.page_count == 2 - assert dm.read_page(0) == page(7) - assert dm.read_page(1) == page(8) + assert dm.read_page(UInt32(0)) == page(7) + assert dm.read_page(UInt32(1)) == page(8) assert path.stat().st_size == 2 * PAGE_SIZE @@ -63,11 +68,11 @@ def test_wrong_size_write_rejected(path, size): with DiskManager(path) as dm: dm.allocate_page() with pytest.raises(DiskManagerError): - dm.write_page(0, bytes(size)) + dm.write_page(UInt32(0), bytes(size)) -@pytest.mark.parametrize("bad", [-1, 1, 99, "0", 1.5]) -def test_bad_page_id_rejected(path, bad): +@pytest.mark.parametrize("bad", [UInt32(1), UInt32(99), UInt32(UInt32.MAX)]) +def test_out_of_range_page_id_rejected(path, bad): with DiskManager(path) as dm: dm.allocate_page() with pytest.raises(DiskManagerError): @@ -79,7 +84,7 @@ def test_bad_page_id_rejected(path, bad): def test_write_never_grows_file(path): with DiskManager(path) as dm: with pytest.raises(DiskManagerError): - dm.write_page(0, bytes(PAGE_SIZE)) + dm.write_page(UInt32(0), bytes(PAGE_SIZE)) assert dm.page_count == 0 @@ -87,7 +92,7 @@ def test_use_after_close_raises(path): dm = DiskManager(path) dm.close() dm.close() # idempotent - for call in (dm.allocate_page, dm.sync, lambda: dm.read_page(0)): + for call in (dm.allocate_page, dm.sync, lambda: dm.read_page(UInt32(0))): with pytest.raises(DiskManagerError): call() @@ -96,7 +101,7 @@ def test_page_count_sees_other_handles_appends(path): with DiskManager(path) as a, DiskManager(path) as b: a.allocate_page() assert b.page_count == 1 - assert b.read_page(0) == bytes(PAGE_SIZE) + assert b.read_page(UInt32(0)) == bytes(PAGE_SIZE) @pytest.mark.skipif(disk_manager._pread is None, reason="needs os.pread") @@ -108,7 +113,7 @@ def test_short_reads_are_retried(path, monkeypatch): ) with DiskManager(path) as dm: dm.write_page(dm.allocate_page(), page(5)) - assert dm.read_page(0) == page(5) + assert dm.read_page(UInt32(0)) == page(5) @pytest.mark.skipif(disk_manager._pwrite is None, reason="needs os.pwrite") @@ -120,7 +125,7 @@ def test_short_writes_are_retried(path, monkeypatch): ) with DiskManager(path) as dm: dm.write_page(dm.allocate_page(), page(9)) - assert dm.read_page(0) == page(9) + assert dm.read_page(UInt32(0)) == page(9) assert path.read_bytes() == page(9) @@ -130,7 +135,7 @@ def test_eof_mid_page_raises(path, monkeypatch): dm.allocate_page() monkeypatch.setattr(disk_manager, "_pread", lambda fd, n, off: b"") with pytest.raises(DiskManagerError): - dm.read_page(0) + dm.read_page(UInt32(0)) def test_concurrent_allocations_get_unique_ids(path): From e36b124a5b1a41d9b41a4c42989aa8d40ef04437 Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Tue, 6 Oct 2026 21:50:56 +0530 Subject: [PATCH 08/18] formatting --- src/seriousdb/disk_manager.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/seriousdb/disk_manager.py b/src/seriousdb/disk_manager.py index 668d209..586ef08 100644 --- a/src/seriousdb/disk_manager.py +++ b/src/seriousdb/disk_manager.py @@ -1,12 +1,10 @@ -#Raw fixed-size page I/O on the .sdb database file. +"""Raw fixed-size page I/O on the .sdb database file.""" from __future__ import annotations -import operator import os from pathlib import Path from threading import RLock -from typing import Self, SupportsIndex from .exceptions import DiskManagerError from .storage_format import PAGE_SIZE From e165644c891639f66332af71cb75fe0b4d6fdf8e Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Wed, 7 Oct 2026 09:55:13 +0530 Subject: [PATCH 09/18] format+generated exception refrence doc --- .../reference/generated/seriousdb.exceptions.md | 6 ++++++ src/seriousdb/disk_manager.py | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/docs/reference/generated/seriousdb.exceptions.md b/docs/reference/generated/seriousdb.exceptions.md index 4f9a65d..4bb532b 100644 --- a/docs/reference/generated/seriousdb.exceptions.md +++ b/docs/reference/generated/seriousdb.exceptions.md @@ -16,6 +16,12 @@ Base class for expected application errors. * **default_detail** (*str*) – Detail used when none is given. * **detail** (*str*) – Human readable description of this error. +### *exception* DiskManagerError(detail=None) + +Bases: [`ApplicationError`](#seriousdb.exceptions.ApplicationError) + +A raw page read, write or allocation on the database file failed. + ### *exception* ResourceNotFoundError(detail=None) Bases: [`ApplicationError`](#seriousdb.exceptions.ApplicationError) diff --git a/src/seriousdb/disk_manager.py b/src/seriousdb/disk_manager.py index 586ef08..ba5174b 100644 --- a/src/seriousdb/disk_manager.py +++ b/src/seriousdb/disk_manager.py @@ -79,8 +79,13 @@ def page_count(self): """Number of whole pages in the file, read from the file size.""" return os.fstat(self._require_open()).st_size // PAGE_SIZE +<<<<<<< HEAD def read_page(self, page_id: UInt32): """Return the PAGE_SIZE bytes of page page_id. +======= + def read_page(self, page_id: UInt32) -> bytes: + """Return the PAGE_SIZE`` bytes of page page_id. +>>>>>>> 06519f2 (format+generated exception refrence doc) Raises ------ @@ -98,7 +103,11 @@ def read_page(self, page_id: UInt32): buf += chunk return bytes(buf) +<<<<<<< HEAD def write_page(self, page_id: UInt32, data: bytes): +======= + def write_page(self, page_id: UInt32, data: bytes) -> None: +>>>>>>> 06519f2 (format+generated exception refrence doc) """Overwrite the existing page page_id with data. Not fsynced; call :meth:sync when the write must be durable. @@ -151,7 +160,11 @@ def __enter__(self): """Return the manager for use in a ``with`` block.""" return self +<<<<<<< HEAD def __exit__(self, *exc_info: object): +======= + def __exit__(self, *exc_info: object) -> None: +>>>>>>> 06519f2 (format+generated exception refrence doc) """Close the file when leaving the ``with`` block.""" self.close() @@ -160,7 +173,11 @@ def _require_open(self): raise DiskManagerError("the database file is closed") return self._fd +<<<<<<< HEAD def _offset(self, fd: int, page_id: UInt32): +======= + def _offset(self, fd: int, page_id: UInt32) -> int: +>>>>>>> 06519f2 (format+generated exception refrence doc) count = os.fstat(fd).st_size // PAGE_SIZE if not 0 <= page_id < count: raise DiskManagerError(f"page {page_id} is out of range (0..{count - 1})") From 9da0477ba22837c9723948191aa8e4685744cf74 Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Wed, 7 Oct 2026 10:00:31 +0530 Subject: [PATCH 10/18] format+generated exception refrence doc --- src/seriousdb/disk_manager.py | 37 ++++++++++------------------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/src/seriousdb/disk_manager.py b/src/seriousdb/disk_manager.py index ba5174b..7971d9c 100644 --- a/src/seriousdb/disk_manager.py +++ b/src/seriousdb/disk_manager.py @@ -48,7 +48,7 @@ class DiskManager: caller must write the metadata page there. """ - def __init__(self, path: str | Path, *, repair_torn_page: bool = False): + def __init__(self, path: str | Path, *, repair_torn_page: bool = False) -> None: self._path = Path(path) self._lock = RLock() self._fd: int | None = None @@ -70,22 +70,17 @@ def __init__(self, path: str | Path, *, repair_torn_page: bool = False): self._fd = fd @property - def path(self): + def path(self) -> Path: """Path of the database file.""" return self._path @property - def page_count(self): + def page_count(self) -> int: """Number of whole pages in the file, read from the file size.""" return os.fstat(self._require_open()).st_size // PAGE_SIZE -<<<<<<< HEAD - def read_page(self, page_id: UInt32): - """Return the PAGE_SIZE bytes of page page_id. -======= def read_page(self, page_id: UInt32) -> bytes: """Return the PAGE_SIZE`` bytes of page page_id. ->>>>>>> 06519f2 (format+generated exception refrence doc) Raises ------ @@ -103,11 +98,7 @@ def read_page(self, page_id: UInt32) -> bytes: buf += chunk return bytes(buf) -<<<<<<< HEAD - def write_page(self, page_id: UInt32, data: bytes): -======= def write_page(self, page_id: UInt32, data: bytes) -> None: ->>>>>>> 06519f2 (format+generated exception refrence doc) """Overwrite the existing page page_id with data. Not fsynced; call :meth:sync when the write must be durable. @@ -125,11 +116,11 @@ def write_page(self, page_id: UInt32, data: bytes) -> None: ) self._write_all(fd, data, self._offset(fd, page_id)) - def allocate_page(self): + def allocate_page(self) -> UInt32: """Allocate a new page at the end of the file and return its ID. The page is zero-filled. Reusing freed pages is the caller's job. - Not fsynced; call :meth:sync before relying on the new size after + Not fsynced; call :meth:`sync` before relying on the new size after a crash. Raises @@ -145,11 +136,11 @@ def allocate_page(self): self._write_all(fd, bytes(PAGE_SIZE), count * PAGE_SIZE) return UInt32(count) - def sync(self): + def sync(self) -> None: """Flush file contents and size to stable storage (fsync).""" os.fsync(self._require_open()) - def close(self): + def close(self) -> None: """Close the file. Safe to call more than once.""" with self._lock: if self._fd is not None: @@ -160,37 +151,29 @@ def __enter__(self): """Return the manager for use in a ``with`` block.""" return self -<<<<<<< HEAD - def __exit__(self, *exc_info: object): -======= def __exit__(self, *exc_info: object) -> None: ->>>>>>> 06519f2 (format+generated exception refrence doc) """Close the file when leaving the ``with`` block.""" self.close() - def _require_open(self): + def _require_open(self) -> int: if self._fd is None: raise DiskManagerError("the database file is closed") return self._fd -<<<<<<< HEAD - def _offset(self, fd: int, page_id: UInt32): -======= def _offset(self, fd: int, page_id: UInt32) -> int: ->>>>>>> 06519f2 (format+generated exception refrence doc) count = os.fstat(fd).st_size // PAGE_SIZE if not 0 <= page_id < count: raise DiskManagerError(f"page {page_id} is out of range (0..{count - 1})") return page_id * PAGE_SIZE - def _read_at(self, fd: int, size: int, offset: int): + def _read_at(self, fd: int, size: int, offset: int) -> bytes: if _pread is not None: return _pread(fd, size, offset) with self._lock: # seek + read must be atomic without pread os.lseek(fd, offset, os.SEEK_SET) return os.read(fd, size) - def _write_all(self, fd: int, data: bytes, offset: int): + def _write_all(self, fd: int, data: bytes, offset: int) -> None: view = memoryview(data) written = 0 while written < len(view): From 1c2a20d9a065b79928ced183d4ab3222c097f3ab Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Wed, 7 Oct 2026 21:11:33 +0530 Subject: [PATCH 11/18] fix: enforce UTF-8 text encoding for disk manager files --- src/seriousdb/disk_manager.py | 1 + tests/test_disk_manager.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/seriousdb/disk_manager.py b/src/seriousdb/disk_manager.py index 7971d9c..16bc414 100644 --- a/src/seriousdb/disk_manager.py +++ b/src/seriousdb/disk_manager.py @@ -186,3 +186,4 @@ def _write_all(self, fd: int, data: bytes, offset: int) -> None: if n <= 0: raise DiskManagerError("write made no progress") written += n + diff --git a/tests/test_disk_manager.py b/tests/test_disk_manager.py index b2fa63c..7496d40 100644 --- a/tests/test_disk_manager.py +++ b/tests/test_disk_manager.py @@ -154,3 +154,4 @@ def work(): t.join() assert sorted(ids) == list(range(200)) assert dm.page_count == 200 + From 5cc35041bc992ba12aac4bbfe9400e9da6db0953 Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Wed, 7 Oct 2026 21:48:57 +0530 Subject: [PATCH 12/18] cache page count for bounds checking to reduce fstat calls --- src/seriousdb/disk_manager.py | 29 ++++++++++++++++++++++------- tests/test_disk_manager.py | 22 +++++++++++++++++++++- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/seriousdb/disk_manager.py b/src/seriousdb/disk_manager.py index 16bc414..86296a9 100644 --- a/src/seriousdb/disk_manager.py +++ b/src/seriousdb/disk_manager.py @@ -1,4 +1,4 @@ -"""Raw fixed-size page I/O on the .sdb database file.""" +"""Raw fixed-size page I/O on the ``.sdb`` database file.""" from __future__ import annotations @@ -52,6 +52,7 @@ def __init__(self, path: str | Path, *, repair_torn_page: bool = False) -> None: self._path = Path(path) self._lock = RLock() self._fd: int | None = None + self._known_pages = 0 fd = os.open(self._path, os.O_RDWR | os.O_CREAT | _O_BINARY, 0o644) try: size = os.fstat(fd).st_size @@ -62,8 +63,10 @@ def __init__(self, path: str | Path, *, repair_torn_page: bool = False) -> None: f"{self._path} is {size} bytes, not a multiple of " f"{PAGE_SIZE}; the last page is incomplete" ) - os.ftruncate(fd, size - tail) + size -= tail + os.ftruncate(fd, size) os.fsync(fd) + self._known_pages = size // PAGE_SIZE except BaseException: os.close(fd) raise @@ -77,7 +80,7 @@ def path(self) -> Path: @property def page_count(self) -> int: """Number of whole pages in the file, read from the file size.""" - return os.fstat(self._require_open()).st_size // PAGE_SIZE + return self._refresh_page_count(self._require_open()) def read_page(self, page_id: UInt32) -> bytes: """Return the PAGE_SIZE`` bytes of page page_id. @@ -130,10 +133,11 @@ def allocate_page(self) -> UInt32: """ fd = self._require_open() with self._lock: - count = os.fstat(fd).st_size // PAGE_SIZE + count = self._refresh_page_count(fd) if count > UInt32.MAX: raise DiskManagerError("page ID would exceed the UInt32 range") self._write_all(fd, bytes(PAGE_SIZE), count * PAGE_SIZE) + self._known_pages = count + 1 return UInt32(count) def sync(self) -> None: @@ -160,10 +164,21 @@ def _require_open(self) -> int: raise DiskManagerError("the database file is closed") return self._fd - def _offset(self, fd: int, page_id: UInt32) -> int: + def _refresh_page_count(self, fd: int) -> int: count = os.fstat(fd).st_size // PAGE_SIZE - if not 0 <= page_id < count: - raise DiskManagerError(f"page {page_id} is out of range (0..{count - 1})") + self._known_pages = max(self._known_pages, count) + return count + + def _offset(self, fd: int, page_id: UInt32) -> int: + # The file only grows while open, so the cached count is a lower + # bound: re-read the file size only when an ID is beyond it, which + # also picks up pages another handle allocated. + if page_id >= self._known_pages: + self._refresh_page_count(fd) + if not 0 <= page_id < self._known_pages: + raise DiskManagerError( + f"page {page_id} is out of range (0..{self._known_pages - 1})" + ) return page_id * PAGE_SIZE def _read_at(self, fd: int, size: int, offset: int) -> bytes: diff --git a/tests/test_disk_manager.py b/tests/test_disk_manager.py index 7496d40..edd4396 100644 --- a/tests/test_disk_manager.py +++ b/tests/test_disk_manager.py @@ -88,6 +88,27 @@ def test_write_never_grows_file(path): assert dm.page_count == 0 +def test_in_range_io_does_not_call_fstat(path, monkeypatch): + with DiskManager(path) as dm: + dm.write_page(dm.allocate_page(), page(1)) + calls: list[int] = [] + real = disk_manager.os.fstat + monkeypatch.setattr( + disk_manager.os, "fstat", lambda fd: calls.append(fd) or real(fd) + ) + dm.read_page(UInt32(0)) + dm.write_page(UInt32(0), page(2)) + assert calls == [] + + +def test_page_allocated_by_another_handle_is_readable(path): + with DiskManager(path) as a, DiskManager(path) as b: + a.write_page(a.allocate_page(), page(4)) + assert b.read_page(UInt32(0)) == page(4) + with pytest.raises(DiskManagerError): + b.read_page(UInt32(1)) + + def test_use_after_close_raises(path): dm = DiskManager(path) dm.close() @@ -154,4 +175,3 @@ def work(): t.join() assert sorted(ids) == list(range(200)) assert dm.page_count == 200 - From ab7b18470b527b1e9864a83accc22fab86c049b7 Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Wed, 7 Oct 2026 22:09:30 +0530 Subject: [PATCH 13/18] fix potential close() race condition with _active counter --- src/seriousdb/disk_manager.py | 62 +++++++++++++++++++++++++---------- tests/test_disk_manager.py | 24 ++++++++++++++ 2 files changed, 68 insertions(+), 18 deletions(-) diff --git a/src/seriousdb/disk_manager.py b/src/seriousdb/disk_manager.py index 86296a9..a7deac2 100644 --- a/src/seriousdb/disk_manager.py +++ b/src/seriousdb/disk_manager.py @@ -3,8 +3,11 @@ from __future__ import annotations import os +from collections.abc import Iterator +from contextlib import contextmanager from pathlib import Path from threading import RLock +from typing import Self from .exceptions import DiskManagerError from .storage_format import PAGE_SIZE @@ -53,6 +56,7 @@ def __init__(self, path: str | Path, *, repair_torn_page: bool = False) -> None: self._lock = RLock() self._fd: int | None = None self._known_pages = 0 + self._active = 0 # I/O calls currently using the fd fd = os.open(self._path, os.O_RDWR | os.O_CREAT | _O_BINARY, 0o644) try: size = os.fstat(fd).st_size @@ -80,7 +84,8 @@ def path(self) -> Path: @property def page_count(self) -> int: """Number of whole pages in the file, read from the file size.""" - return self._refresh_page_count(self._require_open()) + with self._in_flight() as fd: + return self._refresh_page_count(fd) def read_page(self, page_id: UInt32) -> bytes: """Return the PAGE_SIZE`` bytes of page page_id. @@ -91,15 +96,17 @@ def read_page(self, page_id: UInt32) -> bytes: If page_id is out of range, the file is closed, or the file ends before a full page could be read. """ - fd = self._require_open() - offset = self._offset(fd, page_id) - buf = bytearray() - while len(buf) < PAGE_SIZE: - chunk = self._read_at(fd, PAGE_SIZE - len(buf), offset + len(buf)) - if not chunk: - raise DiskManagerError(f"unexpected end of file reading page {page_id}") - buf += chunk - return bytes(buf) + with self._in_flight() as fd: + offset = self._offset(fd, page_id) + buf = bytearray() + while len(buf) < PAGE_SIZE: + chunk = self._read_at(fd, PAGE_SIZE - len(buf), offset + len(buf)) + if not chunk: + raise DiskManagerError( + f"unexpected end of file reading page {page_id}" + ) + buf += chunk + return bytes(buf) def write_page(self, page_id: UInt32, data: bytes) -> None: """Overwrite the existing page page_id with data. @@ -112,12 +119,12 @@ def write_page(self, page_id: UInt32, data: bytes) -> None: If data is not exactly PAGE_SIZE bytes, page_id has not been allocated, or the file is closed. """ - fd = self._require_open() if len(data) != PAGE_SIZE: raise DiskManagerError( f"page data must be {PAGE_SIZE} bytes, got {len(data)}" ) - self._write_all(fd, data, self._offset(fd, page_id)) + with self._in_flight() as fd: + self._write_all(fd, data, self._offset(fd, page_id)) def allocate_page(self) -> UInt32: """Allocate a new page at the end of the file and return its ID. @@ -131,8 +138,7 @@ def allocate_page(self) -> UInt32: DiskManagerError If the file is closed or page IDs would exceed the UInt32 range. """ - fd = self._require_open() - with self._lock: + with self._in_flight() as fd, self._lock: count = self._refresh_page_count(fd) if count > UInt32.MAX: raise DiskManagerError("page ID would exceed the UInt32 range") @@ -142,16 +148,25 @@ def allocate_page(self) -> UInt32: def sync(self) -> None: """Flush file contents and size to stable storage (fsync).""" - os.fsync(self._require_open()) + with self._in_flight() as fd: + os.fsync(fd) def close(self) -> None: - """Close the file. Safe to call more than once.""" + """Close the file. Safe to call more than once. + + Raises + ------ + DiskManagerError + If a read, write, allocation or sync is still running. + """ with self._lock: + if self._active: + raise DiskManagerError("cannot close while I/O is in progress") if self._fd is not None: os.close(self._fd) self._fd = None - def __enter__(self): + def __enter__(self) -> Self: """Return the manager for use in a ``with`` block.""" return self @@ -159,6 +174,18 @@ def __exit__(self, *exc_info: object) -> None: """Close the file when leaving the ``with`` block.""" self.close() + @contextmanager + def _in_flight(self) -> Iterator[int]: + # Counts the call as running so close() cannot free the fd under it. + with self._lock: + fd = self._require_open() + self._active += 1 + try: + yield fd + finally: + with self._lock: + self._active -= 1 + def _require_open(self) -> int: if self._fd is None: raise DiskManagerError("the database file is closed") @@ -201,4 +228,3 @@ def _write_all(self, fd: int, data: bytes, offset: int) -> None: if n <= 0: raise DiskManagerError("write made no progress") written += n - diff --git a/tests/test_disk_manager.py b/tests/test_disk_manager.py index edd4396..7ca04c8 100644 --- a/tests/test_disk_manager.py +++ b/tests/test_disk_manager.py @@ -109,6 +109,30 @@ def test_page_allocated_by_another_handle_is_readable(path): b.read_page(UInt32(1)) +def test_close_during_io_raises(path, monkeypatch): + with DiskManager(path) as dm: + dm.write_page(dm.allocate_page(), page(1)) + real = dm._read_at + + def read_at(fd, size, offset): + with pytest.raises(DiskManagerError): + dm.close() + return real(fd, size, offset) + + monkeypatch.setattr(dm, "_read_at", read_at) + assert dm.read_page(UInt32(0)) == page(1) + # The with-block closed it, so no I/O call leaked its in-flight count. + with pytest.raises(DiskManagerError): + dm.read_page(UInt32(0)) + + +def test_failed_io_does_not_block_close(path): + dm = DiskManager(path) + with pytest.raises(DiskManagerError): + dm.read_page(UInt32(5)) + dm.close() + + def test_use_after_close_raises(path): dm = DiskManager(path) dm.close() From 9d4b6553fbbadacb71f35266aace49f2cc96bb29 Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Wed, 7 Oct 2026 23:11:00 +0530 Subject: [PATCH 14/18] fix: enforce UTF-8 text encoding for disk manager files --- .gitignore | Bin 5102 -> 5104 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/.gitignore b/.gitignore index 9b10f1c77d8b1b0cb48806edabbc71f69dae242b..49f69e23e523841aada2944a8f9dbf30ffb904b8 100644 GIT binary patch delta 12 TcmaE-{y}|%1k+|QrW=9)BXR_* delta 12 TcmeyM{!V>^1k+|IrfY%#BZ362 From 0a4c3660637335cbc6a2c4391917f1353be6f5f0 Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Wed, 7 Oct 2026 23:31:44 +0530 Subject: [PATCH 15/18] fix: restore gitignore as text --- .gitignore | Bin 5104 -> 4838 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/.gitignore b/.gitignore index 49f69e23e523841aada2944a8f9dbf30ffb904b8..4a001109052ef04d2bc57a8ad250ff42f91c1f12 100644 GIT binary patch delta 1863 zcmYjSOK%)S5S9~S<93K&L4hcbL5Yd5vNK3Qkuaykv9XctICf$b0vY$r)b6%-rh78o zyY_IgxPyq)S0uy_;J`h9fCK*kBu<t@S zPQ5Ahq=!^?W8zc$eB!}}z!hAIHfiIYSyJkt05-TFR~^ktQMMq2=GfSGPhFW{cIDL7 z6V^3Fo~i+kHtHCw^eZe>@eZ+YB!$b6UB;PInhcK8;c=0(vEM1n4IE!r1{Z^c-MUn! zOIqOlFuR?x4+J$iA(?gyUiIMg_{iP-Va27+xvP+?TcKP)WqkDxx$LMwn|`~!0yAF3 za+UHt#-$9(sBs3%l2AX(g`r!oiLFD`uO!c1m~~L3hbSNWFh1!rCEVzU5|D?CZZ(eu zb}y}XS&xnkwO5(_<@5#n>*-Cq_{PQwh>_|j9x=x&y6g&h9Cb({lZL=3#M-IvI(E6d zl(mNw=wW!(Airr>>uer9T)uOGT!t0a&xen^#95J3k#?1F-Qn z;IU>DsNI2}`{tA#SiWbC_L1dxL*f zmSE+ojdkSWNg~YF16D||o=2?mhT_>Psg-d92jbjGjTjKX+zZ9CSFSzu-xfFB;7d#t zOI^~`DEp$6Mt z6M@~SD+)}FBRt8Q-oTh-_M3B8R#2?kIh)FO)&?0R{Z63kB2l?!X+wB#^r)SB6?3+* zf1Z16S7smE$%PI3`|P59IU6tHyijB`On!pD0&cUp75n4def#g+2Ul6hz0n8S7rM{$ zV9)e3p@LPFk^N-;U?Q+T%nL+~A+;AaXI#vI#vc$x_Qk@@Tg*^VI`nQ|qWRZd?=*e5 zi*Yj^Bo&lVcI>D_JbOn0Zd zckMw!zHs9Lnkxt3A0Y7uxFIA?+~C3ian4WRtDf1lg^)(wT~%LIKkBQRe=a|{^7WPL zVUQ%c%2+n!Nzx00POPg>GL?V+Db|;C6=wyrHVndGJ=%!)q2tmDC86oWRzd&GqYFo2 z5bsoOsAL$VV^QRYlk*8Ll+Aeqe&geLb04wttD>fLZnb)eCtijni)qR+d58| zfpjqByzuHFGrXt$p5243sk%eKx9 z+bzn3^=tF*&xf^{zlRHT;*v?x=hpd~YMwO?Gk8>}0bW51-Nh<)hX6KW&I|5%*A0d3 z=ECsIDN{`W>(K)#T;vcq>P4BFaKi>XGF&U%I1GlaEV3gtp$j;5NRw=)DHfXPGVYK- zxeEiSEOxcd#)oAQ1~Z74I*kimg;Gkp!gE+@<%Zk}dQuq3J)v+%oD?aS*|210L}u@U znRsyDFlh_sraUN>53KwPtVcKZnH;Fdx^cgMl`ME4WL9OYNN`3csVeUya5_GF*Z-+XDYavtT}_GV|v<;3=rdo$art)nfRI6Uea>7e)04gm_Si?svE>>|7yDOc20zJ%nUdYn?N6o z=?P7AM>(DVjSC6{4=SW2Q?f4x@e;ig zx#)_^Nm&1M`T-8Rc;;3_O3E3)U8IwWvPBB*q2-xJfd1yk?%B_<{_5=I=g4hGXxbHd z$#6*0Ja`z^MX*^1Ph9}+r3FHNv2baPY~#rZpjO{{d;lu-PU;QnU|;na*K6dcAM+A!nD(PA8{xsV!AYynh^CL z0Ym6_&OO4uu_jay)<2y42zjNxr&%^a`B~rk$aMp?Yg+n%^j=Cib6txh=Hmi|cC19e zyt&XWN1r09n>?k7slG3XnVqGnaJTP;K`BvhtamL8jMtTqk@V@Zn={8+HeJu0m~e7o zepNo6S?lkfCSsxim|y0 zIvzx}DpOV9t+J-%ytW1IR2%rkfTb@leSC&Iqx!Fv8}-ji&wx(XR&UiWEYqsDoIHy? zM9ybvORX~urLcSz_CJ^JV(hHEx(*ccrgnpe!j4%H?YVI-RJ5UTjz7-|)xqy80^_yS z1mm}>+ac{9Pb~h)xJCVB_4T*WcX+AEXLl?mH(ON9(?+q2u6x}645Zm(hV|X^&x26F zYC*K1(6^Zp)WFf`G4~SuGo8qA%BiBT8d8!p@~l$!oZ+lQ_YUhH&#!&h1iK0TqA4Sx gAZJqwnBoI!=J8x_E}j!}`5d2Pb74T=|LMZuUr4-%tN;K2 From 32c6bf2977e2af62f6c7e8fb1f114b4a228db071 Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Wed, 7 Oct 2026 23:35:52 +0530 Subject: [PATCH 16/18] added ./idea in gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4a00110..d6d51b3 100644 --- a/.gitignore +++ b/.gitignore @@ -190,7 +190,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -# .idea/ + .idea/ # Abstra # Abstra is an AI-powered process automation framework. From ee2ae47951a0514726227dac33e1e43bc548ad74 Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Sun, 18 Oct 2026 14:50:43 +0530 Subject: [PATCH 17/18] chore: revert unrelated flake.lock change --- flake.lock | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/flake.lock b/flake.lock index 4864c7e..389caae 100644 --- a/flake.lock +++ b/flake.lock @@ -1,23 +1,27 @@ { - "nodes": { - "nixpkgs": { - "locked": { - "lastModified": 1789006805, - "narHash": "sha256-xB8mKMOx1IA9vTDNLmJZ6n4wCMq/cuWBBOzGCRnqxrU=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "8ce4ef6cb6f871616146b9fe26d2a5ae594e94fe", - "type": "github", - }, - "original": { - "owner": "NixOS", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github", - }, - }, - "root": {"inputs": {"nixpkgs": "nixpkgs"}}, + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1789006805, + "narHash": "sha256-xB8mKMOx1IA9vTDNLmJZ6n4wCMq/cuWBBOzGCRnqxrU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "8ce4ef6cb6f871616146b9fe26d2a5ae594e94fe", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } }, - "root": "root", - "version": 7, -} + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} \ No newline at end of file From 69b58731335d7b72f496bd34cf44a972a175774b Mon Sep 17 00:00:00 2001 From: IndicKangaroo Date: Sun, 18 Oct 2026 15:06:05 +0530 Subject: [PATCH 18/18] fix(storage): wrap DiskManager OS errors in DiskManagerError --- src/seriousdb/disk_manager.py | 56 ++++++++++++++++++++++++----------- tests/test_disk_manager.py | 43 +++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 18 deletions(-) diff --git a/src/seriousdb/disk_manager.py b/src/seriousdb/disk_manager.py index a7deac2..96e5391 100644 --- a/src/seriousdb/disk_manager.py +++ b/src/seriousdb/disk_manager.py @@ -23,6 +23,15 @@ _O_BINARY = getattr(os, "O_BINARY", 0) +@contextmanager +def _os_errors(path: Path) -> Iterator[None]: + """Re-raise any OSError as DiskManagerError, keeping it as the cause.""" + try: + yield + except OSError as e: + raise DiskManagerError(f"I/O error on {path}: {e}") from e + + class DiskManager: """Read and write fixed-size pages of the database file by page ID. @@ -57,20 +66,22 @@ def __init__(self, path: str | Path, *, repair_torn_page: bool = False) -> None: self._fd: int | None = None self._known_pages = 0 self._active = 0 # I/O calls currently using the fd - fd = os.open(self._path, os.O_RDWR | os.O_CREAT | _O_BINARY, 0o644) + with _os_errors(self._path): + fd = os.open(self._path, os.O_RDWR | os.O_CREAT | _O_BINARY, 0o644) try: - size = os.fstat(fd).st_size - tail = size % PAGE_SIZE - if tail: - if not repair_torn_page: - raise DiskManagerError( - f"{self._path} is {size} bytes, not a multiple of " - f"{PAGE_SIZE}; the last page is incomplete" - ) - size -= tail - os.ftruncate(fd, size) - os.fsync(fd) - self._known_pages = size // PAGE_SIZE + with _os_errors(self._path): + size = os.fstat(fd).st_size + tail = size % PAGE_SIZE + if tail: + if not repair_torn_page: + raise DiskManagerError( + f"{self._path} is {size} bytes, not a multiple of " + f"{PAGE_SIZE}; the last page is incomplete" + ) + size -= tail + os.ftruncate(fd, size) + os.fsync(fd) + self._known_pages = size // PAGE_SIZE except BaseException: os.close(fd) raise @@ -147,7 +158,13 @@ def allocate_page(self) -> UInt32: return UInt32(count) def sync(self) -> None: - """Flush file contents and size to stable storage (fsync).""" + """Flush file contents and size to stable storage (``fsync``). + + Raises + ------ + DiskManagerError + If the file is closed or the flush fails. + """ with self._in_flight() as fd: os.fsync(fd) @@ -157,14 +174,16 @@ def close(self) -> None: Raises ------ DiskManagerError - If a read, write, allocation or sync is still running. + If a read, write, allocation or sync is still running, or the + file cannot be closed. """ with self._lock: if self._active: raise DiskManagerError("cannot close while I/O is in progress") if self._fd is not None: - os.close(self._fd) - self._fd = None + fd, self._fd = self._fd, None + with _os_errors(self._path): + os.close(fd) def __enter__(self) -> Self: """Return the manager for use in a ``with`` block.""" @@ -181,7 +200,8 @@ def _in_flight(self) -> Iterator[int]: fd = self._require_open() self._active += 1 try: - yield fd + with _os_errors(self._path): + yield fd finally: with self._lock: self._active -= 1 diff --git a/tests/test_disk_manager.py b/tests/test_disk_manager.py index 7ca04c8..aed5f10 100644 --- a/tests/test_disk_manager.py +++ b/tests/test_disk_manager.py @@ -133,6 +133,49 @@ def test_failed_io_does_not_block_close(path): dm.close() +def test_open_failure_is_wrapped(tmp_path): + with pytest.raises(DiskManagerError) as info: + DiskManager(tmp_path / "missing-dir" / "test.sdb") + assert isinstance(info.value.__cause__, OSError) + + +def test_repair_failure_is_wrapped(path, monkeypatch): + path.write_bytes(page(1) + b"x") + + def boom(*args): + raise OSError("disk on fire") + + monkeypatch.setattr(disk_manager.os, "ftruncate", boom) + with pytest.raises(DiskManagerError) as info: + DiskManager(path, repair_torn_page=True) + assert isinstance(info.value.__cause__, OSError) + + +def test_read_write_sync_os_errors_are_wrapped(path, monkeypatch): + def boom(*args): + raise OSError("disk on fire") + + with DiskManager(path) as dm: + dm.write_page(dm.allocate_page(), page(1)) + with monkeypatch.context() as m: + m.setattr(dm, "_read_at", boom) + with pytest.raises(DiskManagerError) as info: + dm.read_page(UInt32(0)) + assert isinstance(info.value.__cause__, OSError) + with monkeypatch.context() as m: + m.setattr(dm, "_write_all", boom) + with pytest.raises(DiskManagerError): + dm.write_page(UInt32(0), page(2)) + with pytest.raises(DiskManagerError): + dm.allocate_page() + with monkeypatch.context() as m: + m.setattr(disk_manager.os, "fsync", boom) + with pytest.raises(DiskManagerError): + dm.sync() + # Failed calls must not leave the in-flight count raised. + assert dm.path == path + + def test_use_after_close_raises(path): dm = DiskManager(path) dm.close()