Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions benchmarks/sqlite_write_throughput.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
"""Compare default SQLite writes with the SnapshotDb import configuration.

Run with ``PYTHONPATH=src python benchmarks/sqlite_write_throughput.py``.
"""

from __future__ import annotations

import argparse
import tempfile
import time
from contextlib import ExitStack
from pathlib import Path

from pt_snap_cli.vendor.memsnapdump.tools.adaptors.database.snapshot_db import SnapshotDb
from pt_snap_cli.vendor.memsnapdump.util.sqlite_meta import SqliteDB


def measure(db: SqliteDB, rows: int, batch_size: int) -> float:
db.conn.execute("CREATE TABLE benchmark (id INTEGER, payload TEXT)")
started = time.perf_counter()
for offset in range(0, rows, batch_size):
batch_end = min(offset + batch_size, rows)
db.conn.executemany(
"INSERT INTO benchmark VALUES (?, ?)",
((index, "x" * 256) for index in range(offset, batch_end)),
)
db.conn.commit()
return time.perf_counter() - started


def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--rows", type=int, default=100_000)
parser.add_argument("--batch-size", type=int, default=1_000)
args = parser.parse_args()

with tempfile.TemporaryDirectory() as tmp_dir, ExitStack() as stack:
root = Path(tmp_dir)
baseline = SqliteDB(str(root / "baseline.db"))
stack.callback(baseline.conn.close)
optimized = SnapshotDb(str(root / "optimized.db"))
Comment thread
codeasier marked this conversation as resolved.
Resolved
stack.callback(optimized.conn.close)
baseline_seconds = measure(baseline, args.rows, args.batch_size)
optimized_seconds = measure(optimized, args.rows, args.batch_size)

print(f"default: {baseline_seconds:.3f}s")
print(f"optimized: {optimized_seconds:.3f}s")
print(f"speedup: {baseline_seconds / optimized_seconds:.2f}x")


if __name__ == "__main__":
main()
6 changes: 6 additions & 0 deletions docs/en/quickstart.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@ Use `--force` when you intentionally need to rebuild it:
pt-snap import snapshot.pkl --force
```

Import currently loads the entire pickle before processing it. For large snapshots, allow
roughly 3-10 times the pickle file size in available memory; the exact peak depends on the
number of frame objects. Run the import on a machine with sufficient memory. `--device`
limits subsequent replay and database writes, but does not reduce the initial pickle-loading
memory peak.

### Step 1: Set the Snapshot Database and Device

Point `pt-snap` to your SQLite snapshot database file:
Expand Down
5 changes: 5 additions & 0 deletions docs/zh/quickstart.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ pt-snap query --list
pt-snap import snapshot.pkl --force
```

当前导入流程会先完整加载 pickle,再开始处理。导入大型快照时,请预留约为 pickle
文件大小 3-10 倍的可用内存;实际峰值取决于 frame 对象的数量。建议在内存充足的机器上
执行导入。`--device` 只能减少后续回放和数据库写入量,无法降低最初加载 pickle 时的
内存峰值。

### 第一步:设置快照数据库和设备

将 `pt-snap` 指向你的 SQLite 快照数据库文件:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ class SnapshotDb(SqliteDB):

def __init__(self, path: str):
super().__init__(path, auto_create=True, with_dictionary_table=True)
# Import builds a replaceable database, so favor write throughput over crash recovery.
self.conn.execute("PRAGMA journal_mode = MEMORY")
self.conn.execute("PRAGMA synchronous = OFF")
self.conn.execute("PRAGMA cache_size = -65536")
# 清理旧版本表格
self._clear_old_tables()

Expand Down
2 changes: 2 additions & 0 deletions tests/core/test_import_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,8 @@ def test_import_snapshot_file_invalid_corrupt_pickle(tmp_path: Path) -> None:
with pytest.raises(ImportExecutionError, match=r"backend (failed|reported failure)"):
service.import_snapshot(ImportOptions(snapshot_file=snapshot_file))

assert not (tmp_path / "corrupt.pkl.db").exists()


def test_failed_reimport_preserves_existing_database(tmp_path: Path) -> None:
import_service_cls = _import_service_type()
Expand Down
32 changes: 32 additions & 0 deletions tests/test_snapshot_db.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
from __future__ import annotations

import sqlite3
from pathlib import Path

from pt_snap_cli.vendor.memsnapdump.tools.adaptors.database.snapshot_db import SnapshotDb
from pt_snap_cli.vendor.memsnapdump.util.sqlite_meta import SqliteDB


def _pragma(conn: sqlite3.Connection, name: str) -> object:
return conn.execute(f"PRAGMA {name}").fetchone()[0]


def test_snapshot_db_optimizes_import_writes(tmp_path: Path) -> None:
db = SnapshotDb(str(tmp_path / "snapshot.db"))
try:
assert _pragma(db.conn, "journal_mode") == "memory"
assert _pragma(db.conn, "synchronous") == 0
assert _pragma(db.conn, "cache_size") == -65536
finally:
db.conn.close()


def test_generic_sqlite_db_keeps_default_write_settings(tmp_path: Path) -> None:
baseline = sqlite3.connect(tmp_path / "baseline.db")
db = SqliteDB(str(tmp_path / "generic.db"))
try:
for name in ("journal_mode", "synchronous", "cache_size"):
assert _pragma(db.conn, name) == _pragma(baseline, name)
finally:
db.conn.close()
baseline.close()
Loading