From f588712545cf97b00d429eb4b3de8747ac08f50e Mon Sep 17 00:00:00 2001 From: codeasier Date: Tue, 28 Jul 2026 14:40:51 +0800 Subject: [PATCH 1/2] perf: tune SQLite snapshot imports --- benchmarks/sqlite_write_throughput.py | 53 +++++++++++++++++++ docs/en/quickstart.md | 6 +++ docs/zh/quickstart.md | 5 ++ .../tools/adaptors/database/snapshot_db.py | 4 ++ tests/core/test_import_service.py | 2 + tests/test_snapshot_db.py | 30 +++++++++++ 6 files changed, 100 insertions(+) create mode 100644 benchmarks/sqlite_write_throughput.py create mode 100644 tests/test_snapshot_db.py diff --git a/benchmarks/sqlite_write_throughput.py b/benchmarks/sqlite_write_throughput.py new file mode 100644 index 0000000..5930fc9 --- /dev/null +++ b/benchmarks/sqlite_write_throughput.py @@ -0,0 +1,53 @@ +"""Compare default SQLite writes with the SnapshotDb import configuration. + +Run with ``PYTHONPATH=src python benchmarks/sqlite_write_throughput.py``. +""" + +from __future__ import annotations + +import argparse +import tempfile +import time +from pathlib import Path + +from pt_snap_cli.vendor.memsnapdump.tools.adaptors.database.snapshot_db import SnapshotDb +from pt_snap_cli.vendor.memsnapdump.util.sqlite_meta import SqliteDB + + +def measure(db: SqliteDB, rows: int, batch_size: int) -> float: + db.conn.execute("CREATE TABLE benchmark (id INTEGER, payload TEXT)") + started = time.perf_counter() + for offset in range(0, rows, batch_size): + batch_end = min(offset + batch_size, rows) + db.conn.executemany( + "INSERT INTO benchmark VALUES (?, ?)", + ((index, "x" * 256) for index in range(offset, batch_end)), + ) + db.conn.commit() + return time.perf_counter() - started + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--rows", type=int, default=100_000) + parser.add_argument("--batch-size", type=int, default=1_000) + args = parser.parse_args() + + with tempfile.TemporaryDirectory() as tmp_dir: + root = Path(tmp_dir) + baseline = SqliteDB(str(root / "baseline.db")) + optimized = SnapshotDb(str(root / "optimized.db")) + try: + baseline_seconds = measure(baseline, args.rows, args.batch_size) + optimized_seconds = measure(optimized, args.rows, args.batch_size) + finally: + baseline.conn.close() + optimized.conn.close() + + print(f"default: {baseline_seconds:.3f}s") + print(f"optimized: {optimized_seconds:.3f}s") + print(f"speedup: {baseline_seconds / optimized_seconds:.2f}x") + + +if __name__ == "__main__": + main() diff --git a/docs/en/quickstart.md b/docs/en/quickstart.md index d2d198b..b98d32a 100644 --- a/docs/en/quickstart.md +++ b/docs/en/quickstart.md @@ -30,6 +30,12 @@ Use `--force` when you intentionally need to rebuild it: pt-snap import snapshot.pkl --force ``` +Import currently loads the entire pickle before processing it. For large snapshots, allow +roughly 3-10 times the pickle file size in available memory; the exact peak depends on the +number of frame objects. Run the import on a machine with sufficient memory. `--device` +limits subsequent replay and database writes, but does not reduce the initial pickle-loading +memory peak. + ### Step 1: Set the Snapshot Database and Device Point `pt-snap` to your SQLite snapshot database file: diff --git a/docs/zh/quickstart.md b/docs/zh/quickstart.md index 5a82258..907fda4 100644 --- a/docs/zh/quickstart.md +++ b/docs/zh/quickstart.md @@ -29,6 +29,11 @@ pt-snap query --list pt-snap import snapshot.pkl --force ``` +当前导入流程会先完整加载 pickle,再开始处理。导入大型快照时,请预留约为 pickle +文件大小 3-10 倍的可用内存;实际峰值取决于 frame 对象的数量。建议在内存充足的机器上 +执行导入。`--device` 只能减少后续回放和数据库写入量,无法降低最初加载 pickle 时的 +内存峰值。 + ### 第一步:设置快照数据库和设备 将 `pt-snap` 指向你的 SQLite 快照数据库文件: diff --git a/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/database/snapshot_db.py b/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/database/snapshot_db.py index 8058fe5..fedf408 100644 --- a/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/database/snapshot_db.py +++ b/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/database/snapshot_db.py @@ -56,6 +56,10 @@ class SnapshotDb(SqliteDB): def __init__(self, path: str): super().__init__(path, auto_create=True, with_dictionary_table=True) + # Import builds a replaceable database, so favor write throughput over crash recovery. + self.conn.execute("PRAGMA journal_mode = MEMORY") + self.conn.execute("PRAGMA synchronous = OFF") + self.conn.execute("PRAGMA cache_size = -65536") # 清理旧版本表格 self._clear_old_tables() diff --git a/tests/core/test_import_service.py b/tests/core/test_import_service.py index 15d3733..c0fec0b 100644 --- a/tests/core/test_import_service.py +++ b/tests/core/test_import_service.py @@ -106,6 +106,8 @@ def test_import_snapshot_file_invalid_corrupt_pickle(tmp_path: Path) -> None: with pytest.raises(ImportExecutionError, match=r"backend (failed|reported failure)"): service.import_snapshot(ImportOptions(snapshot_file=snapshot_file)) + assert not (tmp_path / "corrupt.pkl.db").exists() + def test_failed_reimport_preserves_existing_database(tmp_path: Path) -> None: import_service_cls = _import_service_type() diff --git a/tests/test_snapshot_db.py b/tests/test_snapshot_db.py new file mode 100644 index 0000000..8b84b50 --- /dev/null +++ b/tests/test_snapshot_db.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from pathlib import Path + +from pt_snap_cli.vendor.memsnapdump.tools.adaptors.database.snapshot_db import SnapshotDb +from pt_snap_cli.vendor.memsnapdump.util.sqlite_meta import SqliteDB + + +def _pragma(db: SqliteDB, name: str) -> object: + return db.conn.execute(f"PRAGMA {name}").fetchone()[0] + + +def test_snapshot_db_optimizes_import_writes(tmp_path: Path) -> None: + db = SnapshotDb(str(tmp_path / "snapshot.db")) + try: + assert _pragma(db, "journal_mode") == "memory" + assert _pragma(db, "synchronous") == 0 + assert _pragma(db, "cache_size") == -65536 + finally: + db.conn.close() + + +def test_generic_sqlite_db_keeps_default_write_settings(tmp_path: Path) -> None: + db = SqliteDB(str(tmp_path / "generic.db")) + try: + assert _pragma(db, "journal_mode") == "delete" + assert _pragma(db, "synchronous") == 2 + assert _pragma(db, "cache_size") != -65536 + finally: + db.conn.close() From 5fa9309001a8881d5b80eea1e4e2f35991db6c51 Mon Sep 17 00:00:00 2001 From: codeasier Date: Tue, 28 Jul 2026 15:04:09 +0800 Subject: [PATCH 2/2] test: harden SQLite tuning coverage --- benchmarks/sqlite_write_throughput.py | 13 ++++++------- tests/test_snapshot_db.py | 18 ++++++++++-------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/benchmarks/sqlite_write_throughput.py b/benchmarks/sqlite_write_throughput.py index 5930fc9..8b82078 100644 --- a/benchmarks/sqlite_write_throughput.py +++ b/benchmarks/sqlite_write_throughput.py @@ -8,6 +8,7 @@ import argparse import tempfile import time +from contextlib import ExitStack from pathlib import Path from pt_snap_cli.vendor.memsnapdump.tools.adaptors.database.snapshot_db import SnapshotDb @@ -33,16 +34,14 @@ def main() -> None: parser.add_argument("--batch-size", type=int, default=1_000) args = parser.parse_args() - with tempfile.TemporaryDirectory() as tmp_dir: + with tempfile.TemporaryDirectory() as tmp_dir, ExitStack() as stack: root = Path(tmp_dir) baseline = SqliteDB(str(root / "baseline.db")) + stack.callback(baseline.conn.close) optimized = SnapshotDb(str(root / "optimized.db")) - try: - baseline_seconds = measure(baseline, args.rows, args.batch_size) - optimized_seconds = measure(optimized, args.rows, args.batch_size) - finally: - baseline.conn.close() - optimized.conn.close() + stack.callback(optimized.conn.close) + baseline_seconds = measure(baseline, args.rows, args.batch_size) + optimized_seconds = measure(optimized, args.rows, args.batch_size) print(f"default: {baseline_seconds:.3f}s") print(f"optimized: {optimized_seconds:.3f}s") diff --git a/tests/test_snapshot_db.py b/tests/test_snapshot_db.py index 8b84b50..ba6f0e3 100644 --- a/tests/test_snapshot_db.py +++ b/tests/test_snapshot_db.py @@ -1,30 +1,32 @@ from __future__ import annotations +import sqlite3 from pathlib import Path from pt_snap_cli.vendor.memsnapdump.tools.adaptors.database.snapshot_db import SnapshotDb from pt_snap_cli.vendor.memsnapdump.util.sqlite_meta import SqliteDB -def _pragma(db: SqliteDB, name: str) -> object: - return db.conn.execute(f"PRAGMA {name}").fetchone()[0] +def _pragma(conn: sqlite3.Connection, name: str) -> object: + return conn.execute(f"PRAGMA {name}").fetchone()[0] def test_snapshot_db_optimizes_import_writes(tmp_path: Path) -> None: db = SnapshotDb(str(tmp_path / "snapshot.db")) try: - assert _pragma(db, "journal_mode") == "memory" - assert _pragma(db, "synchronous") == 0 - assert _pragma(db, "cache_size") == -65536 + assert _pragma(db.conn, "journal_mode") == "memory" + assert _pragma(db.conn, "synchronous") == 0 + assert _pragma(db.conn, "cache_size") == -65536 finally: db.conn.close() def test_generic_sqlite_db_keeps_default_write_settings(tmp_path: Path) -> None: + baseline = sqlite3.connect(tmp_path / "baseline.db") db = SqliteDB(str(tmp_path / "generic.db")) try: - assert _pragma(db, "journal_mode") == "delete" - assert _pragma(db, "synchronous") == 2 - assert _pragma(db, "cache_size") != -65536 + for name in ("journal_mode", "synchronous", "cache_size"): + assert _pragma(db.conn, name) == _pragma(baseline, name) finally: db.conn.close() + baseline.close()