diff --git a/.gitattributes b/.gitattributes index 6f484a0..e7d92f7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,9 @@ tests/fixtures/snapshots/snapshot_import_131k_sanitized.pickle filter=lfs diff=lfs merge=lfs -text tests/fixtures/snapshots/snapshot_import_628k_sanitized.pickle filter=lfs diff=lfs merge=lfs -text + +# Exact-hash interop fixture bytes must survive core.autocrlf checkouts. +tests/fixtures/interop/generate_three_shard.py text eol=lf +tests/fixtures/interop/three_shard_free_requested.json text eol=lf +tests/fixtures/interop/checksums.json text eol=lf +tests/fixtures/interop/three_shard_free_requested.pkl.msinsight/manifest.json text eol=lf +tests/fixtures/interop/three_shard_free_requested.pkl.msinsight/device_0/*.db -text diff --git a/docs/en/interop-acceptance.md b/docs/en/interop-acceptance.md index 82749c3..4c25b25 100644 --- a/docs/en/interop-acceptance.md +++ b/docs/en/interop-acceptance.md @@ -144,6 +144,54 @@ Reports use gap/attribution/active coverage at the SAME selected metric event. One query/report shares diminishing time and cumulative 100000 fetched/output rows /64 MiB serialized values, NOT RSS. Failure yields no partial false global result. +## Three-shard free-requested fixture + +[pt-snap-cli #249](https://github.com/codeasier/pt-snap-cli/issues/249) adds a +minimal, synthetic producer fixture under +[`tests/fixtures/interop/`](../../tests/fixtures/interop/README.md): one device, +three real events and three shards, generated by the real CLI with +`--events-per-slice 1`. Allocation 0, free_requested 1 and free_completed 2 each +belong to exactly one shard. There is no new committed executable pickle or +third-party producer code. Source, generation identity, privacy/rights review, +size/hash records and limitations are in the linked provenance record. + +```bash +PYTHONPATH="$PWD/src" python -m pytest tests/test_fixture_provenance.py +PYTHONPATH="$PWD/src" python -m pytest tests/test_interop_three_shard.py +PYTHONPATH="$PWD/src" python tests/fixtures/interop/generate_three_shard.py \ + --output /tmp/NEW-three-shard-interop +``` + +Use an absent output directory. The generator creates its own synthetic pickle +from the inspectable JSON and invokes `pt-snap import` for both `msinsight` +(compatibility-v1, inline callstack) and `pt-snap-native-v2` (interned callstack), +with `--device 0 --events-per-slice 1 --no-focus`. It does not open the GUI or run +an original producer. The committed v1 artifact is checksum-verified before +read-only tests; fresh output reproduces its physical semantics, not necessarily +its timestamp/path/version-dependent bytes. + +The regression checks event ownership, all nine physical event fields, complete +stack text and block references. The free-requested row has ID 1, action 5, +address 4096, size 16, stream 7 and allocated/active/reserved counters 0/16/64 +bytes (the consumer's Caching Allocated/Active/Reserved fields). Block 0 retains +`allocEventId=0`, `freeEventId=2`; `freeEventId` identifies **completed** free. +The existing current-shard-only consumer predicate misses event 1 in shard 0/2. +An independent SQL oracle searches manifest candidates intersecting +`allocEventId < id < freeEventId`, or through the device's final real event when +`freeEventId < 0`, then filters by action/address and the same bounds. Separate +v1/v2 cases prove address reuse cannot attach a later lifetime's request and +open-ended pending-free/allocated lifetimes yield the right event or no event. +The older OOM-containing nine-event `lifecycle_data` is intentionally not used as +this minimal v1 fixture; existing full-replay coverage is retained. + +For a separately authorized GUI retest, use the newly generated original +`three_shard_free_requested.pkl` beside its generated `.msinsight` directory. +Do not expose the immutable committed fixture to a consumer that writes derived +cache tables. Native-v2 is not a claimed msinsight GUI input. This work proves +producer/fixture contracts only: it neither fixes nor verifies +[msinsight #625](https://gitcode.com/Ascend/msinsight/issues/625), and does not +supersede the consumer limitation in the dated acceptance record below. + ## Performance: same reviewed fixture, three modes The bounded measured fixture is `snapshot_expandable.pkl`, 581140 bytes, diff --git a/docs/zh/interop-acceptance.md b/docs/zh/interop-acceptance.md index affab7e..a391028 100644 --- a/docs/zh/interop-acceptance.md +++ b/docs/zh/interop-acceptance.md @@ -114,6 +114,46 @@ missing label、非NULL空文本活动组不能混同。`callstack_analysis` 语 共用递减时间及累计100000 fetched/output行、64 MiB序列化值,**不是 RSS**;失败不能给出 部分的虚假全局结论。 +## 三片 free-requested fixture + +[pt-snap-cli #249](https://github.com/codeasier/pt-snap-cli/issues/249) 在 +[`tests/fixtures/interop/`](../../tests/fixtures/interop/README.md) 增加最小合成生产端 +fixture:单设备、三个真实事件、三个分片,由真实 CLI 配合 `--events-per-slice 1` +生成。allocation 0、free_requested 1、free_completed 2 各只属于一个分片。 +不新增已提交的可执行 pickle,也不引入第三方生产端代码。来源、生成身份、隐私/权利审核、 +大小/hash 和限制均记录在该目录的 provenance 中。 + +```bash +PYTHONPATH="$PWD/src" python -m pytest tests/test_fixture_provenance.py +PYTHONPATH="$PWD/src" python -m pytest tests/test_interop_three_shard.py +PYTHONPATH="$PWD/src" python tests/fixtures/interop/generate_three_shard.py \ + --output /tmp/NEW-three-shard-interop +``` + +输出目录必须不存在。生成器仅从可检查的 JSON 自建合成 pickle,以 +`--device 0 --events-per-slice 1 --no-focus` 分别调用真实 `pt-snap import`: +`msinsight`(compatibility-v1,内联调用栈)与 `pt-snap-native-v2`(interned 调用栈)。 +不打开 GUI、不运行原版生产端。已提交的 v1 产物在只读测试打开前校验 checksum; +新生成结果复现物理语义,不承诺含时间/路径/版本差异的逐字节相等。 + +回归检查事件唯一归属、全部九个物理 event 字段、完整栈文本和 block 引用。 +free-requested 行为 ID 1、action 5、address 4096、size 16、stream 7, +allocated/active/reserved 计数器为 0/16/64 字节(消费端的 +Caching Allocated/Active/Reserved 字段)。Block 0 保留 +`allocEventId=0`、`freeEventId=2`;后者指向 **completed** free。 +现有消费端只查当前片的 predicate 在片 0/2 找不到 event 1。 +独立 SQL oracle 先按 manifest 寻找与 `allocEventId < id < freeEventId` 相交的候选片; +`freeEventId < 0` 时查到该设备最后一个真实事件,再按 action/address 与相同边界过滤。 +另设 v1/v2 地址复用及 open-ended pending-free/allocated 测试,防止关联后一次生命周期, +并校验应返回唯一事件或无事件。原有九事件 `lifecycle_data` 包含 OOM,故不用作此最小 +v1 fixture;既有完整回放覆盖继续保留。 + +另行获准 GUI 复测时,应使用新生成的原始 `three_shard_free_requested.pkl` 及其邻接 +`.msinsight` 目录。不要让会写派生缓存表的消费端修改已提交的只读 fixture。 +不宣称 native-v2 是 msinsight GUI 输入。本项仅验证生产端/fixture 契约, +没有修复或验证 [msinsight #625](https://gitcode.com/Ascend/msinsight/issues/625), +也不覆盖下方带日期验收记录中的消费端限制。 + ## 性能:相同已审小样本,三模式 测量使用 `snapshot_expandable.pkl`,581140字节,SHA256 diff --git a/tests/fixtures/interop/PROVENANCE.md b/tests/fixtures/interop/PROVENANCE.md new file mode 100644 index 0000000..fd9546a --- /dev/null +++ b/tests/fixtures/interop/PROVENANCE.md @@ -0,0 +1,93 @@ +# Three-shard free-requested interoperability fixture + +## Introduction record: 2026-10-11, pt-snap-cli #249 + +Purpose: an independently authored, minimal producer-side regression for +[pt-snap-cli #249](https://github.com/codeasier/pt-snap-cli/issues/249) and a +retest input for [msinsight #625](https://gitcode.com/Ascend/msinsight/issues/625). +This record does **not** establish a consumer fix or a successful GUI run. + +### Source, rights and privacy + +- `three_shard_free_requested.json` is new synthetic data authored for this + repository, under its MIT license. It contains one device, one 64-byte segment, + one 16-byte block, and exactly three events: alloc 0, free_requested 1, + free_completed 2. Stream 7 and address 4096 are invented constants. +- Frame filenames/functions are invented. There are no user captures, model + sessions, account details, host paths from a user machine, or vendor code. + Nothing was copied from or run in msinsight. SQL predicates/schema are + independently expressed interface facts; the fixed target remains + Ascend/msinsight `101f65b877a267ffd5f66ea3834706057ba243e5`. +- The committed artifacts are inspectable JSON/Python and SQLite, **not pickle**. + The executable-snapshot acceptance records in `../snapshots/` are unchanged. + Their historical approvals do not approve new pickle inputs. + +### Generation and reproducibility boundary + +The generator used pt-snap-cli source commit +`55b5981fc03df93a60b809153e8abee5f980ed29`, Python 3.12.14 and SQLite 3.53.1. +It created a new primitive-only protocol-4 pickle from the adjacent JSON in +`/tmp/pt-snap-249-reviewed-fixture/`, then called the actual CLI subprocess: + +```bash +PYTHONPATH="$PWD/src" python tests/fixtures/interop/generate_three_shard.py \ + --output /tmp/pt-snap-249-reviewed-fixture +``` + +The directory must be absent. No external pickle is read by the generator; +imports deserialize only this locally constructed synthetic source. The stored +pickle for that generation was 695 bytes, SHA256 +`3c9513c7a8dca98d589cd82fc90ef7e177eec70b57887a1601b601f418a79f1f`. +Static opcode inspection finds no GLOBAL/STACK_GLOBAL, REDUCE/BUILD, +INST/OBJ/NEWOBJ/NEWOBJ_EX, persistent-ID or extension opcodes. The pickle is not +committed; this observation is not a claim that arbitrary pickles are safe. + +The committed `three_shard_free_requested.pkl.msinsight/` is the unmodified +compatibility-v1 output: three 53,248-byte SQLite shards and a 2,174-byte complete +manifest. Its `sourceFile` is the public, disposable generation path above, not +a relocatable GUI entry or a path expected to exist on the reader's machine. +The CLI timestamp and importer version remain the actual recorded values. +The native-v2 export is generated and tested but not duplicated in the repository. + +Reproduction means equal physical real events, lifecycle references, ownership, +and manifest ranges, verified against explicit expectations in +`tests/test_interop_three_shard.py`. It is not a cross-Python/SQLite byte-for-byte +reproducible-build claim. New generation timestamps, source locations, versions, +SQLite serialization and therefore artifact hashes may differ. Never normalize +or rewrite shipped metadata to hide those differences. `checksums.json` records +exact bytes and SHA256 for the recorded JSON, generator, manifest and every DB; +tests verify them before opening the committed databases and check read-only +inspection leaves them unchanged. Changing an input/artifact requires review and +an explicit checksum/provenance update; do not refresh checksums to silence a +regression. Preserve this introduction record when recording future changes. + +### Expectations and limitations + +- All real event rows appear once, only in their owning shard. Negative synthetic + boundary rows are separate and must not be counted as real events. +- The free-requested row is `(id=1, action=5, address=4096, size=16, stream=7, + allocated=0, active=16, reserved=64)` and has the complete two-frame text stack. + These three counters correspond to the consumer's Caching Allocated/Active/ + Reserved fields, in bytes. No reconstructed ordered-frame extension is claimed. +- Block 0 has `allocEventId=0` and `freeEventId=2` in every shard. The latter refers + to **free_completed**, never free_requested. Its allocation and completed-free + reference text is preserved across shards. +- A current-shard-only free-requested lookup misses event 1 when viewing shard 0 + or 2. Candidate discovery intersects manifest ranges with + `allocEventId < id < freeEventId`; for `freeEventId < 0`, it includes the + device's final real ID. Then action/address and the same bounds select rows. +- Address reuse and open-ended pending-free/allocated lifetimes are separate + generated tests, not extra events hidden in the dedicated three-shard artifact. + The older nine-event `lifecycle_data` includes OOM and cannot be this minimal + v1 fixture. Existing full-replay event coverage remains useful and unchanged. +- Only pt-snap producer/SQL contracts are tested here. The msinsight GUI/C++ fix, + original-producer execution and #625 consumer acceptance remain separate work. + +## Portability follow-up: 2026-10-11 + +Checksum member keys use POSIX separators on every host. Narrow Git attributes +pin the recorded JSON/Python/manifest text to LF and SQLite members to binary, +so `core.autocrlf=true` does not change their accepted bytes. A temporary-repository +checkout regression checks actual Git attributes and byte preservation alongside +an unprotected CRLF-converted control. No recorded artifact bytes or checksums +were changed for this portability correction. diff --git a/tests/fixtures/interop/README.md b/tests/fixtures/interop/README.md new file mode 100644 index 0000000..bba86cf --- /dev/null +++ b/tests/fixtures/interop/README.md @@ -0,0 +1,43 @@ +# Three-shard free-requested fixture + +[Provenance and exact generation identity](PROVENANCE.md) · +[File sizes and SHA256](checksums.json) + +The committed compatibility-v1 dataset is +`three_shard_free_requested.pkl.msinsight/manifest.json`. Device 0 has exactly +three one-event shards: allocation 0, free-requested 1, completed-free 2. +Read it without changing its bytes. The source is synthetic JSON; no executable +pickle is committed. + +Run the producer checks from the repository root: + +```bash +PYTHONPATH="$PWD/src" python -m pytest tests/test_fixture_provenance.py +PYTHONPATH="$PWD/src" python -m pytest tests/test_interop_three_shard.py +``` + +To prepare a new isolated bundle for a separately authorized consumer retest: + +```bash +PYTHONPATH="$PWD/src" python tests/fixtures/interop/generate_three_shard.py \ + --output /tmp/NEW-three-shard-interop +``` + +Use an absent destination. This generates a new local synthetic pickle and both +real CLI exports, with `--device 0 --events-per-slice 1 --no-focus`; it uses no +network, GPU/NPU, model sessions or original producer. The generator does not +launch a GUI and will not replace an existing output directory. + +The GUI's original-source entry requires the **generated** +`three_shard_free_requested.pkl`, adjacent to its generated `.msinsight` directory. +Do not open the committed fixture directly in a consumer that modifies its DBs, +and do not transplant its manifest over another cache. Use the new bundle for +that separate run. Native-v2 output is for pt-snap's reader; it is not a claimed +msinsight input. Record consumer revision and actual observed Free Requested +panel behavior for all three shards before claiming a fix. + +The tests demonstrate the current-shard-only lookup limitation and independently +verify bounded manifest-wide association, including separate address reuse and +open-ended cases in both layouts. See the bilingual +[English acceptance guide](../../../docs/en/interop-acceptance.md#three-shard-free-requested-fixture) +and [Chinese acceptance guide](../../../docs/zh/interop-acceptance.md#三片-free-requested-fixture). diff --git a/tests/fixtures/interop/checksums.json b/tests/fixtures/interop/checksums.json new file mode 100644 index 0000000..08166dc --- /dev/null +++ b/tests/fixtures/interop/checksums.json @@ -0,0 +1,26 @@ +{ + "generate_three_shard.py": { + "bytes": 2462, + "sha256": "b484f505883786d5dec296e97ba20eaba6409628432ed68d263e6afcc1032833" + }, + "three_shard_free_requested.json": { + "bytes": 1667, + "sha256": "1986dbcbe103c84ee209e26be68504c6d7ef6aef7cc5bbf95bdfed0bf57c4eaa" + }, + "three_shard_free_requested.pkl.msinsight/device_0/slice_00000.db": { + "bytes": 53248, + "sha256": "51bc6b5892c2118d9501744a983bd8fe3733446b8ac064d378bfbf3890f240ef" + }, + "three_shard_free_requested.pkl.msinsight/device_0/slice_00001.db": { + "bytes": 53248, + "sha256": "a6afbd48523c489452e1f6e2c6d8eaba4af655a686cbc9343c4ccb56a497c82b" + }, + "three_shard_free_requested.pkl.msinsight/device_0/slice_00002.db": { + "bytes": 53248, + "sha256": "58743c54ff808ee66a461f4e0b78ab60ff020e8291e275e98e2bfcd2877e61f6" + }, + "three_shard_free_requested.pkl.msinsight/manifest.json": { + "bytes": 2174, + "sha256": "c71b91d3792c82f53379d31bc3699ebab38c4620b31ea1678bfb1b55f3eb61fa" + } +} diff --git a/tests/fixtures/interop/generate_three_shard.py b/tests/fixtures/interop/generate_three_shard.py new file mode 100644 index 0000000..e445563 --- /dev/null +++ b/tests/fixtures/interop/generate_three_shard.py @@ -0,0 +1,82 @@ +"""Generate new CLI exports from the adjacent inspectable synthetic JSON only. + +No external pickle, original producer, GUI, network or hardware is used. The +pickle is built locally from primitive JSON values, not supplied by a third party. +""" + +from __future__ import annotations + +import argparse +import json +import os +import pickle +import subprocess +import sys +from pathlib import Path + +SOURCE = Path(__file__).with_name("three_shard_free_requested.json") +FORMATS = ("msinsight", "pt-snap-native-v2") + + +def import_source(source: Path, output_format: str, home: Path) -> dict: + """Exercise the real CLI entry point in a subprocess with isolated focus.""" + env = os.environ.copy() + env.pop("PT_SNAP_DB_PATH", None) + env["HOME"] = str(home) + env["XDG_CONFIG_HOME"] = str(home / ".config") + result = subprocess.run( + [ + sys.executable, + "-m", + "pt_snap_cli.cli", + "import", + str(source), + "--format", + output_format, + "--events-per-slice", + "1", + "--device", + "0", + "--no-focus", + "--json", + ], + cwd=home, + env=env, + capture_output=True, + text=True, + check=False, + timeout=60, + ) + if result.returncode: + raise RuntimeError( + f"CLI import failed ({result.returncode}): {result.stderr}{result.stdout}" + ) + return json.loads(result.stdout) + + +def write_source(directory: Path, data: dict) -> Path: + """Serialize only caller-constructed synthetic data to a NEW local file.""" + path = directory / "three_shard_free_requested.pkl" + with path.open("xb") as stream: + pickle.dump(data, stream, protocol=4) + return path + + +def generate(output: Path) -> dict: + """Keep the generated source adjacent to its v1 GUI cache; never replace.""" + output = output.absolute() + output.mkdir() + source = write_source(output, json.loads(SOURCE.read_text(encoding="utf-8"))) + exports = {fmt: import_source(source, fmt, output) for fmt in FORMATS} + return {"source": str(source), "exports": exports} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True, help="New, absent output directory") + args = parser.parse_args() + print(json.dumps(generate(args.output), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/interop/three_shard_free_requested.json b/tests/fixtures/interop/three_shard_free_requested.json new file mode 100644 index 0000000..3c60d97 --- /dev/null +++ b/tests/fixtures/interop/three_shard_free_requested.json @@ -0,0 +1,82 @@ +{ + "segments": [ + { + "address": 4096, + "total_size": 64, + "stream": 7, + "segment_type": "small", + "allocated_size": 0, + "active_size": 0, + "device": 0, + "is_expandable": false, + "frames": [], + "blocks": [ + { + "address": 4096, + "size": 64, + "requested_size": 0, + "state": "inactive", + "frames": [] + } + ] + } + ], + "device_traces": [ + [ + { + "action": "alloc", + "addr": 4096, + "size": 16, + "stream": 7, + "frames": [ + { + "filename": "synthetic_lifecycle.py", + "line": 10, + "name": "alloc" + }, + { + "filename": "synthetic_entry.py", + "line": 1, + "name": "fixture" + } + ] + }, + { + "action": "free_requested", + "addr": 4096, + "size": 16, + "stream": 7, + "frames": [ + { + "filename": "synthetic_lifecycle.py", + "line": 11, + "name": "free_requested" + }, + { + "filename": "synthetic_entry.py", + "line": 1, + "name": "fixture" + } + ] + }, + { + "action": "free_completed", + "addr": 4096, + "size": 16, + "stream": 7, + "frames": [ + { + "filename": "synthetic_lifecycle.py", + "line": 12, + "name": "free_completed" + }, + { + "filename": "synthetic_entry.py", + "line": 1, + "name": "fixture" + } + ] + } + ] + ] +} diff --git a/tests/fixtures/interop/three_shard_free_requested.pkl.msinsight/device_0/slice_00000.db b/tests/fixtures/interop/three_shard_free_requested.pkl.msinsight/device_0/slice_00000.db new file mode 100644 index 0000000..5315695 Binary files /dev/null and b/tests/fixtures/interop/three_shard_free_requested.pkl.msinsight/device_0/slice_00000.db differ diff --git a/tests/fixtures/interop/three_shard_free_requested.pkl.msinsight/device_0/slice_00001.db b/tests/fixtures/interop/three_shard_free_requested.pkl.msinsight/device_0/slice_00001.db new file mode 100644 index 0000000..240ac70 Binary files /dev/null and b/tests/fixtures/interop/three_shard_free_requested.pkl.msinsight/device_0/slice_00001.db differ diff --git a/tests/fixtures/interop/three_shard_free_requested.pkl.msinsight/device_0/slice_00002.db b/tests/fixtures/interop/three_shard_free_requested.pkl.msinsight/device_0/slice_00002.db new file mode 100644 index 0000000..21f412f Binary files /dev/null and b/tests/fixtures/interop/three_shard_free_requested.pkl.msinsight/device_0/slice_00002.db differ diff --git a/tests/fixtures/interop/three_shard_free_requested.pkl.msinsight/manifest.json b/tests/fixtures/interop/three_shard_free_requested.pkl.msinsight/manifest.json new file mode 100644 index 0000000..a644a15 --- /dev/null +++ b/tests/fixtures/interop/three_shard_free_requested.pkl.msinsight/manifest.json @@ -0,0 +1,72 @@ +{ + "schemaVersion": 1, + "status": "complete", + "sourceFile": "/tmp/pt-snap-249-reviewed-fixture/three_shard_free_requested.pkl", + "cacheHash": "70f880e2ced8dfe4d34f6ee77f43a3969da1c07826a35243271bb02e46ce368a", + "eventsPerSlice": 1, + "devices": { + "0": { + "eventCount": 3, + "sliceCount": 3, + "readySlices": [ + 0, + 1, + 2 + ], + "slices": [ + { + "index": 0, + "startEventId": 0, + "endEventId": 0, + "file": "device_0/slice_00000.db", + "ready": true + }, + { + "index": 1, + "startEventId": 1, + "endEventId": 1, + "file": "device_0/slice_00001.db", + "ready": true + }, + { + "index": 2, + "startEventId": 2, + "endEventId": 2, + "file": "device_0/slice_00002.db", + "ready": true + } + ] + } + }, + "ptSnap": { + "identity": { + "sourceSha256": "3c9513c7a8dca98d589cd82fc90ef7e177eec70b57887a1601b601f418a79f1f", + "requestedDevice": 0, + "eventsPerSlice": 1, + "outputFormat": "compatibility-v1", + "manifestContractVersion": 1, + "exportContractVersion": 1, + "referenceContractVersion": 1, + "metadataSchemaVersion": 1, + "importFormatVersion": 1, + "targetRevision": "101f65b877a267ffd5f66ea3834706057ba243e5" + }, + "metadata": { + "metadata_schema_version": 1, + "import_format_version": 1, + "source_sha256": "3c9513c7a8dca98d589cd82fc90ef7e177eec70b57887a1601b601f418a79f1f", + "source_size": 695, + "source_name": "three_shard_free_requested.pkl", + "requested_device": 0, + "importer_name": "pt-snap-cli", + "importer_version": "0.4.0.post1.dev49", + "completed_at": "2026-10-11T13:37:54.547672+00:00" + }, + "omittedDevices": [], + "members": { + "device_0/slice_00000.db": "51bc6b5892c2118d9501744a983bd8fe3733446b8ac064d378bfbf3890f240ef", + "device_0/slice_00001.db": "a6afbd48523c489452e1f6e2c6d8eaba4af655a686cbc9343c4ccb56a497c82b", + "device_0/slice_00002.db": "58743c54ff808ee66a461f4e0b78ab60ff020e8291e275e98e2bfcd2877e61f6" + } + } +} \ No newline at end of file diff --git a/tests/test_interop_three_shard.py b/tests/test_interop_three_shard.py new file mode 100644 index 0000000..d8d6229 --- /dev/null +++ b/tests/test_interop_three_shard.py @@ -0,0 +1,341 @@ +"""Producer-only three-shard contract for msinsight #625; no GUI acceptance. + +The SQL is an independent interface-facts oracle. The fixed upstream lookup's +lower-bound-only predicate is deliberately tested separately from the proposed +bounded, manifest-wide association. No upstream code is executed or copied. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +import pickle +import shutil +import sqlite3 +import subprocess +from contextlib import closing +from pathlib import Path, PurePosixPath, PureWindowsPath + +import pytest + +from pt_snap_cli.core.dataset_contract import QueryScope +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from tests.fixtures.interop.generate_three_shard import ( + FORMATS, + SOURCE, + generate, + import_source, + write_source, +) + +FIXTURE = SOURCE.parent +COMMITTED = FIXTURE / "three_shard_free_requested.pkl.msinsight" +EXPECTED_EVENTS = [ + ( + 0, + 4, + 4096, + 16, + 7, + 16, + 16, + 64, + "synthetic_entry.py:1 fixture\nsynthetic_lifecycle.py:10 alloc", + ), + ( + 1, + 5, + 4096, + 16, + 7, + 0, + 16, + 64, + "synthetic_entry.py:1 fixture\nsynthetic_lifecycle.py:11 free_requested", + ), + ( + 2, + 6, + 4096, + 16, + 7, + 0, + 0, + 64, + "synthetic_entry.py:1 fixture\nsynthetic_lifecycle.py:12 free_completed", + ), +] + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def fixture_member_name(path, root): + """Checksum manifests use forward slashes on every host OS.""" + return path.relative_to(root).as_posix() + + +def verify_fixture(): + """Check every shipped input/member before any SQLite open, never unpickle.""" + manifest = json.loads((FIXTURE / "checksums.json").read_text()) + expected = {SOURCE.name, "generate_three_shard.py"} | { + fixture_member_name(path, FIXTURE) for path in COMMITTED.rglob("*") if path.is_file() + } + assert set(manifest) == expected + assert not list(FIXTURE.rglob("*.pkl")) + assert not list(FIXTURE.rglob("*.pickle")) + for name, accepted in manifest.items(): + path = FIXTURE / name + assert not path.is_symlink() + assert path.stat().st_size == accepted["bytes"], name + assert digest(path) == accepted["sha256"], name + + +@pytest.fixture(autouse=True) +def check_committed_fixture(): + verify_fixture() + + +@pytest.fixture(params=FORMATS) +def imported(tmp_path, request): + data = json.loads(SOURCE.read_text()) + source = write_source(tmp_path, data) + result = import_source(source, request.param, tmp_path) + assert result["ok"] and result["slice_count"] == 3 + assert result["focus_state"] is None and not (tmp_path / ".pt-snap").exists() + assert not (tmp_path / ".config").exists() + return Path(result["db_path"]), request.param, source + + +def rows(path, sql, params=()): + with closing(sqlite3.connect(path.as_uri() + "?mode=ro&immutable=1", uri=True)) as conn: + return conn.execute(sql, params).fetchall() + + +def events(path, fmt, predicate="t.id>=0", params=()): + stack = "t.callstack" if fmt == "msinsight" else "c.callstack" + join = "" if fmt == "msinsight" else " JOIN callstack c ON c.id=t.callstackId" + return rows( + path, + "SELECT t.id,t.action,t.address,t.size,t.stream,t.allocated,t.active,t.reserved," + f"{stack} FROM trace_entry_0 t{join} WHERE {predicate} ORDER BY t.id", + params, + ) + + +def find_requested(dataset, fmt, *, address, alloc_id, free_id): + """A lifecycle interval selects candidate shards, then exact bounded rows.""" + device = dataset.device(0) + last = device.slices[-1].end_event_id + end = free_id - 1 if free_id >= 0 else last + if alloc_id + 1 > end: + return (), [] + candidates = dataset.paths( + QueryScope("event_range", 0, start_event_id=alloc_id + 1, end_event_id=end) + ) + found = [ + (path.name, event) + for path in candidates + for event in events( + path, + fmt, + "t.action=5 AND t.address=? AND t.id>? AND t.id<=?", + (address, alloc_id, end), + ) + ] + return candidates, found + + +def assert_three_shards(root, fmt): + dataset = DatasetResolver().inspect(root / "manifest.json") + assert dataset is not None and dataset.validation.real_event_count == 3 + device = dataset.device(0) + assert len(device.slices) == 3 and device.ready_slices == (0, 1, 2) + assert [(s.start_event_id, s.end_event_id) for s in device.slices] == [(0, 0), (1, 1), (2, 2)] + observed = [] + hits = [] + for index, shard in enumerate(device.slices): + path = root / shard.file + real = events(path, fmt) + assert real == [EXPECTED_EVENTS[index]] # All nine physical fields and stack text. + observed.extend(real) + assert rows(path, "SELECT allocEventId,freeEventId FROM block_0 WHERE id=0") == [(0, 2)] + assert rows( + path, + "SELECT stream,allocCallstack,freeCallstack FROM pt_snap_block_reference WHERE blockId=0", + ) == [(7, EXPECTED_EVENTS[0][-1], EXPECTED_EVENTS[2][-1])] + # This reproduces the fixed consumer's current-shard, lower-bound-only + # lookup fact. Alloc and completed-free shards cannot find event 1. + hits.append(events(path, fmt, "t.action=5 AND t.address=? AND t.id>?", (4096, 0))) + assert observed == EXPECTED_EVENTS + assert hits == [[], [EXPECTED_EVENTS[1]], []] + candidates, found = find_requested(dataset, fmt, address=4096, alloc_id=0, free_id=2) + assert [p.name for p in candidates] == ["slice_00001.db"] + assert found == [("slice_00001.db", EXPECTED_EVENTS[1])] + + +def test_true_cli_import_keeps_one_event_per_owner_and_discovers_requested(imported): + root, fmt, source = imported + before = {p: digest(p) for p in root.rglob("*") if p.is_file()} + assert_three_shards(root, fmt) + manifest = json.loads((root / "manifest.json").read_text()) + if fmt == "msinsight": + assert ( + manifest["cacheHash"] + == hashlib.sha256(b"mem_snapshot_parser_v2" + source.read_bytes()).hexdigest() + ) + assert {p: digest(p) for p in before} == before + + +def test_shipped_v1_fixture_matches_explicit_physical_oracle_without_mutation(): + before = {p: digest(p) for p in COMMITTED.rglob("*") if p.is_file()} + assert_three_shards(COMMITTED, "msinsight") + assert {p: digest(p) for p in before} == before + + +@pytest.mark.parametrize("fmt", FORMATS) +def test_reused_address_does_not_associate_later_lifetime(tmp_path, fmt): + data = json.loads(SOURCE.read_text()) + second = copy.deepcopy(data["device_traces"][0]) + for event in second: + event["size"] = 24 + event["frames"][0]["name"] = "reused_" + event["action"] + data["device_traces"][0].extend(second) + root = Path(import_source(write_source(tmp_path, data), fmt, tmp_path)["db_path"]) + dataset = DatasetResolver().inspect(root) + assert dataset is not None and dataset.validation.real_event_count == 6 + for alloc_id, requested_id, free_id, size in [(0, 1, 2, 16), (3, 4, 5, 24)]: + block_rows = { + row + for shard in dataset.device(0).slices + for row in rows( + root / shard.file, + "SELECT address,allocEventId,freeEventId,size FROM block_0 WHERE id=?", + (alloc_id,), + ) + } + assert block_rows == {(4096, alloc_id, free_id, size)} + candidates, found = find_requested( + dataset, fmt, address=4096, alloc_id=alloc_id, free_id=free_id + ) + assert [p.name for p in candidates] == [f"slice_{requested_id:05d}.db"] + assert len(found) == 1 and found[0][1][0] == requested_id and found[0][1][3] == size + # Removing the completed-free upper bound would find BOTH lifetimes. + unbounded = [ + row[0] + for shard in dataset.device(0).slices + for row in events(root / shard.file, fmt, "t.action=5 AND t.address=4096 AND t.id>0") + ] + assert unbounded == [1, 4] + + +@pytest.mark.parametrize("fmt", FORMATS) +@pytest.mark.parametrize("pending", [True, False]) +def test_open_ended_lifetime_search_includes_device_last_event(tmp_path, fmt, pending): + data = json.loads(SOURCE.read_text()) + data["device_traces"][0].pop() # No completed-free event exists. + if not pending: + data["device_traces"][0].pop() # Still allocated; no requested-free exists either. + segment = data["segments"][0] + segment["active_size"] = 16 + segment["allocated_size"] = 0 if pending else 16 + segment["blocks"] = [ + { + "address": 4096, + "size": 16, + "requested_size": 16, + "state": "active_pending_free" if pending else "active_allocated", + "frames": [], + }, + {"address": 4112, "size": 48, "requested_size": 0, "state": "inactive", "frames": []}, + ] + root = Path(import_source(write_source(tmp_path, data), fmt, tmp_path)["db_path"]) + dataset = DatasetResolver().inspect(root) + assert dataset is not None and dataset.device(0).event_count == (2 if pending else 1) + for shard in dataset.device(0).slices: + assert rows( + root / shard.file, "SELECT allocEventId,freeEventId FROM block_0 WHERE id=0" + ) == [(0, -1)] + candidates, found = find_requested(dataset, fmt, address=4096, alloc_id=0, free_id=-1) + assert [p.name for p in candidates] == (["slice_00001.db"] if pending else []) + assert found == ([("slice_00001.db", EXPECTED_EVENTS[1])] if pending else []) + + +def test_generator_reproduces_semantics_and_preserves_existing_destination(tmp_path): + output = tmp_path / "new" + result = generate(output) + source = Path(result["source"]) + assert source.is_file() + # This only inspects locally serialized primitive opcodes, never executes pickle. + import pickletools + + assert not {op.name for op, _, _ in pickletools.genops(source.read_bytes())} & { + "GLOBAL", + "STACK_GLOBAL", + "REDUCE", + "BUILD", + "INST", + "OBJ", + "NEWOBJ", + "NEWOBJ_EX", + "PERSID", + "BINPERSID", + "EXT1", + "EXT2", + "EXT4", + } + assert source.read_bytes() == pickle.dumps(json.loads(SOURCE.read_text()), protocol=4) + for fmt, exported in result["exports"].items(): + assert_three_shards(Path(exported["db_path"]), fmt) + before = {p: digest(p) for p in output.rglob("*") if p.is_file()} + with pytest.raises(FileExistsError): + generate(output) + assert {p: digest(p) for p in before} == before + + +@pytest.mark.parametrize("path_class", [PurePosixPath, PureWindowsPath]) +def test_fixture_member_names_are_portable(path_class): + root = path_class("fixture") + path = root / "three_shard_free_requested.pkl.msinsight" / "device_0" / "slice_00000.db" + assert fixture_member_name(path, root) == ( + "three_shard_free_requested.pkl.msinsight/device_0/slice_00000.db" + ) + + +def test_checksum_members_survive_autocrlf_checkout(tmp_path): + """Exercise real Git attributes and checkout conversion without user config edits.""" + repo = tmp_path / "repo" + repo.mkdir() + + def git(*args): + return subprocess.run( + ["git", "-C", str(repo), "-c", "core.autocrlf=true", *args], + check=True, + capture_output=True, + timeout=30, + ).stdout + + git("init", "--quiet") + shutil.copyfile(FIXTURE.parents[2] / ".gitattributes", repo / ".gitattributes") + accepted = json.loads((FIXTURE / "checksums.json").read_text()) + names = [*accepted, "checksums.json"] + prefix = PurePosixPath("tests/fixtures/interop") + for name in names: + target = repo / prefix / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(FIXTURE / name, target) + (repo / "control.txt").write_bytes(b"unprotected\ntext\n") + git("add", ".") + for name in names: + relative = str(prefix / name) + attrs = git("check-attr", "-z", "text", "eol", "--", relative).decode().split("\0") + expected = ("unset", "unspecified") if name.endswith(".db") else ("set", "lf") + assert (attrs[2], attrs[5]) == expected, relative + destination = tmp_path / "checkout" + git("checkout-index", "--all", "--prefix=" + destination.as_posix() + "/") + # Prove autocrlf conversion actually occurred for an unprotected control. + assert (destination / "control.txt").read_bytes() == b"unprotected\r\ntext\r\n" + for name in names: + assert (destination / prefix / name).read_bytes() == (FIXTURE / name).read_bytes(), name