From 047c7f865a3b410c9447c25c1c67272a4be83938 Mon Sep 17 00:00:00 2001 From: codeasier Date: Mon, 5 Oct 2026 23:46:41 +0800 Subject: [PATCH] feat: export explicit msinsight-compatible datasets (#202) --- README.md | 8 + README_zh.md | 6 + docs/en/database.md | 26 +- docs/en/focus-management.md | 11 + docs/en/quickstart.md | 43 ++- docs/en/sharded-snapshotdb.md | 118 ++++++-- docs/zh/database.md | 20 +- docs/zh/focus-management.md | 8 + docs/zh/quickstart.md | 35 ++- docs/zh/sharded-snapshotdb.md | 86 +++++- src/pt_snap_cli/cli.py | 12 +- src/pt_snap_cli/context.py | 10 + src/pt_snap_cli/core/context_cache.py | 14 +- src/pt_snap_cli/core/dataset_contract.py | 16 +- src/pt_snap_cli/core/dataset_files.py | 8 +- src/pt_snap_cli/core/dataset_resolver.py | 8 +- src/pt_snap_cli/core/import_metadata.py | 13 +- src/pt_snap_cli/core/import_service.py | 128 +++++++- src/pt_snap_cli/core/models.py | 5 +- src/pt_snap_cli/core/msinsight_export.py | 258 ++++++++++++++++ src/pt_snap_cli/core/query_service.py | 10 +- tests/core/test_dataset_import.py | 4 +- tests/core/test_msinsight_export.py | 368 +++++++++++++++++++++++ tests/test_contract_cli_api.py | 68 +++++ tests/test_dataset_focus.py | 68 +++-- tests/test_dataset_import_cli.py | 2 +- 26 files changed, 1259 insertions(+), 94 deletions(-) create mode 100644 src/pt_snap_cli/core/msinsight_export.py create mode 100644 tests/core/test_msinsight_export.py diff --git a/README.md b/README.md index c84c9ad..04b29db 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,14 @@ rollback fails (force is not a single crash-atomic swap). See the [native import guide](docs/en/quickstart.md#optional-import-a-native-sharded-dataset). Only bounded dataset `event` queries are currently supported. +Explicit `pt-snap import snapshot.pkl --format msinsight --json` produces +`snapshot.pkl.msinsight/` for the fixed msinsight revision's **same original +pickle + adjacent cache** entrance. Inline stacks cost more space than native +interning. Existing compatible targets are never replaced, even with `--force`; +only fully validated identical pt-snap caches are reused. GUI acceptance remains +**pending/not run**; hash equality alone is not acceptance. See +[compatible export and limits](docs/en/sharded-snapshotdb.md#explicit-msinsight-compatible-export-p2). + ## Commands | Command | Description | diff --git a/README_zh.md b/README_zh.md index 63f29f3..cf2dd8e 100644 --- a/README_zh.md +++ b/README_zh.md @@ -58,6 +58,12 @@ pt-snap split snapshot.pkl --slices 4 --output snapshot-slices (force 不是单次崩溃原子 swap)。详见 [原生分库导入](docs/zh/quickstart.md#可选导入原生分库数据集);当前仅支持有界数据集 `event` 查询。 +显式 `pt-snap import snapshot.pkl --format msinsight --json` 生成 +`snapshot.pkl.msinsight/`,对齐固定 msinsight 版本的**同一原始 pickle + 邻接缓存**入口。 +内联栈比原生去重占更多空间;即使 `--force` 也不替换已有兼容目标,只复用完整校验且 +身份相同的 pt-snap 缓存。GUI **待验收/未运行**,hash 相等不代表验收通过。详见 +[兼容导出及限制](docs/zh/sharded-snapshotdb.md#显式-msinsight-兼容导出p2)。 + ## 命令 | 命令 | 说明 | diff --git a/docs/en/database.md b/docs/en/database.md index 206c908..9b78f0a 100644 --- a/docs/en/database.md +++ b/docs/en/database.md @@ -405,4 +405,28 @@ before bounded `event` addressing; it never repairs or migrates files. Aggregate and lifecycle dataset queries remain unsupported. See the separate [native manifest protocol](sharded-snapshotdb.md#published-native-dataset-p2) and [transaction/recovery guide](quickstart.md#optional-import-a-native-sharded-dataset). -P0 compatibility-v1 table order, inline callstack layout and reject rules are unchanged. +P0 compatibility-v1 table order, inline layout and manifest semantics are unchanged. + +### Explicit compatible export + +`import --format msinsight` (alias `compatibility-v1`) publishes physical v1 +`trace_entry_` tables with the ninth column `callstack TEXT`, not views, +with the exact positional block/dictionary schema. It removes staging's unprefixed +`callstack` table and writes `pt_snap_metadata.import_format_version=1`, retaining +isolated `pt_snap_block_reference` text lifecycle sources. Native standalone and +native datasets keep their interning and v2 metadata. Structured ordered frames +are not emitted or inferred from text. + +Inline stack bytes scale with the sum of stack text **over every event**, not +unique stacks; conversion temporarily holds native and inline tables and may leave +SQLite free pages. Capacity does not bound blocks, loading RSS or total disk. +Compatible datasets additionally carry `ptSnap.identity`, metadata, omitted devices +and full member hashes. Recognized whole-artifact metadata is available only after +all attestation checks; external v1 datasets still need no metadata. + +For finalized compatibility datasets ONLY, bare validation and cached analysis +use `mode=ro&immutable=1` after rejecting all live/dangling sidecars and aliases +before the first SQLite open. Closed/checkpointed original WAL bytes/inventory +remain unchanged. This assumes no active writer; native dataset non-WAL policy and +standalone `mode=ro` defaults are unchanged. See +[compatible protocol and no-replace rules](sharded-snapshotdb.md#explicit-msinsight-compatible-export-p2). diff --git a/docs/en/focus-management.md b/docs/en/focus-management.md index 386cc70..a30b727 100644 --- a/docs/en/focus-management.md +++ b/docs/en/focus-management.md @@ -139,6 +139,17 @@ prior absence, not a valid focus JSON. Preserve evidence for manual recovery; no unconditional I/O/crash atomicity or concurrent-writer lock is promised. See [native import](quickstart.md#optional-import-a-native-sharded-dataset). +Explicit compatible import (`--format msinsight`) has the same requested-focus +compensation, but **never replaces** an existing compatible output, even with +force. Only fully attested identical pt-snap caches can be reused; otherwise use +a new output. GUI still requires the same original pickle plus adjacent cache, +although pt-snap focus needs only the complete artifact. All compatible members +are checked for aliases/live/dangling sidecars before any SQLite open; bare +validation and cached queries use `mode=ro&immutable=1` for finalized closed +members, preserving original checkpointed-WAL bytes/inventory. This is not a live +writer lock; native non-WAL checks and standalone defaults remain unchanged. + + ## Error Handling **No focus configured:** diff --git a/docs/en/quickstart.md b/docs/en/quickstart.md index 39ff874..aa90d08 100644 --- a/docs/en/quickstart.md +++ b/docs/en/quickstart.md @@ -61,8 +61,8 @@ Without `--events-per-slice`, import still produces the existing single DB. With it, the output is `/.pt-snap-native-v2/`, or the same directory beside the source if `--output-dir` is omitted. `--format pt-snap-native-v2` is optional in this mode; `--format single-db` forbids a capacity. -`--format compatibility-v1` and `msinsight` are reserved and explicitly rejected: -native output is **not** an msinsight-compatible export. +Native output is **not** an msinsight-compatible export. Explicit `--format +msinsight` (alias `compatibility-v1`) selects the separate compatible mode below. Capacity counts real chronological positions **per device**, not `max(id)+1` or negative synthetic boundaries. Original sparse/nonzero IDs, OOM and workspace @@ -101,6 +101,45 @@ silently run peak/lifecycle/aggregate queries over one shard. See the [native manifest contract](sharded-snapshotdb.md#published-native-dataset-p2). `SnapshotAnalyzer` remains analysis-only, without import/split methods. +### Optional: Export an msinsight-Compatible Dataset + +```bash +pt-snap import snapshot.pkl --format msinsight --json +pt-snap overview snapshot.pkl.msinsight --json +pt-snap query --template-use event --slice 0 --json +``` + +This **explicit** mode targets `Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5` +and its original-pickle import entrance, not direct directory import. Keep the +same original pickle beside `.msinsight/` for GUI discovery; +pt-snap itself can analyze a relocated complete artifact without pickle. Optional +`--events-per-slice N` defaults to `500000`; `--output-dir` selects another parent, +not an independently supported GUI entrance. `--device` selects an event-bearing +device and omissions are reported. Nonzero/sparse real IDs, OOM/unknown actions, +empty/static-only selections fail, without silent remapping. Native mode still +preserves sparse IDs/OOM/workspace. Trusted pickle and full-input memory costs apply. + +Base tables are physical inline-v1 tables; metadata has import format `1`. +Callstack text is copied once **per event**, not once per distinct stack, so disk +and temporary conversion space can grow substantially. Native interning remains +unchanged. No ordered structured-frame capability is invented from formatted text. +Only bounded `event` dataset queries are currently available. + +Compatible publication is **no-replace**: external/original/user caches are never +overwritten or deleted, even under `--force`. A complete, attested, identical +pt-snap cache is reused (also with force); otherwise choose a new `--output-dir`. +Source content, sourceFile, options, all member hashes and contract versions must +match independently of salted `cacheHash`. A GUI-derived cache modification can +invalidate this attestation, without implying producer rerun. All staged members +close/finalize/validate before exclusive publication and requested focus; ordinary +failure compensates exact old focus bytes, with recovery evidence if rollback fails. +Standalone/native force policies remain unchanged. `SnapshotAnalyzer` is analysis-only. + +**GUI acceptance is pending/not run.** Complete/ready, safe paths/device tables, +and derived allocation-cache availability/buildability also matter; matching hash +alone proves neither GUI reuse nor display parity. Details and fixed source evidence: +[compatible protocol](sharded-snapshotdb.md#explicit-msinsight-compatible-export-p2). + ### Optional: Split a Snapshot Use `pt-snap split` when you need smaller, independently replayable files. Split diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index c26ee75..d24c1f9 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -4,8 +4,8 @@ ## Scope and versions -This freezes an artifact contract and shared validation models, **not a new CLI -exporter or cross-slice query engine**. Existing `import` still defaults to one +This defines artifact contracts, shared validators and an **explicit** compatible +CLI exporter below, not a cross-slice query engine. Existing `import` defaults to one native v2 DB; `split` still produces replayable pickle/JSON. Existing single-DB v1/v2 single-DB analysis is unchanged. The P1 reader below additionally accepts a complete compatibility-v1 manifest or directory as focus. The `complete` gate @@ -21,7 +21,8 @@ Separate version axes: - `schemaVersion=1`: manifest and compatibility base-table contract defined here. - `pt_snap_metadata.import_format_version`: existing pt-snap importer format, - currently 2; **not** the manifest version and not required for external artifacts. + `2` for native and `1` for compatible inline export; **not** the manifest version + and not required for external artifacts. - Future structured-frame extensions have their own version. Unknown extensions do not invalidate base reading or imply structured-frame support. @@ -198,8 +199,9 @@ real/boundary counts, per-device slice paths/ranges and capabilities. Event boun exclude negative synthetic rows (also for standalone DB overview). Text stacks are available, but not necessarily populated; structured frames and cross-slice queries are **unavailable**, including when unknown extensions claim them. -Dataset-wide import metadata is unavailable; per-slice metadata is not a source -attestation for the entire dataset. +External dataset-wide import metadata is unavailable; per-slice metadata alone +is not a source attestation. The compatible exporter below supplies a recognized, +whole-artifact `ptSnap` attestation, making its validated metadata available. Path precedence stays explicit → `PT_SNAP_DB_PATH` → nearest project focus → legacy global config; device precedence stays explicit → focused → first discovered. @@ -238,21 +240,22 @@ lookup even when size/mtime are unchanged. Analyzer-owned caches close on exit; injected caches remain caller-owned. Validation opens/closes slices sequentially. Building, missing/inconsistent slices, unknown base versions and unsafe paths -are rejected before focus writes or query execution. Before opening **any** slice, -the resolver checks every member's canonical non-symlink path, SQLite header and -WAL/journal/SHM sidecars (including dangling symlinks). Persistent WAL mode is -rejected even after checkpointing has removed its sidecars: `mode=ro` can recreate -them. The reader does not repair, checkpoint, delete sidecars or change journal -mode; standalone DB behavior is unchanged. This is read-only inspection, not a -filesystem sandbox or a concurrent producer lock. Native-v2 replay's private staging directory has **no published manifest** -and is rejected as a dataset; its individual DBs remain readable via the existing -single-DB entry. A future compatibility-v1 producer can use this same reader -without producer-specific metadata. The P1 reader does not create a native manifest; publication requires the explicit -P2 native protocol below. Compatibility export and cross-slice lifecycle execution -remain separate work. In particular, the fixed original compatibility producer can -leave finalized, sidecar-free WAL headers, which this reader still rejects. Transport -support for those artifacts is a compatibility prerequisite, not passed upstream/GUI -interoperability; analysis never rewrites them. +are rejected before focus writes or query execution. Before **any** SQLite open, +both the bare P0 validator and resolver check **all** member/parent aliases and +live/dangling WAL, journal and SHM sidecars. Finalized, closed compatibility-v1 +members use `mode=ro&immutable=1`, including cached query Context connections. +This accepts the fixed original producer's checkpointed, sidecar-free WAL headers +without creating sidecars or changing bytes. The immutable flag assumes a closed +finalized artifact, **not a live-WAL shortcut** or concurrent-writer guarantee. +No repair, checkpoint, journal-mode change, migration or deletion occurs. +Native datasets retain persistent-WAL rejection; standalone Context defaults stay +`mode=ro` without immutable. Cache identity includes transport mode and generation. +This is read-only validation, not a filesystem sandbox or a producer lock. + +Native-v2 private staging has **no published manifest**, so it is not a dataset; +individual DBs remain readable via the single-DB entry. Published native format +and compatible export are separate explicit protocols below. Cross-slice lifecycle +execution remains separate work; safe original-WAL reading is not GUI acceptance. ## Published Native Dataset (P2) @@ -297,10 +300,11 @@ The native resolver validates all member hashes, actual schemas/dictionaries, real count/ranges, metadata and finalized references, retaining the P1 pre-open alias/sidecar/persistent-WAL checks and bounded borrowed-cache ownership. Focus, metadata, overview and single-slice `event` addressing work without the source -pickle. Native metadata is available dataset-wide; compatibility metadata remains -unavailable. No migration, repair, cross-slice aggregation/lifecycle execution, -msinsight exporter, salted hash policy or GUI acceptance is implied. The P0 -schema and reject policies above remain unchanged. +pickle. Native metadata is available dataset-wide; external compatibility metadata +remains unavailable unless the recognized compatible attestation below is present. +Native mode implies no migration, repair, cross-slice aggregation/lifecycle execution, +compatible export, salted hash or GUI acceptance. P0 schema/manifest semantics are +unchanged; the compatibility-only finalized-WAL transport is described above. Publication and force/focus compensation are described in the [import guide](quickstart.md#optional-import-a-native-sharded-dataset). A complete @@ -353,6 +357,72 @@ rows, total staged bytes, registry size, or peak RSS. Synthetic regressions and reviewed expandable/multi-device fixtures cover replay equivalence and resource failure paths; no live upstream GUI or performance acceptance is claimed. +## Explicit msinsight-Compatible Export (P2) + +`pt-snap import snapshot.pkl --format msinsight --json` (alias +`--format compatibility-v1`) targets **only** +`Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5`. Its supported entrance is +opening the **same original pickle** in that target with adjacent +`.msinsight/`, not direct standalone-directory GUI import. +`--output-dir` selects another parent for pt-snap; relocation alone does not create +an upstream GUI entrance. `sourceFile` is the absolute original source path. +Capacity defaults to `500000` and can be set with `--events-per-slice`; default +standalone import and explicit native-v2 output remain separate and unchanged. + +The first-party converter consumes closed/finalized native staging, materializes +exact positional inline-v1 **tables**, removes unprefixed native `callstack`, and +writes physical-format metadata `1`. Original real IDs, actions, stream/metrics, +stable device-local block identities, `free_completed` and out-of-slice lifecycle +text references are preserved. No foreign real events are inserted to hide local +JOIN gaps. Unsupported OOM/unknown actions, nonzero/sparse IDs and empty/static-only +selections fail; native retains its wider contract. All selected members finalize, +close and validate before complete publication. `pt_snap_block_reference` v1 is an +isolated text extension, not ordered frames or a claim of cross-source queries. +Those queries and global aggregation remain separate work. Text cannot recover +original ordered frame arrays; original workspace raw-frame loss and string-action +OOM coercion are known upstream limitations, not compatible features to imitate. + +`cacheHash = SHA256(b"mem_snapshot_parser_v2" + raw_pickle_bytes)`: the salt is +**prepended**. Independent `ptSnap.identity` fields are `sourceSha256`, +`requestedDevice`, `eventsPerSlice`, `outputFormat=compatibility-v1`, +`manifestContractVersion=1`, `exportContractVersion=1`, `referenceContractVersion=1`, +`metadataSchemaVersion=1`, `importFormatVersion=1`, and full `targetRevision`. +The `ptSnap` object also contains `metadata`, `omittedDevices`, and `members` +(canonical relative path → finalized SHA256). Whole validated ownership/identity +requires every member/hash and identical per-slice metadata; unknown/external +caches and unrelated files are not adopted. CLI version alone does not invalidate. + +Compatible publication is **no-replace even with force**. Identical recognized +pt-snap caches reuse without loading pickle (force does not bypass this rule). +Changed source path/content, options/contracts, corrupted members or GUI-modified +DBs require a new `--output-dir`; no repair, overwrite or deletion of the old +artifact occurs. Exclusive publication and focus compensation use the existing +checked transaction; source SHA256 is rechecked immediately before publish/reuse. +Native force replacement and recovery semantics are untouched. + +Inline text costs `sum(len(callstack.encode("utf-8")) for each event)` rather than +one copy per distinct stack. Existing native interning remains available. Conversion +temporarily holds both layouts and can leave free SQLite pages; capacity is not an +RSS/registry/block/disk bound. No optional frame enhancement is emitted. + +**GUI remains pending/not run.** Fixed-source parser checks complete manifest, +nonempty identical hash, every ready/resolved/existing member, read-only DB/device +validation and allocation-cache existence **or buildability** before UP_TO_DATE. +Derived allocation caches may modify DBs without invoking the producer: mtime or +new derived tables alone are not rerun evidence. Later GUI acceptance must retain +UP_TO_DATE/no-script-invocation logs and compare curves, blocks and cross-slice +presentation. Source checks and original-artifact production reads are not GUI, +C++ server/compile or full differential/performance acceptance. + +Immutable fixed blob evidence: +[parser predicates](https://api.github.com/repos/Ascend/msinsight/git/blobs/40046e6a92ff5af776ee55bd4cdd4586745af3c2), +[salt order](https://api.github.com/repos/Ascend/msinsight/git/blobs/ca421c7b33f3a0050df9e7a01e666f558801bc2f), +[allocation-cache predicates/build](https://api.github.com/repos/Ascend/msinsight/git/blobs/3d1ecc0ebd0d8e3eb0c984eda6b750ec4ce0e4f1). +The Ascend source is Mulan PSL v2, docs separately CC BY 4.0. This minimal converter +is original interface-facts code, not a copied vendor implementation or automatic +MIT relicensing of Ascend blobs. Existing pt-snap runtime lineage is a separate +license/provenance history. + ## Evidence and limits Interoperability facts were read from fixed upstream diff --git a/docs/zh/database.md b/docs/zh/database.md index 52e96d2..d2f0636 100644 --- a/docs/zh/database.md +++ b/docs/zh/database.md @@ -383,4 +383,22 @@ reader 在有界 `event` 寻址前校验完整成员哈希、最终化身份/引 数据集聚合/生命周期查询仍不支持。详见独立 [原生 manifest 协议](sharded-snapshotdb.md#已发布原生数据集p2)及 [事务恢复指南](quickstart.md#可选导入原生分库数据集)。 -P0 兼容 v1 表序、内联 callstack 布局和拒绝规则不变。 +P0 兼容 v1 表序、内联布局和 manifest 语义不变。 + +### 显式兼容导出 + +`import --format msinsight`(别名 `compatibility-v1`)生成物理 v1 +`trace_entry_` 表,第九列为 `callstack TEXT`,不是 VIEW;block/dictionary +位置 schema 精确匹配。转换去除暂存的未隔离 `callstack` 表,写入 +`pt_snap_metadata.import_format_version=1`,保留隔离的 `pt_snap_block_reference` +文本生命周期来源。原生单库/数据集仍使用去重和 v2 metadata,不生成或从文本推断有序 frames。 + +内联调用栈字节按**所有事件**的文本总和增长,不按不同栈计数;转换暂时并存原生与内联表, +可能保留 SQLite free pages。容量不限制 block、加载 RSS 或总磁盘。兼容 manifest 的 +`ptSnap.identity`、metadata、省略设备和全部成员哈希形成整体证明,完整校验后才报告 +metadata available;外部 v1 数据集仍不要求 metadata。 + +**仅最终化兼容数据集**的裸校验与缓存分析在首个 SQLite 打开前拒绝全部存活/悬空 +sidecar 和别名后使用 `mode=ro&immutable=1`,保留已关闭/checkpoint 原版 WAL 的字节及 +目录清单。前提是无活动 writer;原生数据集非 WAL 策略与单库 `mode=ro` 默认不变。详见 +[兼容协议及 no-replace 规则](sharded-snapshotdb.md#显式-msinsight-兼容导出p2)。 diff --git a/docs/zh/focus-management.md b/docs/zh/focus-management.md index d56324b..1fcf5e3 100644 --- a/docs/zh/focus-management.md +++ b/docs/zh/focus-management.md @@ -132,6 +132,14 @@ pt-snap query --template-use event --slice 0 --json 不存在,不是有效 focus JSON。请保留证据供人工恢复;不承诺任意 I/O/崩溃原子性或并发 writer 锁。详见[原生导入](quickstart.md#可选导入原生分库数据集)。 +显式兼容导入(`--format msinsight`)采用相同的请求 focus 补偿,但即使 force 也 +**不替换**已有兼容输出。仅整体证明完整、身份相同的 pt-snap 缓存可复用,否则换新输出。 +GUI 仍要求同一原始 pickle 与邻接缓存,pt-snap focus 只需完整产物。首个 SQLite 打开 +之前检查全部兼容成员的别名及存活/悬空 sidecar;裸校验和缓存查询对已关闭最终化成员 +使用 `mode=ro&immutable=1`,保留原版 checkpoint WAL 的字节与目录清单。这不是 live +writer 锁,原生非 WAL 检查与单库默认不变。 + + ## 错误处理 **未设置焦点时查询:** diff --git a/docs/zh/quickstart.md b/docs/zh/quickstart.md index 788be60..2bc7dd5 100644 --- a/docs/zh/quickstart.md +++ b/docs/zh/quickstart.md @@ -55,8 +55,8 @@ pt-snap query --template-use event --slice 0 --json 不指定 `--events-per-slice` 时仍生成已有单 DB;指定后生成 `/<完整输入文件名>.pt-snap-native-v2/`,省略 `--output-dir` 则放在源文件旁。 此模式可显式指定 `--format pt-snap-native-v2`;`--format single-db` 不允许容量参数。 -`--format compatibility-v1` 和 `msinsight` 是保留边界并明确拒绝:原生产物**不是** -msinsight 兼容导出。 +原生产物**不是** msinsight 兼容导出;显式 `--format msinsight`(别名 +`compatibility-v1`)选择下述独立兼容模式。 容量按**各设备**的真实时间顺序位置计数,不使用 `max(id)+1`,不计负 ID 合成边界。 保留原始稀疏/非零 ID、OOM 和 workspace 事件。`--device` 选择一个有事件设备,默认 @@ -84,6 +84,37 @@ msinsight 兼容导出。 详见[原生 manifest 合同](sharded-snapshotdb.md#已发布原生数据集p2)。 `SnapshotAnalyzer` 仍只负责分析,不新增 import/split 方法。 +### 可选:导出 msinsight 兼容数据集 + +```bash +pt-snap import snapshot.pkl --format msinsight --json +pt-snap overview snapshot.pkl.msinsight --json +pt-snap query --template-use event --slice 0 --json +``` + +此**显式**模式仅对齐 `Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5` +的原始 pickle 导入入口,不是独立目录入口。GUI 发现要求同一原始 pickle 邻接 +`<完整输入文件名>.msinsight/`;pt-snap 可在没有 pickle 时分析已搬迁完整产物。 +可选 `--events-per-slice N` 默认 `500000`;`--output-dir` 选择其他父目录,不意味着 +GUI 支持独立入口。`--device` 选择有事件设备,并披露省略设备。非零/稀疏真实 ID、 +OOM/未知 action、空/仅静态选择明确失败,不静默映射;原生模式仍保留稀疏 ID、OOM、 +workspace。可信 pickle 与完整加载的内存成本仍适用。 + +基础表是物理内联 v1 表,metadata import format 为 `1`。调用栈文本按**每个事件** +复制,不按不同栈只存一次,磁盘与暂存转换空间可能明显增加;原生去重不变。 +不从格式化文本虚构有序结构化 frames。数据集当前只支持有界 `event` 查询。 + +兼容发布采用 **no-replace**:外部/原版/用户缓存即使 `--force` 也不覆盖、不删除。 +仅完整、已证明且身份相同的 pt-snap 缓存可复用(force 也复用);否则换新 +`--output-dir`。源内容、sourceFile、选项、全部成员哈希和合同版本须独立于 salted +`cacheHash` 匹配。GUI 修改派生缓存可能使该证明失效,但不等于重跑生产者。 +全部暂存成员关闭/最终化/校验后才排他发布并写请求的 focus;普通失败补偿旧 focus +原始字节,回滚失败则保留恢复证据。单库/原生 force 策略不变,`SnapshotAnalyzer` 只分析。 + +**GUI 待验收/未运行。** complete/ready、安全路径/设备表及派生 allocation cache +存在或可构建同样重要;hash 相同不证明 GUI 复用或显示一致。详见 +[兼容协议及固定源码证据](sharded-snapshotdb.md#显式-msinsight-兼容导出p2)。 + ### 可选:拆分快照 如需生成更小、可独立回放的文件,可使用 `pt-snap split`。该命令不会读取或修改 focus: diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index e8cae06..b08827b 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -4,7 +4,7 @@ ## 范围与版本 -本阶段冻结产物协议和共享校验模型,**不新增 CLI 导出器或跨片查询引擎**。现有 +本文定义产物协议、共享校验模型及下述**显式**兼容 CLI 导出器,不实现跨片查询引擎。 `import` 默认仍生成一个原生 v2 DB;`split` 仍输出可回放 pickle/JSON。现有单库 v1/v2 单库分析不变。下述 P1 reader 新增将 complete 兼容 v1 manifest 或目录作为 focus;`complete` 门槛只约束数据集,不约束旧单库分析。 @@ -17,7 +17,7 @@ focus;`complete` 门槛只约束数据集,不约束旧单库分析。 版本轴分别管理: - `schemaVersion=1`:本文的 manifest 和兼容基础表协议。 -- `pt_snap_metadata.import_format_version`:已有 pt-snap 导入格式,当前为 2; +- `pt_snap_metadata.import_format_version`:pt-snap 导入格式,原生为 `2`,兼容内联为 `1`; **不是** manifest 版本,外部产物不要求该表。 - 后续结构化 frame 扩展独立版本化。未知扩展不影响基础读取,也不意味着支持结构化 frames。 @@ -165,7 +165,8 @@ pt-snap query --template-use event --slice 1 --json `overview.dataset` 输出格式、manifest 版本/状态、内容指纹、真实/边界事件数、逐设备 分片路径/范围及能力。事件范围排除负 ID 合成行(单库 overview 也如此)。支持文本调用栈, 不保证每行有栈;结构化 frames 与跨片查询**不可用**,未知扩展的能力声明不会提升它们。 -数据集级导入 metadata 为 unavailable;逐片 metadata 不是整份数据集的来源证明。 +外部数据集级导入 metadata 为 unavailable;逐片 metadata 不足以证明整体来源。 +下述兼容导出器提供可识别的整份 `ptSnap` 证明,其完整校验后的 metadata 为 available。 路径优先级仍为显式 → `PT_SNAP_DB_PATH` → 最近项目 focus → legacy global config; 设备优先级仍为显式 → focused → 首个发现设备。CLI focus 只写选定的项目/全局 focus 文件, @@ -195,16 +196,19 @@ selector。输出 `scope` 标识实际 DB/device/slice、受限真实区间、 最多四个;数据集 generation 变化在下次 lookup 使旧 Context 失效,即使 size/mtime 相同。 analyzer 自有缓存在退出时关闭,注入缓存仍由调用者所有;校验逐片打开并关闭连接。 -building、缺片/矛盾片、未知基础版本、不安全路径在 focus 写入或查询前拒绝。在打开 -**任何**分片前,resolver 先检查全部成员的规范非符号链接路径、SQLite header 及 -WAL/journal/SHM sidecar(包含悬空符号链接)。即使 checkpoint 已移除 sidecar,持久 WAL -模式仍被拒绝:`mode=ro` 也可能重新创建它们。reader 不修复、不执行 checkpoint、不删除 -sidecar、不改变 journal mode;单库行为不变。这是只读检查,不是文件系统沙箱或并发生产端锁。native-v2 replay 的私有 -staging **没有已发布 manifest**,不能作为数据集;其中单个 DB 仍由已有单库入口读取。 -后续兼容 v1 生产端可直接使用同一 reader,不依赖生产端专属 metadata。P1 reader 不创建原生 manifest;发布须采用下述显式 P2 原生协议。兼容导出与跨片 -生命周期执行仍属独立工作。特别是固定版本原始兼容生产端可能留下最终化、无 sidecar -的 WAL header,本 reader 仍拒绝;支持该类产物的只读传输是兼容前提,不代表上游/GUI -互通已验收,分析不会改写它们。 +building、缺片/矛盾片、未知基础版本、不安全路径在 focus 写入或查询前拒绝。 +**任何** SQLite 打开之前,裸 P0 validator 与 resolver 均先检查**全部**成员/父目录别名及 +存活/悬空 WAL、journal、SHM sidecar。最终化、已关闭的兼容 v1 成员使用 +`mode=ro&immutable=1`,缓存查询的 Context 连接也一样;接受固定原始生产端已 checkpoint、 +无 sidecar 的 WAL header,不创建 sidecar、不改变字节。immutable 只适用于已关闭最终化 +产物,**不是读取 live WAL 的捷径**,不保证并发 writer 安全。不修复、不 checkpoint、 +不改变 journal mode、不迁移、不删除。原生数据集仍拒绝持久 WAL,单库 Context 默认仍 +为不带 immutable 的 `mode=ro`。Cache 身份包含传输模式与 generation。这是只读校验, +不是文件系统沙箱或生产端锁。 + +native-v2 私有 staging **没有已发布 manifest**,不能作为数据集;单个 DB 仍由单库 +入口读取。已发布原生格式与兼容导出是下述独立显式协议。跨片生命周期执行仍属独立工作, +安全读取原版 WAL 不等于 GUI 验收。 ## 已发布原生数据集(P2) @@ -242,8 +246,9 @@ trace 列与单库 v2 一致(`callstackId`,非内联文本),保留 OOM a 原生 resolver 校验全部成员哈希、实际 schema/dictionary、真实计数/范围、metadata 和 最终化引用,保留 P1 打开前的别名/sidecar/持久 WAL 检查及有界借用 cache 所有权。 无需源 pickle 即可 focus、metadata、overview 和单片 `event` 寻址。原生数据集 metadata -为 available,兼容数据集仍为 unavailable。不隐含迁移、修复、跨片聚合/生命周期执行、 -msinsight 导出器、salted hash 策略或 GUI 验收;以上 P0 schema 和拒绝规则保持不变。 +为 available,外部兼容数据集为 unavailable(下述可识别兼容证明除外)。原生模式不隐含 +迁移、修复、跨片聚合/生命周期执行、兼容导出、salted hash 或 GUI 验收;P0 schema/manifest +语义不变,仅兼容最终化 WAL 的只读传输修正见上文。 发布、force/focus 补偿见[导入指南](quickstart.md#可选导入原生分库数据集)。完整 manifest 只能与全部最终化成员同时可见,单个 ready 片不构成完整产物。 @@ -284,6 +289,57 @@ pickle 仍可能整体加载。分库只限制**单库真实事件数**,不限 字节、registry 大小或峰值 RSS。合成回归及已审 expandable/多设备 fixture 覆盖回放 等价和资源失败路径;不宣称真实上游 GUI 或性能验收。 +## 显式 msinsight 兼容导出(P2) + +`pt-snap import snapshot.pkl --format msinsight --json`(别名 +`--format compatibility-v1`)**仅**对齐 +`Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5`。支持入口是在该目标版本 +打开**同一原始 pickle**及邻接 `<原始路径>.msinsight/`,不是 GUI 直接导入独立目录。 +`--output-dir` 为 pt-snap 选择其他父目录,搬迁本身不创建上游入口。`sourceFile` 记录 +绝对原始源路径。容量默认 `500000`,可用 `--events-per-slice` 调整;默认单库及显式 +native-v2 输出仍独立且不变。 + +第一方转换器消费已关闭/最终化的原生暂存,物化精确位置内联 v1 **实际表**,去除 +未隔离原生 `callstack`,写入物理格式 metadata `1`。保留原始真实 ID、action、stream/ +指标、设备内稳定 block 身份、`free_completed` 和片外生命周期文本来源引用,不插入 +外片真实事件掩盖片内 JOIN 缺口。OOM/未知 action、非零/稀疏 ID、空/仅静态选择失败; +原生保留更广合同。所有选中成员最终化、关闭、校验后才发布 complete。 +`pt_snap_block_reference` v1 是隔离文本扩展,不是有序 frames 或跨来源查询承诺; +跨来源查询/全局聚合仍属后续独立工作。文本不能恢复原始有序 frame 数组;原版 workspace +raw-frame 丢失与字符串 OOM action 强制转换属于已知局限,不应仿制为兼容能力。 + +`cacheHash = SHA256(b"mem_snapshot_parser_v2" + 原始_pickle_字节)`,salt **前置**。 +独立 `ptSnap.identity` 字段为 `sourceSha256`、`requestedDevice`、`eventsPerSlice`、 +`outputFormat=compatibility-v1`、`manifestContractVersion=1`、`exportContractVersion=1`、 +`referenceContractVersion=1`、`metadataSchemaVersion=1`、`importFormatVersion=1`、完整 +`targetRevision`。`ptSnap` 另含 `metadata`、`omittedDevices` 和 `members`(规范相对路径 +→ 最终化 SHA256)。整体已校验所有权/身份要求每个成员/哈希及逐片 metadata 相同; +不接管未知/外部缓存或无关文件。CLI 版本变化本身不使缓存失效。 + +兼容发布 **force 也 no-replace**。已识别、身份相同的 pt-snap 缓存不加载 pickle 即复用, +force 不绕过该规则。源路径/内容、选项/合同变化、成员损坏或 GUI 修改 DB 后须换新 +`--output-dir`,不修复、不覆盖、不删除旧产物。排他发布与 focus 补偿使用已有受检事务, +发布/复用前再次核对源 SHA256;原生 force 替换与恢复语义不变。 + +内联文本成本按每个事件的 UTF-8 调用栈字节求和,不按不同栈各存一份;原生去重仍可用。 +转换暂时并存两种布局,可能留下 SQLite free pages;容量不是 RSS/registry/block/磁盘 +上限。不生成可选 frame 增强。 + +**GUI 待验收/未运行。** 固定源码 parser 在 UP_TO_DATE 前检查 complete manifest、 +非空相同 hash、每个 ready/可解析/存在成员、只读 DB/设备校验,以及 allocation cache +存在**或可构建**。派生 allocation cache 可在不运行生产者时修改 DB,mtime 或新派生表 +不能单独证明重跑。后续 GUI 验收须保留 UP_TO_DATE/无 script 调用日志并对照曲线、block、 +跨片展示。源码检查及原版产物的生产 reader 读取不是 GUI、C++ server/编译或完整差分/ +性能验收。 + +不可变固定 blob 证据: +[parser 判据](https://api.github.com/repos/Ascend/msinsight/git/blobs/40046e6a92ff5af776ee55bd4cdd4586745af3c2)、 +[salt 顺序](https://api.github.com/repos/Ascend/msinsight/git/blobs/ca421c7b33f3a0050df9e7a01e666f558801bc2f)、 +[allocation cache 判据/构建](https://api.github.com/repos/Ascend/msinsight/git/blobs/3d1ecc0ebd0d8e3eb0c984eda6b750ec4ce0e4f1)。 +Ascend 源码为 Mulan PSL v2,docs 单独为 CC BY 4.0;此最小转换器是本地原创接口事实实现, +未复制 vendor 实现,也不把 Ascend blob 自动重新许可为 MIT。已有 pt-snap 运行时 lineage +属于独立许可证/来源历史。 + ## 来源证据与限制 接口事实已从固定版本真实源码读取: diff --git a/src/pt_snap_cli/cli.py b/src/pt_snap_cli/cli.py index 648533c..2d23442 100644 --- a/src/pt_snap_cli/cli.py +++ b/src/pt_snap_cli/cli.py @@ -389,16 +389,22 @@ def import_snapshot( output_dir: Annotated[Path | None, typer.Option("--output-dir", "-o")] = None, device: Annotated[int | None, typer.Option("--device", "-d")] = None, no_focus: Annotated[bool, typer.Option("--no-focus", help="Skip focus update")] = False, - force: Annotated[bool, typer.Option("--force", help="Rebuild even when cache matches")] = False, + force: Annotated[ + bool, + typer.Option("--force", help="Rebuild native/single DB; never replace compatible output"), + ] = False, events_per_slice: Annotated[ int | None, typer.Option( - "--events-per-slice", help="Real events per device shard; opt into a native dataset" + "--events-per-slice", + help="Real events per device shard; native unless --format msinsight", ), ] = None, output_format: Annotated[ str | None, - typer.Option("--format", help="single-db or pt-snap-native-v2 (compatibility-v1 reserved)"), + typer.Option( + "--format", help="single-db, pt-snap-native-v2, or msinsight (compatibility-v1)" + ), ] = None, json_output: Annotated[bool, _json_flag()] = False, ) -> None: diff --git a/src/pt_snap_cli/context.py b/src/pt_snap_cli/context.py index d11cb0a..140eca4 100644 --- a/src/pt_snap_cli/context.py +++ b/src/pt_snap_cli/context.py @@ -44,12 +44,17 @@ def __init__( devices: list[int] | None = None, *, persistent: bool = False, + immutable: bool = False, ): """Initialize context with database path. Args: db_path: Path to the SQLite database file. devices: Optional list of device IDs to filter. + immutable: Internal opt-in for a closed, finalized compatibility + dataset after ALL member/parent/sidecar checks. Never use this + to read a live WAL or as a concurrency/sandbox guarantee. The + standalone default remains mode=ro without immutable. persistent: When True, the connection stays open after each ``connect()`` call so it can be reused. The caller is responsible for invoking ``close()`` to release it. @@ -61,6 +66,9 @@ def __init__( self.db_path = Path(db_path) self._devices = devices self._persistent = persistent + # Opt-in only after dataset-wide finalized-member checks. Standalone + # analysis keeps mode=ro without immutable and its existing WAL behavior. + self._immutable = immutable self._conn: sqlite3.Connection | None = None self._connect_depth = 0 self._device_ids: list[int] | None = None @@ -260,6 +268,8 @@ def connect(self) -> Generator[sqlite3.Connection, None, None]: # Keep the colon in Windows drive prefixes while encoding SQLite URI metacharacters. encoded_path = quote(self.db_path.as_posix(), safe="/:") uri = f"file:{encoded_path}?mode=ro" + if self._immutable: + uri += "&immutable=1" self._conn = sqlite3.connect(uri, uri=True) self._conn.row_factory = sqlite3.Row diff --git a/src/pt_snap_cli/core/context_cache.py b/src/pt_snap_cli/core/context_cache.py index cf22e0d..c345707 100644 --- a/src/pt_snap_cli/core/context_cache.py +++ b/src/pt_snap_cli/core/context_cache.py @@ -39,7 +39,7 @@ def __init__(self, maxsize: int = 4) -> None: raise ValueError("maxsize must be positive") self._maxsize = maxsize # Insertion order doubles as LRU recency (oldest first). - self._entries: OrderedDict[Path, tuple[Context, tuple[FileSignature, str | None]]] = ( + self._entries: OrderedDict[Path, tuple[Context, tuple[FileSignature, str | None, bool]]] = ( OrderedDict() ) @@ -50,7 +50,9 @@ def maxsize(self) -> int: def __len__(self) -> int: return len(self._entries) - def get(self, db_path: Path | str, *, generation: str | None = None) -> Context: + def get( + self, db_path: Path | str, *, generation: str | None = None, immutable: bool = False + ) -> Context: """Return a cached or freshly opened :class:`Context` for ``db_path``. On a hit the cached entry is promoted to most-recently-used. On a @@ -69,7 +71,7 @@ def get(self, db_path: Path | str, *, generation: str | None = None) -> Context: # consistently whether the cache is hit or missed. # A dataset reader can supply its validated manifest/member content identity. # Single-DB callers retain the original filesystem-signature behavior. - current_signature = (self._file_signature(key), generation) + current_signature = (self._file_signature(key), generation, immutable) cached = self._entries.get(key) if cached is not None: @@ -80,7 +82,11 @@ def get(self, db_path: Path | str, *, generation: str | None = None) -> Context: self._safe_close(ctx) del self._entries[key] - ctx = Context(key, persistent=True) + ctx = ( + Context(key, persistent=True, immutable=True) + if immutable + else Context(key, persistent=True) + ) self._entries[key] = (ctx, current_signature) self._evict_if_needed() return ctx diff --git a/src/pt_snap_cli/core/dataset_contract.py b/src/pt_snap_cli/core/dataset_contract.py index d863c1e..9d8be4c 100644 --- a/src/pt_snap_cli/core/dataset_contract.py +++ b/src/pt_snap_cli/core/dataset_contract.py @@ -16,6 +16,9 @@ from pathlib import Path from typing import Literal, cast +from .dataset_files import require_readonly_member +from .errors import DatabaseSchemaError + MANIFEST_SCHEMA_VERSION = 1 MSINSIGHT_REVISION = "101f65b877a267ffd5f66ea3834706057ba243e5" ACTION_NAMES = ( @@ -470,13 +473,24 @@ def validate_dataset(directory: Path | str) -> DatasetValidation: with path.open(encoding="utf-8") as source: data: object = json.load(source, object_pairs_hook=_unique_object) manifest = parse_manifest(data) + # Preflight ALL members before the first SQLite open, also for callers + # using this bare P0 validator rather than DatasetResolver. + for device in manifest.devices: + for item in device.slices: + member = _safe_file(root, item.file) + try: + require_readonly_member(root, member, immutable=True) + except DatabaseSchemaError as exc: + raise DatasetContractError(item.file, "transport", str(exc)) from exc identities: dict[tuple[int, int], tuple[object, ...]] = {} boundaries = 0 for device in manifest.devices: for item in device.slices: db_path = _safe_file(root, item.file) try: - with closing(sqlite3.connect(db_path.as_uri() + "?mode=ro", uri=True)) as conn: + with closing( + sqlite3.connect(db_path.as_uri() + "?mode=ro&immutable=1", uri=True) + ) as conn: conn.execute("PRAGMA query_only=ON") boundaries += _validate_slice(conn, device, item, identities) except sqlite3.DatabaseError as exc: diff --git a/src/pt_snap_cli/core/dataset_files.py b/src/pt_snap_cli/core/dataset_files.py index 55e7153..e32a6d6 100644 --- a/src/pt_snap_cli/core/dataset_files.py +++ b/src/pt_snap_cli/core/dataset_files.py @@ -8,7 +8,7 @@ from .errors import DatabaseSchemaError -def require_readonly_member(root: Path, member: Path) -> None: +def require_readonly_member(root: Path, member: Path, *, immutable: bool = False) -> None: parts = member.relative_to(root).parts for index in range(1, len(parts) + 1): if root.joinpath(*parts[:index]).is_symlink(): @@ -21,9 +21,9 @@ def require_readonly_member(root: Path, member: Path) -> None: raise DatabaseSchemaError(f"Dataset member has a live SQLite sidecar: {member}") with member.open("rb") as source: header = source.read(20) - # Retain #200's persistent-WAL rejection, including sidecar-free checkpointed - # WAL. Accepting fixed-upstream compatibility producers is a separate boundary. - if header[:16] == b"SQLite format 3\x00" and 2 in header[18:20]: + # Only finalized compatibility datasets use immutable transport. Native + # datasets retain their non-WAL contract; no writer repair/checkpoint occurs. + if not immutable and header[:16] == b"SQLite format 3\x00" and 2 in header[18:20]: raise DatabaseSchemaError(f"Dataset member uses persistent WAL mode: {member}") diff --git a/src/pt_snap_cli/core/dataset_resolver.py b/src/pt_snap_cli/core/dataset_resolver.py index a1d67ed..541e529 100644 --- a/src/pt_snap_cli/core/dataset_resolver.py +++ b/src/pt_snap_cli/core/dataset_resolver.py @@ -142,7 +142,9 @@ def inspect(self, path: Path | str) -> ResolvedDataset | None: # SQLite open, without following aliases or repairing the artifact. for device in planned.devices: for item in device.slices: - _require_readonly_member(root, root / item.file) + _require_readonly_member( + root, root / item.file, immutable=not isinstance(planned, NativeManifest) + ) validation = ( validate_native_dataset(root, planned) if isinstance(planned, NativeManifest) @@ -153,7 +155,9 @@ def inspect(self, path: Path | str) -> ResolvedDataset | None: for device in validation.manifest.devices: for item in device.slices: member = root / item.file - _require_readonly_member(root, member) + _require_readonly_member( + root, member, immutable=not isinstance(planned, NativeManifest) + ) digest.update(item.file.encode("utf-8")) digest.update(_hash_file(member).encode("ascii")) if before != _hash_file(manifest_path): diff --git a/src/pt_snap_cli/core/import_metadata.py b/src/pt_snap_cli/core/import_metadata.py index 07da314..725c4f0 100644 --- a/src/pt_snap_cli/core/import_metadata.py +++ b/src/pt_snap_cli/core/import_metadata.py @@ -1,6 +1,7 @@ from __future__ import annotations import hashlib +import json import sqlite3 from contextlib import closing from dataclasses import asdict @@ -21,6 +22,7 @@ ImportMetadata, MetadataInspection, ) +from pt_snap_cli.core.msinsight_export import inspect_owned_cache from pt_snap_cli.core.native_dataset_contract import NativeManifest METADATA_TABLE = "pt_snap_metadata" @@ -85,7 +87,16 @@ def inspect(self, db_path: Path | str) -> MetadataInspection: if dataset is not None: if isinstance(dataset.validation.manifest, NativeManifest): return MetadataInspection(path, "available", dataset.validation.manifest.metadata) - # Per-slice extensions are not a dataset-wide importer/source attestation. + # External P0 metadata remains optional. Only a recognized complete + # pt-snap manifest attestation can provide dataset-wide metadata. + with (dataset.root / "manifest.json").open(encoding="utf-8") as source: + raw = json.load(source) + if "ptSnap" in raw: + try: + owned = inspect_owned_cache(dataset.root) + except (ValueError, KeyError, TypeError, OSError, sqlite3.DatabaseError): + return MetadataInspection(path, "invalid", reason="metadata_invalid") + return MetadataInspection(path, "available", owned.metadata) return MetadataInspection(path, "unavailable", reason="metadata_missing") try: context = Context(path) diff --git a/src/pt_snap_cli/core/import_service.py b/src/pt_snap_cli/core/import_service.py index 5996794..7e8428f 100644 --- a/src/pt_snap_cli/core/import_service.py +++ b/src/pt_snap_cli/core/import_service.py @@ -2,7 +2,9 @@ import json import logging +import sqlite3 from contextlib import nullcontext +from dataclasses import replace from pathlib import Path from pt_snap_cli.core.dataset_import_backend import DatasetImportBackend @@ -26,6 +28,16 @@ ImportOptions, ImportResult, ) +from pt_snap_cli.core.msinsight_export import ( + COMPATIBILITY_FORMAT, + DEFAULT_CAPACITY, + MsinsightImportBackend, + build_compatible_manifest, + compatible_identity, + convert_shards, + inspect_owned_cache, + salted_hash, +) from pt_snap_cli.core.native_dataset_contract import ( NATIVE_FORMAT, build_native_manifest, @@ -53,10 +65,13 @@ def __init__( self._backend: SnapshotImportBackend = backend or SnapshotImportBackend() self._metadata_service: ImportMetadataService = metadata_service or ImportMetadataService() self._dataset_backend = dataset_backend or DatasetImportBackend() + self._msinsight_backend = MsinsightImportBackend() def import_snapshot(self, options: ImportOptions) -> ImportResult: self._validate_options(options) self._validate_snapshot_file(options.snapshot_file) + if options.format in (COMPATIBILITY_FORMAT, "msinsight"): + return self._import_compatible_dataset(options) if options.events_per_slice is not None: return self._import_dataset(options) output_dir = self._resolve_output_dir(options.snapshot_file, options.output_dir) @@ -151,10 +166,8 @@ def _validate_options(options: ImportOptions) -> None: if options.format is not None else (NATIVE_FORMAT if capacity is not None else "single-db") ) - if selected_format in ("compatibility-v1", "msinsight"): - raise InvalidParameterError( - "Compatibility/msinsight export is not available; use pt-snap-native-v2. No compatible output is generated." - ) + if selected_format in (COMPATIBILITY_FORMAT, "msinsight"): + return if selected_format not in ("single-db", NATIVE_FORMAT): raise InvalidParameterError(f"Unsupported import format: {selected_format}") if (capacity is not None) != (selected_format == NATIVE_FORMAT): @@ -263,6 +276,112 @@ def focus(path: Path) -> None: generated.omitted_devices, ) + def _import_compatible_dataset(self, options: ImportOptions) -> ImportResult: + capacity = options.events_per_slice or DEFAULT_CAPACITY + output = ( + (options.output_dir if options.output_dir is not None else options.snapshot_file.parent) + .expanduser() + .absolute() + ) + target = self._msinsight_backend.target_path(options.snapshot_file, output) + if target.resolve() != target or target.is_symlink(): + raise ImportExecutionError("Compatible output must be canonical and non-symlink.") + source_hash = self._hash_source(options.snapshot_file) + try: + cache_hash = salted_hash(options.snapshot_file) + except OSError as exc: + raise ImportExecutionError(f"Cannot calculate msinsight cacheHash: {exc}") from exc + if target.exists(): + try: + owned = inspect_owned_cache(target) + if ( + owned.identity != compatible_identity(source_hash, options.device, capacity) + or owned.manifest.cache_hash != cache_hash + or owned.manifest.source_file != str(options.snapshot_file.resolve()) + or owned.metadata.source_size != options.snapshot_file.stat().st_size + ): + raise ValueError("Source/options/format identity changed") + except ( + ValueError, + KeyError, + TypeError, + OSError, + sqlite3.DatabaseError, + DatabaseSchemaError, + ) as exc: + raise ImportExecutionError( + f"Existing compatible output is not reusable; preserved even under --force. Choose a new output directory: {exc}" + ) from exc + self._recheck_source(options.snapshot_file, source_hash) + return ImportResult( + target, + options.device, + self._set_focus_if_requested(target, options), + True, + owned.metadata, + None, + target, + tuple(sorted(d.device_id for d in owned.manifest.devices)), + sum(len(d.slices) for d in owned.manifest.devices), + COMPATIBILITY_FORMAT, + owned.omitted_devices, + ) + completed: ImportMetadata | None = None + generated: ShardedReplayResult | None = None + + def finalize(result: ShardedReplayResult) -> None: + nonlocal completed, generated + convert_shards(result) + metadata = replace( + self._metadata_service.build_metadata( + options.snapshot_file, source_hash, options.device + ), + import_format_version=1, + ) + for item in result.slices: + _require_readonly_member(result.directory, item.file) + self._metadata_service.write(item.file, metadata) + manifest = build_compatible_manifest( + result, metadata, options.snapshot_file, capacity, cache_hash + ) + with (result.directory / "manifest.json").open("x", encoding="utf-8") as file: + json.dump(manifest, file, indent=2) + validated = DatasetResolver().inspect(result.directory) + if validated is None or inspect_owned_cache(result.directory).metadata != metadata: + raise ImportExecutionError("Staged compatible artifact validation failed") + completed, generated = metadata, result + + focus_state: FocusState | None = None + + def focus(path: Path) -> None: + nonlocal focus_state + focus_state = self._set_focus_if_requested(path, options) + + self._msinsight_backend.dump_to_dataset( + options.snapshot_file, + target, + capacity=capacity, + device=options.device, + expected=None, + finalize=finalize, + post_publish=focus if options.set_focus else None, + pre_publish=lambda: self._recheck_source(options.snapshot_file, source_hash), + ) + assert completed is not None and generated is not None + return ImportResult( + target, + options.device, + focus_state, + False, + completed, + "database_missing", + target, + tuple(sorted({s.device for s in generated.slices})), + len(generated.slices), + COMPATIBILITY_FORMAT, + generated.omitted_devices, + ) + def _hash_source(self, source: Path) -> str: try: return self._metadata_service.calculate_sha256(source) @@ -288,6 +407,7 @@ def _set_focus_if_requested( transaction = ( self._focus_service.project_focus_transaction() if options.events_per_slice is not None + or options.format in (COMPATIBILITY_FORMAT, "msinsight") else nullcontext() ) with transaction: diff --git a/src/pt_snap_cli/core/models.py b/src/pt_snap_cli/core/models.py index 620eaf3..71030c3 100644 --- a/src/pt_snap_cli/core/models.py +++ b/src/pt_snap_cli/core/models.py @@ -138,9 +138,10 @@ class ImportOptions: output_dir: Parent of the generated standalone DB or native dataset. Defaults to the snapshot file's parent directory. events_per_slice: Positive real-event capacity per device shard; None - preserves standalone behavior. Negative boundaries do not count. + preserves standalone behavior unless msinsight is explicit (500000). + Negative boundaries do not count. format: None selects single-db or pt-snap-native-v2 from capacity; - compatibility-v1/msinsight remain explicitly unavailable. + msinsight/compatibility-v1 explicitly selects fixed-target inline v1. device: Optional device id to focus on. When None, all available devices are imported. set_focus: When True (default), also write project focus so subsequent diff --git a/src/pt_snap_cli/core/msinsight_export.py b/src/pt_snap_cli/core/msinsight_export.py new file mode 100644 index 0000000..834efeb --- /dev/null +++ b/src/pt_snap_cli/core/msinsight_export.py @@ -0,0 +1,258 @@ +"""Original interface-facts converter for fixed msinsight 101f65b8 compatibility. + +No Ascend implementation is copied (its source is Mulan PSL v2). Physical +schema/hash facts: blobs 1fe50a5136f85365db55f4632d58f3cf7090431e, +95612509ef990ac6b7034919dae4938cc6acf3ad and +ca421c7b33f3a0050df9e7a01e666f558801bc2f. See sharded-snapshotdb docs. +Only caller-owned finalized native staging is converted; existing caches are +never mutated, even under force. This is not GUI acceptance. +""" + +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from contextlib import closing +from dataclasses import asdict, dataclass +from pathlib import Path + +from .dataset_contract import ( + ACTION_NAMES, + MANIFEST_SCHEMA_VERSION, + MSINSIGHT_REVISION, + TRACE_COLUMNS, + DatasetManifest, + _unique_object, + validate_dataset, + validate_event_ids, +) +from .dataset_files import hash_file, require_readonly_member +from .dataset_import_backend import DatasetImportBackend +from .errors import ImportExecutionError +from .import_metadata_contract import metadata_from_mapping +from .models import ImportMetadata +from .sharded_replay_service import ShardedReplayResult + +COMPATIBILITY_FORMAT = "compatibility-v1" +DEFAULT_CAPACITY = 500000 +SALT = b"mem_snapshot_parser_v2" +# Independent semantic versions, not the installed CLI version or GUI cacheHash. +EXPORT_VERSION = 1 +REFERENCE_VERSION = 1 + + +def salted_hash(source: Path) -> str: + digest = hashlib.sha256(SALT) + with source.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def compatible_identity(source_hash: str, device: int | None, capacity: int) -> dict[str, object]: + return { + "sourceSha256": source_hash, + "requestedDevice": device, + "eventsPerSlice": capacity, + "outputFormat": COMPATIBILITY_FORMAT, + "manifestContractVersion": MANIFEST_SCHEMA_VERSION, + "exportContractVersion": EXPORT_VERSION, + "referenceContractVersion": REFERENCE_VERSION, + "metadataSchemaVersion": 1, + "importFormatVersion": 1, + "targetRevision": MSINSIGHT_REVISION, + } + + +class MsinsightImportBackend(DatasetImportBackend): + """Use the proven exclusive/compensating publisher, never its replacement path.""" + + @staticmethod + def target_path(source: Path, output_dir: Path) -> Path: + return output_dir / f"{source.name}.msinsight" + + @staticmethod + def inspect_target(target: Path) -> None: + if target.exists() or target.is_symlink(): + raise ImportExecutionError( + "Compatible output already exists; preserved even under --force. Choose a new output directory." + ) + return None + + +def convert_shards(result: ShardedReplayResult) -> None: + """Preserve event/stream/lifecycle values, materialize inline ninth column. + + First validate all selected IDs/actions, without inventing event zero or + coercing unsupported OOM. Block/reference tables are left byte-semantically + intact; no foreign real events are copied into the slice window. + """ + for device in sorted({s.device for s in result.slices}): + ids: list[int] = [] + for item in (s for s in result.slices if s.device == device): + require_readonly_member(result.directory, item.file) + with closing(sqlite3.connect(item.file.as_uri() + "?mode=ro", uri=True)) as conn: + ids.extend( + row[0] + for row in conn.execute( + f"SELECT id FROM trace_entry_{device} WHERE id>=0 ORDER BY id" + ) + ) + if ( + conn.execute( + f"SELECT id FROM trace_entry_{device} WHERE typeof(action)!='integer' " + "OR action NOT BETWEEN 0 AND 7 OR action IS NULL LIMIT 1" + ).fetchone() + is not None + ): + raise ValueError("msinsight compatibility does not support OOM/unknown actions") + validate_event_ids(ids) + for item in result.slices: + device = item.device + trace = f"trace_entry_{device}" + with closing(sqlite3.connect(item.file.as_uri() + "?mode=rw", uri=True)) as conn, conn: + # A missing stack reference must fail, not silently erase an event. + if ( + conn.execute( + f"SELECT t.id FROM {trace} t LEFT JOIN callstack c ON c.id=t.callstackId " + "WHERE c.id IS NULL LIMIT 1" + ).fetchone() + is not None + ): + raise ValueError("Missing finalized callstack reference") + conn.execute(f"ALTER TABLE {trace} RENAME TO pt_snap_native_trace") + columns = ",".join( + f'"{name}" {kind}' + (" PRIMARY KEY" if name == "id" else "") + for name, kind in TRACE_COLUMNS + ) + conn.execute(f"CREATE TABLE {trace} ({columns})") + conn.execute( + f"INSERT INTO {trace} SELECT t.id,t.action,t.address,t.size,t.stream," + "t.allocated,t.active,t.reserved,c.callstack FROM pt_snap_native_trace t " + "JOIN callstack c ON c.id=t.callstackId" + ) + conn.execute("DROP TABLE pt_snap_native_trace") + conn.execute("DROP TABLE callstack") + conn.execute('DELETE FROM dictionary WHERE "table"=? AND "column"=?', (trace, "action")) + conn.executemany( + "INSERT INTO dictionary VALUES (?,?,?,?)", + [(trace, "action", str(key), name) for key, name in enumerate(ACTION_NAMES)], + ) + for metric in ("allocated", "active", "reserved"): + conn.execute(f"CREATE INDEX {trace}_{metric} ON {trace} ({metric})") + + +def build_compatible_manifest( + result: ShardedReplayResult, + metadata: ImportMetadata, + source: Path, + capacity: int, + cache_hash: str, +) -> dict[str, object]: + devices: dict[str, object] = {} + files: dict[str, str] = {} + for device in sorted({s.device for s in result.slices}): + slices = [s for s in result.slices if s.device == device] + devices[str(device)] = { + "eventCount": sum(s.end_position - s.start_position + 1 for s in slices), + "sliceCount": len(slices), + "readySlices": list(range(len(slices))), + "slices": [ + { + "index": s.index, + "startEventId": s.start_event_id, + "endEventId": s.end_event_id, + "file": s.file.relative_to(result.directory).as_posix(), + "ready": True, + } + for s in slices + ], + } + for item in slices: + files[item.file.relative_to(result.directory).as_posix()] = hash_file(item.file) + return { + "schemaVersion": MANIFEST_SCHEMA_VERSION, + "status": "complete", + "sourceFile": str(source.resolve()), + "cacheHash": cache_hash, + "eventsPerSlice": capacity, + "devices": devices, + "ptSnap": { + "identity": compatible_identity( + metadata.source_sha256, metadata.requested_device, capacity + ), + "metadata": asdict(metadata), + "omittedDevices": list(result.omitted_devices), + "members": files, + }, + } + + +@dataclass(frozen=True) +class CompatibleCache: + manifest: DatasetManifest + identity: dict[str, object] + metadata: ImportMetadata + omitted_devices: tuple[int, ...] + + +def inspect_owned_cache(target: Path) -> CompatibleCache: + """Whole valid recognized identity only; salted hash alone is never ownership. + + Reject unexpected files/aliases rather than adopt them. GUI may change + derived tables: that invalidates pt-snap's immutable cache attestation, and + requires a new output, never repair or force-replacement. + """ + validation = validate_dataset(target) + with (target / "manifest.json").open(encoding="utf-8") as file: + data = json.load(file, object_pairs_hook=_unique_object) + extension = data["ptSnap"] + metadata = metadata_from_mapping(extension["metadata"]) + identity = extension["identity"] + expected_identity = compatible_identity( + metadata.source_sha256, metadata.requested_device, validation.manifest.events_per_slice + ) + if identity != expected_identity or any( + type(identity[key]) is not type(value) for key, value in expected_identity.items() + ): + raise ValueError("Unsupported or inconsistent pt-snap compatibility identity") + if ( + metadata.importer_name != "pt-snap-cli" + or metadata.import_format_version != 1 + or metadata.metadata_schema_version != 1 + ): + raise ValueError("Metadata must match physical v1 layout") + paths = {s.file for d in validation.manifest.devices for s in d.slices} + if set(extension["members"]) != paths: + raise ValueError("Incomplete member attestation") + expected = {"manifest.json", *paths, *(str(Path(p).parent) for p in paths)} + actual = {p.relative_to(target).as_posix() for p in target.rglob("*")} + if actual != expected or any(p.is_symlink() for p in target.rglob("*")): + raise ValueError("Unrecognized files or aliases in compatible cache") + for relative in paths: + member = target / relative + if hash_file(member) != extension["members"][relative]: + raise ValueError("Compatible member hash changed") + with closing(sqlite3.connect(member.as_uri() + "?mode=ro&immutable=1", uri=True)) as conn: + conn.row_factory = sqlite3.Row + rows = conn.execute("SELECT * FROM pt_snap_metadata").fetchall() + if ( + len(rows) != 1 + or rows[0]["id"] != 1 + or metadata_from_mapping(dict(rows[0])) != metadata + ): + raise ValueError("Member metadata disagrees with identity") + omitted = extension["omittedDevices"] + devices = {d.device_id for d in validation.manifest.devices} + if ( + not isinstance(omitted, list) + or any(type(v) is not int or v < 0 or v in devices for v in omitted) + or len(set(omitted)) != len(omitted) + ): + raise ValueError("Invalid omitted device inventory") + if metadata.requested_device is not None and devices != {metadata.requested_device}: + raise ValueError("Device selection disagrees with identity") + if Path(validation.manifest.source_file).name != metadata.source_name: + raise ValueError("Source path disagrees with metadata") + return CompatibleCache(validation.manifest, identity, metadata, tuple(omitted)) diff --git a/src/pt_snap_cli/core/query_service.py b/src/pt_snap_cli/core/query_service.py index 2f8f2f5..cf03bb7 100644 --- a/src/pt_snap_cli/core/query_service.py +++ b/src/pt_snap_cli/core/query_service.py @@ -222,7 +222,9 @@ def execute_query( member, query_params, scope = self._dataset_query( dataset, selected.device_id, template, query_params, slice_index ) - ctx = self._validated_context(member, generation=dataset.fingerprint) + ctx = self._validated_context( + member, generation=dataset.fingerprint, immutable=dataset.callstack_layout == "v1" + ) target_device = selected.device_id else: if slice_index is not None: @@ -403,10 +405,12 @@ def _resolve_device_id( raise InvalidDeviceError("No devices found in database.") return ctx.device_ids[0] - def _validated_context(self, db_path: Path, *, generation: str | None = None) -> Context: + def _validated_context( + self, db_path: Path, *, generation: str | None = None, immutable: bool = False + ) -> Context: try: if generation is not None: - return self._context_cache.get(db_path, generation=generation) + return self._context_cache.get(db_path, generation=generation, immutable=immutable) return self._context_cache.get(db_path) except DatabaseNotFoundError as exc: raise DatabaseMissingError(str(exc)) from exc diff --git a/tests/core/test_dataset_import.py b/tests/core/test_dataset_import.py index 54c2970..005c5a4 100644 --- a/tests/core/test_dataset_import.py +++ b/tests/core/test_dataset_import.py @@ -399,8 +399,8 @@ def race(source, dest, identity): (True, None, None), (2, "single-db", None), (None, NATIVE_FORMAT, None), - (2, "compatibility-v1", None), - (2, "msinsight", None), + (2, "compatibility-v2", None), + (2, "inline", None), (2, "unknown", None), (2, None, -1), (2, None, True), diff --git a/tests/core/test_msinsight_export.py b/tests/core/test_msinsight_export.py new file mode 100644 index 0000000..4cd810c --- /dev/null +++ b/tests/core/test_msinsight_export.py @@ -0,0 +1,368 @@ +"""Fixed interface contracts; GUI remains deferred, no vendor code execution.""" + +import copy +import dataclasses +import hashlib +import json +import sqlite3 +from contextlib import closing +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.cli import _safe_call, app +from pt_snap_cli.core.dataset_contract import ( + ACTION_NAMES, + BLOCK_COLUMNS, + BLOCK_STATES, + DICTIONARY_COLUMNS, + MSINSIGHT_REVISION, + TRACE_COLUMNS, + validate_dataset, +) +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from pt_snap_cli.core.errors import ImportExecutionError, SourceChangedError +from pt_snap_cli.core.focus_service import FocusService +from pt_snap_cli.core.import_service import ImportService +from pt_snap_cli.core.models import ImportOptions +from tests.core.test_dataset_import import hashes +from tests.snapshot.test_sharded_replay import lifecycle_data, write_source + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + + +def options(tmp_path, *, multiple=False, capacity=2, focus=True): + data = lifecycle_data() + data["device_traces"][0] = [e for e in data["device_traces"][0] if e["action"] != "oom"] + if multiple: + data["device_traces"] += [copy.deepcopy(data["device_traces"][0]), []] + data["segments"] += [{**copy.deepcopy(s), "device": 1} for s in data["segments"]] + return ImportOptions( + write_source(tmp_path, data), + tmp_path / "output", + set_focus=focus, + events_per_slice=capacity, + format="msinsight", + ) + + +def rows(path, sql): + with closing(sqlite3.connect(path.as_uri() + "?mode=ro&immutable=1", uri=True)) as conn: + return conn.execute(sql).fetchall() + + +@pytest.mark.parametrize("capacity", [1, 2, 100, None]) +@pytest.mark.parametrize("fmt", ["msinsight", "compatibility-v1"]) +def test_physical_schema_inline_events_streams_stable_references_and_native_parity( + tmp_path, capacity, fmt +): + request = dataclasses.replace(options(tmp_path, multiple=True, capacity=capacity), format=fmt) + service = ImportService() + result = service.import_snapshot(request) + assert result.format == "compatibility-v1" and result.metadata.import_format_version == 1 + assert result.devices == (0, 1) and result.omitted_devices == (2,) + assert result.focus_state.callstack_layout == "v1" + manifest = json.loads((result.db_path / "manifest.json").read_text()) + assert manifest["sourceFile"] == str(request.snapshot_file.resolve()) + assert manifest["eventsPerSlice"] == (capacity or 500000) + raw = request.snapshot_file.read_bytes() + assert manifest["cacheHash"] == hashlib.sha256(b"mem_snapshot_parser_v2" + raw).hexdigest() + assert result.metadata.source_sha256 == hashlib.sha256(raw).hexdigest() != manifest["cacheHash"] + assert manifest["ptSnap"]["identity"]["targetRevision"] == MSINSIGHT_REVISION + native = service.import_snapshot( + dataclasses.replace( + request, format=None, events_per_slice=capacity or 500000, set_focus=False + ) + ) + compatible = DatasetResolver().inspect(result.db_path) + native_dataset = DatasetResolver().inspect(native.db_path) + assert validate_dataset(result.db_path).real_event_count == 10 + for device in (0, 1): + lifetimes = {} + for shard, native_shard in zip( + compatible.device(device).slices, native_dataset.device(device).slices, strict=True + ): + path = result.db_path / shard.file + for table, columns in ( + (f"trace_entry_{device}", TRACE_COLUMNS), + (f"block_{device}", BLOCK_COLUMNS), + ("dictionary", DICTIONARY_COLUMNS), + ): + schema = rows(path, f"PRAGMA table_info({table})") + assert tuple((r[1], r[2]) for r in schema) == columns + assert rows(path, f"SELECT type FROM sqlite_master WHERE name='{table}'") == [ + ("table",) + ] + assert not rows(path, "SELECT name FROM sqlite_master WHERE lower(name)='callstack'") + assert rows(path, "SELECT import_format_version FROM pt_snap_metadata") == [(1,)] + actual = rows(path, f"SELECT * FROM trace_entry_{device} ORDER BY id") + expected = rows( + native.db_path / native_shard.file, + f"SELECT t.id,t.action,t.address,t.size,t.stream,t.allocated,t.active,t.reserved,c.callstack FROM trace_entry_{device} t JOIN callstack c ON c.id=t.callstackId ORDER BY t.id", + ) + assert actual == expected + assert [r[0] for r in actual if r[0] >= 0] == list( + range(shard.start_event_id, shard.end_event_id + 1) + ) + assert set( + rows(path, 'SELECT "key","value" FROM dictionary WHERE "column"=\'action\'') + ) == {(str(k), v) for k, v in enumerate(ACTION_NAMES)} + assert set( + rows(path, 'SELECT "key","value" FROM dictionary WHERE "column"=\'state\'') + ) == {(str(k), v) for k, v in BLOCK_STATES} + references = {r[0]: r[1:] for r in rows(path, "SELECT * FROM pt_snap_block_reference")} + for block in rows(path, f"SELECT * FROM block_{device}"): + signature = block[1:4] + block[5:] + assert block[0] not in lifetimes or lifetimes[block[0]] == signature + lifetimes[block[0]] = signature + if block[6] >= 0: + assert block[6] == 3 # free_completed, NOT free_requested=2 + assert references[block[0]][2] == "source.py:4 free_completed" + before = hashes(result.db_path) + with ( + SnapshotAnalyzer(result.db_path) as analyzer, + SnapshotAnalyzer(native.db_path) as native_analyzer, + ): + assert analyzer.get_focus().callstack_layout == "v1" + for device in (0, 1): + for event in range(5): + assert ( + analyzer.execute_query("event", {"id": event}, device_id=device)["rows"] + == native_analyzer.execute_query("event", {"id": event}, device_id=device)[ + "rows" + ] + ) + assert not hasattr(analyzer, "import_snapshot") + assert hashes(result.db_path) == before + + +def test_identical_complete_cache_reuses_without_loading_even_force(tmp_path, monkeypatch): + request = options(tmp_path, focus=False) + service = ImportService() + first = service.import_snapshot(request) + before = hashes(first.db_path) + monkeypatch.setattr( + service._msinsight_backend.replay, + "stage", + lambda *a, **k: pytest.fail("reused cache loaded pickle"), + ) + for force in (False, True): + reused = service.import_snapshot(dataclasses.replace(request, force=force)) + assert reused.reused and reused.metadata == first.metadata + assert hashes(first.db_path) == before + assert not (tmp_path / ".pt-snap").exists() + + +@pytest.mark.parametrize( + "axis", + [ + "source", + "capacity", + "device", + "sourceFile", + "hash", + "version", + "metadata", + "last-member", + "extra", + "missing", + ], +) +@pytest.mark.parametrize("force", [False, True]) +def test_nonidentical_existing_output_never_replaced(tmp_path, axis, force): + service = ImportService() + request = options(tmp_path) + first = service.import_snapshot(request) + focus = (tmp_path / ".pt-snap/focus.json").read_bytes() + manifest_path = first.db_path / "manifest.json" + data = json.loads(manifest_path.read_text()) + if axis == "source": + request.snapshot_file.write_bytes( + request.snapshot_file.read_bytes() + b"trailing-source-bytes" + ) + elif axis == "capacity": + request = dataclasses.replace(request, events_per_slice=3) + elif axis == "device": + request = dataclasses.replace(request, device=0) + elif axis == "sourceFile": + data["sourceFile"] = "/different/input.pkl" + elif axis == "hash": + data["cacheHash"] = "f" * 64 + elif axis == "version": + data["ptSnap"]["identity"]["exportContractVersion"] += 1 + elif axis == "metadata": + data["ptSnap"]["metadata"]["import_format_version"] = 2 + elif axis == "last-member": + (first.db_path / "device_0/slice_00002.db").write_bytes(b"broken") + elif axis == "extra": + (first.db_path / "user-notes").write_text("preserve") + else: + # Test-owned member removal, never an external artifact. + (first.db_path / "device_0/slice_00002.db").unlink() + if axis in ("sourceFile", "hash", "version", "metadata"): + manifest_path.write_text(json.dumps(data)) + before = hashes(first.db_path) + with pytest.raises(ImportExecutionError, match="preserved even under --force"): + service.import_snapshot(dataclasses.replace(request, force=force)) + assert hashes(first.db_path) == before + assert (tmp_path / ".pt-snap/focus.json").read_bytes() == focus + + +@pytest.mark.parametrize("kind", ["empty", "unknown", "external", "file", "symlink", "dangling"]) +def test_original_external_user_targets_preserved_under_force(tmp_path, kind): + request = options(tmp_path) + target = request.output_dir / "input.pkl.msinsight" + target.parent.mkdir() + if kind in ("symlink", "dangling"): + victim = tmp_path / "victim" + if kind == "symlink": + victim.mkdir() + (victim / "owner").write_text("safe") + target.symlink_to(victim, target_is_directory=True) + elif kind == "file": + target.write_bytes(b"user bytes") + else: + target.mkdir() + if kind == "external": + # A genuine valid base artifact still lacks pt-snap ownership. + from tests.test_dataset_focus import make_dataset + + external = make_dataset(tmp_path / "external", devices=(0,), slices=1) + import shutil + + shutil.copytree(external, target, dirs_exist_ok=True) + if kind != "empty": + (target / "owner").write_text("safe") + before = ( + hashes(target) if target.is_dir() else target.read_bytes() if target.is_file() else None + ) + with pytest.raises(ImportExecutionError): + ImportService().import_snapshot(dataclasses.replace(request, force=True)) + assert target.exists() or target.is_symlink() + if target.is_dir(): + assert hashes(target) == before + if kind == "file": + assert target.read_bytes() == before + if kind == "symlink": + assert (victim / "owner").read_text() == "safe" + assert not (tmp_path / ".pt-snap").exists() + + +@pytest.mark.parametrize( + "kind", ["nonzero", "sparse", "oom", "unknown", "empty", "static-only", "empty-selection"] +) +def test_unsupported_input_fails_without_publication_or_focus(tmp_path, kind): + request = options(tmp_path) + data = lifecycle_data() + data["device_traces"][0] = [e for e in data["device_traces"][0] if e["action"] != "oom"] + if kind in ("nonzero", "sparse"): + for index, event in enumerate(data["device_traces"][0]): + event["id"] = index + 1 if kind == "nonzero" else index * 2 + elif kind in ("oom", "unknown"): + data["device_traces"][0][0]["action"] = "oom" if kind == "oom" else "unknown" + elif kind == "empty": + data = {"segments": [], "device_traces": [[]]} + elif kind == "static-only": + data["device_traces"] = [[]] + else: + data["device_traces"].append([]) + request = dataclasses.replace(request, device=1) + write_source(tmp_path, data) + with pytest.raises(ImportExecutionError): + ImportService().import_snapshot(request) + assert not list(request.output_dir.iterdir()) + assert not (tmp_path / ".pt-snap").exists() + + +@pytest.mark.parametrize( + "phase", ["conversion", "metadata", "validation", "source", "focus", "rename", "race"] +) +def test_failures_preserve_old_focus_and_unknown_owner(tmp_path, monkeypatch, phase): + request = options(tmp_path) + old = tmp_path / ".pt-snap/focus.json" + old.parent.mkdir() + old.write_bytes(b'{"db_path":"/old/path","device_id":0}') + before = old.read_bytes() + service = ImportService() + target = request.output_dir / "input.pkl.msinsight" + + def fail(*a, **k): + raise OSError("injected " + phase) + + if phase == "conversion": + monkeypatch.setattr("pt_snap_cli.core.import_service.convert_shards", fail) + elif phase == "metadata": + monkeypatch.setattr(service._metadata_service, "write", fail) + elif phase == "validation": + monkeypatch.setattr("pt_snap_cli.core.import_service.inspect_owned_cache", fail) + elif phase == "source": + monkeypatch.setattr( + service, + "_recheck_source", + lambda *a: (_ for _ in ()).throw(SourceChangedError("source changed")), + ) + elif phase == "focus": + fs = FocusService() + service._focus_service = fs + + def mutate(*a, **k): + old.write_bytes(b"changed") + fail() + + monkeypatch.setattr(fs._config, "write_project_focus", mutate) + else: + move = service._msinsight_backend._move + + def fault(source, destination, identity): + if phase == "race": + destination.mkdir() + (destination / "owner").write_text("safe") + else: + move(source, destination, identity) + fail() + move(source, destination, identity) + + monkeypatch.setattr(service._msinsight_backend, "_move", fault) + with pytest.raises(ImportExecutionError): + service.import_snapshot(request) + assert old.read_bytes() == before + if phase == "race": + assert (target / "owner").read_text() == "safe" + else: + assert not target.exists() + + +def test_cli_json_explicit_mode_default_adjacency_and_api_focus_parity( + tmp_path, monkeypatch, capsys +): + request = options(tmp_path, capacity=None) + args = ["import", str(request.snapshot_file), "--format", "msinsight", "--json"] + runner = CliRunner() + result = runner.invoke(app, args) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data["format"] == "compatibility-v1" and data["slice_count"] == 1 + assert data["db_path"] == str(request.snapshot_file) + ".msinsight" + with SnapshotAnalyzer(data["db_path"]) as analyzer: + assert ( + analyzer.get_focus().callstack_layout == data["focus_state"]["callstack_layout"] == "v1" + ) + cli = runner.invoke( + app, + ["query", data["db_path"], "--template-use", "event", "--params", '{"id":3}', "--json"], + ) + assert cli.exit_code == 0 + assert json.loads(cli.output)["rows"] == analyzer.execute_query("event", {"id": 3})["rows"] + reused = runner.invoke(app, [*args, "--force", "--no-focus"]) + assert reused.exit_code == 0 and json.loads(reused.output)["reused"] + monkeypatch.setattr("sys.argv", ["pt-snap", *args, "--events-per-slice", "0"]) + assert _safe_call() != 0 + assert json.loads(capsys.readouterr().err)["error"]["code"] == "INVALID_PARAMETER" diff --git a/tests/test_contract_cli_api.py b/tests/test_contract_cli_api.py index 4b93bdd..852b764 100644 --- a/tests/test_contract_cli_api.py +++ b/tests/test_contract_cli_api.py @@ -850,6 +850,74 @@ def test_dataset_focus_overview_query_contract(tmp_path: Path, use_manifest: boo analyzer.set_focus(str(path)) +def test_compatible_import_focus_metadata_query_contract(tmp_path: Path, monkeypatch) -> None: + from tests.core.test_msinsight_export import options as compatible_options + + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + request = compatible_options(tmp_path, multiple=True) + imported = _json_stdout( + runner.invoke( + app, + [ + "import", + str(request.snapshot_file), + "--format", + "msinsight", + "--events-per-slice", + "2", + "--json", + ], + ) + ) + assert imported["format"] == "compatibility-v1" + path = imported["dataset_path"] + focused = _json_stdout(runner.invoke(app, ["focus", path, "--device", "1", "--json"])) + with SnapshotAnalyzer(path, device_id=1) as analyzer: + state = analyzer.get_focus() + assert state.available_devices == focused["available_devices"] == [0, 1] + assert state.callstack_layout == focused["callstack_layout"] == "v1" + metadata_result = runner.invoke(app, ["metadata", path, "--json"]) + assert metadata_result.exit_code == 0, metadata_result.output + metadata = json.loads(metadata_result.output) + assert metadata == analyzer.get_database_metadata() + assert metadata["status"] == "available" + assert metadata["metadata"]["import_format_version"] == 1 + overview_result = runner.invoke(app, ["overview", path, "--json"]) + assert overview_result.exit_code == 0, overview_result.output + overview = json.loads(overview_result.output) + api_overview = analyzer.get_database_overview() + assert {key: overview[key] for key in api_overview} == api_overview + for params, selector in [ + ({"id": 3}, []), + ({}, ["--slice", "1"]), + ({"min_id": 2, "max_id": 3}, []), + ]: + cli = _json_stdout( + runner.invoke( + app, + [ + "query", + path, + "--device", + "1", + "--template-use", + "event", + "--params", + json.dumps(params), + "--json", + *selector, + ], + ) + ) + api = analyzer.execute_query("event", params, slice_index=1 if selector else None) + assert {key: cli[key] for key in api} == api + assert cli["scope"]["boundary_events_included"] is False + assert not hasattr(analyzer, "import_snapshot") + + def test_overview_error_contract_maps_four_domain_failures(tmp_path: Path) -> None: missing = runner.invoke(app, ["overview", str(tmp_path / "missing.db"), "--json"]) missing_error = json.loads(missing.stderr)["error"] diff --git a/tests/test_dataset_focus.py b/tests/test_dataset_focus.py index 3c5a489..a066ecd 100644 --- a/tests/test_dataset_focus.py +++ b/tests/test_dataset_focus.py @@ -14,7 +14,13 @@ from pt_snap_cli.cli import app from pt_snap_cli.completion import complete_device_ids from pt_snap_cli.core.context_cache import ContextCache -from pt_snap_cli.core.dataset_contract import ACTION_NAMES, BLOCK_STATES, QueryScope +from pt_snap_cli.core.dataset_contract import ( + ACTION_NAMES, + BLOCK_STATES, + DatasetContractError, + QueryScope, + validate_dataset, +) from pt_snap_cli.core.dataset_resolver import DatasetResolver from pt_snap_cli.core.errors import ( DatabaseSchemaError, @@ -359,7 +365,10 @@ def get(cache, path, **kwargs): @pytest.mark.parametrize("suffix", ["-wal", "-journal", "-shm"]) @pytest.mark.parametrize("dangling", [False, True]) -def test_live_sidecar_rejected_before_any_sqlite_open(tmp_path, monkeypatch, suffix, dangling): +@pytest.mark.parametrize("bare", [False, True]) +def test_live_sidecar_rejected_before_any_sqlite_open( + tmp_path, monkeypatch, suffix, dangling, bare +): root = make_dataset(tmp_path / "dataset") # The last member must be checked before even the first slice is opened. sidecar = root / f"device_2/slice_00002.db{suffix}" @@ -375,8 +384,10 @@ def forbidden(*args, **kwargs): raise AssertionError("SQLite opened before live sidecar rejection") monkeypatch.setattr(sqlite3, "connect", forbidden) - with pytest.raises(DatabaseSchemaError, match="live SQLite sidecar"): - DatasetResolver().inspect(root) + with pytest.raises( + DatasetContractError if bare else DatabaseSchemaError, match="live SQLite sidecar" + ): + validate_dataset(root) if bare else DatasetResolver().inspect(root) assert before == hashes(root) assert inventory == sorted(str(p.relative_to(root)) for p in root.rglob("*")) if dangling: @@ -384,8 +395,8 @@ def forbidden(*args, **kwargs): @pytest.mark.parametrize("manifest_path", [False, True]) -@pytest.mark.parametrize("operation", ["inspect", "focus", "api", "cli"]) -def test_checkpointed_wal_rejected_without_artifact_changes( +@pytest.mark.parametrize("operation", ["inspect", "p0", "focus", "api", "cli"]) +def test_checkpointed_wal_read_without_artifact_changes( tmp_path, monkeypatch, manifest_path, operation ): root = make_dataset(tmp_path / "dataset") @@ -409,28 +420,41 @@ def connect(*args, **kwargs): monkeypatch.setattr(sqlite3, "connect", connect) path = root / "manifest.json" if manifest_path else root if operation == "inspect": - with pytest.raises(DatabaseSchemaError): - DatasetResolver().inspect(path) + assert DatasetResolver().inspect(path).validation.real_event_count == 12 + elif operation == "p0": + assert validate_dataset(root).real_event_count == 12 elif operation == "focus": - with pytest.raises(DatabaseSchemaError): - FocusService().set_project_focus(path) + assert FocusService().set_project_focus(path).callstack_layout == "v1" elif operation == "api": - with SnapshotAnalyzer() as analyzer: - with pytest.raises(ValueError): - analyzer.set_focus(str(path)) - assert analyzer.get_focus().db_path is None + with SnapshotAnalyzer(path) as analyzer: + assert analyzer.execute_query("event", {"id": 5}, device_id=2)["rows"][0]["id"] == 5 + assert analyzer.execute_query("event", {"id": 5}, device_id=2)["rows"][0]["id"] == 5 else: - result = runner.invoke(app, ["focus", str(path), "--json"]) - assert result.exit_code == 1 - assert json.loads(result.stderr)["error"]["code"] == "DATABASE_SCHEMA_INVALID" + result = runner.invoke( + app, + [ + "query", + str(path), + "--template-use", + "event", + "--device", + "2", + "--params", + '{"id":5}', + "--json", + ], + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["rows"][0]["id"] == 5 assert before == hashes(root) assert inventory == sorted(str(p.relative_to(root)) for p in root.rglob("*")) - assert opened == [] - assert not (tmp_path / ".pt-snap/focus.json").exists() + assert opened and all("mode=ro&immutable=1" in str(uri) for uri in opened) + assert not list(root.rglob("*-wal")) and not list(root.rglob("*-shm")) @pytest.mark.parametrize("alias_parent", [False, True]) -def test_member_alias_rejected_before_header_read(tmp_path, monkeypatch, alias_parent): +@pytest.mark.parametrize("bare", [False, True]) +def test_member_alias_rejected_before_header_read(tmp_path, monkeypatch, alias_parent, bare): source = make_dataset(tmp_path / "source", devices=(0,), slices=1) root = tmp_path / "dataset" root.mkdir() @@ -449,8 +473,8 @@ def open_path(path, *args, **kwargs): return original(path, *args, **kwargs) monkeypatch.setattr(Path, "open", open_path) - with pytest.raises(DatabaseSchemaError, match="symlink"): - DatasetResolver().inspect(root) + with pytest.raises(DatasetContractError if bare else DatabaseSchemaError, match="symlink"): + validate_dataset(root) if bare else DatasetResolver().inspect(root) monkeypatch.setattr(Path, "open", original) assert before == hashes(source) diff --git a/tests/test_dataset_import_cli.py b/tests/test_dataset_import_cli.py index 3b16fab..704cdfd 100644 --- a/tests/test_dataset_import_cli.py +++ b/tests/test_dataset_import_cli.py @@ -64,7 +64,7 @@ def test_dataset_cli_json_generation_reuse_force_focus_and_text(tmp_path): ["--format", ""], ["--format", "pt-snap-native-v2"], ["--events-per-slice", "2", "--format", "single-db"], - ["--events-per-slice", "2", "--format", "compatibility-v1"], + ["--events-per-slice", "2", "--format", "compatibility-v2"], ["--device", "-1"], ], )