diff --git a/skills/pt-snap-memory-leak/SKILL.md b/skills/pt-snap-memory-leak/SKILL.md index 9af2fd9..0dbf44e 100644 --- a/skills/pt-snap-memory-leak/SKILL.md +++ b/skills/pt-snap-memory-leak/SKILL.md @@ -183,6 +183,35 @@ This baseline describes successfully freed blocks, bucketed by `freeEventId - al ### 7. Classify findings conservatively +Before any positive or negative per-step / per-iteration conclusion (including +"not a per-step leak"), establish and report iteration evidence: + +- State the trusted iteration count, boundaries mapped to event IDs, source, + and coverage range. Use workload code, logs, or an explicit verified marker + convention to establish the mapping; allocator event counts, waveform minima, + and cleanup counts are not iteration counts. +- `segment_unmap` is only a conditional supporting clue. One `empty_cache` + call can emit multiple unmap events; one iteration can perform multiple + cleanups, or none. Group events and cleanup calls only according to the + verified workload mapping, never by assuming one event or cleanup equals + one step. Do not directly count `segment_unmap` events as steps. +- Check capture start/end and missing markers against that source. Separate + fully covered iterations from partial iterations and uncaptured work; do not + extrapolate a captured count to the whole run. If count, boundaries, source, + or coverage cannot be established, report the unsupported fields and the + per-step conclusion as `unknown`. A verified complete subrange may support + a conclusion scoped only to that subrange, with the rest explicitly unknown. +- Within verified comparable boundaries, correlate allocation identities and + release events with live bytes after expected cleanup. Three surviving blocks + over three verified iterations can support accumulation; three blocks alone + cannot establish a fixed per-step population or rule out accumulation. Keep + observed per-iteration accumulation a candidate, not a confirmed leak; check + application ownership and expected lifetime before assigning a category. +- Jaccard similarity of `(action, callstackId)` sets is only supporting evidence + of similar event kinds. Sets discard multiplicity, ordering, allocation + identity, and byte sizes; even high similarity cannot prove that unreturned + bytes are the only per-iteration difference. + Use these result categories: - `strong leak candidate`: dynamic live bytes grow across comparable captures or phases, persist after expected cleanup, and have ownership evidence. Do not use this category from end-of-trace survival alone. @@ -202,9 +231,10 @@ Report results in this order: 3. `End-of-trace evidence`: dynamic candidate count, dynamic bytes by callstack, static bytes, and representative blocks. 4. `Peak-occupancy evidence`: callstack bytes and block counts at the active peak versus the final event, identity-matched blocks if any, and static plus `[preexisting live]` memory that carries no per-callstack attribution. 5. `Lifecycle evidence`: representative alloc, free-requested, and free-completed event IDs, with ambiguities called out. -6. `Findings`: category, confidence, evidence, inference, and affected callstack or allocation family. -7. `Unknowns`: missing timing, ownership, capture-boundary, callstack, or repeated-capture evidence. -8. `Suggested validation`: targeted experiments that could confirm or reject each leading hypothesis. +6. `Iteration evidence`: trusted iteration count, event-ID boundaries, workload code/log/verified marker source, and coverage range; distinguish complete and partial iterations. Report unsupported fields and per-step conclusions as `unknown` before any positive or negative per-step finding. Include scoped per-iteration live-byte/lifecycle evidence when available. +7. `Findings`: category, confidence, evidence, inference, and affected callstack or allocation family; per-iteration accumulation remains a candidate, not a confirmed leak. +8. `Unknowns`: missing timing, ownership, capture-boundary, callstack, repeated-capture, or iteration-mapping evidence, including uncaptured work. +9. `Suggested validation`: targeted experiments that could confirm or reject each leading hypothesis, including workload logs or explicit marker conventions when iteration mapping is unknown. Useful validation experiments include repeated snapshots at equivalent workload milestones, extending tracing beyond expected cleanup, explicitly releasing suspected application references, synchronizing the device before the capture ends, and comparing behavior before and after allocator cache cleanup. Explain that cache cleanup can change reserved memory without proving that application references were released. @@ -238,5 +268,8 @@ Useful validation experiments include repeated snapshots at equivalent workload - Representative lifecycle events were checked for address reuse and ambiguous pairing. - Dynamic block state was not used as leak evidence. - Event-ID distances were not presented as time durations. +- Every positive or negative per-step conclusion was preceded by a trusted iteration count, event-ID boundaries, source, and coverage range; otherwise the unsupported fields and conclusion were `unknown`. +- Complete iterations were separated from partial or uncaptured work; multiple allocator events or cleanup calls per iteration were reconciled with workload code, logs, or a verified marker convention instead of counted as steps. +- Per-iteration accumulation remained a candidate, not a confirmed leak, and Jaccard set similarity was not used to claim a unique byte difference. - Every finding has a conservative category and confidence level. - Unknowns and validation experiments are included. diff --git a/src/pt_snap_cli/bundled_skills/pt-snap-memory-leak/SKILL.md b/src/pt_snap_cli/bundled_skills/pt-snap-memory-leak/SKILL.md index 9af2fd9..0dbf44e 100644 --- a/src/pt_snap_cli/bundled_skills/pt-snap-memory-leak/SKILL.md +++ b/src/pt_snap_cli/bundled_skills/pt-snap-memory-leak/SKILL.md @@ -183,6 +183,35 @@ This baseline describes successfully freed blocks, bucketed by `freeEventId - al ### 7. Classify findings conservatively +Before any positive or negative per-step / per-iteration conclusion (including +"not a per-step leak"), establish and report iteration evidence: + +- State the trusted iteration count, boundaries mapped to event IDs, source, + and coverage range. Use workload code, logs, or an explicit verified marker + convention to establish the mapping; allocator event counts, waveform minima, + and cleanup counts are not iteration counts. +- `segment_unmap` is only a conditional supporting clue. One `empty_cache` + call can emit multiple unmap events; one iteration can perform multiple + cleanups, or none. Group events and cleanup calls only according to the + verified workload mapping, never by assuming one event or cleanup equals + one step. Do not directly count `segment_unmap` events as steps. +- Check capture start/end and missing markers against that source. Separate + fully covered iterations from partial iterations and uncaptured work; do not + extrapolate a captured count to the whole run. If count, boundaries, source, + or coverage cannot be established, report the unsupported fields and the + per-step conclusion as `unknown`. A verified complete subrange may support + a conclusion scoped only to that subrange, with the rest explicitly unknown. +- Within verified comparable boundaries, correlate allocation identities and + release events with live bytes after expected cleanup. Three surviving blocks + over three verified iterations can support accumulation; three blocks alone + cannot establish a fixed per-step population or rule out accumulation. Keep + observed per-iteration accumulation a candidate, not a confirmed leak; check + application ownership and expected lifetime before assigning a category. +- Jaccard similarity of `(action, callstackId)` sets is only supporting evidence + of similar event kinds. Sets discard multiplicity, ordering, allocation + identity, and byte sizes; even high similarity cannot prove that unreturned + bytes are the only per-iteration difference. + Use these result categories: - `strong leak candidate`: dynamic live bytes grow across comparable captures or phases, persist after expected cleanup, and have ownership evidence. Do not use this category from end-of-trace survival alone. @@ -202,9 +231,10 @@ Report results in this order: 3. `End-of-trace evidence`: dynamic candidate count, dynamic bytes by callstack, static bytes, and representative blocks. 4. `Peak-occupancy evidence`: callstack bytes and block counts at the active peak versus the final event, identity-matched blocks if any, and static plus `[preexisting live]` memory that carries no per-callstack attribution. 5. `Lifecycle evidence`: representative alloc, free-requested, and free-completed event IDs, with ambiguities called out. -6. `Findings`: category, confidence, evidence, inference, and affected callstack or allocation family. -7. `Unknowns`: missing timing, ownership, capture-boundary, callstack, or repeated-capture evidence. -8. `Suggested validation`: targeted experiments that could confirm or reject each leading hypothesis. +6. `Iteration evidence`: trusted iteration count, event-ID boundaries, workload code/log/verified marker source, and coverage range; distinguish complete and partial iterations. Report unsupported fields and per-step conclusions as `unknown` before any positive or negative per-step finding. Include scoped per-iteration live-byte/lifecycle evidence when available. +7. `Findings`: category, confidence, evidence, inference, and affected callstack or allocation family; per-iteration accumulation remains a candidate, not a confirmed leak. +8. `Unknowns`: missing timing, ownership, capture-boundary, callstack, repeated-capture, or iteration-mapping evidence, including uncaptured work. +9. `Suggested validation`: targeted experiments that could confirm or reject each leading hypothesis, including workload logs or explicit marker conventions when iteration mapping is unknown. Useful validation experiments include repeated snapshots at equivalent workload milestones, extending tracing beyond expected cleanup, explicitly releasing suspected application references, synchronizing the device before the capture ends, and comparing behavior before and after allocator cache cleanup. Explain that cache cleanup can change reserved memory without proving that application references were released. @@ -238,5 +268,8 @@ Useful validation experiments include repeated snapshots at equivalent workload - Representative lifecycle events were checked for address reuse and ambiguous pairing. - Dynamic block state was not used as leak evidence. - Event-ID distances were not presented as time durations. +- Every positive or negative per-step conclusion was preceded by a trusted iteration count, event-ID boundaries, source, and coverage range; otherwise the unsupported fields and conclusion were `unknown`. +- Complete iterations were separated from partial or uncaptured work; multiple allocator events or cleanup calls per iteration were reconciled with workload code, logs, or a verified marker convention instead of counted as steps. +- Per-iteration accumulation remained a candidate, not a confirmed leak, and Jaccard set similarity was not used to claim a unique byte difference. - Every finding has a conservative category and confidence level. - Unknowns and validation experiments are included. diff --git a/tests/skills/suites/pt-snap-memory-leak-iterations/cases/incomplete-capture.yaml b/tests/skills/suites/pt-snap-memory-leak-iterations/cases/incomplete-capture.yaml new file mode 100644 index 0000000..b61fcd2 --- /dev/null +++ b/tests/skills/suites/pt-snap-memory-leak-iterations/cases/incomplete-capture.yaml @@ -0,0 +1,61 @@ +schema_version: 1 +kind: skill-eval-case +id: incomplete-capture +title: Partial capture cannot establish whole-run iteration counts +tags: [iterations, partial-capture, unknown] +covers: [iterations.incomplete_capture] +task: + initial_user: > + Diagnose /fixtures/iterations.db device 0. A verified workload log says + events 1-3 are only a trailing fragment of an iteration whose start was + not captured. Inclusive ranges [4,6] and [7,9] are two complete iterations + through their expected cleanup. Their step numbers and the total run + length are unavailable. Tracing ends at event 9 while the workload + continues. There is no ownership evidence for demo.py:11. Can you infer + the whole-run count or rule out per-step leakage from three surviving + blocks? Separate any fully covered subrange evidence from unknown + whole-run conclusions, and request missing capture/ownership evidence. +fixture: + builder: synthetic-snapshotdb + definition: fixtures/accumulation.yaml + mount_path: /fixtures/iterations.db + read_only: true +expected_tools: + actions: + - id: orient + operation: pt_snap.overview + match: {database: /fixtures/iterations.db, json: true} + - id: candidates + operation: pt_snap.query + match: {database: /fixtures/iterations.db, device: 0, template: leak_detection} + expect_output: {allocEventId: 7, size: 41943040} + - id: events + operation: pt_snap.query + match: {database: /fixtures/iterations.db, device: 0, template: event} + expect_output: {id: 9, action: 1} + partial_order: + - {before: orient, after: candidates} + - {before: orient, after: events} + forbidden: [] +oracle: + facts: + whole_run_iteration_count: unknown + whole_run_per_step_conclusion: unknown + complete_iterations: 2 + partial_iterations: 1 + iteration_boundaries: [[4, 6], [7, 9]] + iteration_source: verified workload log + coverage_range: [4, 9] + subrange_per_iteration_candidate_bytes: 41943040 + end_live_candidate_bytes: 125829120 + confirmed_leak: false + result: + allowed_classifications: [inconclusive] + prohibited_classifications: [strong leak candidate] + required_unknowns: [whole-run iteration count, uncaptured work, application ownership, eventual release beyond capture] + prohibited_claims: [not a per-step leak, whole run contains three iterations] + required_evidence: + - {claim: end_live_candidate_bytes, produced_by: candidates} + - {claim: subrange_per_iteration_candidate_bytes, produced_by: events} +scoring: + mandatory_objectives: [safety.read_only, tools.correct_path, evidence.factual_accuracy, evidence.traceability, diagnosis.classification, diagnosis.uncertainty] diff --git a/tests/skills/suites/pt-snap-memory-leak-iterations/cases/multiple-cleanups.yaml b/tests/skills/suites/pt-snap-memory-leak-iterations/cases/multiple-cleanups.yaml new file mode 100644 index 0000000..a3f4a55 --- /dev/null +++ b/tests/skills/suites/pt-snap-memory-leak-iterations/cases/multiple-cleanups.yaml @@ -0,0 +1,62 @@ +schema_version: 1 +kind: skill-eval-case +id: multiple-cleanups +title: Multiple cleanup calls in each verified iteration +tags: [iterations, cleanup-multiplicity] +covers: [iterations.multiple_cleanups] +task: + initial_user: > + Diagnose /fixtures/iterations.db device 0. The verified workload log maps + complete iterations 1, 2, 3 to inclusive event ranges [1,3], [4,6], [7,9], + covering their starts through cleanup with no missing events. Reviewed + code and logs establish TWO empty_cache calls per iteration: each emits + one unmap, at events 2/3, 5/6, and 8/9. A temporary sample appended at + demo.py:11 remains in a user-owned list intended to be cleared each round. + Are the six cleanup calls six steps, so three surviving blocks rule out + per-step growth? Establish count, boundaries, source and coverage before + classifying. Retain the distinction between candidates and confirmed leaks. +fixture: + builder: synthetic-snapshotdb + definition: fixtures/accumulation.yaml + mount_path: /fixtures/iterations.db + read_only: true +expected_tools: + actions: + - id: orient + operation: pt_snap.overview + match: {database: /fixtures/iterations.db, json: true} + - id: candidates + operation: pt_snap.query + match: {database: /fixtures/iterations.db, device: 0, template: leak_detection} + expect_output: {allocEventId: 7, size: 41943040} + - id: events + operation: pt_snap.query + match: {database: /fixtures/iterations.db, device: 0, template: event} + expect_output: {id: 9, action: 1} + partial_order: + - {before: orient, after: candidates} + - {before: orient, after: events} + forbidden: [] +oracle: + facts: + iteration_count: 3 + iteration_boundaries: [[1, 3], [4, 6], [7, 9]] + iteration_source: verified workload log + coverage_range: [1, 9] + complete_iterations: 3 + cleanup_calls: 6 + allocator_events: 9 + unmap_events: 6 + end_live_candidate_bytes: 125829120 + per_iteration_candidate_bytes: 41943040 + confirmed_leak: false + result: + allowed_classifications: [strong leak candidate] + prohibited_classifications: [inconclusive] + required_unknowns: [eventual release beyond capture] + prohibited_claims: [not a per-step leak, six iterations] + required_evidence: + - {claim: end_live_candidate_bytes, produced_by: candidates} + - {claim: unmap_events, produced_by: events} +scoring: + mandatory_objectives: [safety.read_only, tools.correct_path, evidence.factual_accuracy, evidence.traceability, diagnosis.classification, diagnosis.uncertainty] diff --git a/tests/skills/suites/pt-snap-memory-leak-iterations/cases/no-verified-markers.yaml b/tests/skills/suites/pt-snap-memory-leak-iterations/cases/no-verified-markers.yaml new file mode 100644 index 0000000..51dbf9d --- /dev/null +++ b/tests/skills/suites/pt-snap-memory-leak-iterations/cases/no-verified-markers.yaml @@ -0,0 +1,59 @@ +schema_version: 1 +kind: skill-eval-case +id: no-verified-markers +title: Repeating allocator patterns without a verified step mapping +tags: [iterations, unknown, missing-markers] +covers: [iterations.no_verified_markers] +task: + initial_user: > + Diagnose /fixtures/iterations.db device 0. No workload code, logs, or + verified iteration marker convention is available. Capture completeness + is unknown. An analyst says the repeating allocation and unmap pattern + proves three iterations; another guesses eight from waveform minima and + says three survivors disprove per-step leakage. Can either conclusion be + supported? Report iteration count, boundaries, source and coverage, plus + observable retention. Do not invent step boundaries from allocator events. +fixture: + builder: synthetic-snapshotdb + definition: fixtures/accumulation.yaml + mount_path: /fixtures/iterations.db + read_only: true +expected_tools: + actions: + - id: orient + operation: pt_snap.overview + match: {database: /fixtures/iterations.db, json: true} + - id: candidates + operation: pt_snap.query + match: {database: /fixtures/iterations.db, device: 0, template: leak_detection} + expect_output: {allocEventId: 7, size: 41943040} + - id: events + operation: pt_snap.query + match: {database: /fixtures/iterations.db, device: 0, template: event} + expect_output: {id: 9, action: 1} + partial_order: + - {before: orient, after: candidates} + - {before: orient, after: events} + forbidden: [] +oracle: + facts: + iteration_count: unknown + iteration_boundaries: unknown + iteration_source: unknown + iteration_coverage: unknown + per_step_conclusion: unknown + cleanup_calls: unknown + allocator_events: 9 + unmap_events: 6 + end_live_candidate_bytes: 125829120 + confirmed_leak: false + result: + allowed_classifications: [inconclusive] + prohibited_classifications: [strong leak candidate] + required_unknowns: [iteration count, iteration boundaries, iteration source, iteration coverage, application ownership] + prohibited_claims: [not a per-step leak, per-step leak confirmed] + required_evidence: + - {claim: end_live_candidate_bytes, produced_by: candidates} + - {claim: unmap_events, produced_by: events} +scoring: + mandatory_objectives: [safety.read_only, tools.correct_path, evidence.factual_accuracy, evidence.traceability, diagnosis.classification, diagnosis.uncertainty] diff --git a/tests/skills/suites/pt-snap-memory-leak-iterations/cases/trusted-three.yaml b/tests/skills/suites/pt-snap-memory-leak-iterations/cases/trusted-three.yaml new file mode 100644 index 0000000..c24437c --- /dev/null +++ b/tests/skills/suites/pt-snap-memory-leak-iterations/cases/trusted-three.yaml @@ -0,0 +1,66 @@ +schema_version: 1 +kind: skill-eval-case +id: trusted-three +title: Three verified iterations retain one new block each +tags: [iterations, accumulation, external-evidence] +covers: [iterations.trusted_three] +task: + initial_user: > + Diagnose /fixtures/iterations.db device 0. The verified workload log maps + complete iterations 1, 2, 3 to inclusive event ranges [1,3], [4,6], [7,9]; + capture covers all three from start through cleanup with no missing events. + Each iteration calls empty_cache once, emitting two segment_unmap events. + Reviewed workload code says demo.py:11 appends one temporary sample to a + user-owned list intended to be cleared at each iteration end; the list + remains referenced through capture end. Another analyst guessed eight + steps from the waveform and ruled out per-step growth because only three + blocks survived. Evaluate that claim. Jaccard similarity of adjacent + (action, callstackId) sets is 1.0; does that prove the only difference is + unreturned bytes? Report iteration count, boundaries, source, coverage, + per-iteration candidate growth, and confirmation limits. +fixture: + builder: synthetic-snapshotdb + definition: fixtures/accumulation.yaml + mount_path: /fixtures/iterations.db + read_only: true +expected_tools: + actions: + - id: orient + operation: pt_snap.overview + match: {database: /fixtures/iterations.db, json: true} + - id: candidates + operation: pt_snap.query + match: {database: /fixtures/iterations.db, device: 0, template: leak_detection} + expect_output: {allocEventId: 7, size: 41943040} + - id: events + operation: pt_snap.query + match: {database: /fixtures/iterations.db, device: 0, template: event} + expect_output: {id: 9, action: 1} + partial_order: + - {before: orient, after: candidates} + - {before: orient, after: events} + forbidden: [] +oracle: + facts: + iteration_count: 3 + iteration_boundaries: [[1, 3], [4, 6], [7, 9]] + iteration_source: verified workload log + coverage_range: [1, 9] + complete_iterations: 3 + cleanup_calls: 3 + allocator_events: 9 + unmap_events: 6 + end_live_candidate_bytes: 125829120 + per_iteration_candidate_bytes: 41943040 + confirmed_leak: false + jaccard_proves_unique_byte_difference: false + result: + allowed_classifications: [strong leak candidate] + prohibited_classifications: [inconclusive] + required_unknowns: [eventual release beyond capture] + prohibited_claims: [not a per-step leak, eight iterations, Jaccard proves the only difference is unreturned bytes] + required_evidence: + - {claim: end_live_candidate_bytes, produced_by: candidates} + - {claim: unmap_events, produced_by: events} +scoring: + mandatory_objectives: [safety.read_only, tools.correct_path, evidence.factual_accuracy, evidence.traceability, diagnosis.classification, diagnosis.uncertainty] diff --git a/tests/skills/suites/pt-snap-memory-leak-iterations/fixtures/accumulation.yaml b/tests/skills/suites/pt-snap-memory-leak-iterations/fixtures/accumulation.yaml new file mode 100644 index 0000000..a21155f --- /dev/null +++ b/tests/skills/suites/pt-snap-memory-leak-iterations/fixtures/accumulation.yaml @@ -0,0 +1,18 @@ +schema_version: 1 +kind: synthetic-snapshotdb +devices: + 0: + trace: + - {id: 1, action: 4, address: 1048576, size: 41943040, stream: 0, allocated: 41943040, active: 41943040, reserved: 134242304, callstack: "demo.py:11:retain_sample"} + - {id: 2, action: 1, address: 536870912, size: 4096, stream: 0, allocated: 41943040, active: 41943040, reserved: 134238208, callstack: "cleanup.py:20:empty_cache"} + - {id: 3, action: 1, address: 536875008, size: 4096, stream: 0, allocated: 41943040, active: 41943040, reserved: 134234112, callstack: "cleanup.py:20:empty_cache"} + - {id: 4, action: 4, address: 68157440, size: 41943040, stream: 0, allocated: 83886080, active: 83886080, reserved: 134234112, callstack: "demo.py:11:retain_sample"} + - {id: 5, action: 1, address: 536879104, size: 4096, stream: 0, allocated: 83886080, active: 83886080, reserved: 134230016, callstack: "cleanup.py:20:empty_cache"} + - {id: 6, action: 1, address: 536883200, size: 4096, stream: 0, allocated: 83886080, active: 83886080, reserved: 134225920, callstack: "cleanup.py:20:empty_cache"} + - {id: 7, action: 4, address: 135266304, size: 41943040, stream: 0, allocated: 125829120, active: 125829120, reserved: 134225920, callstack: "demo.py:11:retain_sample"} + - {id: 8, action: 1, address: 536887296, size: 4096, stream: 0, allocated: 125829120, active: 125829120, reserved: 134221824, callstack: "cleanup.py:20:empty_cache"} + - {id: 9, action: 1, address: 536891392, size: 4096, stream: 0, allocated: 125829120, active: 125829120, reserved: 134217728, callstack: "cleanup.py:20:empty_cache"} + blocks: + - {id: 1, address: 1048576, size: 41943040, requestedSize: 41943040, state: 1, allocEventId: 1, freeEventId: -1} + - {id: 4, address: 68157440, size: 41943040, requestedSize: 41943040, state: 1, allocEventId: 4, freeEventId: -1} + - {id: 7, address: 135266304, size: 41943040, requestedSize: 41943040, state: 1, allocEventId: 7, freeEventId: -1} diff --git a/tests/skills/suites/pt-snap-memory-leak-iterations/suite.yaml b/tests/skills/suites/pt-snap-memory-leak-iterations/suite.yaml new file mode 100644 index 0000000..c71390c --- /dev/null +++ b/tests/skills/suites/pt-snap-memory-leak-iterations/suite.yaml @@ -0,0 +1,49 @@ +schema_version: 1 +kind: skill-eval-suite +metadata: + id: pt-snap-memory-leak-iterations + title: Iteration evidence before per-step leak conclusions + description: > + Grade externally grounded iteration boundaries, capture coverage, and + conservative accumulation claims without automatic step detection. +skill: + path: skills/pt-snap-memory-leak/SKILL.md + expected_name: pt-snap-memory-leak + profile: diagnostic-readonly +result_contract: + required_fields: [classification, facts, claims, unknowns] + classifications: [strong leak candidate, inconclusive] +objectives: + - {id: safety.read_only, mode: gate, scorer: safety} + - {id: tools.correct_path, mode: score, scorer: tool_path, weight: 30} + - {id: evidence.factual_accuracy, mode: score, scorer: facts, weight: 25} + - {id: evidence.traceability, mode: score, scorer: evidence, weight: 15} + - {id: diagnosis.classification, mode: score, scorer: classification, weight: 20} + - {id: diagnosis.uncertainty, mode: score, scorer: uncertainty, weight: 10} +coverage: + required_branches: + - iterations.trusted_three + - iterations.multiple_cleanups + - iterations.no_verified_markers + - iterations.incomplete_capture +defaults: + runner: local-agent + repetitions: 3 + timeout_seconds: 240 + sandbox: + network: disabled + home: isolated + project: read_only + fixtures: read_only + clear_environment: [PT_SNAP_DB_PATH] + tool_policy: + allowed: [pt_snap.focus_read, pt_snap.capabilities, pt_snap.overview, pt_snap.query] + forbidden: [pt_snap.import, pt_snap.focus_write, pt_snap.split, sqlite.readonly, sqlite.write, filesystem.write, package.install, pickle.access, network.access] + max_calls: 24 + max_failed_calls: 3 + max_repeated_semantic_calls: 2 +cases: + - {id: trusted-three, path: cases/trusted-three.yaml} + - {id: multiple-cleanups, path: cases/multiple-cleanups.yaml} + - {id: no-verified-markers, path: cases/no-verified-markers.yaml} + - {id: incomplete-capture, path: cases/incomplete-capture.yaml} diff --git a/tests/skills/test_memory_leak_iterations_contract.py b/tests/skills/test_memory_leak_iterations_contract.py new file mode 100644 index 0000000..ec47554 --- /dev/null +++ b/tests/skills/test_memory_leak_iterations_contract.py @@ -0,0 +1,158 @@ +"""Iteration contracts and deterministic eval fixtures; no live model is run.""" + +from copy import deepcopy +from pathlib import Path + +import pytest + +from pt_snap_cli.context import Context +from pt_snap_cli.query.executor import QueryExecutor +from pt_snap_cli.query.registry import QueryRegistry, _load_all_templates +from tests.skills.harness.descriptors import load_suite +from tests.skills.harness.fixtures import build_snapshotdb +from tests.skills.harness.grader import RunRecord, ToolCall, grade_run + +SKILL_PATH = Path("skills/pt-snap-memory-leak/SKILL.md") +SUITE_PATH = Path("tests/skills/suites/pt-snap-memory-leak-iterations/suite.yaml") +CASE_IDS = ("trusted-three", "multiple-cleanups", "no-verified-markers", "incomplete-capture") + + +def test_iteration_evidence_precedes_both_positive_and_negative_classification() -> None: + skill = SKILL_PATH.read_text() + step = " ".join(skill.split("### 7. Classify findings conservatively")[1].split()) + step = step.split("## Output Template")[0] + + for required in ( + "Before any positive or negative per-step / per-iteration conclusion", + '"not a per-step leak"', + "trusted iteration count, boundaries mapped to event IDs, source, and coverage range", + "workload code, logs, or an explicit verified marker convention", + "allocator event counts, waveform minima, and cleanup counts are not iteration counts", + "Do not directly count `segment_unmap` events as steps", + "one iteration can perform multiple cleanups, or none", + "fully covered iterations from partial iterations and uncaptured work", + "per-step conclusion as `unknown`", + "a conclusion scoped only to that subrange", + "per-iteration accumulation a candidate, not a confirmed leak", + "Jaccard similarity of `(action, callstackId)` sets", + "Sets discard multiplicity, ordering, allocation identity, and byte sizes", + "cannot prove that unreturned bytes are the only per-iteration difference", + ): + assert required in step + assert step.index("trusted iteration count") < step.index("Use these result categories") + + +def test_iteration_output_and_checklist_require_source_and_coverage() -> None: + skill = SKILL_PATH.read_text() + output = skill.split("## Output Template")[1].split("## Guardrails")[0] + checklist = skill.split("## Verification Checklist")[1] + + assert output.index("`Iteration evidence`") < output.index("`Findings`") + for section in (output, checklist): + for required in ("iteration count", "event-ID boundaries", "source", "coverage range"): + assert required in section + assert "positive or negative per-step" in section + assert "`unknown`" in section + assert "candidate, not a confirmed leak" in section + assert "partial or uncaptured work" in checklist + assert "Jaccard set similarity was not used to claim a unique byte difference" in checklist + + +@pytest.fixture +def iteration_outputs(tmp_path: Path): + QueryRegistry.reset() + _load_all_templates() + database = build_snapshotdb(SUITE_PATH.parent / "fixtures/accumulation.yaml", tmp_path / "i.db") + context = Context(database) + executor = QueryExecutor(context) + try: + yield { + "orient": {"device_ids": context.device_ids}, + "candidates": executor.execute_template("leak_detection", params={}, device_id=0), + "events": executor.execute_template( + "event", params={"order_by": "id", "order_dir": "ASC"}, device_id=0 + ), + } + finally: + context.close() + + +def test_iteration_fixture_exposes_accumulation_but_no_step_markers(iteration_outputs) -> None: + candidates = iteration_outputs["candidates"] + events = iteration_outputs["events"] + assert {row["allocEventId"] for row in candidates} == {1, 4, 7} + assert sum(row["size"] for row in candidates) == 3 * 40 * 1024 * 1024 + assert [row["id"] for row in events if row["action"] == 1] == [2, 3, 5, 6, 8, 9] + assert not any(row["action"] in (5, 6) for row in events) + assert [row["active"] for row in events if row["id"] in (3, 6, 9)] == [ + 41943040, + 83886080, + 125829120, + ] + + +def _oracle_run(case, outputs) -> RunRecord: + """Check the declarative oracle against real queries, not agent behavior.""" + return RunRecord( + tool_calls=tuple( + ToolCall(action.id, action.operation, dict(action.match), output=outputs[action.id]) + for action in case.actions + ), + result={ + "classification": case.allowed_classifications[0], + "facts": deepcopy(case.oracle_facts), + "claims": [ + {"id": claim, "evidence_call_ids": [produced_by]} + for claim, produced_by in case.required_evidence + ], + "unknowns": list(case.required_unknowns), + }, + ) + + +@pytest.mark.parametrize("case_id", CASE_IDS) +def test_iteration_eval_oracles_match_real_fixture_outputs(case_id, iteration_outputs) -> None: + suite = load_suite(SUITE_PATH) + assert {case.id for case in suite.cases} == set(CASE_IDS) + assert set(suite.required_branches) <= { + branch for case in suite.cases for branch in case.covers + } + case = suite.case(case_id) + grade = grade_run(suite, case, _oracle_run(case, iteration_outputs)) + assert grade.passed, grade + assert grade.score == 100 + + +@pytest.mark.parametrize( + ("case_id", "field", "unsupported_value"), + [ + ("trusted-three", "iteration_count", 8), + ("trusted-three", "iteration_count", 9), + ("trusted-three", "iteration_boundaries", [[1, 9]]), + ("trusted-three", "iteration_source", "waveform minima"), + ("trusted-three", "coverage_range", "whole run assumed"), + ("trusted-three", "confirmed_leak", True), + ("trusted-three", "jaccard_proves_unique_byte_difference", True), + ("multiple-cleanups", "iteration_count", 6), + ("no-verified-markers", "iteration_count", 3), + ("no-verified-markers", "per_step_conclusion", "per-step leak"), + ("no-verified-markers", "per_step_conclusion", "not a per-step leak"), + ("incomplete-capture", "whole_run_iteration_count", 3), + ("incomplete-capture", "complete_iterations", 3), + ("incomplete-capture", "whole_run_per_step_conclusion", "not a per-step leak"), + ], +) +def test_iteration_eval_rejects_unsupported_conclusions( + case_id, field, unsupported_value, iteration_outputs +) -> None: + suite = load_suite(SUITE_PATH) + case = suite.case(case_id) + run = _oracle_run(case, iteration_outputs) + run.result["facts"][field] = unsupported_value + grade = grade_run(suite, case, run) + assert not grade.passed + assert not next( + objective + for objective in grade.objectives + if objective.objective_id == "evidence.factual_accuracy" + ).passed