diff --git a/README.md b/README.md index 66fcf16..d932149 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,10 @@ python -m hardsector_tool scan-metadata out_80221_v3 --out scan_80221.json python -m hardsector_tool scan-metadata out_80217_v3 --out scan_80217.json ``` +`scan-metadata` accepts track diagnostics named either `tracks/track_XX.json` or +`tracks/TXX.json`. When per-track JSONs are missing or lack a sector size, +sector payload sizes are sampled to infer the most common logical sector size. + Use the JSON to guide reverse engineering (what sectors look like labels/catalogs/manifests). --- @@ -235,7 +239,7 @@ Optional pppp-based descriptor carving (if your hypothesis suggests names-with-p ### Catalog reporting (evidence-only) -`catalog-report` inventories descriptor-backed module candidates and name lists without writing payloads. It reuses the reconstruction cache used by `qc-capture` so running against an SCP image will transparently reconstruct into `.qc_cache/` (unless `--reconstruct-out` or `--force-reconstruct` is set). Outputs are written to `catalog_report.json` and `catalog_report.txt` inside the `--out` directory. +`catalog-report` inventories descriptor-backed module candidates and name lists without writing payloads. It reuses the reconstruction cache used by `qc-capture` so running against an SCP image will transparently reconstruct into `.qc_cache/` (unless `--reconstruct-out` or `--force-reconstruct` is set). If `--out` is omitted, reports land in `/catalog_report/` (or the cache reconstruction directory for SCP inputs); otherwise `catalog_report.json` and `catalog_report.txt` are written under the chosen `--out` path. Examples: @@ -250,6 +254,11 @@ python -m hardsector_tool catalog-report out_80217_v3 --out reports/out_80217_v3 python -m hardsector_tool catalog-report out_80217_v3 --out reports/out_sysgen --only-prefix SYSGEN. --require-name-in-pppp-list --enable-pppp-descriptors ``` +Notes: + +* SCP inputs reuse the `.qc_cache/` reconstruction cache (shared with `qc-capture`) unless you force a rerun. +* If `--out` is omitted, catalog reports are written under `/catalog_report/` (for SCP inputs this is the cache reconstruction directory). + --- ## Help / discoverability @@ -332,7 +341,7 @@ This is intentional: it prevents name lists from inflating descriptor counts and * `manifest.json` — totals, mapping, track stats * `sectors/Txx_Syy.bin` — recovered logical sector payloads (primary analysis artifact) -* `tracks/track_XX.json` — per-track decode diagnostics +* `tracks/track_XX.json` or `tracks/TXX.json` — per-track decode diagnostics ### Extract output (`derived_*`) diff --git a/src/hardsector_tool/catalog_report.py b/src/hardsector_tool/catalog_report.py index a3d970c..01718d9 100644 --- a/src/hardsector_tool/catalog_report.py +++ b/src/hardsector_tool/catalog_report.py @@ -501,7 +501,7 @@ def _ensure_reconstruction( def catalog_report( input_path: Path, *, - out_dir: Path, + out_dir: Path | None = None, min_refs: int = 3, max_refs: int = 2000, hypotheses: Iterable[str] = ("H1", "H2", "H3", "H4"), @@ -526,8 +526,8 @@ def catalog_report( force: bool = False, ) -> dict: input_path = Path(input_path) - out_dir = Path(out_dir) - _safe_mkdir(out_dir) + output_dir = Path(out_dir) if out_dir is not None else None + cache_root = cache_dir if cache_dir is not None else Path(".qc_cache") if input_path.suffix.lower() == ".scp": recon_dir, used_cache = _ensure_reconstruction( @@ -542,23 +542,25 @@ def catalog_report( similarity_threshold=similarity_threshold, clock_factor=clock_factor, dump_raw_windows=dump_raw_windows, - cache_dir=cache_dir or Path(".qc_cache"), + cache_dir=cache_root, force_reconstruct=force_reconstruct, reconstruct_out=reconstruct_out, force=force, ) input_type = "scp" holes_per_rev = sectors_per_rotation - cache_note = { - "cache_dir": str(cache_dir or Path(".qc_cache")), - "used_cache": used_cache, - } + cache_note = {"cache_dir": str(cache_root), "used_cache": used_cache} else: recon_dir = input_path input_type = "reconstruction" holes_per_rev = sectors_per_rotation cache_note = {"cache_dir": None, "used_cache": False} + report_out_dir = ( + output_dir if output_dir is not None else recon_dir / "catalog_report" + ) + _safe_mkdir(report_out_dir) + report = generate_catalog_report( recon_dir, input_path=input_path, @@ -574,8 +576,8 @@ def catalog_report( holes_per_rev=holes_per_rev, ) - json_path = out_dir / "catalog_report.json" - txt_path = out_dir / "catalog_report.txt" + json_path = report_out_dir / "catalog_report.json" + txt_path = report_out_dir / "catalog_report.txt" json_payload = report["json"] json_payload["reconstruction_cache"] = cache_note @@ -597,7 +599,14 @@ def build_arg_parser( help="Generate an evidence-backed catalog listing from reconstructed sectors", ) parser.add_argument("input", type=Path, help="SCP image or reconstruction output") - parser.add_argument("--out", required=True, type=Path, help="Output directory") + parser.add_argument( + "--out", + type=Path, + help=( + "Output directory. Default: reconstruction path under catalog_report/ " + "(or cache reconstruction for SCP inputs)." + ), + ) parser.add_argument( "--min-refs", type=int, default=3, help="Minimum pointer references to accept" ) @@ -633,7 +642,7 @@ def build_arg_parser( "--cache-dir", type=Path, default=Path(".qc_cache"), - help="Cache directory for reconstruction outputs (SCP inputs)", + help="Cache directory for reconstruction outputs (shared with qc-capture)", ) parser.add_argument( "--force-reconstruct", diff --git a/src/hardsector_tool/scanmeta.py b/src/hardsector_tool/scanmeta.py index f014657..273d022 100644 --- a/src/hardsector_tool/scanmeta.py +++ b/src/hardsector_tool/scanmeta.py @@ -56,15 +56,41 @@ def _load_sector( ) -def _sector_size_for_track(track: int, out_dir: Path, default: int) -> int: - track_path = out_dir / "tracks" / f"T{track:02d}.json" - if track_path.exists(): +def _sector_size_for_track( + out_dir: Path, track: int, default_sector_size: int = 256 +) -> tuple[int, str]: + """Infer sector size for a track using track JSON or sector payloads.""" + + track_json_candidates = [ + out_dir / "tracks" / f"track_{track:02d}.json", + out_dir / "tracks" / f"T{track:02d}.json", + ] + + for track_path in track_json_candidates: + if not track_path.exists(): + continue try: data = json.loads(track_path.read_text()) - return int(data.get("sector_size", default) or default) - except (json.JSONDecodeError, OSError, ValueError): - return default - return default + except (json.JSONDecodeError, OSError): + continue + + for key in ("selected_sector_size", "sector_size", "best_sector_size"): + if key not in data: + continue + try: + return int(data[key]), "track_json" + except (TypeError, ValueError): + continue + + sector_files = list((out_dir / "sectors").glob(f"T{track:02d}_S*.bin")) + if sector_files: + size_counts = Counter(path.stat().st_size for path in sector_files) + max_count = max(size_counts.values()) + candidates = [size for size, count in size_counts.items() if count == max_count] + chosen = max(candidates) + return chosen, "sector_files" + + return default_sector_size, "default" def _slice_context(payload: bytes, offset: int, radius: int = 32) -> tuple[str, str]: @@ -242,8 +268,12 @@ def scan_metadata(out_dir: Path) -> dict: sectors: list[SectorData] = [] missing: list[str] = [] + inferred_sizes: list[tuple[int, str]] = [] for track in expected_tracks: - sector_size = _sector_size_for_track(track, out_dir, default_sector_size) + sector_size, size_source = _sector_size_for_track( + out_dir, track, default_sector_size + ) + inferred_sizes.append((sector_size, size_source)) for sector in range(logical_sectors): entry = _load_sector(track, sector, out_dir, sector_size) if entry is None: @@ -251,10 +281,26 @@ def scan_metadata(out_dir: Path) -> dict: continue sectors.append(entry) + sector_size_inferred = default_sector_size + sector_size_source = "default" + if inferred_sizes: + size_counts = Counter(size for size, _ in inferred_sizes) + max_count = max(size_counts.values()) + candidates = [size for size, count in size_counts.items() if count == max_count] + sector_size_inferred = max(candidates) + source_priority = {"track_json": 2, "sector_files": 1, "default": 0} + for size, source in inferred_sizes: + if size == sector_size_inferred and source_priority.get( + source, -1 + ) >= source_priority.get(sector_size_source, -1): + sector_size_source = source + summary = { "tracks": len(expected_tracks), "sectors_per_track": logical_sectors, - "sector_size": default_sector_size, + "sector_size": sector_size_inferred, + "sector_size_inferred": sector_size_inferred, + "sector_size_source": sector_size_source, "manifest_totals": manifest.get("totals", {}), "missing": missing, } diff --git a/tests/test_catalog_report.py b/tests/test_catalog_report.py index 13c5a3c..71fcfe2 100644 --- a/tests/test_catalog_report.py +++ b/tests/test_catalog_report.py @@ -10,15 +10,26 @@ def _build_recon_fixture(tmp_path: Path) -> Path: sectors_dir.mkdir(parents=True) # Craft a descriptor with three references and a matching pppp entry. - pointer_bytes = (0).to_bytes(2, "little") + (1).to_bytes(2, "little") + (2).to_bytes(2, "little") - sectors_dir.joinpath("T00_S00.bin").write_bytes(b"DATA=HELLO.TXT////" + pointer_bytes) + pointer_bytes = ( + (0).to_bytes(2, "little") + + (1).to_bytes(2, "little") + + (2).to_bytes(2, "little") + ) + sectors_dir.joinpath("T00_S00.bin").write_bytes( + b"DATA=HELLO.TXT////" + pointer_bytes + ) sectors_dir.joinpath("T00_S01.bin").write_bytes(b"pppp=HELLO.TXT////" + b"X" * 16) sectors_dir.joinpath("T00_S02.bin").write_bytes(b"PAYLOAD-02") sectors_dir.joinpath("T00_S03.bin").write_bytes(b"PAYLOAD-03") manifest = { "image": {"path": "fixture.scp"}, - "mapping": {"mode": "dense", "track_step": 1, "side": 0, "present_scp_tracks": [0]}, + "mapping": { + "mode": "dense", + "track_step": 1, + "side": 0, + "present_scp_tracks": [0], + }, "tracks": [ { "track_number": 0, @@ -59,3 +70,23 @@ def test_catalog_report_produces_outputs(tmp_path: Path) -> None: text_contents = txt_path.read_text() assert "HELLO.TXT" in text_contents assert "catalog-report" in text_contents + + +def test_catalog_report_defaults_to_recon_dir_output(tmp_path: Path) -> None: + recon_dir = _build_recon_fixture(tmp_path) + + result = catalog_report.catalog_report( + recon_dir, + out_dir=None, + enable_pppp_descriptors=True, + min_refs=3, + ) + + json_path = recon_dir / "catalog_report" / "catalog_report.json" + txt_path = recon_dir / "catalog_report" / "catalog_report.txt" + + assert json_path.exists() + assert txt_path.exists() + + report = result["report"] + assert report["summary"]["totals"]["descriptors"] == 1 diff --git a/tests/test_scanmeta_sector_size.py b/tests/test_scanmeta_sector_size.py new file mode 100644 index 0000000..3177abb --- /dev/null +++ b/tests/test_scanmeta_sector_size.py @@ -0,0 +1,53 @@ +import json +from pathlib import Path + +from hardsector_tool.scanmeta import scan_metadata + + +def _build_recon(tmp_path: Path) -> Path: + recon_dir = tmp_path / "recon" + sectors_dir = recon_dir / "sectors" + tracks_dir = recon_dir / "tracks" + sectors_dir.mkdir(parents=True, exist_ok=True) + tracks_dir.mkdir(parents=True, exist_ok=True) + + manifest = { + "tracks": [ + { + "track_number": 0, + "sector_size": 128, + "track_score": 1.0, + } + ], + "totals": { + "expected_sectors": 16, + "written_sectors": 16, + "missing_sectors": 0, + }, + } + recon_dir.joinpath("manifest.json").write_text(json.dumps(manifest)) + + for sector in range(16): + payload = bytes([sector]) * 128 + sectors_dir.joinpath(f"T00_S{sector:02d}.bin").write_bytes(payload) + + return recon_dir + + +def test_scan_metadata_prefers_track_json_then_sector_files(tmp_path: Path) -> None: + recon_dir = _build_recon(tmp_path) + track_json = recon_dir / "tracks" / "T00.json" + track_json.write_text(json.dumps({"selected_sector_size": 128})) + + result = scan_metadata(recon_dir) + summary = result["summary"] + assert summary["sector_size"] == 128 + assert summary["sector_size_inferred"] == 128 + assert summary["sector_size_source"] == "track_json" + + track_json.unlink() + result = scan_metadata(recon_dir) + summary = result["summary"] + assert summary["sector_size"] == 128 + assert summary["sector_size_inferred"] == 128 + assert summary["sector_size_source"] == "sector_files"