Improve scan-metadata sector size inference and catalog-report defaults

README.md

@@ -146,6 +146,10 @@ python -m hardsector_tool scan-metadata out_80221_v3 --out scan_80221.json
 python -m hardsector_tool scan-metadata out_80217_v3 --out scan_80217.json
 ```
 
+`scan-metadata` accepts track diagnostics named either `tracks/track_XX.json` or
+`tracks/TXX.json`. When per-track JSONs are missing or lack a sector size,
+sector payload sizes are sampled to infer the most common logical sector size.
+
 Use the JSON to guide reverse engineering (what sectors look like labels/catalogs/manifests).
 
 ---
@@ -235,7 +239,7 @@ Optional pppp-based descriptor carving (if your hypothesis suggests names-with-p
 
 ### Catalog reporting (evidence-only)
 
-`catalog-report` inventories descriptor-backed module candidates and name lists without writing payloads. It reuses the reconstruction cache used by `qc-capture` so running against an SCP image will transparently reconstruct into `.qc_cache/` (unless `--reconstruct-out` or `--force-reconstruct` is set). Outputs are written to `catalog_report.json` and `catalog_report.txt` inside the `--out` directory.
+`catalog-report` inventories descriptor-backed module candidates and name lists without writing payloads. It reuses the reconstruction cache used by `qc-capture` so running against an SCP image will transparently reconstruct into `.qc_cache/` (unless `--reconstruct-out` or `--force-reconstruct` is set). If `--out` is omitted, reports land in `<reconstruction>/catalog_report/` (or the cache reconstruction directory for SCP inputs); otherwise `catalog_report.json` and `catalog_report.txt` are written under the chosen `--out` path.
 
 Examples:
 
@@ -250,6 +254,11 @@ python -m hardsector_tool catalog-report out_80217_v3 --out reports/out_80217_v3
 python -m hardsector_tool catalog-report out_80217_v3 --out reports/out_sysgen --only-prefix SYSGEN. --require-name-in-pppp-list --enable-pppp-descriptors
 ```
 
+Notes:
+
+* SCP inputs reuse the `.qc_cache/` reconstruction cache (shared with `qc-capture`) unless you force a rerun.
+* If `--out` is omitted, catalog reports are written under `<reconstruction>/catalog_report/` (for SCP inputs this is the cache reconstruction directory).
+
 ---
 
 ## Help / discoverability
@@ -332,7 +341,7 @@ This is intentional: it prevents name lists from inflating descriptor counts and
 
 * `manifest.json` — totals, mapping, track stats
 * `sectors/Txx_Syy.bin` — recovered logical sector payloads (primary analysis artifact)
-* `tracks/track_XX.json` — per-track decode diagnostics
+* `tracks/track_XX.json` or `tracks/TXX.json` — per-track decode diagnostics
 
 ### Extract output (`derived_*`)
 

src/hardsector_tool/catalog_report.py

@@ -501,7 +501,7 @@ def _ensure_reconstruction(
 def catalog_report(
     input_path: Path,
     *,
-    out_dir: Path,
+    out_dir: Path | None = None,
     min_refs: int = 3,
     max_refs: int = 2000,
     hypotheses: Iterable[str] = ("H1", "H2", "H3", "H4"),
@@ -526,8 +526,8 @@ def catalog_report(
     force: bool = False,
 ) -> dict:
     input_path = Path(input_path)
-    out_dir = Path(out_dir)
-    _safe_mkdir(out_dir)
+    output_dir = Path(out_dir) if out_dir is not None else None
+    cache_root = cache_dir if cache_dir is not None else Path(".qc_cache")
 
     if input_path.suffix.lower() == ".scp":
         recon_dir, used_cache = _ensure_reconstruction(
@@ -542,23 +542,25 @@ def catalog_report(
             similarity_threshold=similarity_threshold,
             clock_factor=clock_factor,
             dump_raw_windows=dump_raw_windows,
-            cache_dir=cache_dir or Path(".qc_cache"),
+            cache_dir=cache_root,
             force_reconstruct=force_reconstruct,
             reconstruct_out=reconstruct_out,
             force=force,
         )
         input_type = "scp"
         holes_per_rev = sectors_per_rotation
-        cache_note = {
-            "cache_dir": str(cache_dir or Path(".qc_cache")),
-            "used_cache": used_cache,
-        }
+        cache_note = {"cache_dir": str(cache_root), "used_cache": used_cache}
     else:
         recon_dir = input_path
         input_type = "reconstruction"
         holes_per_rev = sectors_per_rotation
         cache_note = {"cache_dir": None, "used_cache": False}
 
+    report_out_dir = (
+        output_dir if output_dir is not None else recon_dir / "catalog_report"
+    )
+    _safe_mkdir(report_out_dir)
+
     report = generate_catalog_report(
         recon_dir,
         input_path=input_path,
@@ -574,8 +576,8 @@ def catalog_report(
         holes_per_rev=holes_per_rev,
     )
 
-    json_path = out_dir / "catalog_report.json"
-    txt_path = out_dir / "catalog_report.txt"
+    json_path = report_out_dir / "catalog_report.json"
+    txt_path = report_out_dir / "catalog_report.txt"
 
     json_payload = report["json"]
     json_payload["reconstruction_cache"] = cache_note
@@ -597,7 +599,14 @@ def build_arg_parser(
         help="Generate an evidence-backed catalog listing from reconstructed sectors",
     )
     parser.add_argument("input", type=Path, help="SCP image or reconstruction output")
-    parser.add_argument("--out", required=True, type=Path, help="Output directory")
+    parser.add_argument(
+        "--out",
+        type=Path,
+        help=(
+            "Output directory. Default: reconstruction path under catalog_report/ "
+            "(or cache reconstruction for SCP inputs)."
+        ),
+    )
     parser.add_argument(
         "--min-refs", type=int, default=3, help="Minimum pointer references to accept"
     )
@@ -633,7 +642,7 @@ def build_arg_parser(
         "--cache-dir",
         type=Path,
         default=Path(".qc_cache"),
-        help="Cache directory for reconstruction outputs (SCP inputs)",
+        help="Cache directory for reconstruction outputs (shared with qc-capture)",
     )
     parser.add_argument(
         "--force-reconstruct",

src/hardsector_tool/scanmeta.py

@@ -56,15 +56,41 @@ def _load_sector(
     )
 
 
-def _sector_size_for_track(track: int, out_dir: Path, default: int) -> int:
-    track_path = out_dir / "tracks" / f"T{track:02d}.json"
-    if track_path.exists():
+def _sector_size_for_track(
+    out_dir: Path, track: int, default_sector_size: int = 256
+) -> tuple[int, str]:
+    """Infer sector size for a track using track JSON or sector payloads."""
+
+    track_json_candidates = [
+        out_dir / "tracks" / f"track_{track:02d}.json",
+        out_dir / "tracks" / f"T{track:02d}.json",
+    ]
+
+    for track_path in track_json_candidates:
+        if not track_path.exists():
+            continue
         try:
             data = json.loads(track_path.read_text())
-            return int(data.get("sector_size", default) or default)
-        except (json.JSONDecodeError, OSError, ValueError):
-            return default
-    return default
+        except (json.JSONDecodeError, OSError):
+            continue
+
+        for key in ("selected_sector_size", "sector_size", "best_sector_size"):
+            if key not in data:
+                continue
+            try:
+                return int(data[key]), "track_json"
+            except (TypeError, ValueError):
+                continue
+
+    sector_files = list((out_dir / "sectors").glob(f"T{track:02d}_S*.bin"))
+    if sector_files:
+        size_counts = Counter(path.stat().st_size for path in sector_files)
+        max_count = max(size_counts.values())
+        candidates = [size for size, count in size_counts.items() if count == max_count]
+        chosen = max(candidates)
+        return chosen, "sector_files"
+
+    return default_sector_size, "default"
 
 
 def _slice_context(payload: bytes, offset: int, radius: int = 32) -> tuple[str, str]:
@@ -242,19 +268,39 @@ def scan_metadata(out_dir: Path) -> dict:
 
     sectors: list[SectorData] = []
     missing: list[str] = []
+    inferred_sizes: list[tuple[int, str]] = []
     for track in expected_tracks:
-        sector_size = _sector_size_for_track(track, out_dir, default_sector_size)
+        sector_size, size_source = _sector_size_for_track(
+            out_dir, track, default_sector_size
+        )
+        inferred_sizes.append((sector_size, size_source))
         for sector in range(logical_sectors):
             entry = _load_sector(track, sector, out_dir, sector_size)
             if entry is None:
                 missing.append(f"T{track:02d}_S{sector:02d}")
                 continue
             sectors.append(entry)
 
+    sector_size_inferred = default_sector_size
+    sector_size_source = "default"
+    if inferred_sizes:
+        size_counts = Counter(size for size, _ in inferred_sizes)
+        max_count = max(size_counts.values())
+        candidates = [size for size, count in size_counts.items() if count == max_count]
+        sector_size_inferred = max(candidates)
+        source_priority = {"track_json": 2, "sector_files": 1, "default": 0}
+        for size, source in inferred_sizes:
+            if size == sector_size_inferred and source_priority.get(
+                source, -1
+            ) >= source_priority.get(sector_size_source, -1):
+                sector_size_source = source
+
     summary = {
         "tracks": len(expected_tracks),
         "sectors_per_track": logical_sectors,
-        "sector_size": default_sector_size,
+        "sector_size": sector_size_inferred,
+        "sector_size_inferred": sector_size_inferred,
+        "sector_size_source": sector_size_source,
         "manifest_totals": manifest.get("totals", {}),
         "missing": missing,
     }

tests/test_catalog_report.py

@@ -10,15 +10,26 @@ def _build_recon_fixture(tmp_path: Path) -> Path:
     sectors_dir.mkdir(parents=True)
 
     # Craft a descriptor with three references and a matching pppp entry.
-    pointer_bytes = (0).to_bytes(2, "little") + (1).to_bytes(2, "little") + (2).to_bytes(2, "little")
-    sectors_dir.joinpath("T00_S00.bin").write_bytes(b"DATA=HELLO.TXT////" + pointer_bytes)
+    pointer_bytes = (
+        (0).to_bytes(2, "little")
+        + (1).to_bytes(2, "little")
+        + (2).to_bytes(2, "little")
+    )
+    sectors_dir.joinpath("T00_S00.bin").write_bytes(
+        b"DATA=HELLO.TXT////" + pointer_bytes
+    )
     sectors_dir.joinpath("T00_S01.bin").write_bytes(b"pppp=HELLO.TXT////" + b"X" * 16)
     sectors_dir.joinpath("T00_S02.bin").write_bytes(b"PAYLOAD-02")
     sectors_dir.joinpath("T00_S03.bin").write_bytes(b"PAYLOAD-03")
 
     manifest = {
         "image": {"path": "fixture.scp"},
-        "mapping": {"mode": "dense", "track_step": 1, "side": 0, "present_scp_tracks": [0]},
+        "mapping": {
+            "mode": "dense",
+            "track_step": 1,
+            "side": 0,
+            "present_scp_tracks": [0],
+        },
         "tracks": [
             {
                 "track_number": 0,
@@ -59,3 +70,23 @@ def test_catalog_report_produces_outputs(tmp_path: Path) -> None:
     text_contents = txt_path.read_text()
     assert "HELLO.TXT" in text_contents
     assert "catalog-report" in text_contents
+
+
+def test_catalog_report_defaults_to_recon_dir_output(tmp_path: Path) -> None:
+    recon_dir = _build_recon_fixture(tmp_path)
+
+    result = catalog_report.catalog_report(
+        recon_dir,
+        out_dir=None,
+        enable_pppp_descriptors=True,
+        min_refs=3,
+    )
+
+    json_path = recon_dir / "catalog_report" / "catalog_report.json"
+    txt_path = recon_dir / "catalog_report" / "catalog_report.txt"
+
+    assert json_path.exists()
+    assert txt_path.exists()
+
+    report = result["report"]
+    assert report["summary"]["totals"]["descriptors"] == 1

tests/test_scanmeta_sector_size.py

@@ -0,0 +1,53 @@
+import json
+from pathlib import Path
+
+from hardsector_tool.scanmeta import scan_metadata
+
+
+def _build_recon(tmp_path: Path) -> Path:
+    recon_dir = tmp_path / "recon"
+    sectors_dir = recon_dir / "sectors"
+    tracks_dir = recon_dir / "tracks"
+    sectors_dir.mkdir(parents=True, exist_ok=True)
+    tracks_dir.mkdir(parents=True, exist_ok=True)
+
+    manifest = {
+        "tracks": [
+            {
+                "track_number": 0,
+                "sector_size": 128,
+                "track_score": 1.0,
+            }
+        ],
+        "totals": {
+            "expected_sectors": 16,
+            "written_sectors": 16,
+            "missing_sectors": 0,
+        },
+    }
+    recon_dir.joinpath("manifest.json").write_text(json.dumps(manifest))
+
+    for sector in range(16):
+        payload = bytes([sector]) * 128
+        sectors_dir.joinpath(f"T00_S{sector:02d}.bin").write_bytes(payload)
+
+    return recon_dir
+
+
+def test_scan_metadata_prefers_track_json_then_sector_files(tmp_path: Path) -> None:
+    recon_dir = _build_recon(tmp_path)
+    track_json = recon_dir / "tracks" / "T00.json"
+    track_json.write_text(json.dumps({"selected_sector_size": 128}))
+
+    result = scan_metadata(recon_dir)
+    summary = result["summary"]
+    assert summary["sector_size"] == 128
+    assert summary["sector_size_inferred"] == 128
+    assert summary["sector_size_source"] == "track_json"
+
+    track_json.unlink()
+    result = scan_metadata(recon_dir)
+    summary = result["summary"]
+    assert summary["sector_size"] == 128
+    assert summary["sector_size_inferred"] == 128
+    assert summary["sector_size_source"] == "sector_files"