diff --git a/_project/TODO/main/planning/tuning-explorer-ingest-mode-extraction-20260712.yaml b/_project/DONE/main/active/tuning-explorer-ingest-mode-extraction-20260712.yaml similarity index 69% rename from _project/TODO/main/planning/tuning-explorer-ingest-mode-extraction-20260712.yaml rename to _project/DONE/main/active/tuning-explorer-ingest-mode-extraction-20260712.yaml index 62fcdc08e..40a6f2cc7 100644 --- a/_project/TODO/main/planning/tuning-explorer-ingest-mode-extraction-20260712.yaml +++ b/_project/DONE/main/active/tuning-explorer-ingest-mode-extraction-20260712.yaml @@ -2,7 +2,7 @@ id: tuning-explorer-ingest-mode-extraction-20260712 title: "Explorer ingest: read tuning_mode from all bundle generations; stop conflating absent with baseline" worktree: main priority: High -status: Not Started +status: Completed category: Core Functionality description: | @@ -28,20 +28,48 @@ description: | work: - id: w0 summary: "Re-validate pinned evidence at current HEAD: rerun the corpus sweep script (scratchpad tuning_sweep.py logic) and confirm field locations; commit compact counts in PR body" - status: pending + status: done needs: [] + notes: | + Re-validated 2026-07-16 against the 528 seed-corpus bundles in + results-data/bundles/ (sweep log: /tmp/ingest-sweep.log). + config.tuning_mode: 0; execution.tuning_mode: 325 (all "tuned"); + absent everywhere: 203. config.tuning_config/config.tuning: absent in + all 528. The repr-string tuning detail lives at + platform.raw_config.tuning_config (python_repr in 468 bundles) and is + never machine-readable in this corpus. - id: w1 summary: "transformer.py: read config.tuning_mode with execution.tuning_mode fallback; represent not-recorded explicitly; tuning_hash only from machine-readable config (never repr strings)" - status: pending + status: done needs: [w0] + notes: | + _tuning_mode: config.tuning_mode first, execution.tuning_mode fallback, + missing stays None (no invented mode). _tuning_hash: dict detail hashed + canonically (json sort_keys); string/repr detail dropped (not canonical); + mode-only hash still emitted. 8 new unit tests in + tests/unit/scripts/explorer_pipeline/test_transformer.py. - id: w2 summary: "facetMatching/receipt: surface not-recorded as its own display state (keep current matching rule until ADR-2; remove the invented 'untuned' default token in favor of the explicit state)" - status: pending + status: done needs: [w1] + notes: | + facetMatching.ts: NOT_RECORDED_TUNING_MODE ("not-recorded") shared + sentinel; null rows match only an explicit not-recorded selection + (legacy "untuned" token kept so existing chips/URLs match the same + rows -- matching semantics unchanged, rule changes gated on ADR-2). + TuningBadge: null/undefined/sentinel render neutral "Not Recorded" + badge instead of "Custom Tuning". ComparabilityReceipt already showed + "Not recorded" -- unchanged. New src/lib/__tests__/facetMatching.test.ts + plus TuningBadge test updates. - id: w3 summary: "Re-ingest the seed corpus and verify counts: 325 tuned / 203 not-recorded now visible in the explorer dataset" - status: pending + status: done needs: [w1] + notes: | + explorer_publish.py build over results-data/ processed 528 results / + 58 cohorts. results.duckdb: tuning_mode tuned=325, NULL(not-recorded)=203 + -- matches w0 exactly. tuning_hash: 325 rows share one mode-only hash, + 203 NULL. deps: needs: [] @@ -81,4 +109,5 @@ metadata: - comparability estimated_effort: Medium created_date: "2026-07-12" + completed_date: "2026-07-16" source_review: "tuning-system deep review 2026-07-12, finding C6" diff --git a/_project/scripts/explorer_pipeline/transformer.py b/_project/scripts/explorer_pipeline/transformer.py index c6ea42b51..9d25264a9 100644 --- a/_project/scripts/explorer_pipeline/transformer.py +++ b/_project/scripts/explorer_pipeline/transformer.py @@ -176,26 +176,50 @@ def _execution_mode(data: dict[str, Any]) -> str | None: def _tuning_mode(data: dict[str, Any]) -> str | None: - """Extract tuning mode from schema-v2 bundle config block.""" + """Extract tuning mode from a schema-v2 bundle. + + Reads ``config.tuning_mode`` first (the current schema location), falling + back to ``execution.tuning_mode`` for the seed-corpus generation that + wrote the value there instead. When neither location carries a value the + result stays ``None`` -- callers must not invent a mode for bundles that + never recorded one (see explorer ingest tuning-mode-extraction TODO). + """ config = data.get("config", {}) - if not isinstance(config, dict): - return None - val = config.get("tuning_mode") - return str(val) if val else None + if isinstance(config, dict): + val = config.get("tuning_mode") + if val: + return str(val) + execution = data.get("execution", {}) + if isinstance(execution, dict): + val = execution.get("tuning_mode") + if val: + return str(val) + return None def _tuning_hash(data: dict[str, Any]) -> str | None: """Compute a stable 8-char hash of the tuning configuration. - Combines tuning_mode and any tuning config dict so that two runs with - identical tuning produce the same hash. Returns None when no tuning info present. + Only machine-readable tuning detail is hashed: a dict is hashed + canonically (``json.dumps(..., sort_keys=True)``), but a string value + (e.g. a Python ``repr()`` of a dataclass, which is not canonical and + varies cosmetically between otherwise-identical configs) is dropped + rather than hashed. The resolved tuning mode (see ``_tuning_mode``, which + includes the ``execution.tuning_mode`` fallback) may still be hashed on + its own when no machine-readable detail is present. Returns None when + there is neither a mode nor machine-readable detail to hash. """ + mode = _tuning_mode(data) config = data.get("config", {}) - if not isinstance(config, dict): - return None - mode = config.get("tuning_mode") - tuning_detail = config.get("tuning_config") or config.get("tuning") - if not mode and not tuning_detail: + tuning_detail: dict[str, Any] | None = None + if isinstance(config, dict): + raw_detail = config.get("tuning_config") or config.get("tuning") + if isinstance(raw_detail, dict): + tuning_detail = raw_detail + # Non-dict detail (e.g. a repr() string) is intentionally not hashed: + # it is not canonical, so cosmetic differences would change the hash + # even when the underlying tuning configuration is identical. + if mode is None and tuning_detail is None: return None payload = json.dumps({"mode": mode, "detail": tuning_detail}, sort_keys=True) return hashlib.sha256(payload.encode()).hexdigest()[:8] diff --git a/results-explorer/src/components/TuningBadge.tsx b/results-explorer/src/components/TuningBadge.tsx index 3fd9cc035..eecab7077 100644 --- a/results-explorer/src/components/TuningBadge.tsx +++ b/results-explorer/src/components/TuningBadge.tsx @@ -2,6 +2,7 @@ // TuningBadge - renders tuning_mode as a StatusBadge with role="computed" // --------------------------------------------------------------------------- +import { NOT_RECORDED_TUNING_MODE } from "@/lib/facetMatching"; import { StatusBadge, type StatusTone } from "./StatusBadge"; interface TuningEntry { @@ -10,6 +11,12 @@ interface TuningEntry { title: string; } +const NOT_RECORDED_CONFIG: TuningEntry = { + label: "Not Recorded", + tone: "neutral", + title: "Tuning state unknown - the run predates tuning metadata or did not record it", +}; + export const TUNING_CONFIG: Record = { tuned: { label: "Tuned", @@ -26,6 +33,7 @@ export const TUNING_CONFIG: Record = { tone: "info", title: "Automatic tuning selected by the platform", }, + [NOT_RECORDED_TUNING_MODE]: NOT_RECORDED_CONFIG, }; const DEFAULT_CONFIG: TuningEntry = { @@ -34,16 +42,21 @@ const DEFAULT_CONFIG: TuningEntry = { title: "Non-standard tuning configuration", }; -export function tuningLabel(tuningMode: string): string { - return (TUNING_CONFIG[tuningMode] ?? DEFAULT_CONFIG).label; +function resolveConfig(tuningMode: string | null | undefined): TuningEntry { + if (tuningMode === null || tuningMode === undefined) return NOT_RECORDED_CONFIG; + return TUNING_CONFIG[tuningMode] ?? DEFAULT_CONFIG; +} + +export function tuningLabel(tuningMode: string | null | undefined): string { + return resolveConfig(tuningMode).label; } interface TuningBadgeProps { - tuningMode: string; + tuningMode: string | null | undefined; } export function TuningBadge({ tuningMode }: TuningBadgeProps) { - const config = TUNING_CONFIG[tuningMode] ?? DEFAULT_CONFIG; + const config = resolveConfig(tuningMode); return ( {config.label} diff --git a/results-explorer/src/components/__tests__/TuningBadge.test.tsx b/results-explorer/src/components/__tests__/TuningBadge.test.tsx index b5a249c99..a8c3ebd10 100644 --- a/results-explorer/src/components/__tests__/TuningBadge.test.tsx +++ b/results-explorer/src/components/__tests__/TuningBadge.test.tsx @@ -7,11 +7,14 @@ * (c) All modes have a title attribute (tooltip) * (d) tuningLabel() helper returns correct label for dropdown reuse * (e) Color semantics: tuned=green, notuning=gray, auto=blue + * (f) null/undefined and the not-recorded sentinel render the neutral + * "Not Recorded" badge (never "Custom Tuning") */ import { render } from "@testing-library/preact"; import { describe, expect, it } from "vitest"; import { TuningBadge, tuningLabel } from "@/components/TuningBadge"; +import { NOT_RECORDED_TUNING_MODE } from "@/lib/facetMatching"; describe("TuningBadge", () => { it("tuned renders with success tone and 'Tuned' label", () => { @@ -71,4 +74,33 @@ describe("TuningBadge", () => { it("tuningLabel() returns 'Custom Tuning' for unknown modes", () => { expect(tuningLabel("mystery")).toBe("Custom Tuning"); }); + + it("null tuning mode renders the neutral 'Not Recorded' badge, not 'Custom Tuning'", () => { + const { container } = render(); + const badge = container.querySelector(".badge"); + expect(badge?.getAttribute("data-tone")).toBe("neutral"); + expect(badge?.textContent).toBe("Not Recorded"); + expect(badge?.getAttribute("title")).toMatch(/predates tuning metadata|did not record/); + }); + + it("undefined tuning mode renders the neutral 'Not Recorded' badge", () => { + const { container } = render(); + const badge = container.querySelector(".badge"); + expect(badge?.getAttribute("data-tone")).toBe("neutral"); + expect(badge?.textContent).toBe("Not Recorded"); + }); + + it("the not-recorded sentinel token renders the same 'Not Recorded' badge", () => { + const { container } = render(); + const badge = container.querySelector(".badge"); + expect(badge?.getAttribute("data-tone")).toBe("neutral"); + expect(badge?.textContent).toBe("Not Recorded"); + expect(badge?.getAttribute("title")).toBeTruthy(); + }); + + it("tuningLabel() returns 'Not Recorded' for null/undefined/sentinel", () => { + expect(tuningLabel(null)).toBe("Not Recorded"); + expect(tuningLabel(undefined)).toBe("Not Recorded"); + expect(tuningLabel(NOT_RECORDED_TUNING_MODE)).toBe("Not Recorded"); + }); }); diff --git a/results-explorer/src/lib/__tests__/facetMatching.test.ts b/results-explorer/src/lib/__tests__/facetMatching.test.ts new file mode 100644 index 000000000..d94f91546 --- /dev/null +++ b/results-explorer/src/lib/__tests__/facetMatching.test.ts @@ -0,0 +1,70 @@ +/** + * Tests for facetMatching tuning-mode semantics. + * + * Rows whose bundle never recorded a tuning mode (tuning_mode null/undefined) + * are an explicit "not-recorded" state: + * (a) they never match a real-mode selection, + * (b) they match a selection that explicitly includes the not-recorded + * sentinel (or the legacy "untuned" token still used by chips/URLs), + * (c) real modes keep matching exactly as before (rule changes are gated + * on ADR-2 and owned by the facet implementation TODO). + */ + +import { describe, expect, it } from "vitest"; +import { + LEGACY_UNLABELLED_TUNING_MODE, + NOT_RECORDED_TUNING_MODE, + matchesFacetRow, + type FacetMatchRow, +} from "@/lib/facetMatching"; +import { DEFAULT_FACETS, type FacetState } from "@/lib/facetModel"; + +function facets(tuningMode: string[]): FacetState { + return { ...DEFAULT_FACETS, tuning_mode: tuningMode }; +} + +const TUNING_ONLY = { keys: ["tuning_mode"] as const }; + +function row(tuningMode: string | null | undefined): FacetMatchRow { + return { tuning_mode: tuningMode }; +} + +describe("facetMatching tuning_mode", () => { + it("empty selection matches every row, including not-recorded", () => { + expect(matchesFacetRow(row("tuned"), facets([]), TUNING_ONLY)).toBe(true); + expect(matchesFacetRow(row(null), facets([]), TUNING_ONLY)).toBe(true); + expect(matchesFacetRow(row(undefined), facets([]), TUNING_ONLY)).toBe(true); + }); + + it("real modes match their own selection", () => { + expect(matchesFacetRow(row("tuned"), facets(["tuned"]), TUNING_ONLY)).toBe(true); + expect(matchesFacetRow(row("auto"), facets(["tuned"]), TUNING_ONLY)).toBe(false); + expect(matchesFacetRow(row("notuning"), facets(["notuning", "auto"]), TUNING_ONLY)).toBe(true); + }); + + it("a not-recorded row never matches a real-mode selection", () => { + for (const selection of [["tuned"], ["notuning"], ["auto"], ["tuned", "auto"]]) { + expect(matchesFacetRow(row(null), facets(selection), TUNING_ONLY)).toBe(false); + expect(matchesFacetRow(row(undefined), facets(selection), TUNING_ONLY)).toBe(false); + } + }); + + it("a not-recorded row matches an explicit not-recorded selection", () => { + expect(matchesFacetRow(row(null), facets([NOT_RECORDED_TUNING_MODE]), TUNING_ONLY)).toBe(true); + expect(matchesFacetRow(row(undefined), facets([NOT_RECORDED_TUNING_MODE]), TUNING_ONLY)).toBe(true); + }); + + it("the legacy 'untuned' token still selects not-recorded rows (chips/URLs)", () => { + expect(matchesFacetRow(row(null), facets([LEGACY_UNLABELLED_TUNING_MODE]), TUNING_ONLY)).toBe(true); + expect(matchesFacetRow(row(undefined), facets([LEGACY_UNLABELLED_TUNING_MODE]), TUNING_ONLY)).toBe(true); + }); + + it("a recorded mode does not match the not-recorded sentinel selection", () => { + expect(matchesFacetRow(row("tuned"), facets([NOT_RECORDED_TUNING_MODE]), TUNING_ONLY)).toBe(false); + expect(matchesFacetRow(row("tuned"), facets([LEGACY_UNLABELLED_TUNING_MODE]), TUNING_ONLY)).toBe(false); + }); + + it("no producer emits the sentinel as a real mode, but if a row carried it verbatim it would match itself", () => { + expect(matchesFacetRow(row(NOT_RECORDED_TUNING_MODE), facets([NOT_RECORDED_TUNING_MODE]), TUNING_ONLY)).toBe(true); + }); +}); diff --git a/results-explorer/src/lib/__tests__/queryFilters.test.ts b/results-explorer/src/lib/__tests__/queryFilters.test.ts index 56c05f8f7..3b3d43f1a 100644 --- a/results-explorer/src/lib/__tests__/queryFilters.test.ts +++ b/results-explorer/src/lib/__tests__/queryFilters.test.ts @@ -104,6 +104,35 @@ describe("buildSelectQuery LIMIT", () => { }); }); +describe("buildWhereClause - tuning_mode not-recorded/untuned sentinels", () => { + it("matches NULL tuning_mode rows for the not-recorded token", () => { + const filters: QueryFilterState = { ...EMPTY_FILTERS, tuningModes: ["not-recorded"] }; + + const { sql, params } = buildSelectQuery(filters, ["result_id"], DEFAULT_SORT, 10); + + expect(sql).toContain("(tuning_mode IS NULL)"); + expect(params).toEqual([]); + }); + + it("still matches NULL tuning_mode rows for the legacy untuned token", () => { + const filters: QueryFilterState = { ...EMPTY_FILTERS, tuningModes: ["untuned"] }; + + const { sql, params } = buildSelectQuery(filters, ["result_id"], DEFAULT_SORT, 10); + + expect(sql).toContain("(tuning_mode IS NULL)"); + expect(params).toEqual([]); + }); + + it("combines a real tuning mode with the not-recorded sentinel via OR", () => { + const filters: QueryFilterState = { ...EMPTY_FILTERS, tuningModes: ["tuned", "not-recorded"] }; + + const { sql, params } = buildSelectQuery(filters, ["result_id"], DEFAULT_SORT, 10); + + expect(sql).toContain("(tuning_mode IN (?) OR tuning_mode IS NULL)"); + expect(params).toEqual(["tuned"]); + }); +}); + describe("buildWhereClause - normalized cost/deployment facets", () => { it("filters by normalized cost status and deployment metadata", () => { const filters: QueryFilterState = { diff --git a/results-explorer/src/lib/facetMatching.ts b/results-explorer/src/lib/facetMatching.ts index 1fd0c1c12..f711cbd92 100644 --- a/results-explorer/src/lib/facetMatching.ts +++ b/results-explorer/src/lib/facetMatching.ts @@ -1,12 +1,22 @@ import { FACET_KEYS, FACET_URL_KEYS, + LEGACY_UNLABELLED_TUNING_MODE, + NOT_RECORDED_TUNING_MODE, + NULL_TUNING_MODE_SENTINELS, dateWindowCutoffIso, type DateWindowFacet, type FacetKey, type FacetState, } from "@/lib/facetModel"; +// Re-exported so existing consumers (TuningBadge, tests) keep importing the +// not-recorded/legacy tuning tokens from here - facetModel.ts is the single +// source of truth (see NULL_TUNING_MODE_SENTINELS, shared with the SQL-side +// facetsToWhereClause/queryFilters filters so in-memory matching here can't +// drift from what the DuckDB-backed pages actually query for). +export { LEGACY_UNLABELLED_TUNING_MODE, NOT_RECORDED_TUNING_MODE }; + export interface FacetMatchRow { benchmark?: string | null; scale_factor?: string | number | null; @@ -96,7 +106,7 @@ function matchesFacetKey(row: FacetMatchRow, facets: FacetState, key: FacetKey, case "execution_mode": return matchesOptional(row.execution_mode, facets.execution_mode); case "tuning_mode": - return matchesRequired(row.tuning_mode ?? "untuned", facets.tuning_mode); + return matchesTuningMode(row.tuning_mode, facets.tuning_mode); case "trust_tier": return matchesRequired(row.trust_label, facets.trust_tier); case "validation_status": @@ -118,6 +128,17 @@ function matchesFacetKey(row: FacetMatchRow, facets: FacetState, key: FacetKey, } } +function matchesTuningMode(value: string | null | undefined, selected: readonly string[]): boolean { + if (selected.length === 0) return true; + if (value === null || value === undefined) { + // Not-recorded rows match only an explicit not-recorded selection -- + // never a real mode. The legacy "untuned" token keeps existing chip + // values and bookmarked URLs matching the same rows as before. + return NULL_TUNING_MODE_SENTINELS.some((sentinel) => selected.includes(sentinel)); + } + return selected.includes(value); +} + function matchesRequired(value: string | null | undefined, selected: readonly string[]): boolean { return selected.length === 0 || (value !== null && value !== undefined && selected.includes(value)); } diff --git a/results-explorer/src/lib/facetModel.ts b/results-explorer/src/lib/facetModel.ts index 5410d7ecd..fb281a156 100644 --- a/results-explorer/src/lib/facetModel.ts +++ b/results-explorer/src/lib/facetModel.ts @@ -22,6 +22,26 @@ export const FACET_KEYS = [ export type FacetKey = (typeof FACET_KEYS)[number]; export type DateWindowFacet = "all" | "30d" | "90d" | "365d"; +/** + * Explicit facet token for rows whose bundle never recorded a tuning mode. + * + * No producer emits this value as a real mode; it exists so "not recorded" + * is a first-class state instead of a null silently coalesced into a + * real-looking mode. Defined here (not in facetMatching.ts, which imports + * from this module) so every SQL- and in-memory-filtering consumer shares + * one source of truth instead of drifting. + */ +export const NOT_RECORDED_TUNING_MODE = "not-recorded"; + +/** Legacy unlabelled-tuning facet token; kept so existing chips and URLs keep matching. */ +export const LEGACY_UNLABELLED_TUNING_MODE = "untuned"; + +/** Every facet token that must match a NULL tuning_mode row, in SQL and in-memory alike. */ +export const NULL_TUNING_MODE_SENTINELS = [ + NOT_RECORDED_TUNING_MODE, + LEGACY_UNLABELLED_TUNING_MODE, +] as const; + export interface FacetState { benchmark: string[]; scale_factor: string[]; @@ -331,7 +351,7 @@ export function facetsToWhereClause( addListClause("test_type", facets.phase, clauses, params); addPlatformClause(facets.platform, clauses, params); addListClause("execution_mode", facets.execution_mode, clauses, params); - addNullableSentinelClause("tuning_mode", facets.tuning_mode, "untuned", clauses, params); + addNullableSentinelClause("tuning_mode", facets.tuning_mode, NULL_TUNING_MODE_SENTINELS, clauses, params); addListClause("trust_label", facets.trust_tier, clauses, params); addListClause("validation_status", facets.validation_status, clauses, params); addListClause(FACET_FILTER_COLUMNS.deployment_class, facets.deployment_class, clauses, params); @@ -471,21 +491,21 @@ function addPlatformClause(values: readonly string[], clauses: string[], params: params.push(...values, ...values); } -function addNullableSentinelClause( +export function addNullableSentinelClause( column: string, values: readonly string[], - nullSentinel: string, + nullSentinels: readonly string[], clauses: string[], params: unknown[], ) { if (values.length === 0) return; - const concreteValues = values.filter((value) => value !== nullSentinel); + const concreteValues = values.filter((value) => !nullSentinels.includes(value)); const subclauses: string[] = []; if (concreteValues.length > 0) { subclauses.push(`${column} IN (${concreteValues.map(() => "?").join(", ")})`); params.push(...concreteValues); } - if (values.includes(nullSentinel)) { + if (values.some((value) => nullSentinels.includes(value))) { subclauses.push(`${column} IS NULL`); } if (subclauses.length > 0) clauses.push(`(${subclauses.join(" OR ")})`); diff --git a/results-explorer/src/lib/queryFilters.ts b/results-explorer/src/lib/queryFilters.ts index d53bb13d7..5c34bc7ff 100644 --- a/results-explorer/src/lib/queryFilters.ts +++ b/results-explorer/src/lib/queryFilters.ts @@ -1,4 +1,9 @@ -import { dateWindowCutoffIso, type DateWindowFacet } from "@/lib/facetModel"; +import { + NULL_TUNING_MODE_SENTINELS, + addNullableSentinelClause, + dateWindowCutoffIso, + type DateWindowFacet, +} from "@/lib/facetModel"; export interface QueryFilterState { benchmarks: string[]; @@ -107,7 +112,7 @@ export function buildWhereClause(filters: QueryFilterState): BuiltQuery { clauses.push(`scale_factor IN (${filters.scaleFactors.map(() => "?").join(", ")})`); params.push(...filters.scaleFactors.map((value) => Number(value))); } - expandListFilter("tuning_mode", filters.tuningModes, clauses, params); + addNullableSentinelClause("tuning_mode", filters.tuningModes, NULL_TUNING_MODE_SENTINELS, clauses, params); expandListFilter("trust_label", filters.trustTiers, clauses, params); expandListFilter("validation_status", filters.validationStatuses, clauses, params); expandListFilter("cost_status", filters.costStatuses, clauses, params); diff --git a/tests/unit/core/tuning/test_ddl_generator_registry.py b/tests/unit/core/tuning/test_ddl_generator_registry.py index e61d8daae..c24891de9 100644 --- a/tests/unit/core/tuning/test_ddl_generator_registry.py +++ b/tests/unit/core/tuning/test_ddl_generator_registry.py @@ -69,7 +69,7 @@ def _discover_concrete_generator_classes() -> dict[str, type[BaseDDLGenerator]]: Walks the actual submodules (not just generators/__init__.py's __all__) so a new generator module is picked up even before anyone remembers to export it. Only classes *defined* in the module (not merely imported, - e.g. PostgreSQLDDLGenerator re-used by timescaledb.py) are counted, and + e.g. PostgreSQLDDLGenerator reused by timescaledb.py) are counted, and abstract classes (e.g. SparkBaseDDLGenerator) are excluded. """ discovered: dict[str, type[BaseDDLGenerator]] = {} @@ -90,7 +90,7 @@ def _registered_generator_class_names() -> set[str]: """Statically extract the class names in get_ddl_generator()'s `generators` mapping literal, by parsing the function's own source with `ast`. - This deliberately does not re-import/re-declare the mapping (that would + This deliberately does not re-import or redeclare the mapping (that would just be a second copy that could drift from the real one) - it inspects the actual dict literal inside get_ddl_generator() itself, so this test fails the moment a class stops being referenced there. diff --git a/tests/unit/scripts/explorer_pipeline/test_transformer.py b/tests/unit/scripts/explorer_pipeline/test_transformer.py index 195453615..433074a19 100644 --- a/tests/unit/scripts/explorer_pipeline/test_transformer.py +++ b/tests/unit/scripts/explorer_pipeline/test_transformer.py @@ -374,6 +374,138 @@ def test_tuning_hash_computed_when_present(self, tmp_path: Path) -> None: assert len(entry.tuning_hash) == 8 assert all(c in "0123456789abcdef" for c in entry.tuning_hash) + def test_tuning_mode_falls_back_to_execution_block(self, tmp_path: Path) -> None: + """Seed-corpus bundles wrote tuning_mode under execution, not config.""" + import copy + + data = copy.deepcopy(MINIMAL_BUNDLE) + data["execution"]["tuning_mode"] = "tuned" + bundle = tmp_path / "exec_tuning_mode.json" + bundle.write_text(json.dumps(data), encoding="utf-8") + + transformer = BundleTransformer() + entry = transformer.to_manifest_entry(bundle) + detail = transformer.to_detail_result(bundle, result_id="exec-tuning-mode") + + assert entry.tuning_mode == "tuned" + assert detail.tuning_mode == "tuned" + + def test_tuning_mode_prefers_config_over_execution(self, tmp_path: Path) -> None: + """When both locations carry a value, config.tuning_mode wins.""" + import copy + + data = copy.deepcopy(MINIMAL_BUNDLE) + data["config"]["tuning_mode"] = "custom" + data["execution"]["tuning_mode"] = "tuned" + bundle = tmp_path / "both_tuning_mode.json" + bundle.write_text(json.dumps(data), encoding="utf-8") + + transformer = BundleTransformer() + entry = transformer.to_manifest_entry(bundle) + + assert entry.tuning_mode == "custom" + + def test_tuning_mode_stays_none_when_absent_everywhere(self, tmp_path: Path) -> None: + """A bundle that never recorded tuning_mode must not be assigned a fake mode.""" + import copy + + data = copy.deepcopy(MINIMAL_BUNDLE) + bundle = tmp_path / "no_tuning_mode.json" + bundle.write_text(json.dumps(data), encoding="utf-8") + + transformer = BundleTransformer() + entry = transformer.to_manifest_entry(bundle) + + assert entry.tuning_mode is None + + def test_tuning_hash_uses_execution_fallback_mode(self, tmp_path: Path) -> None: + """tuning_hash resolves mode via the same execution.tuning_mode fallback.""" + import copy + + data = copy.deepcopy(MINIMAL_BUNDLE) + data["execution"]["tuning_mode"] = "tuned" + bundle = tmp_path / "exec_tuning_hash.json" + bundle.write_text(json.dumps(data), encoding="utf-8") + + transformer = BundleTransformer() + entry = transformer.to_manifest_entry(bundle) + + assert entry.tuning_hash is not None + assert len(entry.tuning_hash) == 8 + + def test_tuning_hash_dict_detail_is_hashed_canonically(self, tmp_path: Path) -> None: + """A dict tuning_config is machine-readable, so key order must not affect the hash.""" + import copy + + data_a = copy.deepcopy(MINIMAL_BUNDLE) + data_a["config"]["tuning_mode"] = "tuned" + data_a["config"]["tuning_config"] = {"a": 1, "b": 2} + bundle_a = tmp_path / "dict_a.json" + bundle_a.write_text(json.dumps(data_a), encoding="utf-8") + + data_b = copy.deepcopy(MINIMAL_BUNDLE) + data_b["config"]["tuning_mode"] = "tuned" + data_b["config"]["tuning_config"] = {"b": 2, "a": 1} + bundle_b = tmp_path / "dict_b.json" + bundle_b.write_text(json.dumps(data_b), encoding="utf-8") + + transformer = BundleTransformer() + entry_a = transformer.to_manifest_entry(bundle_a) + entry_b = transformer.to_manifest_entry(bundle_b) + + assert entry_a.tuning_hash == entry_b.tuning_hash + assert entry_a.tuning_hash is not None + + def test_tuning_hash_string_repr_detail_is_not_hashed(self, tmp_path: Path) -> None: + """A repr() string is not canonical: it must be dropped, not hashed verbatim. + + Two bundles with cosmetically different repr strings for the same mode + must produce the same hash (mode-only), proving the repr text itself + is excluded from the hash payload. + """ + import copy + + data_a = copy.deepcopy(MINIMAL_BUNDLE) + data_a["config"]["tuning_mode"] = "tuned" + data_a["config"]["tuning_config"] = ( + "UnifiedTuningConfiguration(primary_keys=PrimaryKeyConfiguration(enabled=True))" + ) + bundle_a = tmp_path / "repr_a.json" + bundle_a.write_text(json.dumps(data_a), encoding="utf-8") + + data_b = copy.deepcopy(MINIMAL_BUNDLE) + data_b["config"]["tuning_mode"] = "tuned" + data_b["config"]["tuning_config"] = ( + "UnifiedTuningConfiguration(primary_keys=PrimaryKeyConfiguration(enabled=False))" + ) + bundle_b = tmp_path / "repr_b.json" + bundle_b.write_text(json.dumps(data_b), encoding="utf-8") + + transformer = BundleTransformer() + entry_a = transformer.to_manifest_entry(bundle_a) + entry_b = transformer.to_manifest_entry(bundle_b) + mode_only = transformer.to_manifest_entry(bundle_a, data={**data_a, "config": {"tuning_mode": "tuned"}}) + + assert entry_a.tuning_hash is not None + assert entry_a.tuning_hash == entry_b.tuning_hash + assert entry_a.tuning_hash == mode_only.tuning_hash + + def test_tuning_hash_none_when_detail_is_repr_string_and_no_mode(self, tmp_path: Path) -> None: + """No mode + a non-canonical repr string detail: nothing machine-readable to hash.""" + import copy + + data = copy.deepcopy(MINIMAL_BUNDLE) + data["config"]["tuning_config"] = ( + "UnifiedTuningConfiguration(primary_keys=PrimaryKeyConfiguration(enabled=True))" + ) + bundle = tmp_path / "repr_no_mode.json" + bundle.write_text(json.dumps(data), encoding="utf-8") + + transformer = BundleTransformer() + entry = transformer.to_manifest_entry(bundle) + + assert entry.tuning_hash is None + def test_test_type_from_benchmark_block(self, bundle_file: Path) -> None: transformer = BundleTransformer() entry = transformer.to_manifest_entry(bundle_file)