From 459476cb54e39ea8ee149e99146be7df682c8567 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 21:33:47 +0000 Subject: [PATCH 1/3] feat: name leftover-map cosine on leftover pairs (v2.12.24) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Persist leftover-map cosine on leftover post–criterion pairs so a close leftover-map pair is not read as leftover-map alignment. After make seed, closest/farthest pairs sit above the member list with cosine next to leftover-map distance d; click opens that post. --- AGENTS.md | 8 +- ARCHITECTURE.md | 3 +- CHANGELOG.d/2.12.24-leftover-map-cosine.md | 7 ++ CHANGELOG.md | 9 ++ CLAUDE.md | 4 +- backend/app/report_ingestion.py | 13 ++- backend/tests/test_api.py | 12 ++ docker/postgres-init/migrate.sh | 2 +- .../0003-fast-mlsirm-report-integration.md | 5 +- docs/adr/0048-persist-lsirm-leftover-pairs.md | 3 +- docs/adr/0049-leftover-pair-report-ui.md | 4 +- docs/adr/0180-leftover-map-cosine.md | 78 +++++++++++++ frontend/package.json | 2 +- frontend/src/App.test.tsx | 8 +- frontend/src/App.tsx | 32 ++++-- frontend/src/api.ts | 1 + frontend/src/i18n.test.ts | 32 ++++++ frontend/src/i18n.ts | 44 +++++++ frontend/src/leftoverMapCosine.test.ts | 19 +++ frontend/src/leftoverMapCosine.ts | 26 +++++ lineageweave/leftover_pairs.py | 108 +++++++++++++----- lineageweave/period_report.py | 2 +- .../0180_report_leftover_map_cosine.sql | 8 ++ .../0180_report_leftover_map_cosine.sql | 4 + pyproject.toml | 2 +- scripts/seed_demo_data.py | 7 +- tests/test_leftover_pairs.py | 35 +++++- tests/test_migration_replay.py | 1 + tests/test_period_report.py | 3 + tests/test_schema.py | 27 +++++ uv.lock | 2 +- 31 files changed, 451 insertions(+), 60 deletions(-) create mode 100644 CHANGELOG.d/2.12.24-leftover-map-cosine.md create mode 100644 docs/adr/0180-leftover-map-cosine.md create mode 100644 frontend/src/leftoverMapCosine.test.ts create mode 100644 frontend/src/leftoverMapCosine.ts create mode 100644 migrations/0180_report_leftover_map_cosine.sql create mode 100644 migrations/rollback/0180_report_leftover_map_cosine.sql diff --git a/AGENTS.md b/AGENTS.md index 1728f9e61..30b5ef8a3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -186,12 +186,14 @@ in the same spirit) -- never against real data, per the hard rule above. against a live local stack (`make up`) and self-skip without one -- see [README.md](README.md#local-product-stack-docker-compose). -Period leftover pairs (ADR 0017 / 0018) are computed in +Period leftover pairs (ADR 0017 / 0018 / 0180) are computed in `lineageweave/leftover_pairs.py` from the residual after a real GRM/GPCM score, never invented. Missing cells stay out of the Gabriel factorization. Closest and farthest post–criterion pairs -persist to `report_leftover_pair` and sit above the member list so -a click opens that post. +persist to `report_leftover_pair` and sit above the member list with +leftover-map cosine next to leftover-map distance `d` so a click +opens that post. Omit cosine when the complete-case map is missing +or a vector sits at the origin. `frontend/` has its own toolchain (Node pinned via `frontend/mise.toml`, pnpm via Corepack -- do not add a second Node package manager or a diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d0280ff97..6344024d3 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -601,7 +601,8 @@ bank as the dummy high/low band rows, so comparison-strip click through opens those DAG posts. Report members include the earliest open ticket title, status lookup label, and due date when one exists. The home page renders the actual mean θ, the FIPC delta, the CAT-selected item, leftover -closest/farthest pairs above the member list, and the +closest/farthest pairs above the member list with leftover-map cosine +next to leftover-map distance `d`, and the PU / corp / thread comparison -- never a placeholder. TEPP is unchanged. ## Phase 6b: Knowledge Graph as a real Ontology + Semantic Layer diff --git a/CHANGELOG.d/2.12.24-leftover-map-cosine.md b/CHANGELOG.d/2.12.24-leftover-map-cosine.md new file mode 100644 index 000000000..3856da7a3 --- /dev/null +++ b/CHANGELOG.d/2.12.24-leftover-map-cosine.md @@ -0,0 +1,7 @@ +## 2.12.24 — Leftover-map cosine + +- Persist leftover-map cosine on leftover post–criterion pairs + (ADR 0180). After `make seed`, closest and farthest leftover pairs + sit above the member list with cosine next to leftover-map distance + `d`; click opens that post. Omit the badge when cosine is missing. + Never invent a leftover score. diff --git a/CHANGELOG.md b/CHANGELOG.md index c8ed1a099..9e0a53e57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,15 @@ All notable changes to this project are documented here. Format follows environment, so local OIDC and synthetic-data workflows resolve the same pinned dependencies as CI. +## [2.12.24] - 2026-08-24 + +### Added + +- Period leftover pair rows now name leftover-map cosine next to + leftover-map distance `d`, then open that post (Gabriel, 1971; + Jeon et al., 2021, eq. 3; ADR 0180). A missing cosine omits the + badge rather than inventing leftover-map alignment. + ## [2.12.6] - 2026-08-20 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 1bcf50763..15f16e15d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,7 +54,9 @@ chip name contains `Corporate entity: Demo Corp` and the persisted mean θ. The period-report panel says Demo Corp is the opened grouping and to read its mean θ and member posts, then open a post. Those members land immediately under that next action, ahead of Other Corp -and the week strip. Opening Public post names the next action: read +and the week strip. After `make seed`, leftover closest/farthest pairs +sit above the member list with leftover-map cosine next to leftover-map +distance `d`. Opening Public post names the next action: read Event Lineage, Keyman, and evaluation on that post. The popup Event Lineage DAG marks that post current. After that current node, the popup names Keyman and evaluation as the next read. After landed diff --git a/backend/app/report_ingestion.py b/backend/app/report_ingestion.py index 50614b0ad..0fdb31604 100644 --- a/backend/app/report_ingestion.py +++ b/backend/app/report_ingestion.py @@ -443,8 +443,9 @@ async def persist_period_report( """ insert into report_leftover_pair ( grouping_kind, grouping_key, period_code, rubric_version, - pair_kind, post_id, criterion_code, leftover_distance, leftover_residual - ) values ($1,$2,$3,$4,$5,$6,$7,$8,$9) + pair_kind, post_id, criterion_code, leftover_distance, leftover_residual, + leftover_map_cosine + ) values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) """, grouping_kind, grouping_key, @@ -455,6 +456,7 @@ async def persist_period_report( pair.criterion_code, pair.leftover_distance, pair.leftover_residual, + pair.leftover_map_cosine, ) @@ -601,7 +603,7 @@ async def fetch_period_reports( leftover = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli f""" select lp.grouping_key, lp.pair_kind, lp.post_id, lp.criterion_code, - lp.leftover_distance, lp.leftover_residual, p.post_title, + lp.leftover_distance, lp.leftover_residual, lp.leftover_map_cosine, p.post_title, p.visibility_code, p.corporate_entity_id, ({_SOURCE_CONTEXT_PRESENT_SQL}) as has_real_source_context from report_leftover_pair lp @@ -698,6 +700,11 @@ async def fetch_period_reports( "criterion_code": str(row["criterion_code"]), "leftover_distance": float(row["leftover_distance"]), "leftover_residual": float(row["leftover_residual"]), + "leftover_map_cosine": ( + None + if row["leftover_map_cosine"] is None + else float(row["leftover_map_cosine"]) + ), "visibility_code": row["visibility_code"], "corporate_entity_id": str(row["corporate_entity_id"]), "has_real_source_context": bool(row["has_real_source_context"]), diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 438b4786a..dca63b88e 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -113,6 +113,11 @@ / "migrations" / "0102_project_bound_summary_event.sql" ) +_LEFTOVER_MAP_COSINE_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0180_report_leftover_map_cosine.sql" +) def _postgres_available() -> bool: @@ -226,6 +231,7 @@ def seeded_db(demo_analyst_token): cur.execute(_MAJOR_EVENT_ACTION_MIGRATION.read_text()) cur.execute(_PROJECT_BOUND_ACTION_MIGRATION.read_text()) cur.execute(_PROJECT_BOUND_EVENT_MIGRATION.read_text()) + cur.execute(_LEFTOVER_MAP_COSINE_MIGRATION.read_text()) cur.execute( "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) values " "('corporate_entity_level', 'group', 'Group'), " @@ -4518,6 +4524,12 @@ def test_seed_period_report_surfaces_on_get_reports(client, demo_analyst_token, assert leftover_kinds <= {"closest", "farthest"} assert all(pair["post_title"] for pair in high_report.get("leftover_pairs", [])) assert all(pair["leftover_distance"] >= 0 for pair in high_report.get("leftover_pairs", [])) + for pair in high_report.get("leftover_pairs", []): + cosine = pair.get("leftover_map_cosine") + if cosine is None: + continue + assert isinstance(cosine, (int, float)) + assert -1.0 <= float(cosine) <= 1.0 week3 = client.get( "/api/reports/process_unit/2026-W03", diff --git a/docker/postgres-init/migrate.sh b/docker/postgres-init/migrate.sh index f329117d6..780f2ef5a 100644 --- a/docker/postgres-init/migrate.sh +++ b/docker/postgres-init/migrate.sh @@ -18,7 +18,7 @@ for migration in /opt/lineageweave/migrations/*.sql; do migration_name=${migration##*/} case "$migration_name" in 0012_*|0013_*|0014_*|0015_*|0016_*|0017_*|0018_*|0019_*|0020_*|0021_*|0022_*|0023_*|0024_*|0025_*|0026_*|0027_*|0028_*|0029_*|0030_*|0031_*|0032_*|0033_*|0034_*|0035_*|0036_*|0037_*|0038_*|0039_*|0040_*|0041_*|0042_*|0043_*|0044_*|0045_*|0046_*|0047_*|0048_*|0049_*|0050_*) ;; - 0060_*|0100_*|0101_*|0102_*) ;; + 0060_*|0100_*|0101_*|0102_*|0180_*) ;; *) continue ;; esac printf 'Applying %s\n' "$migration_name" diff --git a/docs/adr/0003-fast-mlsirm-report-integration.md b/docs/adr/0003-fast-mlsirm-report-integration.md index bdf234b65..d46e42a74 100644 --- a/docs/adr/0003-fast-mlsirm-report-integration.md +++ b/docs/adr/0003-fast-mlsirm-report-integration.md @@ -100,9 +100,10 @@ than one large PR: `information_polytomous` (Lord, 1980 max-info). Persist the ranking (`report_item_information`) and show the rank-1 item on the Period reports panel. Do not reimplement an information function here. -7. **Leftover-pair slice** (shipped in 0.71.2; ADR 0017 / 0018): after +7. **Leftover-pair slice** (shipped in 0.71.2; ADR 0017 / 0018 / 0180): after IRT main effects, persist closest and farthest post–criterion pairs - from the residual leftover map. Do not fork LSIRM; do not invent a + from the residual leftover map, including leftover-map cosine when + Gabriel coordinates have non-zero norms. Do not fork LSIRM; do not invent a leftover-pair API inside `fast-mlsirm` in this slice. **TEPP boundary.** [ARCHITECTURE.md](../../ARCHITECTURE.md) already diff --git a/docs/adr/0048-persist-lsirm-leftover-pairs.md b/docs/adr/0048-persist-lsirm-leftover-pairs.md index 8e87383f0..80b2dc1cf 100644 --- a/docs/adr/0048-persist-lsirm-leftover-pairs.md +++ b/docs/adr/0048-persist-lsirm-leftover-pairs.md @@ -40,7 +40,8 @@ the IRT matrix is unusable. A rank-0 residual still emits a stable pair so `make seed` is not empty; the stored distance is then zero, not a fabricated interaction. -The UI contract is ADR 0049. +The UI contract is ADR 0049. Leftover-map cosine on those pair rows +is [ADR 0180](0180-leftover-map-cosine.md). ## Consequences diff --git a/docs/adr/0049-leftover-pair-report-ui.md b/docs/adr/0049-leftover-pair-report-ui.md index a93985b40..b9c731e6e 100644 --- a/docs/adr/0049-leftover-pair-report-ui.md +++ b/docs/adr/0049-leftover-pair-report-ui.md @@ -23,7 +23,9 @@ farthest from after main effects.”). Clicking the button opens that post with the same handler as a member row. After `make seed`, closest and farthest leftover pairs sit above the -member list. Click a pair to open that post. +member list with leftover-map cosine next to leftover-map distance +`d` when the complete-case map supplies non-origin coordinates +([ADR 0180](0180-leftover-map-cosine.md)). Click a pair to open that post. Missing leftover rows render nothing — never a placeholder pair. A hidden post never appears as a leftover pair. diff --git a/docs/adr/0180-leftover-map-cosine.md b/docs/adr/0180-leftover-map-cosine.md new file mode 100644 index 000000000..9c7cdbb8d --- /dev/null +++ b/docs/adr/0180-leftover-map-cosine.md @@ -0,0 +1,78 @@ +# ADR 0180 — Name leftover-map cosine on period-report pair rows + +**Decision status:** Accepted +**Date:** 2026-08-24 + +Amends [ADR 0048](0048-persist-lsirm-leftover-pairs.md) and +[ADR 0049](0049-leftover-pair-report-ui.md). + +## Context + +ADR 0048 already persists leftover-map distance `d = ‖ξ_p − ζ_i‖` and +leftover residual `R = Y − E[Y|θ, item]` on `report_leftover_pair`. +ADR 0049 already renders closest and farthest pairs above the member +list and opens the named post. Distance is the Jeon et al. (2021, +eq. 3) map gap. Gabriel (1971) also names leftover-map alignment as +the cosine of the angle between `ξ_p` and `ζ_i`. Hiding that cosine +lets a buyer read a large reconstructed leftover cell, or a close +map pair, as leftover-map alignment without a scale-free value. + +This increment does not persist leftover-map coordinates, does not +name leftover-map inner product, does not name observed `Y` / expected +`E`, does not name leftover-map rank, does not split leftover-map +distance onto two axes, and does not land Post quality on the leftover +criterion. + +The unprotected-stack reconstructions for neighbouring leftover facts +use 0162–0179. This protected-main increment uses **0180** so it does +not collide with leftover-map inner product (0179), leftover residual +disclosure (0178), leftover observed `Y` / expected `E` (0177), +leftover-map rank (0172), two-axis leftover-map distance (0166), +leftover coverage (0168), leftover-map axis share (0148), or leftover +interaction-map persistence (0121). + +## Decision + +Each leftover pair names `leftover_map_cosine` — the cosine of the +angle between leftover-map person and item coordinates that produced +leftover-map distance `d`. Migration `0180` is the single source of +the column on every install path, fresh or existing -- shipped +migrations (`0001` / `0012`) are never edited after the fact. The +column is nullable so older leftover rows keep distance and residual +without fabricating alignment. Fallback pairs that have no +complete-case leftover map, and pairs whose person or item vector +sits at the origin, omit the value rather than inventing one. + +The pair button shows `cos {signed}` next to leftover-map distance +`d` when the value is finite. Next action: leftover-map cosine names +leftover-map alignment independent of distance; open this post to +read the named criterion. A missing or non-finite cosine omits the +badge and keeps the existing closest/farthest next action. Do not +invent a leftover score. Do not invent a theta. + +## Consequences + +`GET /api/reports/{grouping}/{period}` returns `leftover_map_cosine`. +After `make seed`, closest and farthest leftover pairs sit above the +member list with named cosine next to `d`; click opens that post. +Hidden posts stay hidden. + +## Related + +Independent of leftover interaction-map persistence, leftover-criterion +evaluation landing, leftover residual disclosure, leftover observed +`Y` / expected `E`, leftover-map complete-case coverage, leftover-map +axis share, leftover pairs on the grouping comparison strip, two-axis +leftover-map distance, leftover-map rank, and leftover-map inner +product. + +## References + +Gabriel, K. R. (1971). The biplot graphic display of matrices with +application to principal component analysis. *Biometrika, 58*(3), +453–467. https://doi.org/10.1093/biomet/58.3.453 + +Jeon, M., Jin, I. H., Schweinberger, M., & Baugh, S. (2021). Mapping +unobserved item–respondent interactions: A latent space item response +model with interaction map. *Psychometrika, 86*(2), 378–403. +https://doi.org/10.1007/s11336-021-09762-5 diff --git a/frontend/package.json b/frontend/package.json index e2e996bbe..168420b0f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "2.12.6", + "version": "2.12.24", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 7462abd2c..d84e99252 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -932,6 +932,7 @@ describe("App, authenticated", () => { criterion_code: "sales_lead_specificity", leftover_distance: 0.12, leftover_residual: 0.4, + leftover_map_cosine: 0.95, }, { pair_kind: "farthest", @@ -940,6 +941,7 @@ describe("App, authenticated", () => { criterion_code: "general_sentiment_negative", leftover_distance: 1.84, leftover_residual: -1.1, + leftover_map_cosine: -0.91, }, ], members: [ @@ -3357,13 +3359,15 @@ describe("App, authenticated", () => { }); expect(closestPair).toHaveTextContent("Closest leftover: Public post · sales-lead"); expect(closestPair).toHaveTextContent( - "Open this post to read the criterion it sat closest to after main effects.", + "Leftover-map cosine +0.95 names leftover-map alignment independent of distance. Open this post to read sales-lead.", ); + expect(closestPair).toHaveTextContent("cos +0.95"); expect(closestPair).toHaveTextContent("d 0.12"); expect(farthestPair).toHaveTextContent("Farthest leftover: Specification revision requested · negative"); expect(farthestPair).toHaveTextContent( - "Open this post to read the criterion it sat farthest from after main effects.", + "Leftover-map cosine −0.91 names leftover-map alignment independent of distance. Open this post to read negative.", ); + expect(farthestPair).toHaveTextContent("cos −0.91"); expect(farthestPair).toHaveTextContent("d 1.84"); const memberButton = screen.getByRole("button", { name: /open report post: public post/i }); expect(closestPair.compareDocumentPosition(memberButton) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 6fba0dd41..6a63546db 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -102,6 +102,11 @@ import { useLocale, } from "./i18n"; import { rememberOidcReturnUrl, returnUrlFromLocation } from "./oidcReturnUrl"; +import { + formatLeftoverMapCosine, + formatSignedLeftoverValue, + LEFTOVER_MAP_COSINE_ACTION, +} from "./leftoverMapCosine"; import "./App.css"; function orchestratorUnavailableMessage(err: unknown, action: string): string { @@ -3390,15 +3395,21 @@ function ReportsPanel({ )} {report.leftover_pairs && report.leftover_pairs.length > 0 && ( -