Per-level card text: fix the collapse, restore per-level titles, un-break deepseek

tools/audit_stored_article_languages.py

@@ -14,7 +14,7 @@
 parent's — which is what makes this safe for cross-language shares:
 
     article.summary                 on originals
-    ArticleLevelSummary.summary     the per-level feed-card blurbs; these only
+    LevelAdaptedArticleText.summary     the per-level feed-card blurbs; these only
                                     ever exist on originals (the assess step
                                     skips children), so they are always in the
                                     original's language
@@ -45,7 +45,7 @@
   - article.summary is nulled, along with the cached tokenized copy of it (that
     cache is only ever written when empty, so a stale one would outlive the
     regeneration and keep rendering);
-  - bad ArticleLevelSummary rows are deleted, with the bookmark anchors that
+  - bad LevelAdaptedArticleText rows are deleted, with the bookmark anchors that
     reference them (a real FK, no cascade) removed first;
   - child articles are marked broken (LLM_WRONG_LANGUAGE), which takes a
     simplified one out of available_simplified_versions and a translated one out of
@@ -71,8 +71,8 @@
 from zeeguu.core.language.language_check import language_mismatch, describe_mismatches
 from zeeguu.core.model.article import Article
 from zeeguu.core.model.article_broken_code_map import LowQualityTypes
-from zeeguu.core.model.article_level_summary import ArticleLevelSummary
-from zeeguu.core.model.article_level_summary_context import ArticleLevelSummaryContext
+from zeeguu.core.model.level_adapted_article_text import LevelAdaptedArticleText
+from zeeguu.core.model.level_adapted_article_summary_context import LevelAdaptedArticleSummaryContext
 from zeeguu.core.model.article_tokenization_cache import ArticleTokenizationCache
 from zeeguu.core.model.language import Language
 
@@ -188,12 +188,12 @@ def rows_for(articles, rows_by_article):
 
 
 def audit_level_summaries(articles):
-    """ArticleLevelSummary rows — the tappable per-level feed-card blurbs."""
+    """LevelAdaptedArticleText rows — the tappable per-level feed-card blurbs."""
     by_id = {article.id: article for article in articles}
     if not by_id:
         return [], []
-    rows = ArticleLevelSummary.query.filter(
-        ArticleLevelSummary.article_id.in_(list(by_id))
+    rows = LevelAdaptedArticleText.query.filter(
+        LevelAdaptedArticleText.article_id.in_(list(by_id))
     ).all()
 
     wrong, wrong_rows = [], []
@@ -334,7 +334,7 @@ def is_level_adaptation(child):
            "these want marking broken", wrong_language_rows)
 
     wrong_level_summaries, level_summary_rows = audit_level_summaries(originals)
-    report("ArticleLevelSummary.summary", wrong_level_summaries)
+    report("LevelAdaptedArticleText.summary", wrong_level_summaries)
 
     wrong_simplified, simplified_articles = audit_child_articles(simplified)
     report("simplified children (same language as parent)", wrong_simplified)
@@ -399,14 +399,14 @@ def is_level_adaptation(child):
             cache.tokenized_summary = None
             session.add(cache)
 
-    # Bookmarks anchor to a SPECIFIC level summary (ArticleLevelSummaryContext
+    # Bookmarks anchor to a SPECIFIC level summary (LevelAdaptedArticleSummaryContext
     # holds a real FK with no cascade), so the anchors have to go first or the
     # delete fails on the constraint and takes the whole batch down with it.
     # They anchor into text we are throwing away, so there is nothing to keep.
     if level_summary_rows:
         summary_ids = [row.id for row in level_summary_rows]
-        anchors = ArticleLevelSummaryContext.query.filter(
-            ArticleLevelSummaryContext.article_level_summary_id.in_(summary_ids)
+        anchors = LevelAdaptedArticleSummaryContext.query.filter(
+            LevelAdaptedArticleSummaryContext.level_adapted_article_text_id.in_(summary_ids)
         ).all()
         for anchor in anchors:
             session.delete(anchor)

tools/backfill_reassess_summaries.py

@@ -10,7 +10,7 @@
 
 Both are fixed in code now. This tool re-runs the SAME crawl-time assessment
 (assess_summarize_and_classify) on the affected originals, which overwrites
-cefr_level, article_type, summary, and every ArticleLevelSummary row (find_or_
+cefr_level, article_type, summary, and every LevelAdaptedArticleText row (find_or_
 create updates in place) — in the correct language, once 7cd9d6ca is DEPLOYED.
 
 IMPORTANT: run this only AFTER the prompt fix (7cd9d6ca) is live on the server,

tools/migrations/26-08-28--level_adapted_article_text.sql

@@ -0,0 +1,63 @@
+-- Per-level headlines, plus the rename that makes room for them.
+--
+-- WHY THE RENAME. article_level_summary reads as "the article's level" or
+-- "article-level granularity"; the thing it stores is an article's text ADAPTED
+-- TO a level. Adding a title to the same row makes "..._summary" wrong outright,
+-- so the table becomes level_adapted_article_text and the two context types
+-- become LevelAdaptedArticleSummary / LevelAdaptedArticleTitle.
+--
+-- WHY THE TITLES. Before on-demand simplification the feed card DID show a
+-- level-appropriate headline: the overlay borrowed title and summary off the
+-- level-matched simplified child article. When the crawl stopped generating
+-- those children (commit 0d047e32), the per-level summary replaced the summary
+-- half and nothing replaced the title half. That matters more than it sounds:
+-- the default feed view is Headlines, which renders the title and NO summary, so
+-- with no per-level title the CEFR selector changes nothing a default-view
+-- reader can see.
+--
+-- Renaming is cheap here: bookmark_context rows point at context_type by id, and
+-- UPDATE preserves the id, so the existing rows need no migration at all.
+--
+-- IF YOU ALREADY APPLIED 26-08-28--add_article_level_title.sql (an earlier draft
+-- of this file, superseded by the rename), undo it first — it created an empty
+-- table and empty columns that this file recreates under the right names:
+--     DROP TABLE article_level_title_context;
+--     ALTER TABLE article_level_summary DROP COLUMN title, DROP COLUMN tokenized_title;
+--     DELETE FROM context_type WHERE type = 'ArticleLevelTitle';
+
+RENAME TABLE article_level_summary TO level_adapted_article_text;
+RENAME TABLE article_level_summary_context TO level_adapted_article_summary_context;
+
+ALTER TABLE level_adapted_article_summary_context
+    CHANGE COLUMN article_level_summary_id level_adapted_article_text_id INT;
+
+-- The level's headline and its token stream. Nullable on purpose: every row
+-- written before per-level titles has none, and a title dropped by the
+-- wrong-language check leaves its summary in place. Both fall back to the
+-- article's own title.
+ALTER TABLE level_adapted_article_text
+    ADD COLUMN title TEXT AFTER tokenized_summary,
+    ADD COLUMN tokenized_title JSON AFTER title;
+
+-- The tap-to-translate join for those headlines. Separate from the summary
+-- context even though both point at the same level_adapted_article_text row: a
+-- level's title and its summary are two different token streams, and one join
+-- table for both would return the title's bookmarks when highlighting the
+-- summary.
+CREATE TABLE level_adapted_article_title_context (
+    id INT AUTO_INCREMENT PRIMARY KEY,
+    bookmark_id INT NOT NULL,
+    level_adapted_article_text_id INT,
+    CONSTRAINT fk_latc_bookmark FOREIGN KEY (bookmark_id) REFERENCES bookmark (id),
+    CONSTRAINT fk_latc_text FOREIGN KEY (level_adapted_article_text_id) REFERENCES level_adapted_article_text (id) ON DELETE CASCADE,
+    -- One context row per (bookmark, level title): makes a concurrent-insert
+    -- race fail with IntegrityError (which find_or_create catches + re-queries)
+    -- instead of silently creating a duplicate that later breaks .one().
+    UNIQUE KEY uq_latc_bookmark_title (bookmark_id, level_adapted_article_text_id)
+);
+
+-- Same row, new spelling: the id is preserved, so every bookmark_context already
+-- pointing at it stays valid.
+UPDATE context_type SET type = 'LevelAdaptedArticleSummary' WHERE type = 'ArticleLevelSummary';
+
+INSERT INTO context_type (type) VALUES ('LevelAdaptedArticleTitle');

zeeguu/core/account_management/user_account_deletion.py

@@ -39,7 +39,8 @@
 from zeeguu.core.model.user_word_interaction_history import UserWordInteractionHistory
 from zeeguu.core.model.example_sentence_context import ExampleSentenceContext
 from zeeguu.core.model.article_summary_context import ArticleSummaryContext
-from zeeguu.core.model.article_level_summary_context import ArticleLevelSummaryContext
+from zeeguu.core.model.level_adapted_article_summary_context import LevelAdaptedArticleSummaryContext
+from zeeguu.core.model.level_adapted_article_title_context import LevelAdaptedArticleTitleContext
 from zeeguu.core.model.article_fragment_context import ArticleFragmentContext
 from zeeguu.core.model.article_title_context import ArticleTitleContext
 from zeeguu.core.model.video_caption_context import VideoCaptionContext
@@ -68,7 +69,8 @@
 bookmark_context_tables = [
     ExampleSentenceContext,
     ArticleSummaryContext,
-    ArticleLevelSummaryContext,
+    LevelAdaptedArticleSummaryContext,
+    LevelAdaptedArticleTitleContext,
     ArticleFragmentContext,
     ArticleTitleContext,
     VideoCaptionContext,

zeeguu/core/content_recommender/elastic_recommender.py

@@ -531,27 +531,32 @@ def get_user_info_from_content_recommendations(user, content_list):
 def _apply_simplified_display_overlay(user, results):
     """
     Overlay a CEFR-level-matched preview *summary* onto feed-card result dicts
-    that point at an original article, using the per-level ArticleLevelSummary
+    that point at an original article, using the per-level LevelAdaptedArticleText
     rows (on-demand simplification means there are no simplified child articles
     to borrow a summary from anymore).
 
     Sets both the plain ``summary`` teaser (Preview mode) and the tappable
     ``interactiveSummary`` payload (Interactive mode), the latter anchored to the
     specific level-summary row so tap-to-translate and past-bookmark highlighting
-    land on the right tokens. Titles are left as the original — we don't produce
-    per-level titles. Falls back silently to the article's own summary when the
-    learner's level has no simpler summary.
+    land on the right tokens. The level's ``title``/``interactiveTitle`` are
+    overlaid the same way when the row has one — which is what makes the level
+    selector visible in Headlines mode, where the title is the only text on the
+    card. Falls back silently to the article's own title/summary when the
+    learner's level has no simpler one.
 
     Batched in two queries: a columns-only pick of the best level per article,
     then a load of just those chosen rows (so the heavy tokenized_summary JSON is
     deserialized once per article, not once per level).
     """
-    from zeeguu.core.model.article_level_summary import (
-        ArticleLevelSummary,
+    from zeeguu.core.model.level_adapted_article_text import (
+        LevelAdaptedArticleText,
         CEFR_ORDER,
     )
-    from zeeguu.core.model.article_level_summary_context import (
-        ArticleLevelSummaryContext,
+    from zeeguu.core.model.level_adapted_article_summary_context import (
+        LevelAdaptedArticleSummaryContext,
+    )
+    from zeeguu.core.model.level_adapted_article_title_context import (
+        LevelAdaptedArticleTitleContext,
     )
     from zeeguu.core.model.context_identifier import ContextIdentifier
     from zeeguu.core.model.context_type import ContextType
@@ -573,43 +578,54 @@ def _apply_simplified_display_overlay(user, results):
     if not candidate_ids:
         return
 
-    allowed = ArticleLevelSummary.allowed_levels(user_cefr_level)
+    allowed = LevelAdaptedArticleText.allowed_levels(user_cefr_level)
 
     # Two steps so we deserialize the heavy tokenized_summary JSON for only the ONE
     # best row per article, never every level: first a columns-only query to pick
     # the best-matching level per article, then load just those chosen rows.
+    # Article.cefr_level rides along so pick_best can tell "this learner reads at
+    # or above the article's own level" (→ use the article's own summary) from
+    # "this learner needs a simpler one". Joined here rather than read off the
+    # result dicts, whose metrics.cefr_level is the *effective* level and can come
+    # back compound ("B1/B2") — see article_info.
     lightweight = (
-        ArticleLevelSummary.query
+        LevelAdaptedArticleText.query
         .with_entities(
-            ArticleLevelSummary.id,
-            ArticleLevelSummary.article_id,
-            ArticleLevelSummary.cefr_level,
+            LevelAdaptedArticleText.id,
+            LevelAdaptedArticleText.article_id,
+            LevelAdaptedArticleText.cefr_level,
+            Article.cefr_level.label("article_own_level"),
         )
+        .join(Article, Article.id == LevelAdaptedArticleText.article_id)
         .filter(
-            ArticleLevelSummary.article_id.in_(candidate_ids),
-            ArticleLevelSummary.cefr_level.in_(allowed),
+            LevelAdaptedArticleText.article_id.in_(candidate_ids),
+            LevelAdaptedArticleText.cefr_level.in_(allowed),
         )
         .all()
     )
     if not lightweight:
         return
 
     by_article = {}
+    own_level_by_article = {}
     for row in lightweight:
         by_article.setdefault(row.article_id, []).append(row)
+        own_level_by_article[row.article_id] = row.article_own_level
 
     chosen_id_by_article = {}
     for article_id, rows in by_article.items():
-        best_row = ArticleLevelSummary.pick_best(rows, user_cefr_level)
+        best_row = LevelAdaptedArticleText.pick_best(
+            rows, user_cefr_level, own_level_by_article.get(article_id)
+        )
         if best_row:
             chosen_id_by_article[article_id] = best_row.id
     if not chosen_id_by_article:
         return
 
     full_by_id = {
         als.id: als
-        for als in ArticleLevelSummary.query.filter(
-            ArticleLevelSummary.id.in_(chosen_id_by_article.values())
+        for als in LevelAdaptedArticleText.query.filter(
+            LevelAdaptedArticleText.id.in_(chosen_id_by_article.values())
         ).all()
     }
 
@@ -626,28 +642,55 @@ def _apply_simplified_display_overlay(user, results):
         if not display:
             continue
 
+        # The bookmark mapping keys on level_adapted_article_text_id; article_id is
+        # carried only for the client's MWE-ungroup path (parent article id).
+        def _payload(tokens, context_type, past_bookmarks):
+            overrides_by_hash = overrides_by_article.get(display.article_id)
+            if overrides_by_hash:
+                # Returns a cleared copy; never mutates the ORM-loaded token list.
+                tokens = UserArticle._apply_mwe_overrides_to_summary_tokens(
+                    tokens, overrides_by_hash
+                )
+            ctx = ContextIdentifier(
+                context_type,
+                article_id=display.article_id,
+                level_adapted_article_text_id=display.id,
+            )
+            return {
+                "tokens": tokens,
+                "context_identifier": ctx.as_dictionary(),
+                "past_bookmarks": past_bookmarks,
+            }
+
         if display.summary and len(display.summary.strip()) > 10:
             result["summary"] = display.summary.strip()
 
-        tokens = display.get_tokenized_summary()
-        if not tokens:
-            continue
-        # Apply the user's MWE ungroup overrides to the overlaid summary tokens
-        # (returns a cleared copy; never mutates the ORM-loaded token list).
-        overrides_by_hash = overrides_by_article.get(display.article_id)
-        if overrides_by_hash:
-            tokens = UserArticle._apply_mwe_overrides_to_summary_tokens(tokens, overrides_by_hash)
-        # The bookmark mapping keys on article_level_summary_id; article_id is
-        # carried only for the client's MWE-ungroup path (parent article id).
-        ctx = ContextIdentifier(
-            ContextType.ARTICLE_LEVEL_SUMMARY,
-            article_id=display.article_id,
-            article_level_summary_id=display.id,
-        )
-        result["interactiveSummary"] = {
-            "tokens": tokens,
-            "context_identifier": ctx.as_dictionary(),
-            "past_bookmarks": ArticleLevelSummaryContext.get_all_user_bookmarks_for_article_level_summary(
-                user.id, display.id
-            ),
-        }
+        summary_tokens = display.get_tokenized_summary()
+        if summary_tokens:
+            result["interactiveSummary"] = _payload(
+                summary_tokens,
+                ContextType.LEVEL_ADAPTED_ARTICLE_SUMMARY,
+                LevelAdaptedArticleSummaryContext.get_all_user_bookmarks_for_level_adapted_summary(
+                    user.id, display.id
+                ),
+            )
+
+        # Title is overlaid independently of the summary: a row can carry one and
+        # not the other (rows written before per-level titles existed have no
+        # title; the language check can drop a title while keeping its summary).
+        if display.title and display.title.strip():
+            result["title"] = display.title.strip()
+            title_tokens = display.get_tokenized_title()
+            if title_tokens:
+                result["interactiveTitle"] = _payload(
+                    title_tokens,
+                    ContextType.LEVEL_ADAPTED_ARTICLE_TITLE,
+                    LevelAdaptedArticleTitleContext.get_all_user_bookmarks_for_level_adapted_title(
+                        user.id, display.id
+                    ),
+                )
+            else:
+                # Plain title replaced but no tokens to tap: drop any bundled
+                # interactiveTitle rather than leave the ORIGINAL title's tokens
+                # sitting under the level title — they'd translate the wrong words.
+                result.pop("interactiveTitle", None)

zeeguu/core/elastic/elastic_query_builder.py

@@ -95,7 +95,7 @@ def build_elastic_recommender_query(
 
     NOT filtered by CEFR level, deliberately. With on-demand simplification
     every article is readable at the learner's level — the feed card carries a
-    level-appropriate summary (ArticleLevelSummary) and the body simplifies on
+    level-appropriate summary (LevelAdaptedArticleText) and the body simplifies on
     request (POST /simplify_article/<id>) — so an article being "too hard" is no
     longer a reason to hide it. Filtering on available_cefr_levels used to mean
     the opposite: that field lists the levels for which a version was already