From 40cd37a3a54d28d9a58f138c102f9bd935af5f06 Mon Sep 17 00:00:00 2001 From: Rudy Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Tue, 6 Oct 2026 07:35:13 -0400 Subject: [PATCH 1/3] fix(glossary): deduplicate Unicode source case variants Signed-off-by: Rudy Celekli <47457359+rudycelekli@users.noreply.github.com> --- CHANGELOG.md | 2 ++ backend/api/routers/glossary.py | 6 +++-- docs/dubbing/translation-engines.md | 4 +++ tests/test_glossary_scrub.py | 41 +++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 534142f5ab..e9aca80961 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ metadata and the backend fallback mirror it. ### Fixed +- Glossary auto-extraction preserves existing terms across Unicode case variants (#0) — thanks @rudycelekli! + - MCP speech tools wait through model loading and progress-extended CPU renders instead of timing out before the backend (#2609) ## [0.5.7] — 2026-10-05 diff --git a/backend/api/routers/glossary.py b/backend/api/routers/glossary.py index 178c1a172b..ec792fd6a7 100644 --- a/backend/api/routers/glossary.py +++ b/backend/api/routers/glossary.py @@ -270,10 +270,12 @@ def auto_extract(project_id: str, req: AutoExtractRequest): # Dedupe against existing (case-insensitive on source). with db_conn() as conn: existing = conn.execute( - "SELECT LOWER(source) AS src FROM glossary_terms WHERE project_id = ?", + "SELECT source FROM glossary_terms WHERE project_id = ?", (project_id,), ).fetchall() - existing_srcs = {r["src"] for r in existing} + # SQLite LOWER only folds ASCII in the bundled build; apply the same + # Unicode casing to saved and proposed terms so manual entries win. + existing_srcs = {r["source"].lower() for r in existing} inserted = 0 now = time.time() diff --git a/docs/dubbing/translation-engines.md b/docs/dubbing/translation-engines.md index 090effae98..79f6ee90da 100644 --- a/docs/dubbing/translation-engines.md +++ b/docs/dubbing/translation-engines.md @@ -292,3 +292,7 @@ Pasted, picked, and downloaded captions, and SubRip files imported into Stories, Downloaded rolling captions are deduplicated across their brief repeated bridge cues before translation. Ordinary consecutive cues keep intentional repeated words and phrases, even when their timestamps touch. New WebVTT imports retain source cue syntax alongside decoded text, preserving literal escaped tags and entities on unchanged exports. Edited text and older projects retain the legacy markup interpretation. + +### Project glossary deduplication + +Glossary auto-extraction keeps an existing manual or automatic source term when the proposal differs only in Unicode letter case, including accented Latin and Cyrillic names. Deduplication is scoped to the current project, so a term saved in another project does not suppress a new proposal. diff --git a/tests/test_glossary_scrub.py b/tests/test_glossary_scrub.py index 2d02119178..a8d428563e 100644 --- a/tests/test_glossary_scrub.py +++ b/tests/test_glossary_scrub.py @@ -102,3 +102,44 @@ class _Handle: out = glossary.auto_extract("proj-reasoning", _req(target_lang="es", segments=[{"text": "Hello Marcus"}])) assert out["proposed"] == 1 assert [t["target"] for t in out["terms"] if t["source"] == "Marcus"] == ["Marcus"] + + +@pytest.mark.parametrize("source, proposed_source", [ + ("École", "École"), + ("ÉCOLE", "école"), + ("МОСКВА", "Москва"), + ("München", "MÜNCHEN"), + ("Marcus", "MARCUS"), +]) +def test_auto_extract_preserves_existing_unicode_term(monkeypatch, tmp_path, source, proposed_source): + """Both sides of source deduplication must use Unicode-aware casing.""" + import sqlite3 + from types import SimpleNamespace + from api.routers import glossary + from core import db + from services import llm_skills + + path = tmp_path / "glossary.db" + with sqlite3.connect(path) as conn: + conn.executescript(db._BASE_SCHEMA) + monkeypatch.setattr(db, "DB_PATH", path) + original = glossary.add_term("project", glossary.GlossaryTerm( + source=source, target="manual choice", note="keep this")) + other = glossary.add_term("another-project", glossary.GlossaryTerm( + source="Rudy", target="other project")) + body = f"{proposed_source} || automatic choice || candidate\nRudy || Rudy || name\n" + completion = SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=body))]) + handle = SimpleNamespace(model="test", timeout=1, client=SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=lambda **kwargs: completion)))) + monkeypatch.setattr(llm_skills, "resolve_skill_client", lambda sid: handle) + + result = glossary.auto_extract("project", _req(target_lang="en", segments=[{"text": source}])) + + assert result["proposed"] == 2 + assert result["inserted"] == 1 + terms = {term["source"]: term for term in result["terms"]} + assert set(terms) == {source, "Rudy"} + assert terms[source] == original + assert terms[source]["auto"] is False + assert terms["Rudy"]["auto"] is True + assert glossary.list_terms("another-project") == [other] From 82ded33e68af4141c89c0d968b60c49e25e2c731 Mon Sep 17 00:00:00 2001 From: Rudy Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Tue, 6 Oct 2026 07:50:10 -0400 Subject: [PATCH 2/3] docs: link glossary-unicode regression report (#2638) Signed-off-by: Rudy Celekli <47457359+rudycelekli@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9aca80961..7863df9f5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ metadata and the backend fallback mirror it. ### Fixed -- Glossary auto-extraction preserves existing terms across Unicode case variants (#0) — thanks @rudycelekli! +- Glossary auto-extraction preserves existing terms across Unicode case variants (#2638) — thanks @rudycelekli! - MCP speech tools wait through model loading and progress-extended CPU renders instead of timing out before the backend (#2609) From d2c51e93d8cc52649bae2cd4c37404a802474f4d Mon Sep 17 00:00:00 2001 From: Rudy Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Tue, 6 Oct 2026 07:57:23 -0400 Subject: [PATCH 3/3] fix(glossary): fold Unicode source keys symmetrically Signed-off-by: Rudy Celekli <47457359+rudycelekli@users.noreply.github.com> --- backend/api/routers/glossary.py | 8 ++++---- docs/dubbing/translation-engines.md | 2 +- tests/test_glossary_scrub.py | 31 +++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/backend/api/routers/glossary.py b/backend/api/routers/glossary.py index ec792fd6a7..0ab78b2203 100644 --- a/backend/api/routers/glossary.py +++ b/backend/api/routers/glossary.py @@ -274,20 +274,20 @@ def auto_extract(project_id: str, req: AutoExtractRequest): (project_id,), ).fetchall() # SQLite LOWER only folds ASCII in the bundled build; apply the same - # Unicode casing to saved and proposed terms so manual entries win. - existing_srcs = {r["source"].lower() for r in existing} + # Unicode case folding to saved and proposed terms so manual entries win. + existing_srcs = {r["source"].casefold() for r in existing} inserted = 0 now = time.time() for src, tgt, note in proposed: - if src.lower() in existing_srcs: + if src.casefold() in existing_srcs: continue conn.execute( "INSERT INTO glossary_terms (id, project_id, source, target, note, auto, created_at) " "VALUES (?, ?, ?, ?, ?, 1, ?)", (str(uuid.uuid4())[:12], project_id, src, tgt, note, now), ) - existing_srcs.add(src.lower()) + existing_srcs.add(src.casefold()) inserted += 1 rows = conn.execute( diff --git a/docs/dubbing/translation-engines.md b/docs/dubbing/translation-engines.md index 79f6ee90da..b1b57700c7 100644 --- a/docs/dubbing/translation-engines.md +++ b/docs/dubbing/translation-engines.md @@ -295,4 +295,4 @@ New WebVTT imports retain source cue syntax alongside decoded text, preserving l ### Project glossary deduplication -Glossary auto-extraction keeps an existing manual or automatic source term when the proposal differs only in Unicode letter case, including accented Latin and Cyrillic names. Deduplication is scoped to the current project, so a term saved in another project does not suppress a new proposal. +Glossary auto-extraction keeps an existing manual or automatic source term when the proposal differs only in Unicode letter case, including accented Latin and Cyrillic names. Comparison keys use Unicode case folding, so `Straße`/`STRASSE` and Greek sigma variants also match; the saved source spelling and human target remain unchanged. Deduplication is scoped to the current project, so a term saved in another project does not suppress a new proposal. diff --git a/tests/test_glossary_scrub.py b/tests/test_glossary_scrub.py index a8d428563e..7d3399d048 100644 --- a/tests/test_glossary_scrub.py +++ b/tests/test_glossary_scrub.py @@ -110,6 +110,10 @@ class _Handle: ("МОСКВА", "Москва"), ("München", "MÜNCHEN"), ("Marcus", "MARCUS"), + ("Straße", "STRASSE"), + ("STRASSE", "Straße"), + ("ΟΣ", "οσ"), + ("οσ", "ος"), ]) def test_auto_extract_preserves_existing_unicode_term(monkeypatch, tmp_path, source, proposed_source): """Both sides of source deduplication must use Unicode-aware casing.""" @@ -143,3 +147,30 @@ def test_auto_extract_preserves_existing_unicode_term(monkeypatch, tmp_path, sou assert terms[source]["auto"] is False assert terms["Rudy"]["auto"] is True assert glossary.list_terms("another-project") == [other] + + +@pytest.mark.parametrize("first, second", [("Straße", "STRASSE"), ("οσ", "ος")]) +def test_auto_extract_deduplicates_unicode_variants_in_same_batch(monkeypatch, tmp_path, first, second): + import sqlite3 + from types import SimpleNamespace + from api.routers import glossary + from core import db + from services import llm_skills + + path = tmp_path / "glossary.db" + with sqlite3.connect(path) as conn: + conn.executescript(db._BASE_SCHEMA) + monkeypatch.setattr(db, "DB_PATH", path) + body = f"{first} || first choice || term\n{second} || second choice || variant\n" + completion = SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=body))]) + handle = SimpleNamespace(model="test", timeout=1, client=SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=lambda **kwargs: completion)))) + monkeypatch.setattr(llm_skills, "resolve_skill_client", lambda sid: handle) + + result = glossary.auto_extract("project", _req(target_lang="en", segments=[{"text": first}])) + + assert result["proposed"] == 2 + assert result["inserted"] == 1 + assert len(result["terms"]) == 1 + assert result["terms"][0]["source"] == first + assert result["terms"][0]["target"] == "first choice"