From 66305d6a4514d59f317a4e06b613bc5f04367640 Mon Sep 17 00:00:00 2001 From: Rhainland Date: Wed, 18 Nov 2026 19:42:48 +0100 Subject: [PATCH 1/5] fix(boxsets): fetch box sets in the library's metadata language The box set sync asked TMDB for each collection without a language, so box set names, overviews and posters were always English. It now asks in the metadata language of the first box set library holding one of the collection's movies, records that language with the stored collection, and fetches a collection again when the language it is due in changes. Closes #4485 --- internal/api/router.go | 2 +- internal/boxsets/sync.go | 21 +++--- internal/boxsets/sync_test.go | 33 ++++++-- internal/catalog/box_sets.go | 75 ++++++++++++------- internal/catalog/box_sets_db_test.go | 70 ++++++++++++++++- internal/metadata/tmdb/client.go | 35 ++++++++- internal/metadata/tmdb/client_test.go | 26 +++++++ ...1118183932_box_set_collection_language.sql | 11 +++ scripts/ci/db-contracts.txt | 1 + 9 files changed, 225 insertions(+), 49 deletions(-) create mode 100644 migrations/sql/20261118183932_box_set_collection_language.sql diff --git a/internal/api/router.go b/internal/api/router.go index b8e48b3b70..f18b2f6d53 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -3722,7 +3722,7 @@ type tmdbFranchiseAdapter struct { } func (a *tmdbFranchiseAdapter) GetCollection(ctx context.Context, id int) ([]catalog.TMDBCollectionEntry, error) { - collection, err := a.client.GetCollection(ctx, id) + collection, err := a.client.GetCollection(ctx, id, "") if err != nil { return nil, err } diff --git a/internal/boxsets/sync.go b/internal/boxsets/sync.go index 97c3d7fe9d..b60c88058f 100644 --- a/internal/boxsets/sync.go +++ b/internal/boxsets/sync.go @@ -38,16 +38,16 @@ const ( // TMDB is the slice of *tmdb.Client the sync uses. type TMDB interface { GetMovieCollection(ctx context.Context, movieID int) (*tmdb.CollectionRef, error) - GetCollection(ctx context.Context, id int) (*tmdb.Collection, error) + GetCollection(ctx context.Context, id int, language string) (*tmdb.Collection, error) } // Store is the slice of *catalog.BoxSetRepository the sync uses. type Store interface { MoviesNeedingCollectionLookup(ctx context.Context, staleBefore time.Time, limit int, skip []string) ([]catalog.MovieCollectionLookup, error) RecordMovieCollection(ctx context.Context, contentID string, tmdbID int64, collectionID *int64) error - CollectionsNeedingFetch(ctx context.Context, staleBefore time.Time, limit int, skip []int64) ([]int64, error) + CollectionsNeedingFetch(ctx context.Context, staleBefore time.Time, limit int, skip []int64) ([]catalog.CollectionFetch, error) UpsertCollection(ctx context.Context, c catalog.TMDBMovieCollection) error - TouchCollection(ctx context.Context, id int64) error + TouchCollection(ctx context.Context, id int64, language string) error DeleteUnusedCollections(ctx context.Context) (int64, error) CollectionArtworkNeedingCache(ctx context.Context, limit int, skip []int64) ([]catalog.CollectionArtworkDue, error) SetCollectionArtwork(ctx context.Context, id int64, slot catalog.CollectionArtworkSlot, sourcePath, path, thumbhash string) error @@ -165,7 +165,8 @@ func (s *Syncer) lookUpMovies(ctx context.Context, res *Result, failed *[]string } } -// fetchCollections stores every due collection. tried collects the +// fetchCollections stores every due collection in the language it is due +// in. tried collects the // collections this run could not store: failed fetches, and ones TMDB no // longer serves that were never stored. func (s *Syncer) fetchCollections(ctx context.Context, res *Result, tried *[]int64, report func(float64, string)) error { @@ -175,16 +176,17 @@ func (s *Syncer) fetchCollections(ctx context.Context, res *Result, tried *[]int if err != nil || len(due) == 0 { return err } - for _, id := range due { + for _, fetch := range due { if err := ctx.Err(); err != nil { return err } - c, err := s.tmdb.GetCollection(ctx, int(id)) + id := fetch.ID + c, err := s.tmdb.GetCollection(ctx, int(id), fetch.Language) if errors.Is(err, tmdb.ErrNotFound) { // Keep a stored copy rather than refetching a collection TMDB // no longer serves on every run. One never stored stays due // for the next run. - if err := s.store.TouchCollection(ctx, id); err != nil { + if err := s.store.TouchCollection(ctx, id, fetch.Language); err != nil { return err } *tried = append(*tried, id) @@ -199,7 +201,7 @@ func (s *Syncer) fetchCollections(ctx context.Context, res *Result, tried *[]int } continue } - if err := s.store.UpsertCollection(ctx, storedCollection(id, c)); err != nil { + if err := s.store.UpsertCollection(ctx, storedCollection(id, fetch.Language, c)); err != nil { return err } res.CollectionsFetched++ @@ -267,7 +269,7 @@ func tmdbOriginalURL(path string) string { // storedCollection converts a fetched collection, listing its movies in // release order. TMDB returns parts in no reliable order, and a part without // a release date (an announced sequel) goes last. -func storedCollection(id int64, c *tmdb.Collection) catalog.TMDBMovieCollection { +func storedCollection(id int64, language string, c *tmdb.Collection) catalog.TMDBMovieCollection { parts := append([]tmdb.CollectionPart(nil), c.Parts...) sort.SliceStable(parts, func(i, j int) bool { a, b := parts[i].ReleaseDate, parts[j].ReleaseDate @@ -289,5 +291,6 @@ func storedCollection(id int64, c *tmdb.Collection) catalog.TMDBMovieCollection PosterSourcePath: tmdbOriginalURL(c.PosterPath), BackdropSourcePath: tmdbOriginalURL(c.BackdropPath), MemberTMDBIDs: members, + Language: language, } } diff --git a/internal/boxsets/sync_test.go b/internal/boxsets/sync_test.go index 7c68287ed4..dc82fe6546 100644 --- a/internal/boxsets/sync_test.go +++ b/internal/boxsets/sync_test.go @@ -19,6 +19,8 @@ type fakeStore struct { recorded map[string]*int64 collections map[int64]catalog.TMDBMovieCollection touched []int64 + // language is the language every collection is due in. + language string } func (s *fakeStore) MoviesNeedingCollectionLookup(_ context.Context, _ time.Time, limit int, skip []string) ([]catalog.MovieCollectionLookup, error) { @@ -36,17 +38,21 @@ func (s *fakeStore) RecordMovieCollection(_ context.Context, contentID string, _ return nil } -func (s *fakeStore) CollectionsNeedingFetch(_ context.Context, _ time.Time, _ int, skip []int64) ([]int64, error) { - var out []int64 +func (s *fakeStore) CollectionsNeedingFetch(_ context.Context, _ time.Time, _ int, skip []int64) ([]catalog.CollectionFetch, error) { + var ids []int64 for _, id := range s.recorded { - if id == nil || slices.Contains(out, *id) || slices.Contains(s.touched, *id) || slices.Contains(skip, *id) { + if id == nil || slices.Contains(ids, *id) || slices.Contains(s.touched, *id) || slices.Contains(skip, *id) { continue } if _, stored := s.collections[*id]; !stored { - out = append(out, *id) + ids = append(ids, *id) } } - slices.Sort(out) + slices.Sort(ids) + out := make([]catalog.CollectionFetch, len(ids)) + for i, id := range ids { + out[i] = catalog.CollectionFetch{ID: id, Language: s.language} + } return out, nil } @@ -87,7 +93,7 @@ func (s *fakeStore) DeleteUnusedCollections(context.Context) (int64, error) { re // TouchCollection marks only a stored collection fetched, as the // repository's UPDATE does. -func (s *fakeStore) TouchCollection(_ context.Context, id int64) error { +func (s *fakeStore) TouchCollection(_ context.Context, id int64, _ string) error { if _, stored := s.collections[id]; stored { s.touched = append(s.touched, id) } @@ -103,6 +109,8 @@ type fakeTMDB struct { lookups map[int]int // onCollection runs on each collection fetch. onCollection func() + // languages records the language each collection was asked for in. + languages map[int]string } func (f *fakeTMDB) GetMovieCollection(_ context.Context, id int) (*tmdb.CollectionRef, error) { @@ -119,7 +127,10 @@ func (f *fakeTMDB) GetMovieCollection(_ context.Context, id int) (*tmdb.Collecti return ref, nil } -func (f *fakeTMDB) GetCollection(_ context.Context, id int) (*tmdb.Collection, error) { +func (f *fakeTMDB) GetCollection(_ context.Context, id int, language string) (*tmdb.Collection, error) { + if f.languages != nil { + f.languages[id] = language + } if f.onCollection != nil { f.onCollection() } @@ -142,8 +153,10 @@ func TestRunStoresMembershipAndCollectionsInReleaseOrder(t *testing.T) { catalog.MovieCollectionLookup{ContentID: "heat", TMDBID: 949}, catalog.MovieCollectionLookup{ContentID: "gone", TMDBID: 1}, ) + store.language = "de" client := &fakeTMDB{ - movies: map[int]*tmdb.CollectionRef{671: {ID: 1241, Name: "Harry Potter Collection"}, 949: nil}, + languages: map[int]string{}, + movies: map[int]*tmdb.CollectionRef{671: {ID: 1241, Name: "Harry Potter Collection"}, 949: nil}, collections: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Collection", PosterPath: "/p.jpg", Parts: []tmdb.CollectionPart{ {ID: 674, ReleaseDate: "2005-11-16"}, {ID: 999, ReleaseDate: ""}, @@ -169,6 +182,10 @@ func TestRunStoresMembershipAndCollectionsInReleaseOrder(t *testing.T) { } } stored := store.collections[1241] + // The collection is asked for and stored in the language it is due in. + if client.languages[1241] != "de" || stored.Language != "de" { + t.Fatalf("fetched in %q, stored as %q; want de", client.languages[1241], stored.Language) + } if want := []int64{671, 673, 674, 999}; !slices.Equal(stored.MemberTMDBIDs, want) { t.Fatalf("members = %v, want release order %v with the undated part last", stored.MemberTMDBIDs, want) } diff --git a/internal/catalog/box_sets.go b/internal/catalog/box_sets.go index ce296fd3a9..3b59a17cd4 100644 --- a/internal/catalog/box_sets.go +++ b/internal/catalog/box_sets.go @@ -79,7 +79,9 @@ type TMDBMovieCollection struct { BackdropThumbhash string // MemberTMDBIDs lists the collection's movies in release order. MemberTMDBIDs []int64 - FetchedAt time.Time + // Language is the metadata language the collection was fetched in. + Language string + FetchedAt time.Time } // BoxSetGroup is one box set of a library as a viewer sees it: the @@ -197,33 +199,51 @@ const currentBoxSetMember = `EXISTS ( WHERE cmi.content_id = m.content_id AND cmi.tmdb_id = m.checked_tmdb_id::text AND cf.box_sets_enabled AND cf.enabled IS NOT FALSE)` +// CollectionFetch is a collection due for a fetch and the metadata language +// to fetch it in. +type CollectionFetch struct { + ID int64 + Language string +} + // CollectionsNeedingFetch returns up to limit collections a movie of a box -// set library belongs to that are not stored yet or were fetched before -// staleBefore. -func (r *BoxSetRepository) CollectionsNeedingFetch(ctx context.Context, staleBefore time.Time, limit int, skip []int64) ([]int64, error) { +// set library belongs to that are not stored yet, were fetched before +// staleBefore, or were fetched in another language than the one they are +// due in. A collection is due in the metadata language of the first library, +// in library order, that holds one of its movies with box sets on: one +// collection serves every library, so a deployment whose box set libraries +// differ in language shows it in one of them. +func (r *BoxSetRepository) CollectionsNeedingFetch(ctx context.Context, staleBefore time.Time, limit int, skip []int64) ([]CollectionFetch, error) { rows, err := r.pool.Query(ctx, ` - SELECT DISTINCT m.tmdb_collection_id, c.fetched_at - FROM media_item_tmdb_collections m - LEFT JOIN tmdb_movie_collections c ON c.tmdb_collection_id = m.tmdb_collection_id - WHERE m.tmdb_collection_id IS NOT NULL AND (c.tmdb_collection_id IS NULL OR c.fetched_at < $1) - AND `+currentBoxSetMember+` - AND NOT (m.tmdb_collection_id = ANY($3::bigint[])) - ORDER BY c.fetched_at NULLS FIRST, m.tmdb_collection_id + SELECT due.tmdb_collection_id, due.language + FROM ( + SELECT DISTINCT ON (m.tmdb_collection_id) m.tmdb_collection_id, + COALESCE(NULLIF(BTRIM(cf.metadata_language), ''), 'en') AS language + FROM media_item_tmdb_collections m + JOIN media_items cmi ON cmi.content_id = m.content_id AND cmi.tmdb_id = m.checked_tmdb_id::text + JOIN media_item_libraries cmil ON cmil.content_id = cmi.content_id + JOIN media_folders cf ON cf.id = cmil.media_folder_id AND cf.box_sets_enabled AND cf.enabled IS NOT FALSE + WHERE m.tmdb_collection_id IS NOT NULL + AND NOT (m.tmdb_collection_id = ANY($3::bigint[])) + ORDER BY m.tmdb_collection_id, cf.sort_order, cf.id + ) due + LEFT JOIN tmdb_movie_collections c ON c.tmdb_collection_id = due.tmdb_collection_id + WHERE c.tmdb_collection_id IS NULL OR c.fetched_at < $1 OR c.language <> due.language + ORDER BY c.fetched_at NULLS FIRST, due.tmdb_collection_id LIMIT $2`, staleBefore, limit, nonNilSlice(skip)) if err != nil { return nil, fmt.Errorf("querying collections needing a fetch: %w", err) } defer rows.Close() - var ids []int64 - for rows.Next() { - var id int64 - var fetchedAt *time.Time - if err := rows.Scan(&id, &fetchedAt); err != nil { - return nil, fmt.Errorf("scanning collection needing a fetch: %w", err) - } - ids = append(ids, id) + out, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (CollectionFetch, error) { + var c CollectionFetch + err := row.Scan(&c.ID, &c.Language) + return c, err + }) + if err != nil { + return nil, fmt.Errorf("reading collections needing a fetch: %w", err) } - return ids, rows.Err() + return out, nil } // UpsertCollection stores a fetched collection with the provider URLs of its @@ -236,10 +256,11 @@ func (r *BoxSetRepository) UpsertCollection(ctx context.Context, c TMDBMovieColl members = []int64{} } _, err := r.pool.Exec(ctx, ` - INSERT INTO tmdb_movie_collections (tmdb_collection_id, name, overview, poster_path, poster_source_path, backdrop_path, backdrop_source_path, member_tmdb_ids, fetched_at) - VALUES ($1, $2, $3, $4, $4, $5, $5, $6, now()) + INSERT INTO tmdb_movie_collections (tmdb_collection_id, name, overview, poster_path, poster_source_path, backdrop_path, backdrop_source_path, member_tmdb_ids, language, fetched_at) + VALUES ($1, $2, $3, $4, $4, $5, $5, $6, $7, now()) ON CONFLICT (tmdb_collection_id) DO UPDATE SET name = EXCLUDED.name, + language = EXCLUDED.language, overview = EXCLUDED.overview, poster_path = CASE WHEN tmdb_movie_collections.poster_source_path = EXCLUDED.poster_source_path THEN tmdb_movie_collections.poster_path ELSE EXCLUDED.poster_path END, @@ -254,7 +275,7 @@ func (r *BoxSetRepository) UpsertCollection(ctx context.Context, c TMDBMovieColl member_tmdb_ids = EXCLUDED.member_tmdb_ids, fetched_at = EXCLUDED.fetched_at, updated_at = now()`, - c.ID, c.Name, c.Overview, c.PosterSourcePath, c.BackdropSourcePath, members) + c.ID, c.Name, c.Overview, c.PosterSourcePath, c.BackdropSourcePath, members, c.Language) if err != nil { return fmt.Errorf("storing tmdb collection: %w", err) } @@ -323,10 +344,10 @@ func (r *BoxSetRepository) SetCollectionArtwork(ctx context.Context, id int64, s return nil } -// TouchCollection marks a collection fetched without changing it, so one -// TMDB no longer serves is not fetched again on every run. -func (r *BoxSetRepository) TouchCollection(ctx context.Context, id int64) error { - _, err := r.pool.Exec(ctx, `UPDATE tmdb_movie_collections SET fetched_at = now() WHERE tmdb_collection_id = $1`, id) +// TouchCollection marks a collection fetched in language without changing +// it, so one TMDB no longer serves is not fetched again on every run. +func (r *BoxSetRepository) TouchCollection(ctx context.Context, id int64, language string) error { + _, err := r.pool.Exec(ctx, `UPDATE tmdb_movie_collections SET fetched_at = now(), language = $2 WHERE tmdb_collection_id = $1`, id, language) if err != nil { return fmt.Errorf("touching tmdb collection: %w", err) } diff --git a/internal/catalog/box_sets_db_test.go b/internal/catalog/box_sets_db_test.go index 6a286826fe..fc4a87cee2 100644 --- a/internal/catalog/box_sets_db_test.go +++ b/internal/catalog/box_sets_db_test.go @@ -265,11 +265,11 @@ func TestBoxSetCollectionsOfDisabledLibraryDB(t *testing.T) { } refetch := func() bool { t.Helper() - ids, err := f.repo.CollectionsNeedingFetch(ctx, time.Now().Add(-time.Hour), 10_000, nil) + fetches, err := f.repo.CollectionsNeedingFetch(ctx, time.Now().Add(-time.Hour), 10_000, nil) if err != nil { t.Fatal(err) } - return slices.Contains(ids, f.franchise) + return slices.ContainsFunc(fetches, func(c CollectionFetch) bool { return c.ID == f.franchise }) } if !refetch() { t.Fatal("a stale collection of a box set library is not due for a fetch") @@ -294,6 +294,72 @@ func TestBoxSetCollectionsOfDisabledLibraryDB(t *testing.T) { } } +// TestBoxSetCollectionLanguageDB checks that a collection is fetched in the +// metadata language of its box set library, and fetched again once that +// language changes, rather than only when the stored copy goes stale. +func TestBoxSetCollectionLanguageDB(t *testing.T) { + f := seedBoxSets(t) + ctx := t.Context() + exec := func(sql string, args ...any) { + t.Helper() + if _, err := f.pool.Exec(ctx, sql, args...); err != nil { + t.Fatal(err) + } + } + due := func() (string, bool) { + t.Helper() + fetches, err := f.repo.CollectionsNeedingFetch(ctx, time.Now().Add(-time.Hour), 10_000, nil) + if err != nil { + t.Fatal(err) + } + for _, c := range fetches { + if c.ID == f.franchise { + return c.Language, true + } + } + return "", false + } + store := func(language string) { + t.Helper() + if err := f.repo.UpsertCollection(ctx, TMDBMovieCollection{ID: f.franchise, Name: "Saga", Language: language}); err != nil { + t.Fatal(err) + } + } + + store("en") + if language, ok := due(); ok { + t.Fatalf("a fresh collection in its library's language is due for a fetch in %q", language) + } + exec(`UPDATE media_folders SET metadata_language = 'de' WHERE id = $1`, f.lib) + if language, ok := due(); !ok || language != "de" { + t.Fatalf("after the library switched to de, due = %q, %v; want a fetch in de", language, ok) + } + store("de") + if language, ok := due(); ok { + t.Fatalf("a collection stored in de is still due for a fetch in %q", language) + } + + // Another library ahead of it in the library order holds one of the + // collection's movies: its language wins, so every run asks for the same. + var first int + if err := f.pool.QueryRow(ctx, `INSERT INTO media_folders(type,name,enabled,box_sets_enabled,metadata_language,sort_order) VALUES('movies',$1,true,true,'fr',-1) RETURNING id`, f.prefix+"-fr").Scan(&first); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _, _ = f.pool.Exec(context.Background(), `DELETE FROM media_folders WHERE id=$1`, first) }) + exec(`INSERT INTO media_item_libraries(content_id,media_folder_id) VALUES($1,$2)`, f.id("first"), first) + if language, ok := due(); !ok || language != "fr" { + t.Fatalf("with a French library first, due = %q, %v; want a fetch in fr", language, ok) + } + // A collection TMDB no longer serves is not asked for again in the same + // language on every run. + if err := f.repo.TouchCollection(ctx, f.franchise, "fr"); err != nil { + t.Fatal(err) + } + if language, ok := due(); ok { + t.Fatalf("a touched collection is still due for a fetch in %q", language) + } +} + func TestBoxSetCollectionArtworkDB(t *testing.T) { f := seedBoxSets(t) ctx := t.Context() diff --git a/internal/metadata/tmdb/client.go b/internal/metadata/tmdb/client.go index b144d2ad5e..6d1ca59a7a 100644 --- a/internal/metadata/tmdb/client.go +++ b/internal/metadata/tmdb/client.go @@ -831,13 +831,22 @@ func joinIntSlice(values []int, sep string) string { // Universe). An id of 0 is rejected here — placeholder templates with // collection_id=0 are surfaced as failures by the sync path, not silently // turned into an empty fetch. -func (c *Client) GetCollection(ctx context.Context, id int) (*Collection, error) { +// +// language is the Silo metadata language (an ISO 639-1 code such as "de") +// the name, overview and artwork come in; TMDB falls back to its default +// where the collection has no translation or poster in that language. An +// empty language leaves the choice to TMDB, which answers in English. +func (c *Client) GetCollection(ctx context.Context, id int, language string) (*Collection, error) { if id <= 0 { return nil, fmt.Errorf("tmdb: collection id must be > 0 (got %d)", id) } + path := fmt.Sprintf("/collection/%d", id) + if lang := apiLanguage(language); lang != "" { + path += "?language=" + url.QueryEscape(lang) + } var resp collectionResponse - if err := c.doGet(ctx, fmt.Sprintf("/collection/%d", id), &resp); err != nil { + if err := c.doGet(ctx, path, &resp); err != nil { return nil, err } @@ -868,6 +877,28 @@ func (c *Client) GetCollection(ctx context.Context, id int) (*Collection, error) }, nil } +// apiLanguage converts a Silo metadata language into TMDB's language +// parameter the way the TMDB metadata plugin does: English becomes en-US, +// which TMDB prefers, and any other tag passes through normalized (pt-br → +// pt-BR). An empty language stays empty. +func apiLanguage(language string) string { + parts := strings.Split(strings.ReplaceAll(strings.TrimSpace(language), "_", "-"), "-") + if parts[0] == "" { + return "" + } + for i, part := range parts { + if i == 0 { + parts[i] = strings.ToLower(part) + } else { + parts[i] = strings.ToUpper(part) + } + } + if parts[0] == "en" { + return "en-US" + } + return strings.Join(parts, "-") +} + // GetMovieCollection returns the collection a movie belongs to, or nil when // it belongs to none. func (c *Client) GetMovieCollection(ctx context.Context, movieID int) (*CollectionRef, error) { diff --git a/internal/metadata/tmdb/client_test.go b/internal/metadata/tmdb/client_test.go index c627a707c9..270d8c4a17 100644 --- a/internal/metadata/tmdb/client_test.go +++ b/internal/metadata/tmdb/client_test.go @@ -424,3 +424,29 @@ func TestGetMediaDetailFollowerStopsWaitingOnCancel(t *testing.T) { t.Fatalf("upstream calls = %d, want 1 shared fetch", calls.Load()) } } + +// A collection is fetched in the language asked for, the way the TMDB +// metadata plugin asks for titles: English as en-US, any other language as +// given, and no language at all when none is asked for. +func TestGetCollectionLanguage(t *testing.T) { + for _, tc := range []struct{ language, sent string }{ + {"de", "de"}, + {"pt-br", "pt-BR"}, + {"en", "en-US"}, + {"", ""}, + } { + t.Run(tc.language, func(t *testing.T) { + var got url.Values + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + got = r.URL.Query() + _, _ = w.Write([]byte(`{"id":10,"name":"Star Wars - Saga","poster_path":"/fr.jpg","parts":[]}`)) + }) + if _, err := client.GetCollection(context.Background(), 10, tc.language); err != nil { + t.Fatal(err) + } + if lang := got.Get("language"); lang != tc.sent { + t.Errorf("language = %q, want %q", lang, tc.sent) + } + }) + } +} diff --git a/migrations/sql/20261118183932_box_set_collection_language.sql b/migrations/sql/20261118183932_box_set_collection_language.sql new file mode 100644 index 0000000000..9b74e92ec8 --- /dev/null +++ b/migrations/sql/20261118183932_box_set_collection_language.sql @@ -0,0 +1,11 @@ +-- +goose Up +-- The metadata language a TMDB collection's name, overview and artwork were +-- fetched in. The box set sync fetches each collection in the language of its +-- first box set library and fetches it again when that language changes. +-- Collections stored before this column were fetched without a language, +-- which TMDB answers in English. +ALTER TABLE public.tmdb_movie_collections + ADD COLUMN language text NOT NULL DEFAULT 'en'; + +-- +goose Down +ALTER TABLE public.tmdb_movie_collections DROP COLUMN IF EXISTS language; diff --git a/scripts/ci/db-contracts.txt b/scripts/ci/db-contracts.txt index c5e897b21f..39c0dfa6ae 100644 --- a/scripts/ci/db-contracts.txt +++ b/scripts/ci/db-contracts.txt @@ -91,6 +91,7 @@ ./internal/catalog TestBoxSetCollectionsOfDisabledLibraryDB ./internal/catalog TestMovieBoxSetDB ./internal/catalog TestBoxSetCollectionArtworkDB +./internal/catalog TestBoxSetCollectionLanguageDB # Audiobook Series, Authors, and Narrators browse: group counts and durations, # cover order, the viewer's own progress, count-sorted cursor pages, the search From 1e1530a3e79a9db45e7ecdb7c1a0d56ddc19d720 Mon Sep 17 00:00:00 2001 From: Rhainland Date: Wed, 18 Nov 2026 19:47:42 +0100 Subject: [PATCH 2/5] fix(boxsets): fill untranslated box set fields from English TMDB leaves a collection's name or overview empty where its translation lacks them, so the sync now takes those from TMDB's English answer, as item metadata falls back. Also share the box set membership SQL between the fetch and cleanup queries, read the stored language back, simplify the TMDB language mapping, and record the language rule in the box set architecture note. --- docs/architecture/movie-box-sets.md | 12 ++++++--- internal/boxsets/sync.go | 22 +++++++++++++++- internal/boxsets/sync_test.go | 37 +++++++++++++++++++-------- internal/catalog/box_sets.go | 35 +++++++++++++------------ internal/catalog/box_sets_db_test.go | 16 ++++++++---- internal/metadata/tmdb/client.go | 24 +++++------------ internal/metadata/tmdb/client_test.go | 1 - 7 files changed, 94 insertions(+), 53 deletions(-) diff --git a/docs/architecture/movie-box-sets.md b/docs/architecture/movie-box-sets.md index b030f87d6e..6a131a6abb 100644 --- a/docs/architecture/movie-box-sets.md +++ b/docs/architecture/movie-box-sets.md @@ -22,6 +22,11 @@ its own card. This note records the invariants; the wire contract is in collected. TMDB returns a collection's parts in no reliable order, so the sync sorts them by release date, with undated (announced) parts last. +- One row serves every library. It is fetched in the metadata language of + the first library, in library order, that holds one of the collection's + movies with box sets on, and `language` records which. TMDB leaves the + name or overview empty where its translation lacks them; those come from + TMDB's English answer. TMDB picks the poster for the language itself. - The stored name is TMDB's. Reads show it through `catalog.BoxSetTitle` ("… Collection" becomes "… Box Set"). Search matches the name without that suffix (`boxSetStemSQL`, the same rule in SQL; change the two together), @@ -34,9 +39,10 @@ its own card. This note records the invariants; the wire contract is in The `sync_movie_box_sets` task (`internal/boxsets`) is the only code that calls TMDB for box sets; the box set reads use stored rows only. It looks up movies of box set libraries that were never looked up, were rematched, or were -last looked up 30 days ago, then fetches collections that are new or were -fetched 7 days ago, then caches collection artwork still served from TMDB -through the item image cacher, under the same `metadata.cache_images` switch +last looked up 30 days ago, then fetches collections that are new, were +fetched 7 days ago, or were fetched in another language than the one they +are now due in (a library changed its metadata language), then caches +collection artwork still served from TMDB through the item image cacher, under the same `metadata.cache_images` switch as item artwork. It runs at startup, hourly, when a library turns box sets on, and 20 seconds after the node that matched a movie publishes its catalog change. A run stops after 25 failed TMDB calls; the next run resumes. diff --git a/internal/boxsets/sync.go b/internal/boxsets/sync.go index b60c88058f..3a871e283a 100644 --- a/internal/boxsets/sync.go +++ b/internal/boxsets/sync.go @@ -6,6 +6,7 @@ package boxsets import ( + "cmp" "context" "errors" "fmt" @@ -181,7 +182,7 @@ func (s *Syncer) fetchCollections(ctx context.Context, res *Result, tried *[]int return err } id := fetch.ID - c, err := s.tmdb.GetCollection(ctx, int(id), fetch.Language) + c, err := s.fetchCollection(ctx, fetch) if errors.Is(err, tmdb.ErrNotFound) { // Keep a stored copy rather than refetching a collection TMDB // no longer serves on every run. One never stored stays due @@ -210,6 +211,25 @@ func (s *Syncer) fetchCollections(ctx context.Context, res *Result, tried *[]int } } +// fetchCollection fetches a collection in the language it is due in. TMDB +// leaves the name or overview empty where its translation lacks them, so +// those come from its English answer, the way item metadata falls back. +// TMDB falls back for artwork itself. +func (s *Syncer) fetchCollection(ctx context.Context, fetch catalog.CollectionFetch) (*tmdb.Collection, error) { + c, err := s.tmdb.GetCollection(ctx, int(fetch.ID), fetch.Language) + if err != nil || fetch.Language == "en" || (c.Name != "" && c.Overview != "") { + return c, err + } + english, err := s.tmdb.GetCollection(ctx, int(fetch.ID), "") + if err != nil { + return nil, err + } + localized := *c + localized.Name = cmp.Or(c.Name, english.Name) + localized.Overview = cmp.Or(c.Overview, english.Overview) + return &localized, nil +} + // cacheArtwork stores a copy of each collection image still served from // TMDB. A failed image stays on the provider URL and is tried again next run. func (s *Syncer) cacheArtwork(ctx context.Context, res *Result) error { diff --git a/internal/boxsets/sync_test.go b/internal/boxsets/sync_test.go index dc82fe6546..bcc56933c0 100644 --- a/internal/boxsets/sync_test.go +++ b/internal/boxsets/sync_test.go @@ -109,8 +109,11 @@ type fakeTMDB struct { lookups map[int]int // onCollection runs on each collection fetch. onCollection func() - // languages records the language each collection was asked for in. - languages map[int]string + // translated answers collection requests that ask for a language; + // collections answers the others, as TMDB answers in English. + translated map[int]*tmdb.Collection + // languages records the languages each collection was asked for in. + languages map[int][]string } func (f *fakeTMDB) GetMovieCollection(_ context.Context, id int) (*tmdb.CollectionRef, error) { @@ -129,12 +132,15 @@ func (f *fakeTMDB) GetMovieCollection(_ context.Context, id int) (*tmdb.Collecti func (f *fakeTMDB) GetCollection(_ context.Context, id int, language string) (*tmdb.Collection, error) { if f.languages != nil { - f.languages[id] = language + f.languages[id] = append(f.languages[id], language) } if f.onCollection != nil { f.onCollection() } c, ok := f.collections[id] + if t, translated := f.translated[id]; translated && language != "" { + c, ok = t, true + } if !ok { return nil, tmdb.ErrNotFound } @@ -155,9 +161,16 @@ func TestRunStoresMembershipAndCollectionsInReleaseOrder(t *testing.T) { ) store.language = "de" client := &fakeTMDB{ - languages: map[int]string{}, - movies: map[int]*tmdb.CollectionRef{671: {ID: 1241, Name: "Harry Potter Collection"}, 949: nil}, - collections: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Collection", PosterPath: "/p.jpg", Parts: []tmdb.CollectionPart{ + languages: map[int][]string{}, + // TMDB leaves the fields a translation lacks empty. + translated: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Filmreihe", PosterPath: "/de.jpg", Parts: []tmdb.CollectionPart{ + {ID: 674, ReleaseDate: "2005-11-16"}, + {ID: 999, ReleaseDate: ""}, + {ID: 671, ReleaseDate: "2001-11-16"}, + {ID: 673, ReleaseDate: "2004-05-31"}, + }}}, + movies: map[int]*tmdb.CollectionRef{671: {ID: 1241, Name: "Harry Potter Collection"}, 949: nil}, + collections: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Collection", Overview: "The wizarding saga.", PosterPath: "/p.jpg", Parts: []tmdb.CollectionPart{ {ID: 674, ReleaseDate: "2005-11-16"}, {ID: 999, ReleaseDate: ""}, {ID: 671, ReleaseDate: "2001-11-16"}, @@ -182,14 +195,18 @@ func TestRunStoresMembershipAndCollectionsInReleaseOrder(t *testing.T) { } } stored := store.collections[1241] - // The collection is asked for and stored in the language it is due in. - if client.languages[1241] != "de" || stored.Language != "de" { - t.Fatalf("fetched in %q, stored as %q; want de", client.languages[1241], stored.Language) + // The collection is stored in the language it is due in, and the + // overview the translation lacks comes from TMDB's English answer. + if !slices.Equal(client.languages[1241], []string{"de", ""}) || stored.Language != "de" { + t.Fatalf("fetched in %q, stored as %q; want de, then the English fallback", client.languages[1241], stored.Language) + } + if stored.Name != "Harry Potter Filmreihe" || stored.Overview != "The wizarding saga." { + t.Fatalf("name, overview = %q, %q; want the German name and the English overview", stored.Name, stored.Overview) } if want := []int64{671, 673, 674, 999}; !slices.Equal(stored.MemberTMDBIDs, want) { t.Fatalf("members = %v, want release order %v with the undated part last", stored.MemberTMDBIDs, want) } - if stored.PosterSourcePath != "https://image.tmdb.org/t/p/original/p.jpg" || stored.BackdropSourcePath != "" { + if stored.PosterSourcePath != "https://image.tmdb.org/t/p/original/de.jpg" || stored.BackdropSourcePath != "" { t.Fatalf("artwork sources = %q, %q", stored.PosterSourcePath, stored.BackdropSourcePath) } } diff --git a/internal/catalog/box_sets.go b/internal/catalog/box_sets.go index 3b59a17cd4..94be73e4a8 100644 --- a/internal/catalog/box_sets.go +++ b/internal/catalog/box_sets.go @@ -111,7 +111,7 @@ func (r *BoxSetRepository) Collections(ctx context.Context, ids []int64) (map[in return out, nil } rows, err := r.pool.Query(ctx, ` - SELECT tmdb_collection_id, name, overview, poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, member_tmdb_ids, fetched_at + SELECT tmdb_collection_id, name, overview, poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, member_tmdb_ids, language, fetched_at FROM tmdb_movie_collections WHERE tmdb_collection_id = ANY($1)`, ids) if err != nil { return nil, fmt.Errorf("querying tmdb collections: %w", err) @@ -119,7 +119,7 @@ func (r *BoxSetRepository) Collections(ctx context.Context, ids []int64) (map[in defer rows.Close() for rows.Next() { var c TMDBMovieCollection - if err := rows.Scan(&c.ID, &c.Name, &c.Overview, &c.PosterPath, &c.PosterThumbhash, &c.BackdropPath, &c.BackdropThumbhash, &c.MemberTMDBIDs, &c.FetchedAt); err != nil { + if err := rows.Scan(&c.ID, &c.Name, &c.Overview, &c.PosterPath, &c.PosterThumbhash, &c.BackdropPath, &c.BackdropThumbhash, &c.MemberTMDBIDs, &c.Language, &c.FetchedAt); err != nil { return nil, fmt.Errorf("scanning tmdb collection: %w", err) } out[c.ID] = c @@ -187,17 +187,20 @@ func (r *BoxSetRepository) RecordMovieCollection(ctx context.Context, contentID return nil } -// currentBoxSetMember holds when the collection lookup row m is current (its -// movie still has the TMDB ID it was looked up under) and its movie is in an -// enabled library with box sets on. Only such rows keep a collection stored -// and refreshed: a library that turns box sets off, or a rematched movie, -// leaves its lookup rows behind. -const currentBoxSetMember = `EXISTS ( - SELECT 1 FROM media_items cmi +// currentBoxSetMemberLibraries selects, as cf, the libraries that make the +// collection lookup row m current: its movie still has the TMDB ID it was +// looked up under, and they are enabled libraries with box sets on holding +// it. Only such rows keep a collection stored and refreshed: a library that +// turns box sets off, or a rematched movie, leaves its lookup rows behind. +const currentBoxSetMemberLibraries = ` + FROM media_items cmi JOIN media_item_libraries cmil ON cmil.content_id = cmi.content_id JOIN media_folders cf ON cf.id = cmil.media_folder_id WHERE cmi.content_id = m.content_id AND cmi.tmdb_id = m.checked_tmdb_id::text - AND cf.box_sets_enabled AND cf.enabled IS NOT FALSE)` + AND cf.box_sets_enabled AND cf.enabled IS NOT FALSE` + +// currentBoxSetMember holds when the collection lookup row m is current. +const currentBoxSetMember = `EXISTS (SELECT 1 ` + currentBoxSetMemberLibraries + `)` // CollectionFetch is a collection due for a fetch and the metadata language // to fetch it in. @@ -217,15 +220,15 @@ func (r *BoxSetRepository) CollectionsNeedingFetch(ctx context.Context, staleBef rows, err := r.pool.Query(ctx, ` SELECT due.tmdb_collection_id, due.language FROM ( - SELECT DISTINCT ON (m.tmdb_collection_id) m.tmdb_collection_id, - COALESCE(NULLIF(BTRIM(cf.metadata_language), ''), 'en') AS language + SELECT DISTINCT ON (m.tmdb_collection_id) m.tmdb_collection_id, lib.language FROM media_item_tmdb_collections m - JOIN media_items cmi ON cmi.content_id = m.content_id AND cmi.tmdb_id = m.checked_tmdb_id::text - JOIN media_item_libraries cmil ON cmil.content_id = cmi.content_id - JOIN media_folders cf ON cf.id = cmil.media_folder_id AND cf.box_sets_enabled AND cf.enabled IS NOT FALSE + CROSS JOIN LATERAL ( + SELECT COALESCE(NULLIF(BTRIM(cf.metadata_language), ''), 'en') AS language, cf.sort_order, cf.id + `+currentBoxSetMemberLibraries+` + ) lib WHERE m.tmdb_collection_id IS NOT NULL AND NOT (m.tmdb_collection_id = ANY($3::bigint[])) - ORDER BY m.tmdb_collection_id, cf.sort_order, cf.id + ORDER BY m.tmdb_collection_id, lib.sort_order, lib.id ) due LEFT JOIN tmdb_movie_collections c ON c.tmdb_collection_id = due.tmdb_collection_id WHERE c.tmdb_collection_id IS NULL OR c.fetched_at < $1 OR c.language <> due.language diff --git a/internal/catalog/box_sets_db_test.go b/internal/catalog/box_sets_db_test.go index fc4a87cee2..7e416e1cb9 100644 --- a/internal/catalog/box_sets_db_test.go +++ b/internal/catalog/box_sets_db_test.go @@ -95,8 +95,8 @@ func seedBoxSets(t *testing.T) boxSetFixture { exec(`INSERT INTO media_item_tmdb_collections(content_id,checked_tmdb_id,tmdb_collection_id) VALUES($1,$2,$3)`, f.id(m.id), m.checked, collection) } for _, c := range []TMDBMovieCollection{ - {ID: f.franchise, Name: f.prefix + " Saga Collection", MemberTMDBIDs: []int64{101, 102, 103, 104, 105}}, - {ID: f.single, Name: f.prefix + " Lonely Collection", MemberTMDBIDs: []int64{200, 201}}, + {ID: f.franchise, Name: f.prefix + " Saga Collection", MemberTMDBIDs: []int64{101, 102, 103, 104, 105}, Language: "en"}, + {ID: f.single, Name: f.prefix + " Lonely Collection", MemberTMDBIDs: []int64{200, 201}, Language: "en"}, } { if err := f.repo.UpsertCollection(ctx, c); err != nil { t.Fatal(err) @@ -260,9 +260,6 @@ func TestBoxSetSearchAndSyncDB(t *testing.T) { func TestBoxSetCollectionsOfDisabledLibraryDB(t *testing.T) { f := seedBoxSets(t) ctx := t.Context() - if _, err := f.pool.Exec(ctx, `UPDATE tmdb_movie_collections SET fetched_at = now() - interval '30 days' WHERE tmdb_collection_id = $1`, f.franchise); err != nil { - t.Fatal(err) - } refetch := func() bool { t.Helper() fetches, err := f.repo.CollectionsNeedingFetch(ctx, time.Now().Add(-time.Hour), 10_000, nil) @@ -271,6 +268,12 @@ func TestBoxSetCollectionsOfDisabledLibraryDB(t *testing.T) { } return slices.ContainsFunc(fetches, func(c CollectionFetch) bool { return c.ID == f.franchise }) } + if refetch() { + t.Fatal("a fresh collection is due for a fetch") + } + if _, err := f.pool.Exec(ctx, `UPDATE tmdb_movie_collections SET fetched_at = now() - interval '30 days' WHERE tmdb_collection_id = $1`, f.franchise); err != nil { + t.Fatal(err) + } if !refetch() { t.Fatal("a stale collection of a box set library is not due for a fetch") } @@ -338,6 +341,9 @@ func TestBoxSetCollectionLanguageDB(t *testing.T) { if language, ok := due(); ok { t.Fatalf("a collection stored in de is still due for a fetch in %q", language) } + if stored, err := f.repo.Collections(ctx, []int64{f.franchise}); err != nil || stored[f.franchise].Language != "de" { + t.Fatalf("stored collection = %+v, %v; want language de", stored[f.franchise], err) + } // Another library ahead of it in the library order holds one of the // collection's movies: its language wins, so every run asks for the same. diff --git a/internal/metadata/tmdb/client.go b/internal/metadata/tmdb/client.go index 6d1ca59a7a..b49ccd7bf1 100644 --- a/internal/metadata/tmdb/client.go +++ b/internal/metadata/tmdb/client.go @@ -877,26 +877,16 @@ func (c *Client) GetCollection(ctx context.Context, id int, language string) (*C }, nil } -// apiLanguage converts a Silo metadata language into TMDB's language -// parameter the way the TMDB metadata plugin does: English becomes en-US, -// which TMDB prefers, and any other tag passes through normalized (pt-br → -// pt-BR). An empty language stays empty. +// apiLanguage converts a Silo metadata language (an ISO 639-1 code) into +// TMDB's language parameter the way the TMDB metadata plugin does: English +// becomes en-US, which TMDB prefers, and any other code passes through. An +// empty language stays empty. func apiLanguage(language string) string { - parts := strings.Split(strings.ReplaceAll(strings.TrimSpace(language), "_", "-"), "-") - if parts[0] == "" { - return "" - } - for i, part := range parts { - if i == 0 { - parts[i] = strings.ToLower(part) - } else { - parts[i] = strings.ToUpper(part) - } - } - if parts[0] == "en" { + language = strings.ToLower(strings.TrimSpace(language)) + if language == "en" { return "en-US" } - return strings.Join(parts, "-") + return language } // GetMovieCollection returns the collection a movie belongs to, or nil when diff --git a/internal/metadata/tmdb/client_test.go b/internal/metadata/tmdb/client_test.go index 270d8c4a17..2ca372c32d 100644 --- a/internal/metadata/tmdb/client_test.go +++ b/internal/metadata/tmdb/client_test.go @@ -431,7 +431,6 @@ func TestGetMediaDetailFollowerStopsWaitingOnCancel(t *testing.T) { func TestGetCollectionLanguage(t *testing.T) { for _, tc := range []struct{ language, sent string }{ {"de", "de"}, - {"pt-br", "pt-BR"}, {"en", "en-US"}, {"", ""}, } { From 1e8af961186bb02820a00981deb78d1ad6f6ad0f Mon Sep 17 00:00:00 2001 From: Rhainland Date: Wed, 18 Nov 2026 19:49:26 +0100 Subject: [PATCH 3/5] fix(boxsets): keep a translated box set when its English fallback fails A failed English request no longer holds back a collection TMDB already returned in the library's language; it is stored as translated unless it has no name. --- docs/architecture/movie-box-sets.md | 9 +++++---- internal/boxsets/sync.go | 7 ++++++- internal/boxsets/sync_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/architecture/movie-box-sets.md b/docs/architecture/movie-box-sets.md index 6a131a6abb..94cc0a5d43 100644 --- a/docs/architecture/movie-box-sets.md +++ b/docs/architecture/movie-box-sets.md @@ -23,10 +23,11 @@ its own card. This note records the invariants; the wire contract is in parts in no reliable order, so the sync sorts them by release date, with undated (announced) parts last. - One row serves every library. It is fetched in the metadata language of - the first library, in library order, that holds one of the collection's - movies with box sets on, and `language` records which. TMDB leaves the - name or overview empty where its translation lacks them; those come from - TMDB's English answer. TMDB picks the poster for the language itself. + the first enabled library, in library order, that holds one of the + collection's movies with box sets on, and `language` records which. TMDB + leaves the name or overview empty where its translation lacks them; those + come from TMDB's English answer. TMDB picks the poster for the language + itself. - The stored name is TMDB's. Reads show it through `catalog.BoxSetTitle` ("… Collection" becomes "… Box Set"). Search matches the name without that suffix (`boxSetStemSQL`, the same rule in SQL; change the two together), diff --git a/internal/boxsets/sync.go b/internal/boxsets/sync.go index 3a871e283a..4daa246062 100644 --- a/internal/boxsets/sync.go +++ b/internal/boxsets/sync.go @@ -214,7 +214,8 @@ func (s *Syncer) fetchCollections(ctx context.Context, res *Result, tried *[]int // fetchCollection fetches a collection in the language it is due in. TMDB // leaves the name or overview empty where its translation lacks them, so // those come from its English answer, the way item metadata falls back. -// TMDB falls back for artwork itself. +// TMDB falls back for artwork itself. When that English answer fails, the +// translation is kept as it is unless it has no name to show. func (s *Syncer) fetchCollection(ctx context.Context, fetch catalog.CollectionFetch) (*tmdb.Collection, error) { c, err := s.tmdb.GetCollection(ctx, int(fetch.ID), fetch.Language) if err != nil || fetch.Language == "en" || (c.Name != "" && c.Overview != "") { @@ -222,6 +223,10 @@ func (s *Syncer) fetchCollection(ctx context.Context, fetch catalog.CollectionFe } english, err := s.tmdb.GetCollection(ctx, int(fetch.ID), "") if err != nil { + if c.Name != "" { + slog.WarnContext(ctx, "box sets: English collection fallback failed; storing the translation as it is", "component", "boxsets", "tmdb_collection_id", fetch.ID, "error", err) + return c, nil + } return nil, err } localized := *c diff --git a/internal/boxsets/sync_test.go b/internal/boxsets/sync_test.go index bcc56933c0..9e012970a8 100644 --- a/internal/boxsets/sync_test.go +++ b/internal/boxsets/sync_test.go @@ -114,6 +114,8 @@ type fakeTMDB struct { translated map[int]*tmdb.Collection // languages records the languages each collection was asked for in. languages map[int][]string + // englishErr answers collection requests that ask for no language. + englishErr error } func (f *fakeTMDB) GetMovieCollection(_ context.Context, id int) (*tmdb.CollectionRef, error) { @@ -137,6 +139,9 @@ func (f *fakeTMDB) GetCollection(_ context.Context, id int, language string) (*t if f.onCollection != nil { f.onCollection() } + if language == "" && f.englishErr != nil { + return nil, f.englishErr + } c, ok := f.collections[id] if t, translated := f.translated[id]; translated && language != "" { c, ok = t, true @@ -211,6 +216,25 @@ func TestRunStoresMembershipAndCollectionsInReleaseOrder(t *testing.T) { } } +// A translation is stored as it is when the English answer that would fill +// its missing overview fails, rather than holding the box set back. +func TestRunKeepsTheTranslationWhenTheEnglishFallbackFails(t *testing.T) { + store := newFakeStore(catalog.MovieCollectionLookup{ContentID: "hp1", TMDBID: 671}) + store.language = "de" + client := &fakeTMDB{ + movies: map[int]*tmdb.CollectionRef{671: {ID: 1241}}, + translated: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Filmreihe", Parts: []tmdb.CollectionPart{{ID: 671}}}}, + englishErr: errors.New("tmdb timeout"), + } + res, err := NewSyncer(store, client).Run(t.Context(), noReport) + if err != nil { + t.Fatal(err) + } + if stored, ok := store.collections[1241]; !ok || stored.Name != "Harry Potter Filmreihe" || res.Failures != 0 { + t.Fatalf("stored = %+v, %v; failures = %d; want the German collection stored without a failure", stored, ok, res.Failures) + } +} + type fakeCacher struct { requests []metadata.CacheImageRequest fail bool From 4ab7be87db7ef3f980ebd3ae1f4266b7f8c98691 Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:12:59 +0000 Subject: [PATCH 4/5] fix(boxsets): fill untranslated box set text in the fallback language Item metadata fills text a translation lacks from metadata.fallback_language and leaves it empty when that setting is the original language. The box set sync always asked TMDB for English instead. It now reads the same setting, hot-reloaded through the metadata service. Co-Authored-By: Claude Opus 5.5 --- cmd/silo/main.go | 2 +- docs/architecture/movie-box-sets.md | 7 ++-- internal/boxsets/sync.go | 37 +++++++++++++------- internal/boxsets/sync_test.go | 35 ++++++++++++++---- internal/metadata/language_fallback.go | 6 ++-- internal/metadata/local_artwork_discovery.go | 2 +- internal/metadata/service.go | 4 +-- 7 files changed, 64 insertions(+), 29 deletions(-) diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 67b00ab3cd..c105872a5b 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -3397,7 +3397,7 @@ func main() { taskMgr.Register(tasks.NewRefreshSeriesAirDatesTask(catalog.NewEpisodeRepository(deps.DB))) boxSetSyncer := boxsets.NewSyncer(catalog.NewBoxSetRepository(deps.DB), tmdbClient) if metadataService != nil { - boxSetSyncer.WithImages(metadataService.AutoImageCacher) + boxSetSyncer.WithImages(metadataService.AutoImageCacher).WithFallbackLanguage(metadataService.FallbackLanguage) } taskMgr.Register(tasks.NewSyncMovieBoxSetsTask(deps.DB, boxSetSyncer)) go boxsets.Watch(appCtx, deps.EventsHub, func() { diff --git a/docs/architecture/movie-box-sets.md b/docs/architecture/movie-box-sets.md index 94cc0a5d43..cfeb19cf4f 100644 --- a/docs/architecture/movie-box-sets.md +++ b/docs/architecture/movie-box-sets.md @@ -25,9 +25,10 @@ its own card. This note records the invariants; the wire contract is in - One row serves every library. It is fetched in the metadata language of the first enabled library, in library order, that holds one of the collection's movies with box sets on, and `language` records which. TMDB - leaves the name or overview empty where its translation lacks them; those - come from TMDB's English answer. TMDB picks the poster for the language - itself. + leaves the overview (and possibly the name) empty where its translation + lacks it; those come from its answer in `metadata.fallback_language`, as + item metadata does, and stay empty when that is the original language. + TMDB picks the poster for the language itself. - The stored name is TMDB's. Reads show it through `catalog.BoxSetTitle` ("… Collection" becomes "… Box Set"). Search matches the name without that suffix (`boxSetStemSQL`, the same rule in SQL; change the two together), diff --git a/internal/boxsets/sync.go b/internal/boxsets/sync.go index 4daa246062..7fd1b11ef7 100644 --- a/internal/boxsets/sync.go +++ b/internal/boxsets/sync.go @@ -69,12 +69,15 @@ type Syncer struct { // images returns the image cacher, or nil while artwork is served from // the provider (no artwork store, or metadata.cache_images off). images func() metadata.ImageCacher - now func() time.Time + // fallbackLanguage returns metadata.fallback_language, empty when off. + fallbackLanguage func() string + now func() time.Time } -// NewSyncer returns a syncer over store and client. +// NewSyncer returns a syncer over store and client. Until WithFallbackLanguage +// says otherwise, it falls back to English, the setting's default. func NewSyncer(store Store, client TMDB) *Syncer { - return &Syncer{store: store, tmdb: client, now: time.Now} + return &Syncer{store: store, tmdb: client, fallbackLanguage: func() string { return "en" }, now: time.Now} } // WithImages caches collection artwork through the cacher images returns, @@ -84,6 +87,14 @@ func (s *Syncer) WithImages(images func() metadata.ImageCacher) *Syncer { return s } +// WithFallbackLanguage fills the text a collection's translation lacks in the +// language fallback returns (metadata.fallback_language), the way item +// metadata falls back; empty fills nothing. +func (s *Syncer) WithFallbackLanguage(fallback func() string) *Syncer { + s.fallbackLanguage = fallback + return s +} + // Run deletes the collections no movie belongs to any more, looks up every // movie of a box set library that is due, then fetches every collection that // is due and caches their artwork. It repeats while a lookup finds movies @@ -212,26 +223,28 @@ func (s *Syncer) fetchCollections(ctx context.Context, res *Result, tried *[]int } // fetchCollection fetches a collection in the language it is due in. TMDB -// leaves the name or overview empty where its translation lacks them, so -// those come from its English answer, the way item metadata falls back. -// TMDB falls back for artwork itself. When that English answer fails, the -// translation is kept as it is unless it has no name to show. +// leaves the overview empty where its translation lacks one (and may do the +// same for the name), so those come from its answer in the fallback language, +// the way item metadata falls back. TMDB falls back for artwork itself. When +// that answer fails, the translation is kept as it is unless it has no name to +// show. func (s *Syncer) fetchCollection(ctx context.Context, fetch catalog.CollectionFetch) (*tmdb.Collection, error) { c, err := s.tmdb.GetCollection(ctx, int(fetch.ID), fetch.Language) - if err != nil || fetch.Language == "en" || (c.Name != "" && c.Overview != "") { + fallback := s.fallbackLanguage() + if err != nil || fallback == "" || fallback == fetch.Language || (c.Name != "" && c.Overview != "") { return c, err } - english, err := s.tmdb.GetCollection(ctx, int(fetch.ID), "") + other, err := s.tmdb.GetCollection(ctx, int(fetch.ID), fallback) if err != nil { if c.Name != "" { - slog.WarnContext(ctx, "box sets: English collection fallback failed; storing the translation as it is", "component", "boxsets", "tmdb_collection_id", fetch.ID, "error", err) + slog.WarnContext(ctx, "box sets: fallback language collection fetch failed; storing the translation as it is", "component", "boxsets", "tmdb_collection_id", fetch.ID, "fallback_language", fallback, "error", err) return c, nil } return nil, err } localized := *c - localized.Name = cmp.Or(c.Name, english.Name) - localized.Overview = cmp.Or(c.Overview, english.Overview) + localized.Name = cmp.Or(c.Name, other.Name) + localized.Overview = cmp.Or(c.Overview, other.Overview) return &localized, nil } diff --git a/internal/boxsets/sync_test.go b/internal/boxsets/sync_test.go index 9e012970a8..16af6d2398 100644 --- a/internal/boxsets/sync_test.go +++ b/internal/boxsets/sync_test.go @@ -109,12 +109,12 @@ type fakeTMDB struct { lookups map[int]int // onCollection runs on each collection fetch. onCollection func() - // translated answers collection requests that ask for a language; - // collections answers the others, as TMDB answers in English. + // translated answers collection requests in a language other than + // English; collections answers the others. translated map[int]*tmdb.Collection // languages records the languages each collection was asked for in. languages map[int][]string - // englishErr answers collection requests that ask for no language. + // englishErr answers collection requests in English. englishErr error } @@ -139,11 +139,12 @@ func (f *fakeTMDB) GetCollection(_ context.Context, id int, language string) (*t if f.onCollection != nil { f.onCollection() } - if language == "" && f.englishErr != nil { + english := language == "" || language == "en" + if english && f.englishErr != nil { return nil, f.englishErr } c, ok := f.collections[id] - if t, translated := f.translated[id]; translated && language != "" { + if t, translated := f.translated[id]; translated && !english { c, ok = t, true } if !ok { @@ -201,8 +202,9 @@ func TestRunStoresMembershipAndCollectionsInReleaseOrder(t *testing.T) { } stored := store.collections[1241] // The collection is stored in the language it is due in, and the - // overview the translation lacks comes from TMDB's English answer. - if !slices.Equal(client.languages[1241], []string{"de", ""}) || stored.Language != "de" { + // overview the translation lacks comes from the fallback language, + // English by default. + if !slices.Equal(client.languages[1241], []string{"de", "en"}) || stored.Language != "de" { t.Fatalf("fetched in %q, stored as %q; want de, then the English fallback", client.languages[1241], stored.Language) } if stored.Name != "Harry Potter Filmreihe" || stored.Overview != "The wizarding saga." { @@ -235,6 +237,25 @@ func TestRunKeepsTheTranslationWhenTheEnglishFallbackFails(t *testing.T) { } } +// With the fallback language set to the original language, the text a +// translation lacks stays empty, as it does for item metadata. +func TestRunFillsNothingWhenTheFallbackIsOff(t *testing.T) { + store := newFakeStore(catalog.MovieCollectionLookup{ContentID: "hp1", TMDBID: 671}) + store.language = "de" + client := &fakeTMDB{ + languages: map[int][]string{}, + movies: map[int]*tmdb.CollectionRef{671: {ID: 1241}}, + translated: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Filmreihe", Parts: []tmdb.CollectionPart{{ID: 671}}}}, + collections: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Collection", Overview: "The wizarding saga.", Parts: []tmdb.CollectionPart{{ID: 671}}}}, + } + if _, err := NewSyncer(store, client).WithFallbackLanguage(func() string { return "" }).Run(t.Context(), noReport); err != nil { + t.Fatal(err) + } + if stored := store.collections[1241]; !slices.Equal(client.languages[1241], []string{"de"}) || stored.Overview != "" { + t.Fatalf("asked in %q, stored overview %q; want only de and no overview", client.languages[1241], stored.Overview) + } +} + type fakeCacher struct { requests []metadata.CacheImageRequest fail bool diff --git a/internal/metadata/language_fallback.go b/internal/metadata/language_fallback.go index f8c0d379a4..8ff8d4564d 100644 --- a/internal/metadata/language_fallback.go +++ b/internal/metadata/language_fallback.go @@ -36,8 +36,8 @@ func (s *MetadataService) SetFallbackLanguage(language string) { s.fallbackLanguage.Store(baseMetadataLanguage(language)) } -// configuredFallbackLanguage is metadata.fallback_language, empty when off. -func (s *MetadataService) configuredFallbackLanguage() string { +// FallbackLanguage is metadata.fallback_language, empty when off. +func (s *MetadataService) FallbackLanguage() string { language, _ := s.fallbackLanguage.Load().(string) return language } @@ -45,7 +45,7 @@ func (s *MetadataService) configuredFallbackLanguage() string { // fallbackLanguageFor is the language a refresh in requested fetches gaps in, // or empty when the fallback is off or is the requested language itself. func (s *MetadataService) fallbackLanguageFor(requested string) string { - fallback := s.configuredFallbackLanguage() + fallback := s.FallbackLanguage() requested = baseMetadataLanguage(requested) if fallback == "" || requested == "" || fallback == requested { return "" diff --git a/internal/metadata/local_artwork_discovery.go b/internal/metadata/local_artwork_discovery.go index 1627ce9e36..d947fa0252 100644 --- a/internal/metadata/local_artwork_discovery.go +++ b/internal/metadata/local_artwork_discovery.go @@ -179,7 +179,7 @@ func (s *MetadataService) saveProvisionalLocalArtwork(ctx context.Context, conte return } item := *existing - fallback := s.configuredFallbackLanguage() + fallback := s.FallbackLanguage() applyBestImages(&item, images, MergeFillEmpty, language, fallback) // Local artwork is only found with image caching on, so stored copies keep // serving until the new copy lands. diff --git a/internal/metadata/service.go b/internal/metadata/service.go index 31702982fb..bfedbdb897 100644 --- a/internal/metadata/service.go +++ b/internal/metadata/service.go @@ -2934,7 +2934,7 @@ func (s *MetadataService) mergeAndPersist( // localization tables like any other non-canonical refresh. canonicalLanguage, isCanonicalWrite := canonicalMetadataLanguage(req, existingItem, locked) // Read once, so artwork and text in this write follow the same setting. - fallbackLanguage := s.configuredFallbackLanguage() + fallbackLanguage := s.FallbackLanguage() switch { case isCanonicalWrite: // Fallback text belongs to the provider IDs it was fetched with, so it @@ -4805,7 +4805,7 @@ func (s *MetadataService) refreshSeriesChildTarget( if err := s.fillTargetChildText(ctx, series, providerIDs, folderID, language, seasonNumber, episodeNumber, childCtx, seasons, episodes, mode, mergeMode, reset); err != nil { return err } - } else if s.configuredFallbackLanguage() != "" { + } else if s.FallbackLanguage() != "" { // Text a provider marked as another language is no translation. dropForeignChildText(seasons, episodes, language) } From 45c12e082ca33682bcc285663a60454ff1f9e83c Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:18:31 +0000 Subject: [PATCH 5/5] fix(boxsets): choose a box set's language from a library that shows it A library holding fewer of a collection's movies than its box set minimum does not show the box set, so it no longer picks the language the shared row is fetched in while another library shows it. A failed fallback-language answer now fails the fetch. Storing the translation without it wrote a blank overview over the stored one, marked the row fresh for a week, and kept the failure out of the run's budget. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/movie-box-sets.md | 12 +++++++----- internal/boxsets/sync.go | 10 +++------- internal/boxsets/sync_test.go | 10 +++++----- internal/catalog/box_sets.go | 15 +++++++++------ internal/catalog/box_sets_db_test.go | 9 +++++++-- 5 files changed, 31 insertions(+), 25 deletions(-) diff --git a/docs/architecture/movie-box-sets.md b/docs/architecture/movie-box-sets.md index cfeb19cf4f..4a6be236be 100644 --- a/docs/architecture/movie-box-sets.md +++ b/docs/architecture/movie-box-sets.md @@ -23,11 +23,13 @@ its own card. This note records the invariants; the wire contract is in parts in no reliable order, so the sync sorts them by release date, with undated (announced) parts last. - One row serves every library. It is fetched in the metadata language of - the first enabled library, in library order, that holds one of the - collection's movies with box sets on, and `language` records which. TMDB - leaves the overview (and possibly the name) empty where its translation - lacks it; those come from its answer in `metadata.fallback_language`, as - item metadata does, and stay empty when that is the original language. + the first enabled library with box sets on, in library order, that holds + at least its box set minimum of the collection's movies (failing that, the + first that holds one), and `language` records which. TMDB leaves the + overview (and possibly the name) empty where its translation lacks it; + those come from its answer in `metadata.fallback_language`, as item + metadata does, and stay empty when that is the original language. When + that answer fails, the fetch fails and the stored row is left as it was. TMDB picks the poster for the language itself. - The stored name is TMDB's. Reads show it through `catalog.BoxSetTitle` ("… Collection" becomes "… Box Set"). Search matches the name without that diff --git a/internal/boxsets/sync.go b/internal/boxsets/sync.go index 7fd1b11ef7..25f73787d6 100644 --- a/internal/boxsets/sync.go +++ b/internal/boxsets/sync.go @@ -226,8 +226,8 @@ func (s *Syncer) fetchCollections(ctx context.Context, res *Result, tried *[]int // leaves the overview empty where its translation lacks one (and may do the // same for the name), so those come from its answer in the fallback language, // the way item metadata falls back. TMDB falls back for artwork itself. When -// that answer fails, the translation is kept as it is unless it has no name to -// show. +// that answer fails, the fetch fails: the stored copy keeps its text and the +// collection stays due for the next run. func (s *Syncer) fetchCollection(ctx context.Context, fetch catalog.CollectionFetch) (*tmdb.Collection, error) { c, err := s.tmdb.GetCollection(ctx, int(fetch.ID), fetch.Language) fallback := s.fallbackLanguage() @@ -236,11 +236,7 @@ func (s *Syncer) fetchCollection(ctx context.Context, fetch catalog.CollectionFe } other, err := s.tmdb.GetCollection(ctx, int(fetch.ID), fallback) if err != nil { - if c.Name != "" { - slog.WarnContext(ctx, "box sets: fallback language collection fetch failed; storing the translation as it is", "component", "boxsets", "tmdb_collection_id", fetch.ID, "fallback_language", fallback, "error", err) - return c, nil - } - return nil, err + return nil, fmt.Errorf("fetching in fallback language %s: %v", fallback, err) //nolint:errorlint // not ErrNotFound: a collection TMDB serves in one language is not gone } localized := *c localized.Name = cmp.Or(c.Name, other.Name) diff --git a/internal/boxsets/sync_test.go b/internal/boxsets/sync_test.go index 16af6d2398..db26b2821e 100644 --- a/internal/boxsets/sync_test.go +++ b/internal/boxsets/sync_test.go @@ -218,9 +218,9 @@ func TestRunStoresMembershipAndCollectionsInReleaseOrder(t *testing.T) { } } -// A translation is stored as it is when the English answer that would fill -// its missing overview fails, rather than holding the box set back. -func TestRunKeepsTheTranslationWhenTheEnglishFallbackFails(t *testing.T) { +// A failed fallback answer fails the fetch rather than storing the +// translation without its overview, so the collection is tried again. +func TestRunRetriesWhenTheFallbackFails(t *testing.T) { store := newFakeStore(catalog.MovieCollectionLookup{ContentID: "hp1", TMDBID: 671}) store.language = "de" client := &fakeTMDB{ @@ -232,8 +232,8 @@ func TestRunKeepsTheTranslationWhenTheEnglishFallbackFails(t *testing.T) { if err != nil { t.Fatal(err) } - if stored, ok := store.collections[1241]; !ok || stored.Name != "Harry Potter Filmreihe" || res.Failures != 0 { - t.Fatalf("stored = %+v, %v; failures = %d; want the German collection stored without a failure", stored, ok, res.Failures) + if _, stored := store.collections[1241]; stored || res.Failures != 1 { + t.Fatalf("stored = %v, failures = %d; want nothing stored and one failure", stored, res.Failures) } } diff --git a/internal/catalog/box_sets.go b/internal/catalog/box_sets.go index 94be73e4a8..93de67f65c 100644 --- a/internal/catalog/box_sets.go +++ b/internal/catalog/box_sets.go @@ -213,9 +213,10 @@ type CollectionFetch struct { // set library belongs to that are not stored yet, were fetched before // staleBefore, or were fetched in another language than the one they are // due in. A collection is due in the metadata language of the first library, -// in library order, that holds one of its movies with box sets on: one -// collection serves every library, so a deployment whose box set libraries -// differ in language shows it in one of them. +// in library order, that shows it: one with box sets on holding at least its +// box set minimum of the collection's movies, or failing that any library +// holding one of them. One collection serves every library, so a deployment +// whose box set libraries differ in language shows it in one of them. func (r *BoxSetRepository) CollectionsNeedingFetch(ctx context.Context, staleBefore time.Time, limit int, skip []int64) ([]CollectionFetch, error) { rows, err := r.pool.Query(ctx, ` SELECT due.tmdb_collection_id, due.language @@ -223,17 +224,19 @@ func (r *BoxSetRepository) CollectionsNeedingFetch(ctx context.Context, staleBef SELECT DISTINCT ON (m.tmdb_collection_id) m.tmdb_collection_id, lib.language FROM media_item_tmdb_collections m CROSS JOIN LATERAL ( - SELECT COALESCE(NULLIF(BTRIM(cf.metadata_language), ''), 'en') AS language, cf.sort_order, cf.id + SELECT cf.id, cf.sort_order, cf.box_set_min_movies, + COALESCE(NULLIF(BTRIM(cf.metadata_language), ''), 'en') AS language `+currentBoxSetMemberLibraries+` ) lib WHERE m.tmdb_collection_id IS NOT NULL AND NOT (m.tmdb_collection_id = ANY($3::bigint[])) - ORDER BY m.tmdb_collection_id, lib.sort_order, lib.id + GROUP BY m.tmdb_collection_id, lib.id, lib.sort_order, lib.box_set_min_movies, lib.language + ORDER BY m.tmdb_collection_id, count(*) >= GREATEST(lib.box_set_min_movies, $4) DESC, lib.sort_order, lib.id ) due LEFT JOIN tmdb_movie_collections c ON c.tmdb_collection_id = due.tmdb_collection_id WHERE c.tmdb_collection_id IS NULL OR c.fetched_at < $1 OR c.language <> due.language ORDER BY c.fetched_at NULLS FIRST, due.tmdb_collection_id - LIMIT $2`, staleBefore, limit, nonNilSlice(skip)) + LIMIT $2`, staleBefore, limit, nonNilSlice(skip), MinBoxSetMinMovies) if err != nil { return nil, fmt.Errorf("querying collections needing a fetch: %w", err) } diff --git a/internal/catalog/box_sets_db_test.go b/internal/catalog/box_sets_db_test.go index 7e416e1cb9..d8b83fb3e5 100644 --- a/internal/catalog/box_sets_db_test.go +++ b/internal/catalog/box_sets_db_test.go @@ -345,14 +345,19 @@ func TestBoxSetCollectionLanguageDB(t *testing.T) { t.Fatalf("stored collection = %+v, %v; want language de", stored[f.franchise], err) } - // Another library ahead of it in the library order holds one of the - // collection's movies: its language wins, so every run asks for the same. + // Another library ahead of it in the library order holds the + // collection's movies: its language wins once it holds enough of them to + // show the box set, and every run asks for the same. var first int if err := f.pool.QueryRow(ctx, `INSERT INTO media_folders(type,name,enabled,box_sets_enabled,metadata_language,sort_order) VALUES('movies',$1,true,true,'fr',-1) RETURNING id`, f.prefix+"-fr").Scan(&first); err != nil { t.Fatal(err) } t.Cleanup(func() { _, _ = f.pool.Exec(context.Background(), `DELETE FROM media_folders WHERE id=$1`, first) }) exec(`INSERT INTO media_item_libraries(content_id,media_folder_id) VALUES($1,$2)`, f.id("first"), first) + if language, ok := due(); ok { + t.Fatalf("a French library below its box set minimum made the collection due in %q", language) + } + exec(`INSERT INTO media_item_libraries(content_id,media_folder_id) VALUES($1,$2)`, f.id("second"), first) if language, ok := due(); !ok || language != "fr" { t.Fatalf("with a French library first, due = %q, %v; want a fetch in fr", language, ok) }