diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 67b00ab3cd..c105872a5b 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -3397,7 +3397,7 @@ func main() { taskMgr.Register(tasks.NewRefreshSeriesAirDatesTask(catalog.NewEpisodeRepository(deps.DB))) boxSetSyncer := boxsets.NewSyncer(catalog.NewBoxSetRepository(deps.DB), tmdbClient) if metadataService != nil { - boxSetSyncer.WithImages(metadataService.AutoImageCacher) + boxSetSyncer.WithImages(metadataService.AutoImageCacher).WithFallbackLanguage(metadataService.FallbackLanguage) } taskMgr.Register(tasks.NewSyncMovieBoxSetsTask(deps.DB, boxSetSyncer)) go boxsets.Watch(appCtx, deps.EventsHub, func() { diff --git a/docs/architecture/movie-box-sets.md b/docs/architecture/movie-box-sets.md index b030f87d6e..4a6be236be 100644 --- a/docs/architecture/movie-box-sets.md +++ b/docs/architecture/movie-box-sets.md @@ -22,6 +22,15 @@ its own card. This note records the invariants; the wire contract is in collected. TMDB returns a collection's parts in no reliable order, so the sync sorts them by release date, with undated (announced) parts last. +- One row serves every library. It is fetched in the metadata language of + the first enabled library with box sets on, in library order, that holds + at least its box set minimum of the collection's movies (failing that, the + first that holds one), and `language` records which. TMDB leaves the + overview (and possibly the name) empty where its translation lacks it; + those come from its answer in `metadata.fallback_language`, as item + metadata does, and stay empty when that is the original language. When + that answer fails, the fetch fails and the stored row is left as it was. + TMDB picks the poster for the language itself. - The stored name is TMDB's. Reads show it through `catalog.BoxSetTitle` ("… Collection" becomes "… Box Set"). Search matches the name without that suffix (`boxSetStemSQL`, the same rule in SQL; change the two together), @@ -34,9 +43,10 @@ its own card. This note records the invariants; the wire contract is in The `sync_movie_box_sets` task (`internal/boxsets`) is the only code that calls TMDB for box sets; the box set reads use stored rows only. It looks up movies of box set libraries that were never looked up, were rematched, or were -last looked up 30 days ago, then fetches collections that are new or were -fetched 7 days ago, then caches collection artwork still served from TMDB -through the item image cacher, under the same `metadata.cache_images` switch +last looked up 30 days ago, then fetches collections that are new, were +fetched 7 days ago, or were fetched in another language than the one they +are now due in (a library changed its metadata language), then caches +collection artwork still served from TMDB through the item image cacher, under the same `metadata.cache_images` switch as item artwork. It runs at startup, hourly, when a library turns box sets on, and 20 seconds after the node that matched a movie publishes its catalog change. A run stops after 25 failed TMDB calls; the next run resumes. diff --git a/internal/api/router.go b/internal/api/router.go index b8e48b3b70..f18b2f6d53 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -3722,7 +3722,7 @@ type tmdbFranchiseAdapter struct { } func (a *tmdbFranchiseAdapter) GetCollection(ctx context.Context, id int) ([]catalog.TMDBCollectionEntry, error) { - collection, err := a.client.GetCollection(ctx, id) + collection, err := a.client.GetCollection(ctx, id, "") if err != nil { return nil, err } diff --git a/internal/boxsets/sync.go b/internal/boxsets/sync.go index 97c3d7fe9d..25f73787d6 100644 --- a/internal/boxsets/sync.go +++ b/internal/boxsets/sync.go @@ -6,6 +6,7 @@ package boxsets import ( + "cmp" "context" "errors" "fmt" @@ -38,16 +39,16 @@ const ( // TMDB is the slice of *tmdb.Client the sync uses. type TMDB interface { GetMovieCollection(ctx context.Context, movieID int) (*tmdb.CollectionRef, error) - GetCollection(ctx context.Context, id int) (*tmdb.Collection, error) + GetCollection(ctx context.Context, id int, language string) (*tmdb.Collection, error) } // Store is the slice of *catalog.BoxSetRepository the sync uses. type Store interface { MoviesNeedingCollectionLookup(ctx context.Context, staleBefore time.Time, limit int, skip []string) ([]catalog.MovieCollectionLookup, error) RecordMovieCollection(ctx context.Context, contentID string, tmdbID int64, collectionID *int64) error - CollectionsNeedingFetch(ctx context.Context, staleBefore time.Time, limit int, skip []int64) ([]int64, error) + CollectionsNeedingFetch(ctx context.Context, staleBefore time.Time, limit int, skip []int64) ([]catalog.CollectionFetch, error) UpsertCollection(ctx context.Context, c catalog.TMDBMovieCollection) error - TouchCollection(ctx context.Context, id int64) error + TouchCollection(ctx context.Context, id int64, language string) error DeleteUnusedCollections(ctx context.Context) (int64, error) CollectionArtworkNeedingCache(ctx context.Context, limit int, skip []int64) ([]catalog.CollectionArtworkDue, error) SetCollectionArtwork(ctx context.Context, id int64, slot catalog.CollectionArtworkSlot, sourcePath, path, thumbhash string) error @@ -68,12 +69,15 @@ type Syncer struct { // images returns the image cacher, or nil while artwork is served from // the provider (no artwork store, or metadata.cache_images off). images func() metadata.ImageCacher - now func() time.Time + // fallbackLanguage returns metadata.fallback_language, empty when off. + fallbackLanguage func() string + now func() time.Time } -// NewSyncer returns a syncer over store and client. +// NewSyncer returns a syncer over store and client. Until WithFallbackLanguage +// says otherwise, it falls back to English, the setting's default. func NewSyncer(store Store, client TMDB) *Syncer { - return &Syncer{store: store, tmdb: client, now: time.Now} + return &Syncer{store: store, tmdb: client, fallbackLanguage: func() string { return "en" }, now: time.Now} } // WithImages caches collection artwork through the cacher images returns, @@ -83,6 +87,14 @@ func (s *Syncer) WithImages(images func() metadata.ImageCacher) *Syncer { return s } +// WithFallbackLanguage fills the text a collection's translation lacks in the +// language fallback returns (metadata.fallback_language), the way item +// metadata falls back; empty fills nothing. +func (s *Syncer) WithFallbackLanguage(fallback func() string) *Syncer { + s.fallbackLanguage = fallback + return s +} + // Run deletes the collections no movie belongs to any more, looks up every // movie of a box set library that is due, then fetches every collection that // is due and caches their artwork. It repeats while a lookup finds movies @@ -165,7 +177,8 @@ func (s *Syncer) lookUpMovies(ctx context.Context, res *Result, failed *[]string } } -// fetchCollections stores every due collection. tried collects the +// fetchCollections stores every due collection in the language it is due +// in. tried collects the // collections this run could not store: failed fetches, and ones TMDB no // longer serves that were never stored. func (s *Syncer) fetchCollections(ctx context.Context, res *Result, tried *[]int64, report func(float64, string)) error { @@ -175,16 +188,17 @@ func (s *Syncer) fetchCollections(ctx context.Context, res *Result, tried *[]int if err != nil || len(due) == 0 { return err } - for _, id := range due { + for _, fetch := range due { if err := ctx.Err(); err != nil { return err } - c, err := s.tmdb.GetCollection(ctx, int(id)) + id := fetch.ID + c, err := s.fetchCollection(ctx, fetch) if errors.Is(err, tmdb.ErrNotFound) { // Keep a stored copy rather than refetching a collection TMDB // no longer serves on every run. One never stored stays due // for the next run. - if err := s.store.TouchCollection(ctx, id); err != nil { + if err := s.store.TouchCollection(ctx, id, fetch.Language); err != nil { return err } *tried = append(*tried, id) @@ -199,7 +213,7 @@ func (s *Syncer) fetchCollections(ctx context.Context, res *Result, tried *[]int } continue } - if err := s.store.UpsertCollection(ctx, storedCollection(id, c)); err != nil { + if err := s.store.UpsertCollection(ctx, storedCollection(id, fetch.Language, c)); err != nil { return err } res.CollectionsFetched++ @@ -208,6 +222,28 @@ func (s *Syncer) fetchCollections(ctx context.Context, res *Result, tried *[]int } } +// fetchCollection fetches a collection in the language it is due in. TMDB +// leaves the overview empty where its translation lacks one (and may do the +// same for the name), so those come from its answer in the fallback language, +// the way item metadata falls back. TMDB falls back for artwork itself. When +// that answer fails, the fetch fails: the stored copy keeps its text and the +// collection stays due for the next run. +func (s *Syncer) fetchCollection(ctx context.Context, fetch catalog.CollectionFetch) (*tmdb.Collection, error) { + c, err := s.tmdb.GetCollection(ctx, int(fetch.ID), fetch.Language) + fallback := s.fallbackLanguage() + if err != nil || fallback == "" || fallback == fetch.Language || (c.Name != "" && c.Overview != "") { + return c, err + } + other, err := s.tmdb.GetCollection(ctx, int(fetch.ID), fallback) + if err != nil { + return nil, fmt.Errorf("fetching in fallback language %s: %v", fallback, err) //nolint:errorlint // not ErrNotFound: a collection TMDB serves in one language is not gone + } + localized := *c + localized.Name = cmp.Or(c.Name, other.Name) + localized.Overview = cmp.Or(c.Overview, other.Overview) + return &localized, nil +} + // cacheArtwork stores a copy of each collection image still served from // TMDB. A failed image stays on the provider URL and is tried again next run. func (s *Syncer) cacheArtwork(ctx context.Context, res *Result) error { @@ -267,7 +303,7 @@ func tmdbOriginalURL(path string) string { // storedCollection converts a fetched collection, listing its movies in // release order. TMDB returns parts in no reliable order, and a part without // a release date (an announced sequel) goes last. -func storedCollection(id int64, c *tmdb.Collection) catalog.TMDBMovieCollection { +func storedCollection(id int64, language string, c *tmdb.Collection) catalog.TMDBMovieCollection { parts := append([]tmdb.CollectionPart(nil), c.Parts...) sort.SliceStable(parts, func(i, j int) bool { a, b := parts[i].ReleaseDate, parts[j].ReleaseDate @@ -289,5 +325,6 @@ func storedCollection(id int64, c *tmdb.Collection) catalog.TMDBMovieCollection PosterSourcePath: tmdbOriginalURL(c.PosterPath), BackdropSourcePath: tmdbOriginalURL(c.BackdropPath), MemberTMDBIDs: members, + Language: language, } } diff --git a/internal/boxsets/sync_test.go b/internal/boxsets/sync_test.go index 7c68287ed4..db26b2821e 100644 --- a/internal/boxsets/sync_test.go +++ b/internal/boxsets/sync_test.go @@ -19,6 +19,8 @@ type fakeStore struct { recorded map[string]*int64 collections map[int64]catalog.TMDBMovieCollection touched []int64 + // language is the language every collection is due in. + language string } func (s *fakeStore) MoviesNeedingCollectionLookup(_ context.Context, _ time.Time, limit int, skip []string) ([]catalog.MovieCollectionLookup, error) { @@ -36,17 +38,21 @@ func (s *fakeStore) RecordMovieCollection(_ context.Context, contentID string, _ return nil } -func (s *fakeStore) CollectionsNeedingFetch(_ context.Context, _ time.Time, _ int, skip []int64) ([]int64, error) { - var out []int64 +func (s *fakeStore) CollectionsNeedingFetch(_ context.Context, _ time.Time, _ int, skip []int64) ([]catalog.CollectionFetch, error) { + var ids []int64 for _, id := range s.recorded { - if id == nil || slices.Contains(out, *id) || slices.Contains(s.touched, *id) || slices.Contains(skip, *id) { + if id == nil || slices.Contains(ids, *id) || slices.Contains(s.touched, *id) || slices.Contains(skip, *id) { continue } if _, stored := s.collections[*id]; !stored { - out = append(out, *id) + ids = append(ids, *id) } } - slices.Sort(out) + slices.Sort(ids) + out := make([]catalog.CollectionFetch, len(ids)) + for i, id := range ids { + out[i] = catalog.CollectionFetch{ID: id, Language: s.language} + } return out, nil } @@ -87,7 +93,7 @@ func (s *fakeStore) DeleteUnusedCollections(context.Context) (int64, error) { re // TouchCollection marks only a stored collection fetched, as the // repository's UPDATE does. -func (s *fakeStore) TouchCollection(_ context.Context, id int64) error { +func (s *fakeStore) TouchCollection(_ context.Context, id int64, _ string) error { if _, stored := s.collections[id]; stored { s.touched = append(s.touched, id) } @@ -103,6 +109,13 @@ type fakeTMDB struct { lookups map[int]int // onCollection runs on each collection fetch. onCollection func() + // translated answers collection requests in a language other than + // English; collections answers the others. + translated map[int]*tmdb.Collection + // languages records the languages each collection was asked for in. + languages map[int][]string + // englishErr answers collection requests in English. + englishErr error } func (f *fakeTMDB) GetMovieCollection(_ context.Context, id int) (*tmdb.CollectionRef, error) { @@ -119,11 +132,21 @@ func (f *fakeTMDB) GetMovieCollection(_ context.Context, id int) (*tmdb.Collecti return ref, nil } -func (f *fakeTMDB) GetCollection(_ context.Context, id int) (*tmdb.Collection, error) { +func (f *fakeTMDB) GetCollection(_ context.Context, id int, language string) (*tmdb.Collection, error) { + if f.languages != nil { + f.languages[id] = append(f.languages[id], language) + } if f.onCollection != nil { f.onCollection() } + english := language == "" || language == "en" + if english && f.englishErr != nil { + return nil, f.englishErr + } c, ok := f.collections[id] + if t, translated := f.translated[id]; translated && !english { + c, ok = t, true + } if !ok { return nil, tmdb.ErrNotFound } @@ -142,9 +165,18 @@ func TestRunStoresMembershipAndCollectionsInReleaseOrder(t *testing.T) { catalog.MovieCollectionLookup{ContentID: "heat", TMDBID: 949}, catalog.MovieCollectionLookup{ContentID: "gone", TMDBID: 1}, ) + store.language = "de" client := &fakeTMDB{ + languages: map[int][]string{}, + // TMDB leaves the fields a translation lacks empty. + translated: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Filmreihe", PosterPath: "/de.jpg", Parts: []tmdb.CollectionPart{ + {ID: 674, ReleaseDate: "2005-11-16"}, + {ID: 999, ReleaseDate: ""}, + {ID: 671, ReleaseDate: "2001-11-16"}, + {ID: 673, ReleaseDate: "2004-05-31"}, + }}}, movies: map[int]*tmdb.CollectionRef{671: {ID: 1241, Name: "Harry Potter Collection"}, 949: nil}, - collections: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Collection", PosterPath: "/p.jpg", Parts: []tmdb.CollectionPart{ + collections: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Collection", Overview: "The wizarding saga.", PosterPath: "/p.jpg", Parts: []tmdb.CollectionPart{ {ID: 674, ReleaseDate: "2005-11-16"}, {ID: 999, ReleaseDate: ""}, {ID: 671, ReleaseDate: "2001-11-16"}, @@ -169,14 +201,61 @@ func TestRunStoresMembershipAndCollectionsInReleaseOrder(t *testing.T) { } } stored := store.collections[1241] + // The collection is stored in the language it is due in, and the + // overview the translation lacks comes from the fallback language, + // English by default. + if !slices.Equal(client.languages[1241], []string{"de", "en"}) || stored.Language != "de" { + t.Fatalf("fetched in %q, stored as %q; want de, then the English fallback", client.languages[1241], stored.Language) + } + if stored.Name != "Harry Potter Filmreihe" || stored.Overview != "The wizarding saga." { + t.Fatalf("name, overview = %q, %q; want the German name and the English overview", stored.Name, stored.Overview) + } if want := []int64{671, 673, 674, 999}; !slices.Equal(stored.MemberTMDBIDs, want) { t.Fatalf("members = %v, want release order %v with the undated part last", stored.MemberTMDBIDs, want) } - if stored.PosterSourcePath != "https://image.tmdb.org/t/p/original/p.jpg" || stored.BackdropSourcePath != "" { + if stored.PosterSourcePath != "https://image.tmdb.org/t/p/original/de.jpg" || stored.BackdropSourcePath != "" { t.Fatalf("artwork sources = %q, %q", stored.PosterSourcePath, stored.BackdropSourcePath) } } +// A failed fallback answer fails the fetch rather than storing the +// translation without its overview, so the collection is tried again. +func TestRunRetriesWhenTheFallbackFails(t *testing.T) { + store := newFakeStore(catalog.MovieCollectionLookup{ContentID: "hp1", TMDBID: 671}) + store.language = "de" + client := &fakeTMDB{ + movies: map[int]*tmdb.CollectionRef{671: {ID: 1241}}, + translated: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Filmreihe", Parts: []tmdb.CollectionPart{{ID: 671}}}}, + englishErr: errors.New("tmdb timeout"), + } + res, err := NewSyncer(store, client).Run(t.Context(), noReport) + if err != nil { + t.Fatal(err) + } + if _, stored := store.collections[1241]; stored || res.Failures != 1 { + t.Fatalf("stored = %v, failures = %d; want nothing stored and one failure", stored, res.Failures) + } +} + +// With the fallback language set to the original language, the text a +// translation lacks stays empty, as it does for item metadata. +func TestRunFillsNothingWhenTheFallbackIsOff(t *testing.T) { + store := newFakeStore(catalog.MovieCollectionLookup{ContentID: "hp1", TMDBID: 671}) + store.language = "de" + client := &fakeTMDB{ + languages: map[int][]string{}, + movies: map[int]*tmdb.CollectionRef{671: {ID: 1241}}, + translated: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Filmreihe", Parts: []tmdb.CollectionPart{{ID: 671}}}}, + collections: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Collection", Overview: "The wizarding saga.", Parts: []tmdb.CollectionPart{{ID: 671}}}}, + } + if _, err := NewSyncer(store, client).WithFallbackLanguage(func() string { return "" }).Run(t.Context(), noReport); err != nil { + t.Fatal(err) + } + if stored := store.collections[1241]; !slices.Equal(client.languages[1241], []string{"de"}) || stored.Overview != "" { + t.Fatalf("asked in %q, stored overview %q; want only de and no overview", client.languages[1241], stored.Overview) + } +} + type fakeCacher struct { requests []metadata.CacheImageRequest fail bool diff --git a/internal/catalog/box_sets.go b/internal/catalog/box_sets.go index ce296fd3a9..93de67f65c 100644 --- a/internal/catalog/box_sets.go +++ b/internal/catalog/box_sets.go @@ -79,7 +79,9 @@ type TMDBMovieCollection struct { BackdropThumbhash string // MemberTMDBIDs lists the collection's movies in release order. MemberTMDBIDs []int64 - FetchedAt time.Time + // Language is the metadata language the collection was fetched in. + Language string + FetchedAt time.Time } // BoxSetGroup is one box set of a library as a viewer sees it: the @@ -109,7 +111,7 @@ func (r *BoxSetRepository) Collections(ctx context.Context, ids []int64) (map[in return out, nil } rows, err := r.pool.Query(ctx, ` - SELECT tmdb_collection_id, name, overview, poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, member_tmdb_ids, fetched_at + SELECT tmdb_collection_id, name, overview, poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, member_tmdb_ids, language, fetched_at FROM tmdb_movie_collections WHERE tmdb_collection_id = ANY($1)`, ids) if err != nil { return nil, fmt.Errorf("querying tmdb collections: %w", err) @@ -117,7 +119,7 @@ func (r *BoxSetRepository) Collections(ctx context.Context, ids []int64) (map[in defer rows.Close() for rows.Next() { var c TMDBMovieCollection - if err := rows.Scan(&c.ID, &c.Name, &c.Overview, &c.PosterPath, &c.PosterThumbhash, &c.BackdropPath, &c.BackdropThumbhash, &c.MemberTMDBIDs, &c.FetchedAt); err != nil { + if err := rows.Scan(&c.ID, &c.Name, &c.Overview, &c.PosterPath, &c.PosterThumbhash, &c.BackdropPath, &c.BackdropThumbhash, &c.MemberTMDBIDs, &c.Language, &c.FetchedAt); err != nil { return nil, fmt.Errorf("scanning tmdb collection: %w", err) } out[c.ID] = c @@ -185,45 +187,69 @@ func (r *BoxSetRepository) RecordMovieCollection(ctx context.Context, contentID return nil } -// currentBoxSetMember holds when the collection lookup row m is current (its -// movie still has the TMDB ID it was looked up under) and its movie is in an -// enabled library with box sets on. Only such rows keep a collection stored -// and refreshed: a library that turns box sets off, or a rematched movie, -// leaves its lookup rows behind. -const currentBoxSetMember = `EXISTS ( - SELECT 1 FROM media_items cmi +// currentBoxSetMemberLibraries selects, as cf, the libraries that make the +// collection lookup row m current: its movie still has the TMDB ID it was +// looked up under, and they are enabled libraries with box sets on holding +// it. Only such rows keep a collection stored and refreshed: a library that +// turns box sets off, or a rematched movie, leaves its lookup rows behind. +const currentBoxSetMemberLibraries = ` + FROM media_items cmi JOIN media_item_libraries cmil ON cmil.content_id = cmi.content_id JOIN media_folders cf ON cf.id = cmil.media_folder_id WHERE cmi.content_id = m.content_id AND cmi.tmdb_id = m.checked_tmdb_id::text - AND cf.box_sets_enabled AND cf.enabled IS NOT FALSE)` + AND cf.box_sets_enabled AND cf.enabled IS NOT FALSE` + +// currentBoxSetMember holds when the collection lookup row m is current. +const currentBoxSetMember = `EXISTS (SELECT 1 ` + currentBoxSetMemberLibraries + `)` + +// CollectionFetch is a collection due for a fetch and the metadata language +// to fetch it in. +type CollectionFetch struct { + ID int64 + Language string +} // CollectionsNeedingFetch returns up to limit collections a movie of a box -// set library belongs to that are not stored yet or were fetched before -// staleBefore. -func (r *BoxSetRepository) CollectionsNeedingFetch(ctx context.Context, staleBefore time.Time, limit int, skip []int64) ([]int64, error) { +// set library belongs to that are not stored yet, were fetched before +// staleBefore, or were fetched in another language than the one they are +// due in. A collection is due in the metadata language of the first library, +// in library order, that shows it: one with box sets on holding at least its +// box set minimum of the collection's movies, or failing that any library +// holding one of them. One collection serves every library, so a deployment +// whose box set libraries differ in language shows it in one of them. +func (r *BoxSetRepository) CollectionsNeedingFetch(ctx context.Context, staleBefore time.Time, limit int, skip []int64) ([]CollectionFetch, error) { rows, err := r.pool.Query(ctx, ` - SELECT DISTINCT m.tmdb_collection_id, c.fetched_at - FROM media_item_tmdb_collections m - LEFT JOIN tmdb_movie_collections c ON c.tmdb_collection_id = m.tmdb_collection_id - WHERE m.tmdb_collection_id IS NOT NULL AND (c.tmdb_collection_id IS NULL OR c.fetched_at < $1) - AND `+currentBoxSetMember+` - AND NOT (m.tmdb_collection_id = ANY($3::bigint[])) - ORDER BY c.fetched_at NULLS FIRST, m.tmdb_collection_id - LIMIT $2`, staleBefore, limit, nonNilSlice(skip)) + SELECT due.tmdb_collection_id, due.language + FROM ( + SELECT DISTINCT ON (m.tmdb_collection_id) m.tmdb_collection_id, lib.language + FROM media_item_tmdb_collections m + CROSS JOIN LATERAL ( + SELECT cf.id, cf.sort_order, cf.box_set_min_movies, + COALESCE(NULLIF(BTRIM(cf.metadata_language), ''), 'en') AS language + `+currentBoxSetMemberLibraries+` + ) lib + WHERE m.tmdb_collection_id IS NOT NULL + AND NOT (m.tmdb_collection_id = ANY($3::bigint[])) + GROUP BY m.tmdb_collection_id, lib.id, lib.sort_order, lib.box_set_min_movies, lib.language + ORDER BY m.tmdb_collection_id, count(*) >= GREATEST(lib.box_set_min_movies, $4) DESC, lib.sort_order, lib.id + ) due + LEFT JOIN tmdb_movie_collections c ON c.tmdb_collection_id = due.tmdb_collection_id + WHERE c.tmdb_collection_id IS NULL OR c.fetched_at < $1 OR c.language <> due.language + ORDER BY c.fetched_at NULLS FIRST, due.tmdb_collection_id + LIMIT $2`, staleBefore, limit, nonNilSlice(skip), MinBoxSetMinMovies) if err != nil { return nil, fmt.Errorf("querying collections needing a fetch: %w", err) } defer rows.Close() - var ids []int64 - for rows.Next() { - var id int64 - var fetchedAt *time.Time - if err := rows.Scan(&id, &fetchedAt); err != nil { - return nil, fmt.Errorf("scanning collection needing a fetch: %w", err) - } - ids = append(ids, id) + out, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (CollectionFetch, error) { + var c CollectionFetch + err := row.Scan(&c.ID, &c.Language) + return c, err + }) + if err != nil { + return nil, fmt.Errorf("reading collections needing a fetch: %w", err) } - return ids, rows.Err() + return out, nil } // UpsertCollection stores a fetched collection with the provider URLs of its @@ -236,10 +262,11 @@ func (r *BoxSetRepository) UpsertCollection(ctx context.Context, c TMDBMovieColl members = []int64{} } _, err := r.pool.Exec(ctx, ` - INSERT INTO tmdb_movie_collections (tmdb_collection_id, name, overview, poster_path, poster_source_path, backdrop_path, backdrop_source_path, member_tmdb_ids, fetched_at) - VALUES ($1, $2, $3, $4, $4, $5, $5, $6, now()) + INSERT INTO tmdb_movie_collections (tmdb_collection_id, name, overview, poster_path, poster_source_path, backdrop_path, backdrop_source_path, member_tmdb_ids, language, fetched_at) + VALUES ($1, $2, $3, $4, $4, $5, $5, $6, $7, now()) ON CONFLICT (tmdb_collection_id) DO UPDATE SET name = EXCLUDED.name, + language = EXCLUDED.language, overview = EXCLUDED.overview, poster_path = CASE WHEN tmdb_movie_collections.poster_source_path = EXCLUDED.poster_source_path THEN tmdb_movie_collections.poster_path ELSE EXCLUDED.poster_path END, @@ -254,7 +281,7 @@ func (r *BoxSetRepository) UpsertCollection(ctx context.Context, c TMDBMovieColl member_tmdb_ids = EXCLUDED.member_tmdb_ids, fetched_at = EXCLUDED.fetched_at, updated_at = now()`, - c.ID, c.Name, c.Overview, c.PosterSourcePath, c.BackdropSourcePath, members) + c.ID, c.Name, c.Overview, c.PosterSourcePath, c.BackdropSourcePath, members, c.Language) if err != nil { return fmt.Errorf("storing tmdb collection: %w", err) } @@ -323,10 +350,10 @@ func (r *BoxSetRepository) SetCollectionArtwork(ctx context.Context, id int64, s return nil } -// TouchCollection marks a collection fetched without changing it, so one -// TMDB no longer serves is not fetched again on every run. -func (r *BoxSetRepository) TouchCollection(ctx context.Context, id int64) error { - _, err := r.pool.Exec(ctx, `UPDATE tmdb_movie_collections SET fetched_at = now() WHERE tmdb_collection_id = $1`, id) +// TouchCollection marks a collection fetched in language without changing +// it, so one TMDB no longer serves is not fetched again on every run. +func (r *BoxSetRepository) TouchCollection(ctx context.Context, id int64, language string) error { + _, err := r.pool.Exec(ctx, `UPDATE tmdb_movie_collections SET fetched_at = now(), language = $2 WHERE tmdb_collection_id = $1`, id, language) if err != nil { return fmt.Errorf("touching tmdb collection: %w", err) } diff --git a/internal/catalog/box_sets_db_test.go b/internal/catalog/box_sets_db_test.go index 6a286826fe..d8b83fb3e5 100644 --- a/internal/catalog/box_sets_db_test.go +++ b/internal/catalog/box_sets_db_test.go @@ -95,8 +95,8 @@ func seedBoxSets(t *testing.T) boxSetFixture { exec(`INSERT INTO media_item_tmdb_collections(content_id,checked_tmdb_id,tmdb_collection_id) VALUES($1,$2,$3)`, f.id(m.id), m.checked, collection) } for _, c := range []TMDBMovieCollection{ - {ID: f.franchise, Name: f.prefix + " Saga Collection", MemberTMDBIDs: []int64{101, 102, 103, 104, 105}}, - {ID: f.single, Name: f.prefix + " Lonely Collection", MemberTMDBIDs: []int64{200, 201}}, + {ID: f.franchise, Name: f.prefix + " Saga Collection", MemberTMDBIDs: []int64{101, 102, 103, 104, 105}, Language: "en"}, + {ID: f.single, Name: f.prefix + " Lonely Collection", MemberTMDBIDs: []int64{200, 201}, Language: "en"}, } { if err := f.repo.UpsertCollection(ctx, c); err != nil { t.Fatal(err) @@ -260,16 +260,19 @@ func TestBoxSetSearchAndSyncDB(t *testing.T) { func TestBoxSetCollectionsOfDisabledLibraryDB(t *testing.T) { f := seedBoxSets(t) ctx := t.Context() - if _, err := f.pool.Exec(ctx, `UPDATE tmdb_movie_collections SET fetched_at = now() - interval '30 days' WHERE tmdb_collection_id = $1`, f.franchise); err != nil { - t.Fatal(err) - } refetch := func() bool { t.Helper() - ids, err := f.repo.CollectionsNeedingFetch(ctx, time.Now().Add(-time.Hour), 10_000, nil) + fetches, err := f.repo.CollectionsNeedingFetch(ctx, time.Now().Add(-time.Hour), 10_000, nil) if err != nil { t.Fatal(err) } - return slices.Contains(ids, f.franchise) + return slices.ContainsFunc(fetches, func(c CollectionFetch) bool { return c.ID == f.franchise }) + } + if refetch() { + t.Fatal("a fresh collection is due for a fetch") + } + if _, err := f.pool.Exec(ctx, `UPDATE tmdb_movie_collections SET fetched_at = now() - interval '30 days' WHERE tmdb_collection_id = $1`, f.franchise); err != nil { + t.Fatal(err) } if !refetch() { t.Fatal("a stale collection of a box set library is not due for a fetch") @@ -294,6 +297,80 @@ func TestBoxSetCollectionsOfDisabledLibraryDB(t *testing.T) { } } +// TestBoxSetCollectionLanguageDB checks that a collection is fetched in the +// metadata language of its box set library, and fetched again once that +// language changes, rather than only when the stored copy goes stale. +func TestBoxSetCollectionLanguageDB(t *testing.T) { + f := seedBoxSets(t) + ctx := t.Context() + exec := func(sql string, args ...any) { + t.Helper() + if _, err := f.pool.Exec(ctx, sql, args...); err != nil { + t.Fatal(err) + } + } + due := func() (string, bool) { + t.Helper() + fetches, err := f.repo.CollectionsNeedingFetch(ctx, time.Now().Add(-time.Hour), 10_000, nil) + if err != nil { + t.Fatal(err) + } + for _, c := range fetches { + if c.ID == f.franchise { + return c.Language, true + } + } + return "", false + } + store := func(language string) { + t.Helper() + if err := f.repo.UpsertCollection(ctx, TMDBMovieCollection{ID: f.franchise, Name: "Saga", Language: language}); err != nil { + t.Fatal(err) + } + } + + store("en") + if language, ok := due(); ok { + t.Fatalf("a fresh collection in its library's language is due for a fetch in %q", language) + } + exec(`UPDATE media_folders SET metadata_language = 'de' WHERE id = $1`, f.lib) + if language, ok := due(); !ok || language != "de" { + t.Fatalf("after the library switched to de, due = %q, %v; want a fetch in de", language, ok) + } + store("de") + if language, ok := due(); ok { + t.Fatalf("a collection stored in de is still due for a fetch in %q", language) + } + if stored, err := f.repo.Collections(ctx, []int64{f.franchise}); err != nil || stored[f.franchise].Language != "de" { + t.Fatalf("stored collection = %+v, %v; want language de", stored[f.franchise], err) + } + + // Another library ahead of it in the library order holds the + // collection's movies: its language wins once it holds enough of them to + // show the box set, and every run asks for the same. + var first int + if err := f.pool.QueryRow(ctx, `INSERT INTO media_folders(type,name,enabled,box_sets_enabled,metadata_language,sort_order) VALUES('movies',$1,true,true,'fr',-1) RETURNING id`, f.prefix+"-fr").Scan(&first); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _, _ = f.pool.Exec(context.Background(), `DELETE FROM media_folders WHERE id=$1`, first) }) + exec(`INSERT INTO media_item_libraries(content_id,media_folder_id) VALUES($1,$2)`, f.id("first"), first) + if language, ok := due(); ok { + t.Fatalf("a French library below its box set minimum made the collection due in %q", language) + } + exec(`INSERT INTO media_item_libraries(content_id,media_folder_id) VALUES($1,$2)`, f.id("second"), first) + if language, ok := due(); !ok || language != "fr" { + t.Fatalf("with a French library first, due = %q, %v; want a fetch in fr", language, ok) + } + // A collection TMDB no longer serves is not asked for again in the same + // language on every run. + if err := f.repo.TouchCollection(ctx, f.franchise, "fr"); err != nil { + t.Fatal(err) + } + if language, ok := due(); ok { + t.Fatalf("a touched collection is still due for a fetch in %q", language) + } +} + func TestBoxSetCollectionArtworkDB(t *testing.T) { f := seedBoxSets(t) ctx := t.Context() diff --git a/internal/metadata/language_fallback.go b/internal/metadata/language_fallback.go index f8c0d379a4..8ff8d4564d 100644 --- a/internal/metadata/language_fallback.go +++ b/internal/metadata/language_fallback.go @@ -36,8 +36,8 @@ func (s *MetadataService) SetFallbackLanguage(language string) { s.fallbackLanguage.Store(baseMetadataLanguage(language)) } -// configuredFallbackLanguage is metadata.fallback_language, empty when off. -func (s *MetadataService) configuredFallbackLanguage() string { +// FallbackLanguage is metadata.fallback_language, empty when off. +func (s *MetadataService) FallbackLanguage() string { language, _ := s.fallbackLanguage.Load().(string) return language } @@ -45,7 +45,7 @@ func (s *MetadataService) configuredFallbackLanguage() string { // fallbackLanguageFor is the language a refresh in requested fetches gaps in, // or empty when the fallback is off or is the requested language itself. func (s *MetadataService) fallbackLanguageFor(requested string) string { - fallback := s.configuredFallbackLanguage() + fallback := s.FallbackLanguage() requested = baseMetadataLanguage(requested) if fallback == "" || requested == "" || fallback == requested { return "" diff --git a/internal/metadata/local_artwork_discovery.go b/internal/metadata/local_artwork_discovery.go index 1627ce9e36..d947fa0252 100644 --- a/internal/metadata/local_artwork_discovery.go +++ b/internal/metadata/local_artwork_discovery.go @@ -179,7 +179,7 @@ func (s *MetadataService) saveProvisionalLocalArtwork(ctx context.Context, conte return } item := *existing - fallback := s.configuredFallbackLanguage() + fallback := s.FallbackLanguage() applyBestImages(&item, images, MergeFillEmpty, language, fallback) // Local artwork is only found with image caching on, so stored copies keep // serving until the new copy lands. diff --git a/internal/metadata/service.go b/internal/metadata/service.go index 31702982fb..bfedbdb897 100644 --- a/internal/metadata/service.go +++ b/internal/metadata/service.go @@ -2934,7 +2934,7 @@ func (s *MetadataService) mergeAndPersist( // localization tables like any other non-canonical refresh. canonicalLanguage, isCanonicalWrite := canonicalMetadataLanguage(req, existingItem, locked) // Read once, so artwork and text in this write follow the same setting. - fallbackLanguage := s.configuredFallbackLanguage() + fallbackLanguage := s.FallbackLanguage() switch { case isCanonicalWrite: // Fallback text belongs to the provider IDs it was fetched with, so it @@ -4805,7 +4805,7 @@ func (s *MetadataService) refreshSeriesChildTarget( if err := s.fillTargetChildText(ctx, series, providerIDs, folderID, language, seasonNumber, episodeNumber, childCtx, seasons, episodes, mode, mergeMode, reset); err != nil { return err } - } else if s.configuredFallbackLanguage() != "" { + } else if s.FallbackLanguage() != "" { // Text a provider marked as another language is no translation. dropForeignChildText(seasons, episodes, language) } diff --git a/internal/metadata/tmdb/client.go b/internal/metadata/tmdb/client.go index b144d2ad5e..b49ccd7bf1 100644 --- a/internal/metadata/tmdb/client.go +++ b/internal/metadata/tmdb/client.go @@ -831,13 +831,22 @@ func joinIntSlice(values []int, sep string) string { // Universe). An id of 0 is rejected here — placeholder templates with // collection_id=0 are surfaced as failures by the sync path, not silently // turned into an empty fetch. -func (c *Client) GetCollection(ctx context.Context, id int) (*Collection, error) { +// +// language is the Silo metadata language (an ISO 639-1 code such as "de") +// the name, overview and artwork come in; TMDB falls back to its default +// where the collection has no translation or poster in that language. An +// empty language leaves the choice to TMDB, which answers in English. +func (c *Client) GetCollection(ctx context.Context, id int, language string) (*Collection, error) { if id <= 0 { return nil, fmt.Errorf("tmdb: collection id must be > 0 (got %d)", id) } + path := fmt.Sprintf("/collection/%d", id) + if lang := apiLanguage(language); lang != "" { + path += "?language=" + url.QueryEscape(lang) + } var resp collectionResponse - if err := c.doGet(ctx, fmt.Sprintf("/collection/%d", id), &resp); err != nil { + if err := c.doGet(ctx, path, &resp); err != nil { return nil, err } @@ -868,6 +877,18 @@ func (c *Client) GetCollection(ctx context.Context, id int) (*Collection, error) }, nil } +// apiLanguage converts a Silo metadata language (an ISO 639-1 code) into +// TMDB's language parameter the way the TMDB metadata plugin does: English +// becomes en-US, which TMDB prefers, and any other code passes through. An +// empty language stays empty. +func apiLanguage(language string) string { + language = strings.ToLower(strings.TrimSpace(language)) + if language == "en" { + return "en-US" + } + return language +} + // GetMovieCollection returns the collection a movie belongs to, or nil when // it belongs to none. func (c *Client) GetMovieCollection(ctx context.Context, movieID int) (*CollectionRef, error) { diff --git a/internal/metadata/tmdb/client_test.go b/internal/metadata/tmdb/client_test.go index c627a707c9..2ca372c32d 100644 --- a/internal/metadata/tmdb/client_test.go +++ b/internal/metadata/tmdb/client_test.go @@ -424,3 +424,28 @@ func TestGetMediaDetailFollowerStopsWaitingOnCancel(t *testing.T) { t.Fatalf("upstream calls = %d, want 1 shared fetch", calls.Load()) } } + +// A collection is fetched in the language asked for, the way the TMDB +// metadata plugin asks for titles: English as en-US, any other language as +// given, and no language at all when none is asked for. +func TestGetCollectionLanguage(t *testing.T) { + for _, tc := range []struct{ language, sent string }{ + {"de", "de"}, + {"en", "en-US"}, + {"", ""}, + } { + t.Run(tc.language, func(t *testing.T) { + var got url.Values + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + got = r.URL.Query() + _, _ = w.Write([]byte(`{"id":10,"name":"Star Wars - Saga","poster_path":"/fr.jpg","parts":[]}`)) + }) + if _, err := client.GetCollection(context.Background(), 10, tc.language); err != nil { + t.Fatal(err) + } + if lang := got.Get("language"); lang != tc.sent { + t.Errorf("language = %q, want %q", lang, tc.sent) + } + }) + } +} diff --git a/migrations/sql/20261118183932_box_set_collection_language.sql b/migrations/sql/20261118183932_box_set_collection_language.sql new file mode 100644 index 0000000000..9b74e92ec8 --- /dev/null +++ b/migrations/sql/20261118183932_box_set_collection_language.sql @@ -0,0 +1,11 @@ +-- +goose Up +-- The metadata language a TMDB collection's name, overview and artwork were +-- fetched in. The box set sync fetches each collection in the language of its +-- first box set library and fetches it again when that language changes. +-- Collections stored before this column were fetched without a language, +-- which TMDB answers in English. +ALTER TABLE public.tmdb_movie_collections + ADD COLUMN language text NOT NULL DEFAULT 'en'; + +-- +goose Down +ALTER TABLE public.tmdb_movie_collections DROP COLUMN IF EXISTS language; diff --git a/scripts/ci/db-contracts.txt b/scripts/ci/db-contracts.txt index c5e897b21f..39c0dfa6ae 100644 --- a/scripts/ci/db-contracts.txt +++ b/scripts/ci/db-contracts.txt @@ -91,6 +91,7 @@ ./internal/catalog TestBoxSetCollectionsOfDisabledLibraryDB ./internal/catalog TestMovieBoxSetDB ./internal/catalog TestBoxSetCollectionArtworkDB +./internal/catalog TestBoxSetCollectionLanguageDB # Audiobook Series, Authors, and Narrators browse: group counts and durations, # cover order, the viewer's own progress, count-sorted cursor pages, the search