Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/silo/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -3397,7 +3397,7 @@ func main() {
taskMgr.Register(tasks.NewRefreshSeriesAirDatesTask(catalog.NewEpisodeRepository(deps.DB)))
boxSetSyncer := boxsets.NewSyncer(catalog.NewBoxSetRepository(deps.DB), tmdbClient)
if metadataService != nil {
boxSetSyncer.WithImages(metadataService.AutoImageCacher)
boxSetSyncer.WithImages(metadataService.AutoImageCacher).WithFallbackLanguage(metadataService.FallbackLanguage)
}
taskMgr.Register(tasks.NewSyncMovieBoxSetsTask(deps.DB, boxSetSyncer))
go boxsets.Watch(appCtx, deps.EventsHub, func() {
Expand Down
16 changes: 13 additions & 3 deletions docs/architecture/movie-box-sets.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,15 @@ its own card. This note records the invariants; the wire contract is in
collected. TMDB returns a collection's
parts in no reliable order, so the sync sorts them by release date, with
undated (announced) parts last.
- One row serves every library. It is fetched in the metadata language of
the first enabled library with box sets on, in library order, that holds
at least its box set minimum of the collection's movies (failing that, the
first that holds one), and `language` records which. TMDB leaves the
overview (and possibly the name) empty where its translation lacks it;
those come from its answer in `metadata.fallback_language`, as item
metadata does, and stay empty when that is the original language. When
that answer fails, the fetch fails and the stored row is left as it was.
TMDB picks the poster for the language itself.
- The stored name is TMDB's. Reads show it through `catalog.BoxSetTitle`
("… Collection" becomes "… Box Set"). Search matches the name without that
suffix (`boxSetStemSQL`, the same rule in SQL; change the two together),
Expand All @@ -34,9 +43,10 @@ its own card. This note records the invariants; the wire contract is in
The `sync_movie_box_sets` task (`internal/boxsets`) is the only code that calls
TMDB for box sets; the box set reads use stored rows only. It looks up
movies of box set libraries that were never looked up, were rematched, or were
last looked up 30 days ago, then fetches collections that are new or were
fetched 7 days ago, then caches collection artwork still served from TMDB
through the item image cacher, under the same `metadata.cache_images` switch
last looked up 30 days ago, then fetches collections that are new, were
fetched 7 days ago, or were fetched in another language than the one they
are now due in (a library changed its metadata language), then caches
collection artwork still served from TMDB through the item image cacher, under the same `metadata.cache_images` switch
as item artwork. It runs at startup, hourly, when a library turns box sets
on, and 20 seconds after the node that matched a movie publishes its catalog
change. A run stops after 25 failed TMDB calls; the next run resumes.
Expand Down
2 changes: 1 addition & 1 deletion internal/api/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -3722,7 +3722,7 @@ type tmdbFranchiseAdapter struct {
}

func (a *tmdbFranchiseAdapter) GetCollection(ctx context.Context, id int) ([]catalog.TMDBCollectionEntry, error) {
collection, err := a.client.GetCollection(ctx, id)
collection, err := a.client.GetCollection(ctx, id, "")
if err != nil {
return nil, err
}
Expand Down
61 changes: 49 additions & 12 deletions internal/boxsets/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package boxsets

import (
"cmp"
"context"
"errors"
"fmt"
Expand Down Expand Up @@ -38,16 +39,16 @@ const (
// TMDB is the slice of *tmdb.Client the sync uses.
type TMDB interface {
GetMovieCollection(ctx context.Context, movieID int) (*tmdb.CollectionRef, error)
GetCollection(ctx context.Context, id int) (*tmdb.Collection, error)
GetCollection(ctx context.Context, id int, language string) (*tmdb.Collection, error)
}

// Store is the slice of *catalog.BoxSetRepository the sync uses.
type Store interface {
MoviesNeedingCollectionLookup(ctx context.Context, staleBefore time.Time, limit int, skip []string) ([]catalog.MovieCollectionLookup, error)
RecordMovieCollection(ctx context.Context, contentID string, tmdbID int64, collectionID *int64) error
CollectionsNeedingFetch(ctx context.Context, staleBefore time.Time, limit int, skip []int64) ([]int64, error)
CollectionsNeedingFetch(ctx context.Context, staleBefore time.Time, limit int, skip []int64) ([]catalog.CollectionFetch, error)
UpsertCollection(ctx context.Context, c catalog.TMDBMovieCollection) error
TouchCollection(ctx context.Context, id int64) error
TouchCollection(ctx context.Context, id int64, language string) error
DeleteUnusedCollections(ctx context.Context) (int64, error)
CollectionArtworkNeedingCache(ctx context.Context, limit int, skip []int64) ([]catalog.CollectionArtworkDue, error)
SetCollectionArtwork(ctx context.Context, id int64, slot catalog.CollectionArtworkSlot, sourcePath, path, thumbhash string) error
Expand All @@ -68,12 +69,15 @@ type Syncer struct {
// images returns the image cacher, or nil while artwork is served from
// the provider (no artwork store, or metadata.cache_images off).
images func() metadata.ImageCacher
now func() time.Time
// fallbackLanguage returns metadata.fallback_language, empty when off.
fallbackLanguage func() string
now func() time.Time
}

// NewSyncer returns a syncer over store and client.
// NewSyncer returns a syncer over store and client. Until WithFallbackLanguage
// says otherwise, it falls back to English, the setting's default.
func NewSyncer(store Store, client TMDB) *Syncer {
return &Syncer{store: store, tmdb: client, now: time.Now}
return &Syncer{store: store, tmdb: client, fallbackLanguage: func() string { return "en" }, now: time.Now}
}

// WithImages caches collection artwork through the cacher images returns,
Expand All @@ -83,6 +87,14 @@ func (s *Syncer) WithImages(images func() metadata.ImageCacher) *Syncer {
return s
}

// WithFallbackLanguage fills the text a collection's translation lacks in the
// language fallback returns (metadata.fallback_language), the way item
// metadata falls back; empty fills nothing.
func (s *Syncer) WithFallbackLanguage(fallback func() string) *Syncer {
s.fallbackLanguage = fallback
return s
}

// Run deletes the collections no movie belongs to any more, looks up every
// movie of a box set library that is due, then fetches every collection that
// is due and caches their artwork. It repeats while a lookup finds movies
Expand Down Expand Up @@ -165,7 +177,8 @@ func (s *Syncer) lookUpMovies(ctx context.Context, res *Result, failed *[]string
}
}

// fetchCollections stores every due collection. tried collects the
// fetchCollections stores every due collection in the language it is due
// in. tried collects the
// collections this run could not store: failed fetches, and ones TMDB no
// longer serves that were never stored.
func (s *Syncer) fetchCollections(ctx context.Context, res *Result, tried *[]int64, report func(float64, string)) error {
Expand All @@ -175,16 +188,17 @@ func (s *Syncer) fetchCollections(ctx context.Context, res *Result, tried *[]int
if err != nil || len(due) == 0 {
return err
}
for _, id := range due {
for _, fetch := range due {
if err := ctx.Err(); err != nil {
return err
}
c, err := s.tmdb.GetCollection(ctx, int(id))
id := fetch.ID
c, err := s.fetchCollection(ctx, fetch)
if errors.Is(err, tmdb.ErrNotFound) {
// Keep a stored copy rather than refetching a collection TMDB
// no longer serves on every run. One never stored stays due
// for the next run.
if err := s.store.TouchCollection(ctx, id); err != nil {
if err := s.store.TouchCollection(ctx, id, fetch.Language); err != nil {
return err
}
*tried = append(*tried, id)
Expand All @@ -199,7 +213,7 @@ func (s *Syncer) fetchCollections(ctx context.Context, res *Result, tried *[]int
}
continue
}
if err := s.store.UpsertCollection(ctx, storedCollection(id, c)); err != nil {
if err := s.store.UpsertCollection(ctx, storedCollection(id, fetch.Language, c)); err != nil {
return err
}
res.CollectionsFetched++
Expand All @@ -208,6 +222,28 @@ func (s *Syncer) fetchCollections(ctx context.Context, res *Result, tried *[]int
}
}

// fetchCollection fetches a collection in the language it is due in. TMDB
// leaves the overview empty where its translation lacks one (and may do the
// same for the name), so those come from its answer in the fallback language,
// the way item metadata falls back. TMDB falls back for artwork itself. When
// that answer fails, the fetch fails: the stored copy keeps its text and the
// collection stays due for the next run.
func (s *Syncer) fetchCollection(ctx context.Context, fetch catalog.CollectionFetch) (*tmdb.Collection, error) {
c, err := s.tmdb.GetCollection(ctx, int(fetch.ID), fetch.Language)
fallback := s.fallbackLanguage()
if err != nil || fallback == "" || fallback == fetch.Language || (c.Name != "" && c.Overview != "") {
return c, err
}
other, err := s.tmdb.GetCollection(ctx, int(fetch.ID), fallback)
if err != nil {
return nil, fmt.Errorf("fetching in fallback language %s: %v", fallback, err) //nolint:errorlint // not ErrNotFound: a collection TMDB serves in one language is not gone
}
localized := *c
localized.Name = cmp.Or(c.Name, other.Name)
localized.Overview = cmp.Or(c.Overview, other.Overview)
return &localized, nil
}

// cacheArtwork stores a copy of each collection image still served from
// TMDB. A failed image stays on the provider URL and is tried again next run.
func (s *Syncer) cacheArtwork(ctx context.Context, res *Result) error {
Expand Down Expand Up @@ -267,7 +303,7 @@ func tmdbOriginalURL(path string) string {
// storedCollection converts a fetched collection, listing its movies in
// release order. TMDB returns parts in no reliable order, and a part without
// a release date (an announced sequel) goes last.
func storedCollection(id int64, c *tmdb.Collection) catalog.TMDBMovieCollection {
func storedCollection(id int64, language string, c *tmdb.Collection) catalog.TMDBMovieCollection {
parts := append([]tmdb.CollectionPart(nil), c.Parts...)
sort.SliceStable(parts, func(i, j int) bool {
a, b := parts[i].ReleaseDate, parts[j].ReleaseDate
Expand All @@ -289,5 +325,6 @@ func storedCollection(id int64, c *tmdb.Collection) catalog.TMDBMovieCollection
PosterSourcePath: tmdbOriginalURL(c.PosterPath),
BackdropSourcePath: tmdbOriginalURL(c.BackdropPath),
MemberTMDBIDs: members,
Language: language,
}
}
97 changes: 88 additions & 9 deletions internal/boxsets/sync_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ type fakeStore struct {
recorded map[string]*int64
collections map[int64]catalog.TMDBMovieCollection
touched []int64
// language is the language every collection is due in.
language string
}

func (s *fakeStore) MoviesNeedingCollectionLookup(_ context.Context, _ time.Time, limit int, skip []string) ([]catalog.MovieCollectionLookup, error) {
Expand All @@ -36,17 +38,21 @@ func (s *fakeStore) RecordMovieCollection(_ context.Context, contentID string, _
return nil
}

func (s *fakeStore) CollectionsNeedingFetch(_ context.Context, _ time.Time, _ int, skip []int64) ([]int64, error) {
var out []int64
func (s *fakeStore) CollectionsNeedingFetch(_ context.Context, _ time.Time, _ int, skip []int64) ([]catalog.CollectionFetch, error) {
var ids []int64
for _, id := range s.recorded {
if id == nil || slices.Contains(out, *id) || slices.Contains(s.touched, *id) || slices.Contains(skip, *id) {
if id == nil || slices.Contains(ids, *id) || slices.Contains(s.touched, *id) || slices.Contains(skip, *id) {
continue
}
if _, stored := s.collections[*id]; !stored {
out = append(out, *id)
ids = append(ids, *id)
}
}
slices.Sort(out)
slices.Sort(ids)
out := make([]catalog.CollectionFetch, len(ids))
for i, id := range ids {
out[i] = catalog.CollectionFetch{ID: id, Language: s.language}
}
return out, nil
}

Expand Down Expand Up @@ -87,7 +93,7 @@ func (s *fakeStore) DeleteUnusedCollections(context.Context) (int64, error) { re

// TouchCollection marks only a stored collection fetched, as the
// repository's UPDATE does.
func (s *fakeStore) TouchCollection(_ context.Context, id int64) error {
func (s *fakeStore) TouchCollection(_ context.Context, id int64, _ string) error {
if _, stored := s.collections[id]; stored {
s.touched = append(s.touched, id)
}
Expand All @@ -103,6 +109,13 @@ type fakeTMDB struct {
lookups map[int]int
// onCollection runs on each collection fetch.
onCollection func()
// translated answers collection requests in a language other than
// English; collections answers the others.
translated map[int]*tmdb.Collection
// languages records the languages each collection was asked for in.
languages map[int][]string
// englishErr answers collection requests in English.
englishErr error
}

func (f *fakeTMDB) GetMovieCollection(_ context.Context, id int) (*tmdb.CollectionRef, error) {
Expand All @@ -119,11 +132,21 @@ func (f *fakeTMDB) GetMovieCollection(_ context.Context, id int) (*tmdb.Collecti
return ref, nil
}

func (f *fakeTMDB) GetCollection(_ context.Context, id int) (*tmdb.Collection, error) {
func (f *fakeTMDB) GetCollection(_ context.Context, id int, language string) (*tmdb.Collection, error) {
if f.languages != nil {
f.languages[id] = append(f.languages[id], language)
}
if f.onCollection != nil {
f.onCollection()
}
english := language == "" || language == "en"
if english && f.englishErr != nil {
return nil, f.englishErr
}
c, ok := f.collections[id]
if t, translated := f.translated[id]; translated && !english {
c, ok = t, true
}
if !ok {
return nil, tmdb.ErrNotFound
}
Expand All @@ -142,9 +165,18 @@ func TestRunStoresMembershipAndCollectionsInReleaseOrder(t *testing.T) {
catalog.MovieCollectionLookup{ContentID: "heat", TMDBID: 949},
catalog.MovieCollectionLookup{ContentID: "gone", TMDBID: 1},
)
store.language = "de"
client := &fakeTMDB{
languages: map[int][]string{},
// TMDB leaves the fields a translation lacks empty.
translated: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Filmreihe", PosterPath: "/de.jpg", Parts: []tmdb.CollectionPart{
{ID: 674, ReleaseDate: "2005-11-16"},
{ID: 999, ReleaseDate: ""},
{ID: 671, ReleaseDate: "2001-11-16"},
{ID: 673, ReleaseDate: "2004-05-31"},
}}},
movies: map[int]*tmdb.CollectionRef{671: {ID: 1241, Name: "Harry Potter Collection"}, 949: nil},
collections: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Collection", PosterPath: "/p.jpg", Parts: []tmdb.CollectionPart{
collections: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Collection", Overview: "The wizarding saga.", PosterPath: "/p.jpg", Parts: []tmdb.CollectionPart{
{ID: 674, ReleaseDate: "2005-11-16"},
{ID: 999, ReleaseDate: ""},
{ID: 671, ReleaseDate: "2001-11-16"},
Expand All @@ -169,14 +201,61 @@ func TestRunStoresMembershipAndCollectionsInReleaseOrder(t *testing.T) {
}
}
stored := store.collections[1241]
// The collection is stored in the language it is due in, and the
// overview the translation lacks comes from the fallback language,
// English by default.
if !slices.Equal(client.languages[1241], []string{"de", "en"}) || stored.Language != "de" {
t.Fatalf("fetched in %q, stored as %q; want de, then the English fallback", client.languages[1241], stored.Language)
}
if stored.Name != "Harry Potter Filmreihe" || stored.Overview != "The wizarding saga." {
t.Fatalf("name, overview = %q, %q; want the German name and the English overview", stored.Name, stored.Overview)
}
if want := []int64{671, 673, 674, 999}; !slices.Equal(stored.MemberTMDBIDs, want) {
t.Fatalf("members = %v, want release order %v with the undated part last", stored.MemberTMDBIDs, want)
}
if stored.PosterSourcePath != "https://image.tmdb.org/t/p/original/p.jpg" || stored.BackdropSourcePath != "" {
if stored.PosterSourcePath != "https://image.tmdb.org/t/p/original/de.jpg" || stored.BackdropSourcePath != "" {
t.Fatalf("artwork sources = %q, %q", stored.PosterSourcePath, stored.BackdropSourcePath)
}
}

// A failed fallback answer fails the fetch rather than storing the
// translation without its overview, so the collection is tried again.
func TestRunRetriesWhenTheFallbackFails(t *testing.T) {
store := newFakeStore(catalog.MovieCollectionLookup{ContentID: "hp1", TMDBID: 671})
store.language = "de"
client := &fakeTMDB{
movies: map[int]*tmdb.CollectionRef{671: {ID: 1241}},
translated: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Filmreihe", Parts: []tmdb.CollectionPart{{ID: 671}}}},
englishErr: errors.New("tmdb timeout"),
}
res, err := NewSyncer(store, client).Run(t.Context(), noReport)
if err != nil {
t.Fatal(err)
}
if _, stored := store.collections[1241]; stored || res.Failures != 1 {
t.Fatalf("stored = %v, failures = %d; want nothing stored and one failure", stored, res.Failures)
}
}

// With the fallback language set to the original language, the text a
// translation lacks stays empty, as it does for item metadata.
func TestRunFillsNothingWhenTheFallbackIsOff(t *testing.T) {
store := newFakeStore(catalog.MovieCollectionLookup{ContentID: "hp1", TMDBID: 671})
store.language = "de"
client := &fakeTMDB{
languages: map[int][]string{},
movies: map[int]*tmdb.CollectionRef{671: {ID: 1241}},
translated: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Filmreihe", Parts: []tmdb.CollectionPart{{ID: 671}}}},
collections: map[int]*tmdb.Collection{1241: {ID: 1241, Name: "Harry Potter Collection", Overview: "The wizarding saga.", Parts: []tmdb.CollectionPart{{ID: 671}}}},
}
if _, err := NewSyncer(store, client).WithFallbackLanguage(func() string { return "" }).Run(t.Context(), noReport); err != nil {
t.Fatal(err)
}
if stored := store.collections[1241]; !slices.Equal(client.languages[1241], []string{"de"}) || stored.Overview != "" {
t.Fatalf("asked in %q, stored overview %q; want only de and no overview", client.languages[1241], stored.Overview)
}
}

type fakeCacher struct {
requests []metadata.CacheImageRequest
fail bool
Expand Down
Loading
Loading