Retry the remaining plex.tv calls that had no retry

core/plex_api.py

@@ -1558,9 +1558,20 @@ def _fetch_user_watchlist(self, user, valid_sections: List[int], watchlist_episo
             fresh_session = requests.Session()
 
             if user is None:
-                # Main account - use the main token with a fresh session
-                self._rate_limited_api_call()
-                account = MyPlexAccount(token=self.plex_token, session=fresh_session)
+                # Main account - use the main token with a fresh session.
+                # Retried like its home-user sibling below: constructing the
+                # account is a plex.tv round trip, and on an install with no
+                # home users it is the only one, so a single read timeout here
+                # marks the whole watchlist incomplete and skips array restore
+                # for the run.
+                def _fresh_main_account():
+                    self._rate_limited_api_call()
+                    return MyPlexAccount(token=self.plex_token, session=fresh_session)
+
+                account = _retry_plextv_call(
+                    _fresh_main_account,
+                    label=f"main account for {current_username}",
+                )
                 logging.debug(f"[USER:{current_username}] Created fresh MyPlexAccount (main user)")
             else:
                 # Home/managed user - create fresh admin account then switch to home user

tests/test_plex_api_main_account_retry.py

@@ -0,0 +1,174 @@
+"""The main-account watchlist path retries plex.tv like its home-user sibling.
+
+Observed 2026-08-13 20:32 on a single-user install:
+
+    Processing 1 users for watchlist (main + 0 home users)
+    [USER:Brandon] Fetching watchlist media
+    ERROR [PLEX API] Error (get Plex account for Brandon):
+        HTTPSConnectionPool(host='plex.tv', port=443): Read timed out. (read timeout=30)
+    WARNING Skipping array restore - watchlist data incomplete (plex.tv unreachable)
+
+`MyPlexAccount(...)` in the `user is None` branch was the one plex.tv entry
+point left unwrapped — `_get_main_account()` and the `switchHomeUser` path had
+both been given retries already. On an install with no home users that branch is
+the only watchlist fetch there is, so "this user failed" and "watchlist data is
+incomplete" are the same event, and the run skipped array restore entirely.
+
+Retry semantics themselves live in `test_plex_api_retry.py`; this module only
+checks that this call site is wired into the helper.
+"""
+
+import os
+import sys
+import threading
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+sys.modules['fcntl'] = MagicMock()
+for _mod in [
+    'apscheduler', 'apscheduler.schedulers',
+    'apscheduler.schedulers.background', 'apscheduler.triggers',
+    'apscheduler.triggers.cron', 'apscheduler.triggers.interval',
+    'plexapi', 'plexapi.server', 'plexapi.video', 'plexapi.myplex',
+    'plexapi.library',
+]:
+    sys.modules.setdefault(_mod, MagicMock())
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import requests
+
+from core.plex_api import PlexManager, PLEXTV_MAX_RETRIES
+
+# Verbatim from the log line above — a read timeout, not a connection reset.
+# The two arrive as different exception classes and only ReadTimeout was seen
+# in production.
+READ_TIMEOUT = requests.exceptions.ReadTimeout(
+    "HTTPSConnectionPool(host='plex.tv', port=443): "
+    "Read timed out. (read timeout=30)"
+)
+
+
+def _bare_api(username="Brandon"):
+    """A PlexManager with just the state the main-account branch touches."""
+    api = PlexManager.__new__(PlexManager)
+    api.plex_url = "http://localhost:32400"
+    api.plex_token = "ADMIN_TOKEN"
+    api._user_tokens = {}
+    api._user_is_home = {}
+    api._ondeck_data_complete = True
+    api._watchlist_data_complete = True
+    api._token_lock = threading.Lock()
+    api._rate_limited_api_call = MagicMock()
+    api._token_cache = MagicMock()
+    # user is None → the username comes from the local server, not plex.tv
+    api.plex = MagicMock()
+    api.plex.myPlexAccount.return_value.title = username
+    api.mark_watchlist_incomplete = MagicMock(
+        side_effect=lambda: setattr(api, '_watchlist_data_complete', False)
+    )
+    return api
+
+
+def _fetch(api):
+    """Drain the generator for the main account (user=None), no RSS."""
+    return list(api._fetch_user_watchlist(
+        user=None,
+        valid_sections=[1],
+        watchlist_episodes=3,
+        skip_watchlist=[],
+        rss_url=None,
+        filtered_sections=[1],
+    ))
+
+
+class TestMainAccountRetry:
+
+    def test_transient_timeout_then_success_reaches_the_watchlist(self):
+        """The case from the log: one blip must not cost the run."""
+        api = _bare_api()
+        account = MagicMock()
+        account.watchlist.return_value = []
+
+        with patch('core.plex_api.MyPlexAccount',
+                   side_effect=[READ_TIMEOUT, account]) as mock_acct, \
+             patch('core.plex_api.requests.Session'), \
+             patch('core.plex_api.time.sleep'):
+            _fetch(api)
+
+        assert mock_acct.call_count == 2
+        assert api.mark_watchlist_incomplete.call_count == 0
+        assert api.is_watchlist_data_complete() is True
+        account.watchlist.assert_called_once()
+
+    def test_three_attempts_before_giving_up(self):
+        """Persistent timeout → PLEXTV_MAX_RETRIES tries, then mark incomplete."""
+        api = _bare_api()
+
+        with patch('core.plex_api.MyPlexAccount',
+                   side_effect=READ_TIMEOUT) as mock_acct, \
+             patch('core.plex_api.requests.Session'), \
+             patch('core.plex_api.time.sleep'):
+            _fetch(api)
+
+        assert mock_acct.call_count == PLEXTV_MAX_RETRIES == 3
+        api.mark_watchlist_incomplete.assert_called_once()
+
+    def test_backs_off_between_attempts(self):
+        """2s then 4s, so a brief plex.tv blip has time to clear."""
+        api = _bare_api()
+
+        with patch('core.plex_api.MyPlexAccount', side_effect=READ_TIMEOUT), \
+             patch('core.plex_api.requests.Session'), \
+             patch('core.plex_api.time.sleep') as mock_sleep:
+            _fetch(api)
+
+        assert [c.args[0] for c in mock_sleep.call_args_list] == [2, 4]
+
+    def test_connection_errors_are_retried_too(self):
+        """DNS failures surface as ConnectionError — see issue #197."""
+        api = _bare_api()
+        account = MagicMock()
+        account.watchlist.return_value = []
+
+        with patch('core.plex_api.MyPlexAccount',
+                   side_effect=[requests.ConnectionError(
+                       "Failed to resolve 'plex.tv'"), account]) as mock_acct, \
+             patch('core.plex_api.requests.Session'), \
+             patch('core.plex_api.time.sleep'):
+            _fetch(api)
+
+        assert mock_acct.call_count == 2
+        assert api.mark_watchlist_incomplete.call_count == 0
+
+    def test_auth_failure_is_not_retried(self):
+        """A revoked token fails identically on every attempt; retrying only stalls."""
+        api = _bare_api()
+
+        with patch('core.plex_api.MyPlexAccount',
+                   side_effect=ValueError("(401) Unauthorized")) as mock_acct, \
+             patch('core.plex_api.requests.Session'), \
+             patch('core.plex_api.time.sleep') as mock_sleep:
+            _fetch(api)
+
+        assert mock_acct.call_count == 1
+        mock_sleep.assert_not_called()
+        api.mark_watchlist_incomplete.assert_called_once()
+
+
+class TestFailureStillGuardsArrayRestore:
+    """Retrying changes how often we give up, never what giving up means."""
+
+    def test_exhausted_retries_still_flag_incomplete_data(self):
+        api = _bare_api()
+
+        with patch('core.plex_api.MyPlexAccount', side_effect=READ_TIMEOUT), \
+             patch('core.plex_api.requests.Session'), \
+             patch('core.plex_api.time.sleep'):
+            yielded = _fetch(api)
+
+        assert yielded == []
+        assert api.is_watchlist_data_complete() is False, (
+            "array restore must stay blocked when the watchlist never loaded"
+        )

tests/test_settings_service_plextv_retry.py

@@ -0,0 +1,179 @@
+"""The Settings user list survives a transient plex.tv timeout.
+
+Observed 2026-08-12 15:02 during the hourly cache refresh:
+
+    Refreshing Plex data cache...
+    WARNING Could not get main account: HTTPSConnectionPool(host='plex.tv',
+        port=443): Read timed out. (read timeout=30)
+    INFO Fetched 24 shared users
+
+The admin silently vanished from the Settings user list for that refresh
+cycle while the shared users loaded fine — the very next plex.tv call
+succeeded. That independence is why the two blocks retry separately instead
+of sharing one fetched account.
+
+Retries here are capped at 2 attempts, not the PLEXTV_MAX_RETRIES of 3 used
+by the caching engine: this code is reachable from GET /settings/plex/users,
+so attempts are paid in page latency, and losing is a stale user list rather
+than a skipped array restore.
+"""
+
+import os
+import sys
+import threading
+from datetime import datetime
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+sys.modules.setdefault('fcntl', MagicMock())
+for _mod in [
+    'apscheduler', 'apscheduler.schedulers',
+    'apscheduler.schedulers.background', 'apscheduler.triggers',
+    'apscheduler.triggers.cron', 'apscheduler.triggers.interval',
+    'plexapi', 'plexapi.server', 'plexapi.video', 'plexapi.myplex',
+    'plexapi.library',
+]:
+    sys.modules.setdefault(_mod, MagicMock())
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import requests
+
+from web.services.settings_service import SettingsService
+
+READ_TIMEOUT = requests.exceptions.ReadTimeout(
+    "HTTPSConnectionPool(host='plex.tv', port=443): "
+    "Read timed out. (read timeout=30)"
+)
+
+UI_ATTEMPTS = 2
+
+
+def _bare_service():
+    """A SettingsService with only the state get_plex_users() touches."""
+    svc = SettingsService.__new__(SettingsService)
+    svc._cache_lock = threading.Lock()
+    svc._plex_users_cache = []
+    svc._plex_cache_time = datetime.now()
+    svc._last_plex_error = None
+    svc._prefetched_users = None
+    svc._save_plex_cache_to_file = MagicMock()
+    svc._is_plex_cache_valid = MagicMock(return_value=False)
+    return svc
+
+
+def _shared_user(title, home=False):
+    user = MagicMock()
+    user.title = title
+    user.home = home
+    return user
+
+
+def _account(username="Brandon", shared=()):
+    account = MagicMock()
+    account.username = username
+    account.title = username
+    account.users.return_value = list(shared)
+    return account
+
+
+def _fetch(svc):
+    """Explicit credentials bypass the cache branch and force a live fetch."""
+    return svc.get_plex_users(plex_url="http://localhost:32400",
+                              plex_token="TOKEN")
+
+
+class TestTransientTimeoutIsRetried:
+
+    def test_main_account_recovers_on_the_second_attempt(self):
+        account = _account(shared=[_shared_user("Paige")])
+        plex = MagicMock()
+        plex.myPlexAccount.side_effect = [READ_TIMEOUT, account, account]
+        svc = _bare_service()
+
+        with patch('plexapi.server.PlexServer', return_value=plex), \
+             patch('core.plex_api.time.sleep'):
+            users = _fetch(svc)
+
+        assert [u["username"] for u in users] == ["Brandon", "Paige"]
+        assert svc.get_last_plex_error() is None
+
+    def test_gives_up_after_two_attempts(self):
+        """Capped lower than the engine's three — see the module docstring."""
+        account = _account(shared=[_shared_user("Paige")])
+        plex = MagicMock()
+        # Fail the two main-account attempts, then let the shared-user
+        # block's own fetch succeed.
+        plex.myPlexAccount.side_effect = [READ_TIMEOUT, READ_TIMEOUT, account]
+        svc = _bare_service()
+
+        with patch('plexapi.server.PlexServer', return_value=plex), \
+             patch('core.plex_api.time.sleep'):
+            users = _fetch(svc)
+
+        assert plex.myPlexAccount.call_count == UI_ATTEMPTS + 1
+        assert [u["username"] for u in users] == ["Paige"]
+
+    def test_the_user_list_call_is_retried_too(self):
+        """account.users() is the actual shared-user round trip."""
+        account = _account(shared=[_shared_user("Paige")])
+        account.users.side_effect = [READ_TIMEOUT, [_shared_user("Paige")]]
+        plex = MagicMock()
+        plex.myPlexAccount.return_value = account
+        svc = _bare_service()
+
+        with patch('plexapi.server.PlexServer', return_value=plex), \
+             patch('core.plex_api.time.sleep'):
+            users = _fetch(svc)
+
+        assert account.users.call_count == 2
+        assert [u["username"] for u in users] == ["Brandon", "Paige"]
+
+
+class TestTheTwoBlocksStayIndependent:
+    """The exact shape of the logged incident."""
+
+    def test_a_lost_main_account_does_not_cost_the_shared_users(self):
+        account = _account(shared=[_shared_user("Paige"), _shared_user("Alex")])
+        plex = MagicMock()
+        # Both main-account attempts time out; the shared-user block then
+        # fetches its own account successfully, as happened in the log.
+        plex.myPlexAccount.side_effect = [READ_TIMEOUT, READ_TIMEOUT, account]
+        svc = _bare_service()
+
+        with patch('plexapi.server.PlexServer', return_value=plex), \
+             patch('core.plex_api.time.sleep'):
+            users = _fetch(svc)
+
+        assert [u["username"] for u in users] == ["Paige", "Alex"]
+        assert all(u["is_admin"] is False for u in users)
+
+    def test_total_plextv_failure_reports_an_error(self):
+        plex = MagicMock()
+        plex.myPlexAccount.side_effect = READ_TIMEOUT
+        svc = _bare_service()
+
+        with patch('plexapi.server.PlexServer', return_value=plex), \
+             patch('core.plex_api.time.sleep'):
+            users = _fetch(svc)
+
+        assert users == []
+        assert "Could not get account info" in svc.get_last_plex_error()
+
+
+class TestNonTransientFailures:
+
+    def test_auth_failure_is_not_retried(self):
+        """A revoked token fails the same way every time."""
+        plex = MagicMock()
+        plex.myPlexAccount.side_effect = ValueError("(401) Unauthorized")
+        svc = _bare_service()
+
+        with patch('plexapi.server.PlexServer', return_value=plex), \
+             patch('core.plex_api.time.sleep') as mock_sleep:
+            _fetch(svc)
+
+        # One attempt per block, no backoff.
+        assert plex.myPlexAccount.call_count == 2
+        mock_sleep.assert_not_called()