From 471c11a6631d8d8c2ba018627792cf277b488fac Mon Sep 17 00:00:00 2001
From: Tobias Schlottke <tobias@saas.group>
Date: Sun, 10 May 2026 11:42:49 +0200
Subject: [PATCH] fix(spotify): bypass per-zone cooldown on cross-zone error
 bursts (#252)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The restartStreak counter in SpotifyConnectInstance.scheduleRestart conflates
two failure modes: a single zone stuck in a tight reconnect loop (the case #244
was built for) and account-side Spotify rebalances that knock every
offload=false zone offline simultaneously (issue #252). For the latter,
applying the 5-minute per-zone cooldown is by-construction wrong — every zone
trips it independently after the same single account-side event, so the user
sees ~5 minutes of unplayable state per burst, multiple bursts per day.

This change adds a static recentBurstErrors registry on SpotifyConnectInstance
and detects the synchronized pattern (>= burstMinZones distinct zones report
Spotify errors within burstWindowMs). When detected, the per-zone restartStreak
is reset and a short, jittered, zone-staggered restart is scheduled instead of
the cooldown. Single-zone reconnect loops still trip the existing streak path
unchanged.

Defaults: 3 zones / 3s window, 8-25s reconnect with deterministic per-zone
stagger by zoneId. The threshold can be raised if the detector ever fires on
something that isn't a real burst.

This is a sketch rather than a final patch — three open questions that need
the maintainer's input are listed in the PR body.

Refs: #252
---
 .../inputs/spotify/spotifyInputService.ts     | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/src/adapters/inputs/spotify/spotifyInputService.ts b/src/adapters/inputs/spotify/spotifyInputService.ts
index 8f24ba8c..961ac207 100644
--- a/src/adapters/inputs/spotify/spotifyInputService.ts
+++ b/src/adapters/inputs/spotify/spotifyInputService.ts
@@ -76,6 +76,14 @@ class SpotifyConnectInstance {
   private readonly restartCooldownMs = 5 * 60 * 1000; // 5 minutes
   private readonly restartStreakWindowMs = 30 * 1000; // 30 seconds
   static accountCredentials = new Map<string, string>();
+  // [issue #252] Cross-zone error burst detector. Account-side Spotify rebalances
+  // disconnect every offload=false zone simultaneously; that's one event, not N
+  // independent failures. Detect the synchronized pattern (>= burstMinZones zones
+  // reporting Spotify errors within burstWindowMs) so we can bypass the per-zone
+  // cooldown that's appropriate for single-zone reconnect loops but punishing here.
+  private static recentBurstErrors: Array<{ zoneId: number; at: number }> = [];
+  private static readonly burstWindowMs = 3000;
+  private static readonly burstMinZones = 3;
   private readonly pipeId: string;
 
   constructor(
@@ -369,6 +377,36 @@ class SpotifyConnectInstance {
         lowerMessage.includes('429') ||
         lowerMessage.includes('too many requests') ||
         lowerMessage.includes('rate limit');
+      // [issue #252] If multiple zones report Spotify errors within a small
+      // window, classify as an account-side burst (not a per-zone reconnect
+      // loop). Bypass the cooldown the streak counter is otherwise built to
+      // trigger and reschedule with a short jittered, zone-staggered delay.
+      const burstNow = Date.now();
+      SpotifyConnectInstance.recentBurstErrors = SpotifyConnectInstance.recentBurstErrors.filter(
+        (e) => burstNow - e.at <= SpotifyConnectInstance.burstWindowMs,
+      );
+      SpotifyConnectInstance.recentBurstErrors.push({ zoneId: this.zoneId, at: burstNow });
+      const distinctZones = new Set(
+        SpotifyConnectInstance.recentBurstErrors.map((e) => e.zoneId),
+      ).size;
+      if (distinctZones >= SpotifyConnectInstance.burstMinZones) {
+        this.log.warn('synchronized cross-zone error burst; bypassing per-zone cooldown', {
+          zoneId: this.zoneId,
+          zonesAffected: distinctZones,
+          windowMs: SpotifyConnectInstance.burstWindowMs,
+          errorCode: ev.errorCode,
+          message,
+        });
+        this.restartStreak = { count: 0, firstAt: burstNow };
+        this.notifyOutputError(this.zoneId, `spotify ${message}`);
+        this.stopConnectHost();
+        // 8-25s with deterministic per-zone stagger; reconnects don't dogpile
+        // against Spotify's just-recovered backend.
+        const burstDelay =
+          8000 + (this.zoneId % 11) * 750 + Math.floor(Math.random() * 5000);
+        this.scheduleRestart({ minDelayMs: burstDelay });
+        return;
+      }
       if (ev.errorCode === 'audio_key_error') {
         // Audio key errors usually mean the session is unhealthy; trigger a cool-down.
         this.restartStreak = { count: 10, firstAt: Date.now() };