feat: auto-reconnect and rejoin line after manager restart

src/components/production-line/production-line.tsx

@@ -42,6 +42,8 @@ import { useLinePolling } from "./use-line-polling.ts";
 import { useMasterInputMute } from "./use-master-input-mute.ts";
 import { useMuteInput } from "./use-mute-input.tsx";
 import { useUpdateCallDevice } from "./use-update-call-device.tsx";
+import { useSessionRecovery } from "./use-session-recovery.ts";
+import { ReconnectBanner } from "./reconnect-banner.tsx";
 import { useVolumeReducer } from "./use-volume-reducer.tsx";
 import { UserControls } from "./user-controls.tsx";
 import { UserList } from "./user-list.tsx";
@@ -426,6 +428,21 @@ export const ProductionLine = ({
 
   const { loading, connectionError } = useIsLoading({ connectionState });
 
+  const handleRejoin = useCallback(() => {
+    dispatch({
+      type: "UPDATE_CALL",
+      payload: { id, updates: { connectionState: null } },
+    });
+  }, [dispatch, id]);
+
+  const {
+    phase: recoveryPhase,
+    attempt: connectionAttempt,
+    rejoin,
+    reportHeartbeat,
+    reportEstablishError,
+  } = useSessionRecovery({ connectionState, onRejoin: handleRejoin });
+
   useCheckBadLineData({
     joinProductionOptions,
     paramLineId,
@@ -504,8 +521,21 @@ export const ProductionLine = ({
             />
           </ConnectionErrorWrapper>
         ))}
+      {joinProductionOptions && !loading && (
+        <ReconnectBanner phase={recoveryPhase} onRejoin={rejoin} />
+      )}
       {connectionActive && (
         <SymphonyRtcConnectionComponent
+          key={connectionAttempt}
+          onHeartbeatStatus={reportHeartbeat}
+          // Only route establish failures into the recovery flow (retry with
+          // backoff) once we are rejoining (attempt > 0). On the very first
+          // attempt there is no established call to recover, so a failure keeps
+          // the original fatal ERROR path in useEstablishSession, which surfaces
+          // the "Connection failed" screen instead of silently retrying.
+          onEstablishError={
+            connectionAttempt > 0 ? reportEstablishError : undefined
+          }
           joinProductionOptions={joinProductionOptions}
           audiooutput={audiooutput || undefined}
           inputAudioStream={inputAudioStream}

src/components/production-line/reconnect-banner.test.tsx

@@ -0,0 +1,26 @@
+import { describe, it, expect, vi } from "vitest";
+import { render, screen, fireEvent } from "@testing-library/react";
+import { ReconnectBanner } from "./reconnect-banner.tsx";
+
+describe("ReconnectBanner", () => {
+  it("renders nothing when healthy", () => {
+    const { container } = render(
+      <ReconnectBanner phase="ok" onRejoin={vi.fn()} />
+    );
+    expect(container).toBeEmptyDOMElement();
+  });
+
+  it("shows a reconnecting status without a rejoin button", () => {
+    render(<ReconnectBanner phase="reconnecting" onRejoin={vi.fn()} />);
+    expect(screen.getByRole("status").textContent).toMatch(/Reconnecting/);
+    expect(screen.queryByRole("button")).toBeNull();
+  });
+
+  it("shows an error with a manual rejoin button when failed", () => {
+    const onRejoin = vi.fn();
+    render(<ReconnectBanner phase="failed" onRejoin={onRejoin} />);
+    expect(screen.getByRole("alert")).toBeTruthy();
+    fireEvent.click(screen.getByRole("button", { name: "Rejoin" }));
+    expect(onRejoin).toHaveBeenCalledTimes(1);
+  });
+});

src/components/production-line/reconnect-banner.tsx

@@ -0,0 +1,54 @@
+import styled from "@emotion/styled";
+import { TRecoveryPhase } from "./use-session-recovery.ts";
+
+const Banner = styled.div<{ failed: boolean }>`
+  position: fixed;
+  top: 0;
+  left: 0;
+  right: 0;
+  z-index: 1000;
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  gap: 1.5rem;
+  padding: 1rem 2rem;
+  font-size: 1.6rem;
+  color: #fff;
+  background: ${({ failed }) => (failed ? "#b3261e" : "#8a5a00")};
+`;
+
+const RejoinButton = styled.button`
+  font-size: 1.4rem;
+  padding: 0.4rem 1.2rem;
+  border-radius: 0.4rem;
+  border: 1px solid #fff;
+  background: transparent;
+  color: #fff;
+  cursor: pointer;
+`;
+
+type TProps = {
+  phase: TRecoveryPhase;
+  onRejoin: () => void;
+};
+
+export const ReconnectBanner = ({ phase, onRejoin }: TProps) => {
+  if (phase === "ok") return null;
+
+  if (phase === "reconnecting") {
+    return (
+      <Banner failed={false} role="status">
+        Connection to the server was lost. Reconnecting...
+      </Banner>
+    );
+  }
+
+  return (
+    <Banner failed role="alert">
+      Could not reconnect to the line.
+      <RejoinButton type="button" onClick={onRejoin}>
+        Rejoin
+      </RejoinButton>
+    </Banner>
+  );
+};

src/components/production-line/symphony-rtc-connection-component.tsx

@@ -2,7 +2,7 @@ import { TJoinProductionOptions } from "./types";
 import { TGlobalStateAction } from "../../global-state/global-state-actions";
 import { useEstablishSession } from "./use-establish-session";
 import { useRtcConnection } from "./use-rtc-connection";
-import { useHeartbeat } from "./use-heartbeat";
+import { THeartbeatStatus, useHeartbeat } from "./use-heartbeat";
 import { TUseAudioInputValues } from "./use-audio-input";
 import { useCallStateSync } from "./use-call-state-sync";
 
@@ -12,6 +12,8 @@ type SymphonyRtcConnectionComponentProps = {
   inputAudioStream: TUseAudioInputValues;
   callId: string;
   dispatch: React.Dispatch<TGlobalStateAction>;
+  onHeartbeatStatus?: (status: THeartbeatStatus) => void;
+  onEstablishError?: (error: Error) => void;
 };
 
 export const SymphonyRtcConnectionComponent = ({
@@ -20,11 +22,14 @@ export const SymphonyRtcConnectionComponent = ({
   inputAudioStream,
   callId,
   dispatch,
+  onHeartbeatStatus,
+  onEstablishError,
 }: SymphonyRtcConnectionComponentProps) => {
   const { sessionId, sdpOffer } = useEstablishSession({
     joinProductionOptions,
     callId,
     dispatch,
+    onError: onEstablishError,
   });
 
   const { connectionState, audioElements } = useRtcConnection({
@@ -36,7 +41,7 @@ export const SymphonyRtcConnectionComponent = ({
     callId,
   });
 
-  useHeartbeat({ sessionId });
+  useHeartbeat({ sessionId, onStatusChange: onHeartbeatStatus });
 
   // Sync all state properties with the global state
   useCallStateSync({

src/components/production-line/use-establish-session.ts

@@ -9,14 +9,20 @@ type TUseGetRtcOfferOptions = {
   joinProductionOptions: TJoinProductionOptions | null;
   callId: string;
   dispatch: Dispatch<TGlobalStateAction>;
+  // When provided, a failed session request is reported here instead of being
+  // dispatched as a call error (used while rejoining after a connection loss).
+  onError?: (error: Error) => void;
 };
 
 // A hook for fetching the web rtc sdp offer from the backend
 export const useEstablishSession = ({
   joinProductionOptions,
   callId,
   dispatch,
+  onError,
 }: TUseGetRtcOfferOptions) => {
+  const onErrorRef = useRef(onError);
+  onErrorRef.current = onError;
   const [sdpOffer, setSdpOffer] = useState<string | null>(null);
   const [sessionId, setSessionId] = useState<string | null>(null);
   const reqRef = useRef<{
@@ -63,6 +69,14 @@ export const useEstablishSession = ({
       .catch((e) => {
         if (!state.cancelled) {
           reqRef.current = null;
+          if (onErrorRef.current) {
+            onErrorRef.current(
+              e instanceof Error
+                ? e
+                : new Error("Failed to establish audio session.")
+            );
+            return;
+          }
           dispatch({
             type: "ERROR",
             payload: {

src/components/production-line/use-heartbeat.test.ts

@@ -126,6 +126,91 @@ describe("useHeartbeat", () => {
     expect(mockHeartbeat).toHaveBeenCalledTimes(2);
   });
 
+  it("reports unreachable on network errors and ok once the manager is back", async () => {
+    const onStatusChange = vi.fn();
+    mockHeartbeat
+      .mockRejectedValueOnce(new TypeError("Failed to fetch"))
+      .mockResolvedValue("ok");
+    renderHook(() => useHeartbeat({ sessionId: "s1", onStatusChange }), {
+      wrapper,
+    });
+
+    await act(async () => {
+      await vi.advanceTimersByTimeAsync(10000);
+    });
+    expect(onStatusChange).toHaveBeenLastCalledWith("unreachable");
+
+    await act(async () => {
+      await vi.runOnlyPendingTimersAsync();
+    });
+    expect(onStatusChange).toHaveBeenLastCalledWith("ok");
+  });
+
+  it("reports unreachable on 5xx responses", async () => {
+    const onStatusChange = vi.fn();
+    mockHeartbeat.mockRejectedValue(error500());
+    renderHook(() => useHeartbeat({ sessionId: "s1", onStatusChange }), {
+      wrapper,
+    });
+    await act(async () => {
+      await vi.advanceTimersByTimeAsync(10000);
+    });
+    expect(onStatusChange).toHaveBeenCalledWith("unreachable");
+  });
+
+  it("reports unreachable on 429 (rate limited) responses", async () => {
+    const onStatusChange = vi.fn();
+    mockHeartbeat.mockRejectedValue(
+      Object.assign(new Error("too many requests"), { status: 429 })
+    );
+    renderHook(() => useHeartbeat({ sessionId: "s1", onStatusChange }), {
+      wrapper,
+    });
+    await act(async () => {
+      await vi.advanceTimersByTimeAsync(10000);
+    });
+    expect(onStatusChange).toHaveBeenCalledWith("unreachable");
+  });
+
+  it("reports lost and stops when the session is gone (404)", async () => {
+    const onStatusChange = vi.fn();
+    mockHeartbeat.mockRejectedValue(
+      Object.assign(new Error("not found"), { status: 404 })
+    );
+    renderHook(() => useHeartbeat({ sessionId: "s1", onStatusChange }), {
+      wrapper,
+    });
+    await act(async () => {
+      await vi.advanceTimersByTimeAsync(10000);
+    });
+    expect(onStatusChange).toHaveBeenCalledWith("lost");
+
+    // The loop stops once the session is gone — no further heartbeats.
+    await act(async () => {
+      await vi.advanceTimersByTimeAsync(120000);
+    });
+    expect(mockHeartbeat).toHaveBeenCalledTimes(1);
+  });
+
+  it("reports lost and stops when the session is gone (410)", async () => {
+    const onStatusChange = vi.fn();
+    mockHeartbeat.mockRejectedValue(
+      Object.assign(new Error("gone"), { status: 410 })
+    );
+    renderHook(() => useHeartbeat({ sessionId: "s1", onStatusChange }), {
+      wrapper,
+    });
+    await act(async () => {
+      await vi.advanceTimersByTimeAsync(10000);
+    });
+    expect(onStatusChange).toHaveBeenCalledWith("lost");
+
+    await act(async () => {
+      await vi.advanceTimersByTimeAsync(120000);
+    });
+    expect(mockHeartbeat).toHaveBeenCalledTimes(1);
+  });
+
   it("stops and dispatches HEARTBEAT_ERROR after 3 consecutive 401s", async () => {
     mockHeartbeat.mockRejectedValue(error401());
     renderHook(() => useHeartbeat({ sessionId: "s1" }), { wrapper });

src/components/production-line/use-heartbeat.ts

@@ -1,18 +1,37 @@
-import { useEffect } from "react";
+import { useEffect, useRef } from "react";
 import { API } from "../../api/api.ts";
 import { noop } from "../../helpers.ts";
 import logger from "../../utils/logger.ts";
 import { useGlobalState } from "../../global-state/context-provider.tsx";
 import { backoffDelayMs } from "../../utils/backoff.ts";
 import { authCircuitBreaker } from "../../api/auth-circuit-breaker.ts";
 
-type TProps = { sessionId: string | null };
+// ok          - the manager acknowledged the heartbeat.
+// unreachable - the manager could not be reached (network error / 5xx), e.g.
+//               while it restarts. The loop keeps retrying with backoff.
+// lost        - the manager is up but no longer knows the session (404/410).
+//               The loop stops; the session has to be re-established.
+export type THeartbeatStatus = "ok" | "unreachable" | "lost";
+
+type TProps = {
+  sessionId: string | null;
+  onStatusChange?: (status: THeartbeatStatus) => void;
+};
+
+const isSessionGone = (status: number | undefined) =>
+  status === 404 || status === 410;
+
+const isManagerUnreachable = (status: number | undefined) =>
+  status === undefined || status === 429 || status >= 500;
 
 // Normal cadence between heartbeats while the session is healthy.
 const HEARTBEAT_INTERVAL_MS = 10_000;
 
-export const useHeartbeat = ({ sessionId }: TProps) => {
+export const useHeartbeat = ({ sessionId, onStatusChange }: TProps) => {
   const [, dispatch] = useGlobalState();
+  // Kept in a ref so a new callback identity never restarts the loop.
+  const onStatusChangeRef = useRef(onStatusChange);
+  onStatusChangeRef.current = onStatusChange;
 
   useEffect(() => {
     if (!sessionId) return noop;
@@ -67,11 +86,22 @@ export const useHeartbeat = ({ sessionId }: TProps) => {
           if (cancelled) return;
           failure401Count = 0;
           consecutiveFailureCount = 0;
+          onStatusChangeRef.current?.("ok");
           schedule(HEARTBEAT_INTERVAL_MS);
         })
         .catch((err) => {
           if (cancelled) return;
+          if (isSessionGone(err.status)) {
+            logger.red(
+              `Session ${sessionId} is no longer known by the server.`
+            );
+            onStatusChangeRef.current?.("lost");
+            return; // stop the loop — the session must be re-established
+          }
           consecutiveFailureCount += 1;
+          if (isManagerUnreachable(err.status)) {
+            onStatusChangeRef.current?.("unreachable");
+          }
           if (err.status === 401) {
             failure401Count += 1;
           }