diff --git a/src/Puppis.Driver/Sessions/FaultedTunerRecheck.cs b/src/Puppis.Driver/Sessions/FaultedTunerRecheck.cs
index 11ceed4..f897d23 100644
--- a/src/Puppis.Driver/Sessions/FaultedTunerRecheck.cs
+++ b/src/Puppis.Driver/Sessions/FaultedTunerRecheck.cs
@@ -5,6 +5,11 @@
namespace Puppis.Driver.Sessions;
+///
+/// When a faulted device is next tried, and whether it failed again soon after it was last handed back out.
+///
+public sealed record RecheckPlan(DateTimeOffset NextTryAt, bool Relapsed);
+
public sealed class FaultedTunerRecheck(
ITunerDeviceCheck check,
TimeProvider timeProvider,
@@ -23,11 +28,11 @@ ILogger logger
private readonly Lock gate = new();
private readonly Dictionary pending = new(StringComparer.Ordinal);
- private readonly Dictionary restoredAt = new(StringComparer.Ordinal);
+ private readonly Dictionary restored = new(StringComparer.Ordinal);
private bool disposed;
- public DateTimeOffset? Schedule(DeviceSettings device, Action restore)
+ public RecheckPlan? Schedule(DeviceSettings device, Action restore)
{
string deviceId = device.Id!;
DateTimeOffset now = timeProvider.GetUtcNow();
@@ -39,26 +44,24 @@ ILogger logger
return null;
}
- if (restoredAt.TryGetValue(deviceId, out DateTimeOffset restored) && now - restored <= RelapseWindow)
- {
- return null;
- }
-
if (pending.TryGetValue(deviceId, out Pending? waiting))
{
- return waiting.NextTryAt;
+ return new RecheckPlan(waiting.NextTryAt, false);
}
- var entry = new Pending(device, restore, now + Intervals[0]);
+ int relapse = restored.TryGetValue(deviceId, out Restoration? last) && now - last.At <= RelapseWindow
+ ? Math.Min(last.Relapse + 1, Intervals.Count - 1)
+ : 0;
+ Pending entry = new(device, restore, now + Intervals[relapse], relapse) { Tries = relapse };
entry.Timer = timeProvider.CreateTimer(
_ => TryAgain(entry),
null,
- Intervals[0],
+ Intervals[relapse],
Timeout.InfiniteTimeSpan
);
pending[deviceId] = entry;
- return entry.NextTryAt;
+ return new RecheckPlan(entry.NextTryAt, relapse > 0);
}
}
@@ -135,7 +138,7 @@ private void TryAgain(Pending entry)
}
pending.Remove(deviceId);
- restoredAt[deviceId] = timeProvider.GetUtcNow();
+ restored[deviceId] = new Restoration(timeProvider.GetUtcNow(), entry.Relapse);
}
entry.Timer?.Dispose();
@@ -167,8 +170,12 @@ private void StillFaulted(Pending entry, Exception error)
);
}
- private sealed class Pending(DeviceSettings device, Action restore, DateTimeOffset nextTryAt)
+ private sealed record Restoration(DateTimeOffset At, int Relapse);
+
+ private sealed class Pending(DeviceSettings device, Action restore, DateTimeOffset nextTryAt, int relapse)
{
+ public int Relapse { get; } = relapse;
+
public DeviceSettings Device { get; } = device;
public Action Restore { get; } = restore;
diff --git a/src/Puppis.Driver/Sessions/TunerSessionManager.cs b/src/Puppis.Driver/Sessions/TunerSessionManager.cs
index c635ac2..d5a250a 100644
--- a/src/Puppis.Driver/Sessions/TunerSessionManager.cs
+++ b/src/Puppis.Driver/Sessions/TunerSessionManager.cs
@@ -33,7 +33,7 @@ public sealed class TunerSessionManager(
{
public const int RetainedSessions = 64;
- public const int RepeatedTuneFailureCeiling = 3;
+ public const int DifferentChannelsThatFaultTheDevice = 2;
public static readonly TimeSpan DefaultHardStopLimit = TimeSpan.FromSeconds(30);
@@ -57,9 +57,7 @@ public sealed class TunerSessionManager(
private readonly ConcurrentQueue ended = new();
private readonly ConcurrentDictionary tunings = [];
private readonly ConcurrentDictionary writing = new(StringComparer.Ordinal);
- private readonly Dictionary> tuneFailureStreaks = new(
- StringComparer.Ordinal
- );
+ private readonly Dictionary> tuneFailures = new(StringComparer.Ordinal);
private readonly Lock streakGate = new();
private readonly TimeSpan drainCap = TimeSpan.FromHours(
Math.Max(0, configuration.ShutdownGraceHours)
@@ -713,19 +711,19 @@ error is DvbDeviceException
} failed
)
{
- RecordTuneFailure(deviceId, TuningKey.Of(request), failed);
+ RecordTuneFailure(deviceId, TuningKey.Of(request), failed, request.Purpose);
}
refusal = error switch
{
DvbDeviceException { Failure: TuningFailure.NoLock } => SessionStart.Refused(
SessionRefusal.NoLock,
- $"The device '{deviceId}' opened but the frontend did not lock: {error.Message}"
+ $"The device '{deviceId}' opened but the frontend did not lock on {TuningKey.Of(request)}: {error.Message}"
),
DvbDeviceException { Failure: TuningFailure.LockedWithoutData } =>
SessionStart.Refused(
SessionRefusal.NoData,
- $"The device '{deviceId}' locked but delivered nothing: {error.Message}"
+ $"The device '{deviceId}' locked on {TuningKey.Of(request)} but delivered nothing: {error.Message}"
),
_ => SessionStart.Refused(
SessionRefusal.DeviceUnavailable,
@@ -1382,7 +1380,7 @@ session.FailureCause is DvbDeviceException
} channel
)
{
- RecordTuneFailure(session.DeviceId, tuning, channel);
+ RecordTuneFailure(session.DeviceId, tuning, channel, session.Purpose);
}
}
@@ -1399,46 +1397,78 @@ private void FaultAfterFailing(TunerSession session)
$"The device failed while serving '{session.SessionId}': "
+ (session.FailureCause?.Message ?? "no cause was recorded.");
- faultedDevices[deviceId] = new DeviceFault(fault, null, DeviceFaultKind.Failed);
- healthChangedAt[deviceId] = timeProvider.GetUtcNow();
-
- pool.Discard(deviceId);
-
- if (recheck is null || draining || DeviceNamed(deviceId) is not { } device)
+ if (FaultUntilItOpens(deviceId, new DeviceFault(fault, null, DeviceFaultKind.Failed)) is not { } plan)
{
+ logger.LogError(
+ "The device {DeviceId} failed while serving {SessionId} and {NotTriedAgain}",
+ deviceId,
+ session.SessionId.Value,
+ NotTriedAgain()
+ );
+
return;
}
- if (recheck.Schedule(device, Restore) is { } nextTryAt)
+ if (!plan.Relapsed)
{
logger.LogWarning(
"The device {DeviceId} failed while serving {SessionId}, so it is not handed out until it opens again; it is tried again at {NextTryAt}.",
deviceId,
session.SessionId.Value,
- nextTryAt
+ plan.NextTryAt
);
return;
}
- faultedDevices[deviceId] = new DeviceFault(
- fault
- + $" It had been handed back out less than {RelapseMinutes} minutes earlier, so it is not tried again.",
- null,
- DeviceFaultKind.FailedAgain);
-
logger.LogError(
- "The device {DeviceId} failed again while serving {SessionId} within {RelapseMinutes} minutes of being handed back out, so it is not tried again and stays faulted until the driver restarts.",
+ "The device {DeviceId} failed again while serving {SessionId} within {RelapseMinutes} minutes of being handed back out, so it waits longer than last time and is tried again at {NextTryAt}.",
deviceId,
session.SessionId.Value,
- RelapseMinutes
+ RelapseMinutes,
+ plan.NextTryAt
);
}
+ private RecheckPlan? FaultUntilItOpens(string deviceId, DeviceFault fault)
+ {
+ faultedDevices[deviceId] = fault;
+ healthChangedAt[deviceId] = timeProvider.GetUtcNow();
+
+ pool.Discard(deviceId);
+
+ events?.Signal(DriverEvents.TunerHealthChanged);
+ events?.Signal(DriverEvents.Tuners);
+
+ if (recheck is null || draining || DeviceNamed(deviceId) is not { } device)
+ {
+ return null;
+ }
+
+ RecheckPlan? plan = recheck.Schedule(device, Restore);
+
+ if (plan is { Relapsed: true } && fault.Kind.Kind is TunerFaultKind.DeviceFailed)
+ {
+ faultedDevices[deviceId] = fault with
+ {
+ Detail = fault.Detail
+ + $" It had been handed back out less than {RelapseMinutes} minutes earlier, so it waits longer than last time before it is tried again.",
+ Kind = DeviceFaultKind.FailedAgain,
+ };
+ }
+
+ return plan;
+ }
+
private static double RelapseMinutes => FaultedTunerRecheck.RelapseWindow.TotalMinutes;
private void Restore(string deviceId)
{
+ lock (streakGate)
+ {
+ tuneFailures.Remove(deviceId);
+ }
+
faultedDevices.TryRemove(deviceId, out _);
healthChangedAt[deviceId] = timeProvider.GetUtcNow();
@@ -1481,25 +1511,32 @@ private void Announce()
events?.Signal(DriverEvents.Tuners);
}
- private void RecordTuneFailure(string deviceId, TuningKey tuning, DvbDeviceException cause)
+ private void RecordTuneFailure(
+ string deviceId,
+ TuningKey tuning,
+ DvbDeviceException cause,
+ SessionPurpose purpose
+ )
{
- int streak;
+ bool first;
+ TuningKey[] channels;
lock (streakGate)
{
- if (!tuneFailureStreaks.TryGetValue(deviceId, out Dictionary? perChannel))
+ first = !tuneFailures.TryGetValue(deviceId, out List? counted);
+ counted ??= tuneFailures[deviceId] = [];
+
+ if (purpose is not SessionPurpose.Scan && !counted.Any(seen => OnTheSameChannel(seen, tuning)))
{
- perChannel = tuneFailureStreaks[deviceId] = [];
+ counted.Add(tuning);
}
- streak = perChannel[tuning] = perChannel.TryGetValue(tuning, out int seen)
- ? seen + 1
- : 1;
+ channels = [.. counted];
}
- if (streak < RepeatedTuneFailureCeiling)
+ if (channels.Length < DifferentChannelsThatFaultTheDevice)
{
- if (streak is 1)
+ if (first)
{
healthChangedAt[deviceId] = timeProvider.GetUtcNow();
events?.Signal(DriverEvents.TunerHealthChanged);
@@ -1509,24 +1546,49 @@ private void RecordTuneFailure(string deviceId, TuningKey tuning, DvbDeviceExcep
return;
}
- Fault(
+ string fault =
+ $"The device failed to receive {string.Join(", ", channels)} one after another without delivering"
+ + $" anything in between; the last failure was: {cause.Message}";
+ RecheckPlan? plan = FaultUntilItOpens(
deviceId,
- new DeviceFault(
- $"The device failed to receive {tuning} {streak} times in a row without delivering"
- + $" anything in between; the last failure was: {cause.Message}",
- TuningFailureTitles.Of(cause.Failure),
- DeviceFaultKind.RepeatedTuneFailure
- )
+ new DeviceFault(fault, TuningFailureTitles.Of(cause.Failure), DeviceFaultKind.RepeatedTuneFailure)
+ );
+
+ if (plan is null)
+ {
+ logger.LogError(
+ "The device {DeviceId} failed to receive {Channels} one after another and {NotTriedAgain}",
+ deviceId,
+ string.Join(", ", channels),
+ NotTriedAgain()
+ );
+
+ return;
+ }
+
+ logger.LogWarning(
+ "The device {DeviceId} failed to receive {Channels} one after another, so it is not handed out until it opens again; it is tried again at {NextTryAt}.",
+ deviceId,
+ string.Join(", ", channels),
+ plan.NextTryAt
);
}
+ private string NotTriedAgain() =>
+ draining
+ ? "the driver is shutting down, so it is not tried again."
+ : "nothing is set to try it again, so it stays faulted until the driver restarts.";
+
+ private static bool OnTheSameChannel(TuningKey one, TuningKey other) =>
+ one.Kind == other.Kind && one.PhysicalChannel == other.PhysicalChannel;
+
private void ForgiveTuneFailures(string deviceId)
{
bool forgiven;
lock (streakGate)
{
- forgiven = tuneFailureStreaks.Remove(deviceId);
+ forgiven = tuneFailures.Remove(deviceId);
}
if (forgiven && !faultedDevices.ContainsKey(deviceId))
@@ -1538,15 +1600,13 @@ private void ForgiveTuneFailures(string deviceId)
}
///
- /// Whether a channel has failed to tune on this device since it last delivered a session through to its end,
- /// fewer times than it takes to fault the device.
+ /// Whether a channel has failed to tune on this device since it last delivered a session through to its end.
///
public bool IsFailingToTune(string deviceId)
{
lock (streakGate)
{
- return tuneFailureStreaks.TryGetValue(deviceId, out Dictionary? perChannel)
- && perChannel.Values.Any(streak => streak > 0);
+ return tuneFailures.ContainsKey(deviceId);
}
}
diff --git a/src/Puppis.Driver/Tuning/Dvb/DvbTunerDevice.cs b/src/Puppis.Driver/Tuning/Dvb/DvbTunerDevice.cs
index b38849e..2417413 100644
--- a/src/Puppis.Driver/Tuning/Dvb/DvbTunerDevice.cs
+++ b/src/Puppis.Driver/Tuning/Dvb/DvbTunerDevice.cs
@@ -42,6 +42,7 @@ public sealed class DvbTunerDevice : ITunerDevice, ISignalQualitySource
private readonly TimeProvider time;
private readonly DvbDevicePaths paths;
private readonly DvbTunerSettings settings;
+ private readonly DvbChannel channel;
private readonly DvbFrontend frontend;
private readonly int demux;
private readonly int dvr;
@@ -54,6 +55,7 @@ private DvbTunerDevice(
TimeProvider time,
DvbDevicePaths paths,
DvbTunerSettings settings,
+ DvbChannel channel,
DvbFrontend frontend,
int demux,
int dvr
@@ -63,6 +65,7 @@ int dvr
this.time = time;
this.paths = paths;
this.settings = settings;
+ this.channel = channel;
this.frontend = frontend;
this.demux = demux;
this.dvr = dvr;
@@ -126,7 +129,7 @@ out FrontendStatus lastSeen
(demux, dvr) = OpenTheStream(calls, paths, settings);
}
- return new DvbTunerDevice(calls, time, paths, settings, frontend, demux, dvr);
+ return new DvbTunerDevice(calls, time, paths, settings, channel, frontend, demux, dvr);
}
catch
{
@@ -232,19 +235,19 @@ private DvbDeviceException NothingArrived()
if (!frontend.TryStatus(out FrontendStatus status))
{
return DvbFailure.Refused(
- $"{paths.Dvr}: no transport stream bytes arrived within {waited}, and the frontend would not say whether it is still locked. This is left unclassified rather than recorded as a tuner that was locked and delivering nothing."
+ $"{paths.Dvr}: no transport stream bytes arrived from {channel} within {waited}, and the frontend would not say whether it is still locked. This is left unclassified rather than recorded as a tuner that was locked and delivering nothing."
);
}
if (!status.HasFlag(FrontendStatus.Lock))
{
return DvbFailure.NoLock(
- $"{paths.Dvr}: no transport stream bytes arrived within {waited}, and the frontend is no longer locked; its status is now {status}."
+ $"{paths.Dvr}: no transport stream bytes arrived from {channel} within {waited}, and the frontend is no longer locked; its status is now {status}."
);
}
return DvbFailure.LockedWithoutData(
- $"{paths.Dvr}: the frontend is still locked ({status}) and no transport stream bytes arrived within {waited}. The tuner is synchronised and the demux is delivering nothing, which is a different fault from failing to lock."
+ $"{paths.Dvr}: the frontend is still locked onto {channel} ({status}) and no transport stream bytes arrived within {waited}. The tuner is synchronised and the demux is delivering nothing, which is a different fault from failing to lock."
);
}
diff --git a/src/Puppis.Driver/Tuning/Usb/CechZd1/CechZd1TunerDevice.cs b/src/Puppis.Driver/Tuning/Usb/CechZd1/CechZd1TunerDevice.cs
index d3279d7..328f0f2 100644
--- a/src/Puppis.Driver/Tuning/Usb/CechZd1/CechZd1TunerDevice.cs
+++ b/src/Puppis.Driver/Tuning/Usb/CechZd1/CechZd1TunerDevice.cs
@@ -34,6 +34,7 @@ public sealed class CechZd1TunerDevice : ITunerDevice, ISignalQualitySource
private readonly Lock gate = new();
private ILogger? logger;
private long opened;
+ private long hertz;
private bool firstBytesReported;
private bool closed;
@@ -215,6 +216,7 @@ string how
var device = new CechZd1TunerDevice(kept, reader, pacing, bytePatience, standby);
device.logger = logger;
device.opened = stages.Started;
+ device.hertz = hertz;
device.Report(logger, hertz, stages, how);
return device;
@@ -329,16 +331,16 @@ private DvbDeviceException NothingArrived()
catch (DvbDeviceException)
{
return DvbFailure.Refused(
- $"{usb.Path}: no transport stream arrived within {waited}, and the demodulator would not say whether it is still locked."
+ $"{usb.Path}: no transport stream arrived on {hertz / 1000} kHz within {waited}, and the demodulator would not say whether it is still locked."
);
}
return CechZd1Signal.LockedWithData(registers.StatusAfter)
? DvbFailure.LockedWithoutData(
- $"{usb.Path}: the demodulator is locked and no transport stream arrived within {waited}."
+ $"{usb.Path}: the demodulator is locked on {hertz / 1000} kHz and no transport stream arrived within {waited}."
)
: DvbFailure.NoLock(
- $"{usb.Path}: no transport stream arrived within {waited}, and the demodulator is no longer locked (0x{registers.StatusAfter:X4})."
+ $"{usb.Path}: no transport stream arrived on {hertz / 1000} kHz within {waited}, and the demodulator is no longer locked (0x{registers.StatusAfter:X4})."
);
}
diff --git a/tests/Puppis.Driver.Tests/CechZd1Tests.cs b/tests/Puppis.Driver.Tests/CechZd1Tests.cs
index 4637114..9349d55 100644
--- a/tests/Puppis.Driver.Tests/CechZd1Tests.cs
+++ b/tests/Puppis.Driver.Tests/CechZd1Tests.cs
@@ -460,6 +460,29 @@ public void ALockedTunerThatSendsNothingSaysSo()
DvbDeviceException refused = UsbTestSupport.Refusal(() => device.Read(188, CancellationToken.None));
Assert.Equal(TuningFailure.LockedWithoutData, refused.Failure);
+ Assert.Contains($"{Channel22 / 1000} kHz", refused.Message, StringComparison.Ordinal);
+ device.Dispose();
+ }
+
+ [Fact]
+ public void TheWaitForBytesStartsWhenTheyAreAskedForNotWhenTheTunerWasOpened()
+ {
+ (CechZd1TunerDevice device, FakeBridge bridge, _, ManualTimeProvider clock) = Opened();
+ clock.Advance(TimeSpan.FromSeconds(4));
+ int idle = 0;
+ bridge.Idle = waited =>
+ {
+ clock.Advance(waited);
+
+ if (++idle * waited >= TimeSpan.FromSeconds(4))
+ {
+ bridge.Transfers.Enqueue(UsbTestSupport.Units(50, skip: 30));
+ }
+ };
+
+ byte[] read = device.Read(188, CancellationToken.None);
+
+ Assert.Equal(188, read.Length);
device.Dispose();
}
diff --git a/tests/Puppis.Driver.Tests/DvbTunerDeviceTests.cs b/tests/Puppis.Driver.Tests/DvbTunerDeviceTests.cs
index 753faff..538a96b 100644
--- a/tests/Puppis.Driver.Tests/DvbTunerDeviceTests.cs
+++ b/tests/Puppis.Driver.Tests/DvbTunerDeviceTests.cs
@@ -287,6 +287,7 @@ public void ALockedTunerThatDeliversNothingIsADistinctFailureFromNotLocking()
Assert.Contains("still locked", refusal.Message, StringComparison.Ordinal);
Assert.Contains("/dev/dvb/adapter0/dvr0", refusal.Message, StringComparison.Ordinal);
+ Assert.Contains(DvbChannel.Terrestrial(55).ToString(), refusal.Message, StringComparison.Ordinal);
Assert.Equal(TuningFailure.LockedWithoutData, refusal.Failure);
}
diff --git a/tests/Puppis.Driver.Tests/FaultedTunerRecheckTests.cs b/tests/Puppis.Driver.Tests/FaultedTunerRecheckTests.cs
index d68b6e2..e5cdb7c 100644
--- a/tests/Puppis.Driver.Tests/FaultedTunerRecheckTests.cs
+++ b/tests/Puppis.Driver.Tests/FaultedTunerRecheckTests.cs
@@ -51,6 +51,24 @@ private static StartSessionRequest Watching(string sessionId) =>
DeviceId = "adapter0",
};
+ private static StartSessionRequest Surveying(string sessionId, int channel) =>
+ new()
+ {
+ SessionId = SessionId.Parse(sessionId),
+ Purpose = SessionPurpose.Survey,
+ Tuning = new TuningRequest(TunerKind.Terrestrial, channel, 50001),
+ DeviceId = "adapter0",
+ };
+
+ private SessionRefusal Refusal(TunerSessionManager manager, StartSessionRequest request)
+ {
+ SessionRefusal refusal = manager.Begin(request).Refusal;
+
+ clock.Advance(TimeSpan.FromSeconds(6));
+
+ return refusal;
+ }
+
private static void FailOn(TunerSessionManager manager, string sessionId)
{
SessionStart start = manager.Begin(Watching(sessionId));
@@ -120,7 +138,7 @@ public void ATunerThatStillWillNotOpenIsTriedAfterFiveAndFifteenMinutesAndThenEv
}
[Fact]
- public void ATunerThatFailsAgainWithinHalfAnHourOfComingBackIsNotTriedAgain()
+ public void ATunerThatFailsAgainWithinHalfAnHourOfComingBackIsTriedAgainAfterALongerWait()
{
TunerSessionManager manager = Manager();
@@ -131,19 +149,94 @@ public void ATunerThatFailsAgainWithinHalfAnHourOfComingBackIsNotTriedAgain()
clock.Advance(TimeSpan.FromMinutes(30));
FailOn(manager, "s-2");
- foreach (int minutes in new[] { 1, 5, 15, 60, 60 })
- {
- clock.Advance(TimeSpan.FromMinutes(minutes));
- }
-
- Assert.Single(check.Tried);
- Assert.True(manager.IsFaulted("adapter0", out _));
Assert.Equal(TunerFaultKind.DeviceFailedAgain, manager.FaultKindOf("adapter0")?.Kind);
SessionStart refused = manager.Begin(Watching("s-3"));
Assert.Equal(SessionRefusal.FaultedDevice, refused.Refusal);
- Assert.Contains("until the driver restarts", refused.Detail, StringComparison.Ordinal);
+ Assert.Contains((clock.GetUtcNow() + TimeSpan.FromMinutes(5)).ToString("O"), refused.Detail, StringComparison.Ordinal);
+ Assert.DoesNotContain("restart", refused.Detail, StringComparison.Ordinal);
+
+ Wait(TimeSpan.FromMinutes(5), 1, manager);
+
+ Assert.False(manager.IsFaulted("adapter0", out _));
+ }
+
+ [Fact]
+ public void ATunerThatTookTwoFailedTriesToComeBackAndFailsAgainSoonWaitsFiveMinutes()
+ {
+ TunerSessionManager manager = Manager();
+ check.RefuseTheNext(2);
+
+ FailOn(manager, "s-1");
+ Wait(TimeSpan.FromMinutes(1), 0, manager);
+ Wait(TimeSpan.FromMinutes(5), 1, manager);
+ Wait(TimeSpan.FromMinutes(15), 2, manager);
+
+ Assert.False(manager.IsFaulted("adapter0", out _));
+
+ clock.Advance(TimeSpan.FromMinutes(10));
+ FailOn(manager, "s-2");
+
+ Assert.Equal(TunerFaultKind.DeviceFailedAgain, manager.FaultKindOf("adapter0")?.Kind);
+
+ Wait(TimeSpan.FromMinutes(5), 3, manager);
+
+ Assert.False(manager.IsFaulted("adapter0", out _));
+ }
+
+ [Fact]
+ public void ATunerThatKeepsFailingRightAfterComingBackWaitsLongerEachTimeAndIsNeverGivenUpOn()
+ {
+ TunerSessionManager manager = Manager();
+ TimeSpan[] waits =
+ [
+ TimeSpan.FromMinutes(1),
+ TimeSpan.FromMinutes(5),
+ TimeSpan.FromMinutes(15),
+ TimeSpan.FromMinutes(60),
+ TimeSpan.FromMinutes(60),
+ ];
+
+ for (int round = 0; round < waits.Length; round++)
+ {
+ FailOn(manager, $"s-{round}");
+ Wait(waits[round], round, manager);
+
+ Assert.False(manager.IsFaulted("adapter0", out _));
+ }
+ }
+
+ [Fact]
+ public void ATunerFaultedForFailingOnTwoChannelsIsTriedAgainAndServesOnceItOpens()
+ {
+ TunerSessionManager manager = new(
+ Configuration,
+ new FailingChannelDeviceFactory(deadChannels: [14, 15]),
+ clock,
+ NullLogger.Instance,
+ events: hub,
+ deviceCheck: check
+ );
+
+ Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Surveying("survey-14", 14)));
+ Assert.Equal(SessionRefusal.NoLock, manager.Begin(Surveying("survey-15", 15)).Refusal);
+
+ Assert.Equal(TunerFaultKind.RepeatedTuneFailure, manager.FaultKindOf("adapter0")?.Kind);
+ Assert.DoesNotContain("restart", manager.Begin(Surveying("survey-20", 20)).Detail, StringComparison.Ordinal);
+
+ Wait(TimeSpan.FromMinutes(1), 0, manager);
+
+ Assert.False(manager.IsFaulted("adapter0", out _));
+ Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Surveying("survey-14-again", 14)));
+ Assert.False(manager.IsFaulted("adapter0", out _));
+
+ Assert.True(manager.Begin(Surveying("survey-20-again", 20)).TryGetSession(out TunerSession? served));
+
+ served.Stop();
+ served.WaitForEnd(TimeSpan.FromSeconds(10));
+
+ Assert.Equal(SessionState.Stopped, served.State);
}
[Fact]
@@ -176,6 +269,11 @@ public void ATunerFaultedForDisagreeingWithTheLedgerIsNeverTriedAgain()
Assert.Empty(check.Tried);
Assert.True(manager.IsFaulted("adapter0", out _));
+
+ SessionStart refused = manager.Begin(Watching("s-1"));
+
+ Assert.Equal(SessionRefusal.FaultedDevice, refused.Refusal);
+ Assert.Contains("until the driver restarts", refused.Detail, StringComparison.Ordinal);
}
[Fact]
diff --git a/tests/Puppis.Driver.Tests/SessionTestSupport.cs b/tests/Puppis.Driver.Tests/SessionTestSupport.cs
index 60d05dc..fdab22d 100644
--- a/tests/Puppis.Driver.Tests/SessionTestSupport.cs
+++ b/tests/Puppis.Driver.Tests/SessionTestSupport.cs
@@ -7,6 +7,7 @@
using Puppis.Driver.Descrambling;
using Puppis.Driver.Recording;
using Puppis.Driver.Tuning;
+using Puppis.Driver.Tuning.Dvb;
namespace Puppis.Driver.Tests;
@@ -193,6 +194,14 @@ public ITunerDevice Create(DeviceSettings device, TuningRequest tuning, TunePara
: new ScriptedTunerDevice();
}
+public sealed class FailingChannelDeviceFactory(int[] deadChannels) : ITunerDeviceFactory
+{
+ public ITunerDevice Create(DeviceSettings device, TuningRequest tuning, TuneParams? tune, HeadHold hold) =>
+ deadChannels.Contains(tuning.PhysicalChannel)
+ ? throw DvbFailure.NoLock("the frontend did not lock within 5 seconds.")
+ : new ScriptedTunerDevice();
+}
+
public sealed class CountingRecordingWriter : IRecordingWriter
{
private long bytesWritten;
diff --git a/tests/Puppis.Driver.Tests/TuneFailureClassificationTests.cs b/tests/Puppis.Driver.Tests/TuneFailureClassificationTests.cs
index 1823493..58ca889 100644
--- a/tests/Puppis.Driver.Tests/TuneFailureClassificationTests.cs
+++ b/tests/Puppis.Driver.Tests/TuneFailureClassificationTests.cs
@@ -26,6 +26,8 @@ public sealed class TuneFailureClassificationTests
[new DeviceSettings("adapter0", DeviceKind.Terrestrial)]
);
+ private static readonly DriverHello Hello = new(DriverProtocol.Version, "instance-1", []);
+
private readonly ManualTimeProvider clock = new(Start);
[Fact]
@@ -92,7 +94,7 @@ public void AChannelThatDeliveredNoBytesEndsItsSessionWithoutCondemningTheTuner(
Assert.Equal(
SessionRefusalTitles.NoData,
- SessionViews.Of(silent, new DriverHello(DriverProtocol.Version, "instance-1", [])).FailureTitle
+ SessionViews.Of(silent, Hello).FailureTitle
);
SessionStart second = manager.Begin(Request("scan-15", 15));
@@ -105,134 +107,186 @@ public void AChannelThatDeliveredNoBytesEndsItsSessionWithoutCondemningTheTuner(
}
[Fact]
- public void AChannelThatKeepsFailingTheSameWayEndsInAFaultTheHealthSurfacesCanSee()
+ public void AChannelThatKeepsDeliveringNothingAfterLockingLeavesTheTunerServingOtherChannels()
{
- TunerSessionManager manager = Manager(new ThrowingDeviceFactory(() => DvbFailure.NoLock(
- "the frontend did not lock within 5 seconds."
- )));
+ TunerSessionManager manager = Manager(new SilentChannelDeviceFactory(silentChannels: [27]));
- for (int attempt = 1; attempt <= TunerSessionManager.RepeatedTuneFailureCeiling; attempt++)
+ for (int attempt = 1; attempt <= 4; attempt++)
{
- SessionStart start = manager.Begin(Request($"scan-{attempt}", 14));
+ SessionStart start = manager.Begin(Request($"survey-{attempt}", 27, SessionPurpose.Survey));
- Assert.Equal(SessionRefusal.NoLock, start.Refusal);
+ Assert.True(start.TryGetSession(out TunerSession? session));
+
+ session.WaitForEnd(Deadlock);
+
+ Assert.Equal(SessionState.Failed, session.State);
+ Assert.Equal(SessionRefusalTitles.NoData, SessionViews.Of(session, Hello).FailureTitle);
+ Assert.False(manager.IsFaulted("adapter0", out _));
clock.Advance(TimeSpan.FromSeconds(6));
}
- Assert.True(manager.IsFaulted("adapter0", out string? detail));
- Assert.Contains("channel 14", detail, StringComparison.Ordinal);
- Assert.Contains(
- TunerSessionManager.RepeatedTuneFailureCeiling.ToString(),
- detail,
- StringComparison.Ordinal
- );
+ TunerSnapshot tuner = Assert.Single(SessionViews.Tuners(Configuration, manager));
- SessionStart refused = manager.Begin(Request("scan-after", 14));
+ Assert.Equal(TunerState.Idle, tuner.State);
+ Assert.Equal(TunerHealthLevel.Degraded, tuner.Health?.Level);
+ Assert.Equal(TunerDegradedKind.TuneFailing, tuner.Health?.DegradedKind);
+
+ SessionStart other = manager.Begin(Request("survey-other", 21, SessionPurpose.Survey));
- Assert.Equal(SessionRefusal.FaultedDevice, refused.Refusal);
+ Assert.Equal(SessionRefusal.None, other.Refusal);
+ Assert.True(other.TryGetSession(out TunerSession? served));
+
+ served.Stop();
+ served.WaitForEnd(Deadlock);
+
+ Assert.Equal(SessionState.Stopped, served.State);
}
[Fact]
- public void ASweepAcrossManyEmptyChannelsLeavesTheTunerInGoodStanding()
+ public void AChannelThatKeepsFailingToLockIsRefusedEachTimeAndTheTunerStaysInService()
{
- TunerSessionManager manager = Manager(new ThrowingDeviceFactory(() => DvbFailure.NoLock(
- "the frontend did not lock within 5 seconds."
- )));
+ TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannels: [14]));
- for (int channel = 13; channel <= 22; channel++)
+ for (int attempt = 1; attempt <= 4; attempt++)
{
- SessionStart start = manager.Begin(Request($"scan-{channel}", channel));
+ SessionStart start = manager.Begin(Request($"survey-{attempt}", 14, SessionPurpose.Survey));
Assert.Equal(SessionRefusal.NoLock, start.Refusal);
+ Assert.Contains("terrestrial channel 14", start.Detail, StringComparison.Ordinal);
clock.Advance(TimeSpan.FromSeconds(6));
}
Assert.False(manager.IsFaulted("adapter0", out _));
+
+ SessionStart other = manager.Begin(Request("survey-other", 20, SessionPurpose.Survey));
+
+ Assert.True(other.TryGetSession(out TunerSession? served));
+
+ served.Stop();
+ served.WaitForEnd(Deadlock);
}
[Fact]
- public void ASessionThatDeliveredResetsTheFailureStreakOfItsDevice()
+ public void ARefusalForDeliveringNothingNamesTheChannel()
{
- TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannel: 14));
+ TunerSessionManager manager = Manager(new ThrowingDeviceFactory(() =>
+ DvbFailure.LockedWithoutData("the frontend is still locked and nothing arrived.")));
- for (int round = 1; round < TunerSessionManager.RepeatedTuneFailureCeiling; round++)
- {
- Assert.Equal(
- SessionRefusal.NoLock,
- manager.Begin(Request($"before-{round}", 14)).Refusal
- );
+ SessionStart start = manager.Begin(Request("survey-1", 14, SessionPurpose.Survey));
- clock.Advance(TimeSpan.FromSeconds(6));
- }
+ Assert.Equal(SessionRefusal.NoData, start.Refusal);
+ Assert.Contains("terrestrial channel 14", start.Detail, StringComparison.Ordinal);
+ }
- SessionStart delivered = manager.Begin(Request("delivering", 20));
+ [Fact]
+ public void FailingOnTwoDifferentChannelsInARowFaultsTheTuner()
+ {
+ TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannels: [14, 15]));
- Assert.True(delivered.TryGetSession(out TunerSession? session));
+ Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("survey-1", 14, SessionPurpose.Survey)));
+ Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("survey-2", 14, SessionPurpose.Survey)));
- session.Stop();
- session.WaitForEnd(Deadlock);
+ Assert.False(manager.IsFaulted("adapter0", out _));
- Assert.Equal(SessionState.Stopped, session.State);
+ Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("survey-3", 15, SessionPurpose.Survey)));
- for (int round = 1; round < TunerSessionManager.RepeatedTuneFailureCeiling; round++)
- {
- Assert.Equal(
- SessionRefusal.NoLock,
- manager.Begin(Request($"after-{round}", 14)).Refusal
- );
+ Assert.True(manager.IsFaulted("adapter0", out string? detail));
+ Assert.Contains("terrestrial channel 14", detail, StringComparison.Ordinal);
+ Assert.Contains("terrestrial channel 15", detail, StringComparison.Ordinal);
+ Assert.Equal(
+ SessionRefusal.FaultedDevice,
+ manager.Begin(Request("survey-4", 20, SessionPurpose.Survey)).Refusal
+ );
+ }
- clock.Advance(TimeSpan.FromSeconds(6));
- }
+ [Fact]
+ public void ASessionThatDeliveredBetweenTwoFailingChannelsKeepsTheTunerInService()
+ {
+ TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannels: [14, 15, 16]));
+ Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("before", 14, SessionPurpose.Survey)));
+
+ Assert.True(manager.Begin(Request("delivering", 20, SessionPurpose.Survey)).TryGetSession(out TunerSession? session));
+
+ session.Stop();
+ session.WaitForEnd(Deadlock);
+
+ Assert.Equal(SessionState.Stopped, session.State);
+ Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("after", 15, SessionPurpose.Survey)));
Assert.False(manager.IsFaulted("adapter0", out _));
- Assert.Equal(
- SessionRefusal.NoLock,
- manager.Begin(Request("the-last-straw", 14)).Refusal
- );
+ Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("the-last-straw", 16, SessionPurpose.Survey)));
Assert.True(manager.IsFaulted("adapter0", out _));
}
[Fact]
- public void AChannelThatKeepsGoingSilentAfterLockAlsoEndsInAFault()
+ public void TwoDifferentChannelsThatGoSilentAfterLockingInARowFaultTheTuner()
{
- TunerSessionManager manager = Manager(new QueuedDeviceFactory(new ConcurrentQueue(
- [new SilentAfterLockDevice(), new SilentAfterLockDevice(), new SilentAfterLockDevice()]
- )));
+ TunerSessionManager manager = Manager(new SilentChannelDeviceFactory(silentChannels: [14, 15]));
- for (int attempt = 1; attempt <= TunerSessionManager.RepeatedTuneFailureCeiling; attempt++)
+ foreach (int channel in new[] { 14, 15 })
{
- SessionStart start = manager.Begin(Request($"scan-{attempt}", 14));
-
- Assert.True(start.TryGetSession(out TunerSession? session));
+ Assert.True(
+ manager.Begin(Request($"survey-{channel}", channel, SessionPurpose.Survey))
+ .TryGetSession(out TunerSession? session)
+ );
session.WaitForEnd(Deadlock);
Assert.Equal(SessionState.Failed, session.State);
+ Assert.NotEqual(SessionStopReason.DeviceFailed, session.StopReason);
clock.Advance(TimeSpan.FromSeconds(6));
}
Assert.True(manager.IsFaulted("adapter0", out string? detail));
- Assert.Contains("channel 14", detail, StringComparison.Ordinal);
+ Assert.Contains("terrestrial channel 15", detail, StringComparison.Ordinal);
}
[Fact]
- public void ATunerFaultedForNotLockingSaysSoInItsHealth()
+ public void EmptyChannelsAScanMeetsNeitherCountTowardsAFaultNorExcuseTheOthers()
+ {
+ TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannels: [14, 15, 16]));
+
+ Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("survey-14", 14, SessionPurpose.Survey)));
+ Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("scan-15", 15)));
+
+ Assert.False(manager.IsFaulted("adapter0", out _));
+
+ Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("survey-16", 16, SessionPurpose.Survey)));
+
+ Assert.True(manager.IsFaulted("adapter0", out _));
+ }
+
+ [Fact]
+ public void ASweepAcrossManyEmptyChannelsLeavesTheTunerInGoodStanding()
{
TunerSessionManager manager = Manager(new ThrowingDeviceFactory(() => DvbFailure.NoLock(
"the frontend did not lock within 5 seconds."
)));
- for (int attempt = 1; attempt <= TunerSessionManager.RepeatedTuneFailureCeiling; attempt++)
+ for (int channel = 13; channel <= 22; channel++)
{
- manager.Begin(Request($"scan-{attempt}", 14));
+ SessionStart start = manager.Begin(Request($"scan-{channel}", channel));
+
+ Assert.Equal(SessionRefusal.NoLock, start.Refusal);
clock.Advance(TimeSpan.FromSeconds(6));
}
+ Assert.False(manager.IsFaulted("adapter0", out _));
+ }
+
+ [Fact]
+ public void ATunerFaultedForNotLockingSaysSoInItsHealth()
+ {
+ TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannels: [14, 15]));
+
+ Refusal(manager, Request("survey-14", 14, SessionPurpose.Survey));
+ Refusal(manager, Request("survey-15", 15, SessionPurpose.Survey));
+
TunerSnapshot tuner = Assert.Single(SessionViews.Tuners(Configuration, manager));
Assert.Equal(TunerState.Faulted, tuner.State);
@@ -243,13 +297,14 @@ public void ATunerFaultedForNotLockingSaysSoInItsHealth()
[Fact]
public void ATunerFaultedForDeliveringNothingAfterLockingSaysSoInItsHealth()
{
- TunerSessionManager manager = Manager(new QueuedDeviceFactory(new ConcurrentQueue(
- [new SilentAfterLockDevice(), new SilentAfterLockDevice(), new SilentAfterLockDevice()]
- )));
+ TunerSessionManager manager = Manager(new SilentChannelDeviceFactory(silentChannels: [14, 15]));
- for (int attempt = 1; attempt <= TunerSessionManager.RepeatedTuneFailureCeiling; attempt++)
+ foreach (int channel in new[] { 14, 15 })
{
- Assert.True(manager.Begin(Request($"scan-{attempt}", 14)).TryGetSession(out TunerSession? session));
+ Assert.True(
+ manager.Begin(Request($"survey-{channel}", channel, SessionPurpose.Survey))
+ .TryGetSession(out TunerSession? session)
+ );
session.WaitForEnd(Deadlock);
@@ -299,9 +354,9 @@ public void ATunerThatHasNotFaultedNamesNoTuningFailureEvenAfterFailingToLock()
[Fact]
public void ATunerThatDeliversASessionThroughAfterFailingToTuneIsHealthyAgain()
{
- TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannel: 14));
+ TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannels: [14]));
- Assert.Equal(SessionRefusal.NoLock, manager.Begin(Request("scan-1", 14)).Refusal);
+ Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("scan-1", 14)));
clock.Advance(TimeSpan.FromSeconds(6));
Assert.Equal(
@@ -322,16 +377,10 @@ public void ATunerThatDeliversASessionThroughAfterFailingToTuneIsHealthyAgain()
[Fact]
public void ATunerThatFaultedIsFaultedRatherThanDegraded()
{
- TunerSessionManager manager = Manager(new ThrowingDeviceFactory(() => DvbFailure.NoLock(
- "the frontend did not lock within 5 seconds."
- )));
-
- for (int attempt = 1; attempt <= TunerSessionManager.RepeatedTuneFailureCeiling; attempt++)
- {
- manager.Begin(Request($"scan-{attempt}", 14));
+ TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannels: [14, 15]));
- clock.Advance(TimeSpan.FromSeconds(6));
- }
+ Refusal(manager, Request("survey-14", 14, SessionPurpose.Survey));
+ Refusal(manager, Request("survey-15", 15, SessionPurpose.Survey));
TunerSnapshot tuner = Assert.Single(SessionViews.Tuners(Configuration, manager));
@@ -339,14 +388,23 @@ public void ATunerThatFaultedIsFaultedRatherThanDegraded()
Assert.Equal(TunerDegradedKind.Unspecified, tuner.Health?.DegradedKind);
}
- private sealed class FailingChannelDeviceFactory(int deadChannel) : ITunerDeviceFactory
+ private sealed class SilentChannelDeviceFactory(int[] silentChannels) : ITunerDeviceFactory
{
public ITunerDevice Create(DeviceSettings device, TuningRequest tuning, TuneParams? tune, HeadHold hold) =>
- tuning.PhysicalChannel == deadChannel
- ? throw DvbFailure.NoLock("the frontend did not lock within 5 seconds.")
+ silentChannels.Contains(tuning.PhysicalChannel)
+ ? new SilentAfterLockDevice()
: new ScriptedTunerDevice();
}
+ private SessionRefusal Refusal(TunerSessionManager manager, StartSessionRequest request)
+ {
+ SessionRefusal refusal = manager.Begin(request).Refusal;
+
+ clock.Advance(TimeSpan.FromSeconds(6));
+
+ return refusal;
+ }
+
private TunerSessionManager Manager(ITunerDeviceFactory factory) =>
new(
Configuration,
@@ -355,11 +413,15 @@ private TunerSessionManager Manager(ITunerDeviceFactory factory) =>
NullLogger.Instance
);
- private static StartSessionRequest Request(string sessionId, int channel) =>
+ private static StartSessionRequest Request(
+ string sessionId,
+ int channel,
+ SessionPurpose purpose = SessionPurpose.Scan
+ ) =>
new()
{
SessionId = SessionId.Parse(sessionId),
- Purpose = SessionPurpose.Scan,
+ Purpose = purpose,
Tuning = new TuningRequest(TunerKind.Terrestrial, channel, 50001),
EndsAt = Start.AddHours(1),
};