From 9f0dc093bd7382530a3913d2695e59996a010ab8 Mon Sep 17 00:00:00 2001 From: fyui001 Date: Fri, 13 Nov 2026 22:04:06 +0900 Subject: [PATCH 1/2] =?UTF-8?q?1=20=E3=81=A4=E3=81=AE=E3=83=81=E3=83=A3?= =?UTF-8?q?=E3=83=B3=E3=83=8D=E3=83=AB=E3=81=8C=E5=8F=97=E4=BF=A1=E3=81=A7?= =?UTF-8?q?=E3=81=8D=E3=81=AA=E3=81=84=E3=81=A0=E3=81=91=E3=81=A7=E3=81=AF?= =?UTF-8?q?=E3=83=81=E3=83=A5=E3=83=BC=E3=83=8A=E3=83=BC=E3=82=92=E6=AD=A2?= =?UTF-8?q?=E3=82=81=E3=81=9A=E3=80=81=E7=95=B0=E3=81=AA=E3=82=8B=202=20?= =?UTF-8?q?=E3=81=A4=E3=81=AE=E3=83=81=E3=83=A3=E3=83=B3=E3=83=8D=E3=83=AB?= =?UTF-8?q?=E3=81=A7=E7=B6=9A=E3=81=91=E3=81=A6=E5=A4=B1=E6=95=97=E3=81=97?= =?UTF-8?q?=E3=81=9F=E3=81=A8=E3=81=8D=E3=81=A0=E3=81=91=E6=AD=A2=E3=82=81?= =?UTF-8?q?=E3=81=A6=E3=80=81=E6=AD=A2=E3=82=81=E3=81=9F=E3=83=81=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E3=83=8A=E3=83=BC=E3=81=AF=E9=96=93=E9=9A=94=E3=82=92?= =?UTF-8?q?=E5=BB=B6=E3=81=B0=E3=81=97=E3=81=AA=E3=81=8C=E3=82=89=E8=A9=A6?= =?UTF-8?q?=E3=81=97=E7=9B=B4=E3=81=99=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude --- .../Sessions/FaultedTunerRecheck.cs | 29 ++- .../Sessions/TunerSessionManager.cs | 126 ++++++---- .../Tuning/Dvb/DvbTunerDevice.cs | 11 +- .../Tuning/Usb/CechZd1/CechZd1TunerDevice.cs | 8 +- tests/Puppis.Driver.Tests/CechZd1Tests.cs | 23 ++ .../DvbTunerDeviceTests.cs | 1 + .../FaultedTunerRecheckTests.cs | 98 +++++++- .../TuneFailureClassificationTests.cs | 234 ++++++++++++------ 8 files changed, 375 insertions(+), 155 deletions(-) diff --git a/src/Puppis.Driver/Sessions/FaultedTunerRecheck.cs b/src/Puppis.Driver/Sessions/FaultedTunerRecheck.cs index 11ceed4..6c2cbfa 100644 --- a/src/Puppis.Driver/Sessions/FaultedTunerRecheck.cs +++ b/src/Puppis.Driver/Sessions/FaultedTunerRecheck.cs @@ -5,6 +5,11 @@ namespace Puppis.Driver.Sessions; +/// +/// When a faulted device is next tried, and whether it failed again soon after it was last handed back out. +/// +public sealed record RecheckPlan(DateTimeOffset NextTryAt, bool Relapsed); + public sealed class FaultedTunerRecheck( ITunerDeviceCheck check, TimeProvider timeProvider, @@ -23,11 +28,11 @@ ILogger logger private readonly Lock gate = new(); private readonly Dictionary pending = new(StringComparer.Ordinal); - private readonly Dictionary restoredAt = new(StringComparer.Ordinal); + private readonly Dictionary restored = new(StringComparer.Ordinal); private bool disposed; - public DateTimeOffset? Schedule(DeviceSettings device, Action restore) + public RecheckPlan? Schedule(DeviceSettings device, Action restore) { string deviceId = device.Id!; DateTimeOffset now = timeProvider.GetUtcNow(); @@ -39,26 +44,24 @@ ILogger logger return null; } - if (restoredAt.TryGetValue(deviceId, out DateTimeOffset restored) && now - restored <= RelapseWindow) - { - return null; - } - if (pending.TryGetValue(deviceId, out Pending? waiting)) { - return waiting.NextTryAt; + return new RecheckPlan(waiting.NextTryAt, false); } - var entry = new Pending(device, restore, now + Intervals[0]); + int tries = restored.TryGetValue(deviceId, out Restoration? last) && now - last.At <= RelapseWindow + ? Math.Min(last.Tries + 1, Intervals.Count - 1) + : 0; + var entry = new Pending(device, restore, now + Intervals[tries]) { Tries = tries }; entry.Timer = timeProvider.CreateTimer( _ => TryAgain(entry), null, - Intervals[0], + Intervals[tries], Timeout.InfiniteTimeSpan ); pending[deviceId] = entry; - return entry.NextTryAt; + return new RecheckPlan(entry.NextTryAt, tries > 0); } } @@ -135,7 +138,7 @@ private void TryAgain(Pending entry) } pending.Remove(deviceId); - restoredAt[deviceId] = timeProvider.GetUtcNow(); + restored[deviceId] = new Restoration(timeProvider.GetUtcNow(), entry.Tries); } entry.Timer?.Dispose(); @@ -167,6 +170,8 @@ private void StillFaulted(Pending entry, Exception error) ); } + private sealed record Restoration(DateTimeOffset At, int Tries); + private sealed class Pending(DeviceSettings device, Action restore, DateTimeOffset nextTryAt) { public DeviceSettings Device { get; } = device; diff --git a/src/Puppis.Driver/Sessions/TunerSessionManager.cs b/src/Puppis.Driver/Sessions/TunerSessionManager.cs index c635ac2..1c3df22 100644 --- a/src/Puppis.Driver/Sessions/TunerSessionManager.cs +++ b/src/Puppis.Driver/Sessions/TunerSessionManager.cs @@ -33,7 +33,7 @@ public sealed class TunerSessionManager( { public const int RetainedSessions = 64; - public const int RepeatedTuneFailureCeiling = 3; + public const int DifferentChannelsThatFaultTheDevice = 2; public static readonly TimeSpan DefaultHardStopLimit = TimeSpan.FromSeconds(30); @@ -57,9 +57,7 @@ public sealed class TunerSessionManager( private readonly ConcurrentQueue ended = new(); private readonly ConcurrentDictionary tunings = []; private readonly ConcurrentDictionary writing = new(StringComparer.Ordinal); - private readonly Dictionary> tuneFailureStreaks = new( - StringComparer.Ordinal - ); + private readonly Dictionary> tuneFailures = new(StringComparer.Ordinal); private readonly Lock streakGate = new(); private readonly TimeSpan drainCap = TimeSpan.FromHours( Math.Max(0, configuration.ShutdownGraceHours) @@ -713,19 +711,19 @@ error is DvbDeviceException } failed ) { - RecordTuneFailure(deviceId, TuningKey.Of(request), failed); + RecordTuneFailure(deviceId, TuningKey.Of(request), failed, request.Purpose); } refusal = error switch { DvbDeviceException { Failure: TuningFailure.NoLock } => SessionStart.Refused( SessionRefusal.NoLock, - $"The device '{deviceId}' opened but the frontend did not lock: {error.Message}" + $"The device '{deviceId}' opened but the frontend did not lock on {TuningKey.Of(request)}: {error.Message}" ), DvbDeviceException { Failure: TuningFailure.LockedWithoutData } => SessionStart.Refused( SessionRefusal.NoData, - $"The device '{deviceId}' locked but delivered nothing: {error.Message}" + $"The device '{deviceId}' locked on {TuningKey.Of(request)} but delivered nothing: {error.Message}" ), _ => SessionStart.Refused( SessionRefusal.DeviceUnavailable, @@ -1382,7 +1380,7 @@ session.FailureCause is DvbDeviceException } channel ) { - RecordTuneFailure(session.DeviceId, tuning, channel); + RecordTuneFailure(session.DeviceId, tuning, channel, session.Purpose); } } @@ -1399,46 +1397,71 @@ private void FaultAfterFailing(TunerSession session) $"The device failed while serving '{session.SessionId}': " + (session.FailureCause?.Message ?? "no cause was recorded."); - faultedDevices[deviceId] = new DeviceFault(fault, null, DeviceFaultKind.Failed); - healthChangedAt[deviceId] = timeProvider.GetUtcNow(); - - pool.Discard(deviceId); - - if (recheck is null || draining || DeviceNamed(deviceId) is not { } device) + if (FaultUntilItOpens(deviceId, new DeviceFault(fault, null, DeviceFaultKind.Failed)) is not { } plan) { return; } - if (recheck.Schedule(device, Restore) is { } nextTryAt) + if (!plan.Relapsed) { logger.LogWarning( "The device {DeviceId} failed while serving {SessionId}, so it is not handed out until it opens again; it is tried again at {NextTryAt}.", deviceId, session.SessionId.Value, - nextTryAt + plan.NextTryAt ); return; } - faultedDevices[deviceId] = new DeviceFault( - fault - + $" It had been handed back out less than {RelapseMinutes} minutes earlier, so it is not tried again.", - null, - DeviceFaultKind.FailedAgain); - logger.LogError( - "The device {DeviceId} failed again while serving {SessionId} within {RelapseMinutes} minutes of being handed back out, so it is not tried again and stays faulted until the driver restarts.", + "The device {DeviceId} failed again while serving {SessionId} within {RelapseMinutes} minutes of being handed back out, so it waits longer than last time and is tried again at {NextTryAt}.", deviceId, session.SessionId.Value, - RelapseMinutes + RelapseMinutes, + plan.NextTryAt ); } + private RecheckPlan? FaultUntilItOpens(string deviceId, DeviceFault fault) + { + faultedDevices[deviceId] = fault; + healthChangedAt[deviceId] = timeProvider.GetUtcNow(); + + pool.Discard(deviceId); + + events?.Signal(DriverEvents.TunerHealthChanged); + events?.Signal(DriverEvents.Tuners); + + if (recheck is null || draining || DeviceNamed(deviceId) is not { } device) + { + return null; + } + + RecheckPlan? plan = recheck.Schedule(device, Restore); + + if (plan is { Relapsed: true } && fault.Kind.Kind is TunerFaultKind.DeviceFailed) + { + faultedDevices[deviceId] = fault with + { + Detail = fault.Detail + + $" It had been handed back out less than {RelapseMinutes} minutes earlier, so it waits longer than last time before it is tried again.", + Kind = DeviceFaultKind.FailedAgain, + }; + } + + return plan; + } + private static double RelapseMinutes => FaultedTunerRecheck.RelapseWindow.TotalMinutes; private void Restore(string deviceId) { + lock (streakGate) + { + tuneFailures.Remove(deviceId); + } + faultedDevices.TryRemove(deviceId, out _); healthChangedAt[deviceId] = timeProvider.GetUtcNow(); @@ -1481,25 +1504,32 @@ private void Announce() events?.Signal(DriverEvents.Tuners); } - private void RecordTuneFailure(string deviceId, TuningKey tuning, DvbDeviceException cause) + private void RecordTuneFailure( + string deviceId, + TuningKey tuning, + DvbDeviceException cause, + SessionPurpose purpose + ) { - int streak; + bool first; + TuningKey[] channels; lock (streakGate) { - if (!tuneFailureStreaks.TryGetValue(deviceId, out Dictionary? perChannel)) + first = !tuneFailures.TryGetValue(deviceId, out List? counted); + counted ??= tuneFailures[deviceId] = []; + + if (purpose is not SessionPurpose.Scan && !counted.Any(seen => OnTheSameChannel(seen, tuning))) { - perChannel = tuneFailureStreaks[deviceId] = []; + counted.Add(tuning); } - streak = perChannel[tuning] = perChannel.TryGetValue(tuning, out int seen) - ? seen + 1 - : 1; + channels = [.. counted]; } - if (streak < RepeatedTuneFailureCeiling) + if (channels.Length < DifferentChannelsThatFaultTheDevice) { - if (streak is 1) + if (first) { healthChangedAt[deviceId] = timeProvider.GetUtcNow(); events?.Signal(DriverEvents.TunerHealthChanged); @@ -1509,24 +1539,32 @@ private void RecordTuneFailure(string deviceId, TuningKey tuning, DvbDeviceExcep return; } - Fault( + string fault = + $"The device failed to receive {string.Join(", ", channels)} one after another without delivering" + + $" anything in between; the last failure was: {cause.Message}"; + RecheckPlan? plan = FaultUntilItOpens( deviceId, - new DeviceFault( - $"The device failed to receive {tuning} {streak} times in a row without delivering" - + $" anything in between; the last failure was: {cause.Message}", - TuningFailureTitles.Of(cause.Failure), - DeviceFaultKind.RepeatedTuneFailure - ) + new DeviceFault(fault, TuningFailureTitles.Of(cause.Failure), DeviceFaultKind.RepeatedTuneFailure) + ); + + logger.LogWarning( + "The device {DeviceId} failed to receive {Channels} one after another, so it is not handed out until it opens again; it is tried again at {NextTryAt}.", + deviceId, + string.Join(", ", channels), + plan?.NextTryAt ); } + private static bool OnTheSameChannel(TuningKey one, TuningKey other) => + one.Kind == other.Kind && one.PhysicalChannel == other.PhysicalChannel; + private void ForgiveTuneFailures(string deviceId) { bool forgiven; lock (streakGate) { - forgiven = tuneFailureStreaks.Remove(deviceId); + forgiven = tuneFailures.Remove(deviceId); } if (forgiven && !faultedDevices.ContainsKey(deviceId)) @@ -1538,15 +1576,13 @@ private void ForgiveTuneFailures(string deviceId) } /// - /// Whether a channel has failed to tune on this device since it last delivered a session through to its end, - /// fewer times than it takes to fault the device. + /// Whether a channel has failed to tune on this device since it last delivered a session through to its end. /// public bool IsFailingToTune(string deviceId) { lock (streakGate) { - return tuneFailureStreaks.TryGetValue(deviceId, out Dictionary? perChannel) - && perChannel.Values.Any(streak => streak > 0); + return tuneFailures.ContainsKey(deviceId); } } diff --git a/src/Puppis.Driver/Tuning/Dvb/DvbTunerDevice.cs b/src/Puppis.Driver/Tuning/Dvb/DvbTunerDevice.cs index b38849e..2417413 100644 --- a/src/Puppis.Driver/Tuning/Dvb/DvbTunerDevice.cs +++ b/src/Puppis.Driver/Tuning/Dvb/DvbTunerDevice.cs @@ -42,6 +42,7 @@ public sealed class DvbTunerDevice : ITunerDevice, ISignalQualitySource private readonly TimeProvider time; private readonly DvbDevicePaths paths; private readonly DvbTunerSettings settings; + private readonly DvbChannel channel; private readonly DvbFrontend frontend; private readonly int demux; private readonly int dvr; @@ -54,6 +55,7 @@ private DvbTunerDevice( TimeProvider time, DvbDevicePaths paths, DvbTunerSettings settings, + DvbChannel channel, DvbFrontend frontend, int demux, int dvr @@ -63,6 +65,7 @@ int dvr this.time = time; this.paths = paths; this.settings = settings; + this.channel = channel; this.frontend = frontend; this.demux = demux; this.dvr = dvr; @@ -126,7 +129,7 @@ out FrontendStatus lastSeen (demux, dvr) = OpenTheStream(calls, paths, settings); } - return new DvbTunerDevice(calls, time, paths, settings, frontend, demux, dvr); + return new DvbTunerDevice(calls, time, paths, settings, channel, frontend, demux, dvr); } catch { @@ -232,19 +235,19 @@ private DvbDeviceException NothingArrived() if (!frontend.TryStatus(out FrontendStatus status)) { return DvbFailure.Refused( - $"{paths.Dvr}: no transport stream bytes arrived within {waited}, and the frontend would not say whether it is still locked. This is left unclassified rather than recorded as a tuner that was locked and delivering nothing." + $"{paths.Dvr}: no transport stream bytes arrived from {channel} within {waited}, and the frontend would not say whether it is still locked. This is left unclassified rather than recorded as a tuner that was locked and delivering nothing." ); } if (!status.HasFlag(FrontendStatus.Lock)) { return DvbFailure.NoLock( - $"{paths.Dvr}: no transport stream bytes arrived within {waited}, and the frontend is no longer locked; its status is now {status}." + $"{paths.Dvr}: no transport stream bytes arrived from {channel} within {waited}, and the frontend is no longer locked; its status is now {status}." ); } return DvbFailure.LockedWithoutData( - $"{paths.Dvr}: the frontend is still locked ({status}) and no transport stream bytes arrived within {waited}. The tuner is synchronised and the demux is delivering nothing, which is a different fault from failing to lock." + $"{paths.Dvr}: the frontend is still locked onto {channel} ({status}) and no transport stream bytes arrived within {waited}. The tuner is synchronised and the demux is delivering nothing, which is a different fault from failing to lock." ); } diff --git a/src/Puppis.Driver/Tuning/Usb/CechZd1/CechZd1TunerDevice.cs b/src/Puppis.Driver/Tuning/Usb/CechZd1/CechZd1TunerDevice.cs index d3279d7..328f0f2 100644 --- a/src/Puppis.Driver/Tuning/Usb/CechZd1/CechZd1TunerDevice.cs +++ b/src/Puppis.Driver/Tuning/Usb/CechZd1/CechZd1TunerDevice.cs @@ -34,6 +34,7 @@ public sealed class CechZd1TunerDevice : ITunerDevice, ISignalQualitySource private readonly Lock gate = new(); private ILogger? logger; private long opened; + private long hertz; private bool firstBytesReported; private bool closed; @@ -215,6 +216,7 @@ string how var device = new CechZd1TunerDevice(kept, reader, pacing, bytePatience, standby); device.logger = logger; device.opened = stages.Started; + device.hertz = hertz; device.Report(logger, hertz, stages, how); return device; @@ -329,16 +331,16 @@ private DvbDeviceException NothingArrived() catch (DvbDeviceException) { return DvbFailure.Refused( - $"{usb.Path}: no transport stream arrived within {waited}, and the demodulator would not say whether it is still locked." + $"{usb.Path}: no transport stream arrived on {hertz / 1000} kHz within {waited}, and the demodulator would not say whether it is still locked." ); } return CechZd1Signal.LockedWithData(registers.StatusAfter) ? DvbFailure.LockedWithoutData( - $"{usb.Path}: the demodulator is locked and no transport stream arrived within {waited}." + $"{usb.Path}: the demodulator is locked on {hertz / 1000} kHz and no transport stream arrived within {waited}." ) : DvbFailure.NoLock( - $"{usb.Path}: no transport stream arrived within {waited}, and the demodulator is no longer locked (0x{registers.StatusAfter:X4})." + $"{usb.Path}: no transport stream arrived on {hertz / 1000} kHz within {waited}, and the demodulator is no longer locked (0x{registers.StatusAfter:X4})." ); } diff --git a/tests/Puppis.Driver.Tests/CechZd1Tests.cs b/tests/Puppis.Driver.Tests/CechZd1Tests.cs index 4637114..9349d55 100644 --- a/tests/Puppis.Driver.Tests/CechZd1Tests.cs +++ b/tests/Puppis.Driver.Tests/CechZd1Tests.cs @@ -460,6 +460,29 @@ public void ALockedTunerThatSendsNothingSaysSo() DvbDeviceException refused = UsbTestSupport.Refusal(() => device.Read(188, CancellationToken.None)); Assert.Equal(TuningFailure.LockedWithoutData, refused.Failure); + Assert.Contains($"{Channel22 / 1000} kHz", refused.Message, StringComparison.Ordinal); + device.Dispose(); + } + + [Fact] + public void TheWaitForBytesStartsWhenTheyAreAskedForNotWhenTheTunerWasOpened() + { + (CechZd1TunerDevice device, FakeBridge bridge, _, ManualTimeProvider clock) = Opened(); + clock.Advance(TimeSpan.FromSeconds(4)); + int idle = 0; + bridge.Idle = waited => + { + clock.Advance(waited); + + if (++idle * waited >= TimeSpan.FromSeconds(4)) + { + bridge.Transfers.Enqueue(UsbTestSupport.Units(50, skip: 30)); + } + }; + + byte[] read = device.Read(188, CancellationToken.None); + + Assert.Equal(188, read.Length); device.Dispose(); } diff --git a/tests/Puppis.Driver.Tests/DvbTunerDeviceTests.cs b/tests/Puppis.Driver.Tests/DvbTunerDeviceTests.cs index 753faff..538a96b 100644 --- a/tests/Puppis.Driver.Tests/DvbTunerDeviceTests.cs +++ b/tests/Puppis.Driver.Tests/DvbTunerDeviceTests.cs @@ -287,6 +287,7 @@ public void ALockedTunerThatDeliversNothingIsADistinctFailureFromNotLocking() Assert.Contains("still locked", refusal.Message, StringComparison.Ordinal); Assert.Contains("/dev/dvb/adapter0/dvr0", refusal.Message, StringComparison.Ordinal); + Assert.Contains(DvbChannel.Terrestrial(55).ToString(), refusal.Message, StringComparison.Ordinal); Assert.Equal(TuningFailure.LockedWithoutData, refusal.Failure); } diff --git a/tests/Puppis.Driver.Tests/FaultedTunerRecheckTests.cs b/tests/Puppis.Driver.Tests/FaultedTunerRecheckTests.cs index d68b6e2..55cd3d9 100644 --- a/tests/Puppis.Driver.Tests/FaultedTunerRecheckTests.cs +++ b/tests/Puppis.Driver.Tests/FaultedTunerRecheckTests.cs @@ -4,9 +4,11 @@ using Puppis.Contracts; using Puppis.Driver.Configuration; +using Puppis.Driver.Descrambling; using Puppis.Driver.Events; using Puppis.Driver.Sessions; using Puppis.Driver.Tuning; +using Puppis.Driver.Tuning.Dvb; namespace Puppis.Driver.Tests; @@ -51,6 +53,24 @@ private static StartSessionRequest Watching(string sessionId) => DeviceId = "adapter0", }; + private static StartSessionRequest Surveying(string sessionId, int channel) => + new() + { + SessionId = SessionId.Parse(sessionId), + Purpose = SessionPurpose.Survey, + Tuning = new TuningRequest(TunerKind.Terrestrial, channel, 50001), + DeviceId = "adapter0", + }; + + private SessionRefusal Refusal(TunerSessionManager manager, StartSessionRequest request) + { + SessionRefusal refusal = manager.Begin(request).Refusal; + + clock.Advance(TimeSpan.FromSeconds(6)); + + return refusal; + } + private static void FailOn(TunerSessionManager manager, string sessionId) { SessionStart start = manager.Begin(Watching(sessionId)); @@ -120,7 +140,7 @@ public void ATunerThatStillWillNotOpenIsTriedAfterFiveAndFifteenMinutesAndThenEv } [Fact] - public void ATunerThatFailsAgainWithinHalfAnHourOfComingBackIsNotTriedAgain() + public void ATunerThatFailsAgainWithinHalfAnHourOfComingBackIsTriedAgainAfterALongerWait() { TunerSessionManager manager = Manager(); @@ -131,19 +151,71 @@ public void ATunerThatFailsAgainWithinHalfAnHourOfComingBackIsNotTriedAgain() clock.Advance(TimeSpan.FromMinutes(30)); FailOn(manager, "s-2"); - foreach (int minutes in new[] { 1, 5, 15, 60, 60 }) - { - clock.Advance(TimeSpan.FromMinutes(minutes)); - } - - Assert.Single(check.Tried); - Assert.True(manager.IsFaulted("adapter0", out _)); Assert.Equal(TunerFaultKind.DeviceFailedAgain, manager.FaultKindOf("adapter0")?.Kind); SessionStart refused = manager.Begin(Watching("s-3")); Assert.Equal(SessionRefusal.FaultedDevice, refused.Refusal); - Assert.Contains("until the driver restarts", refused.Detail, StringComparison.Ordinal); + Assert.Contains((clock.GetUtcNow() + TimeSpan.FromMinutes(5)).ToString("O"), refused.Detail, StringComparison.Ordinal); + Assert.DoesNotContain("restart", refused.Detail, StringComparison.Ordinal); + + Wait(TimeSpan.FromMinutes(5), 1, manager); + + Assert.False(manager.IsFaulted("adapter0", out _)); + } + + [Fact] + public void ATunerThatKeepsFailingRightAfterComingBackWaitsLongerEachTimeAndIsNeverGivenUpOn() + { + TunerSessionManager manager = Manager(); + TimeSpan[] waits = + [ + TimeSpan.FromMinutes(1), + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(15), + TimeSpan.FromMinutes(60), + TimeSpan.FromMinutes(60), + ]; + + for (int round = 0; round < waits.Length; round++) + { + FailOn(manager, $"s-{round}"); + Wait(waits[round], round, manager); + + Assert.False(manager.IsFaulted("adapter0", out _)); + } + } + + [Fact] + public void ATunerFaultedForFailingOnTwoChannelsIsTriedAgainAndServesOnceItOpens() + { + var manager = new TunerSessionManager( + Configuration, + new DeadChannelsDeviceFactory(14, 15), + clock, + NullLogger.Instance, + events: hub, + deviceCheck: check + ); + + Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Surveying("survey-14", 14))); + Assert.Equal(SessionRefusal.NoLock, manager.Begin(Surveying("survey-15", 15)).Refusal); + + Assert.Equal(TunerFaultKind.RepeatedTuneFailure, manager.FaultKindOf("adapter0")?.Kind); + Assert.DoesNotContain("restart", manager.Begin(Surveying("survey-20", 20)).Detail, StringComparison.Ordinal); + + Wait(TimeSpan.FromMinutes(1), 0, manager); + + Assert.False(manager.IsFaulted("adapter0", out _)); + Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Surveying("survey-14-again", 14))); + Assert.False(manager.IsFaulted("adapter0", out _)); + + Assert.True(manager.Begin(Surveying("survey-20-again", 20)).TryGetSession(out TunerSession? served)); + + served.Stop(); + served.WaitForEnd(TimeSpan.FromSeconds(10)); + + Assert.Equal(SessionState.Stopped, served.State); } [Fact] @@ -263,6 +335,14 @@ public void WithNothingToCheckADeviceWithAFailedTunerStaysFaultedUntilTheDriverR Assert.Contains("until the driver restarts", refused.Detail, StringComparison.Ordinal); } + private sealed class DeadChannelsDeviceFactory(params int[] deadChannels) : ITunerDeviceFactory + { + public ITunerDevice Create(DeviceSettings device, TuningRequest tuning, TuneParams? tune, HeadHold hold) => + deadChannels.Contains(tuning.PhysicalChannel) + ? throw DvbFailure.NoLock("the frontend did not lock within 5 seconds.") + : new ScriptedTunerDevice(); + } + private sealed class ScriptedTunerDeviceCheck : ITunerDeviceCheck { private readonly ConcurrentQueue tried = new(); diff --git a/tests/Puppis.Driver.Tests/TuneFailureClassificationTests.cs b/tests/Puppis.Driver.Tests/TuneFailureClassificationTests.cs index 1823493..2f6f918 100644 --- a/tests/Puppis.Driver.Tests/TuneFailureClassificationTests.cs +++ b/tests/Puppis.Driver.Tests/TuneFailureClassificationTests.cs @@ -26,6 +26,8 @@ public sealed class TuneFailureClassificationTests [new DeviceSettings("adapter0", DeviceKind.Terrestrial)] ); + private static readonly DriverHello Hello = new(DriverProtocol.Version, "instance-1", []); + private readonly ManualTimeProvider clock = new(Start); [Fact] @@ -92,7 +94,7 @@ public void AChannelThatDeliveredNoBytesEndsItsSessionWithoutCondemningTheTuner( Assert.Equal( SessionRefusalTitles.NoData, - SessionViews.Of(silent, new DriverHello(DriverProtocol.Version, "instance-1", [])).FailureTitle + SessionViews.Of(silent, Hello).FailureTitle ); SessionStart second = manager.Begin(Request("scan-15", 15)); @@ -105,134 +107,186 @@ public void AChannelThatDeliveredNoBytesEndsItsSessionWithoutCondemningTheTuner( } [Fact] - public void AChannelThatKeepsFailingTheSameWayEndsInAFaultTheHealthSurfacesCanSee() + public void AChannelThatKeepsDeliveringNothingAfterLockingLeavesTheTunerServingOtherChannels() { - TunerSessionManager manager = Manager(new ThrowingDeviceFactory(() => DvbFailure.NoLock( - "the frontend did not lock within 5 seconds." - ))); + TunerSessionManager manager = Manager(new SilentChannelDeviceFactory(silentChannels: [27])); - for (int attempt = 1; attempt <= TunerSessionManager.RepeatedTuneFailureCeiling; attempt++) + for (int attempt = 1; attempt <= 4; attempt++) { - SessionStart start = manager.Begin(Request($"scan-{attempt}", 14)); + SessionStart start = manager.Begin(Request($"survey-{attempt}", 27, SessionPurpose.Survey)); - Assert.Equal(SessionRefusal.NoLock, start.Refusal); + Assert.True(start.TryGetSession(out TunerSession? session)); + + session.WaitForEnd(Deadlock); + + Assert.Equal(SessionState.Failed, session.State); + Assert.Equal(SessionRefusalTitles.NoData, SessionViews.Of(session, Hello).FailureTitle); + Assert.False(manager.IsFaulted("adapter0", out _)); clock.Advance(TimeSpan.FromSeconds(6)); } - Assert.True(manager.IsFaulted("adapter0", out string? detail)); - Assert.Contains("channel 14", detail, StringComparison.Ordinal); - Assert.Contains( - TunerSessionManager.RepeatedTuneFailureCeiling.ToString(), - detail, - StringComparison.Ordinal - ); + TunerSnapshot tuner = Assert.Single(SessionViews.Tuners(Configuration, manager)); - SessionStart refused = manager.Begin(Request("scan-after", 14)); + Assert.Equal(TunerState.Idle, tuner.State); + Assert.Equal(TunerHealthLevel.Degraded, tuner.Health?.Level); + Assert.Equal(TunerDegradedKind.TuneFailing, tuner.Health?.DegradedKind); + + SessionStart other = manager.Begin(Request("survey-other", 21, SessionPurpose.Survey)); + + Assert.Equal(SessionRefusal.None, other.Refusal); + Assert.True(other.TryGetSession(out TunerSession? served)); - Assert.Equal(SessionRefusal.FaultedDevice, refused.Refusal); + served.Stop(); + served.WaitForEnd(Deadlock); + + Assert.Equal(SessionState.Stopped, served.State); } [Fact] - public void ASweepAcrossManyEmptyChannelsLeavesTheTunerInGoodStanding() + public void AChannelThatKeepsFailingToLockIsRefusedEachTimeAndTheTunerStaysInService() { - TunerSessionManager manager = Manager(new ThrowingDeviceFactory(() => DvbFailure.NoLock( - "the frontend did not lock within 5 seconds." - ))); + TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannels: [14])); - for (int channel = 13; channel <= 22; channel++) + for (int attempt = 1; attempt <= 4; attempt++) { - SessionStart start = manager.Begin(Request($"scan-{channel}", channel)); + SessionStart start = manager.Begin(Request($"survey-{attempt}", 14, SessionPurpose.Survey)); Assert.Equal(SessionRefusal.NoLock, start.Refusal); + Assert.Contains("terrestrial channel 14", start.Detail, StringComparison.Ordinal); clock.Advance(TimeSpan.FromSeconds(6)); } Assert.False(manager.IsFaulted("adapter0", out _)); + + SessionStart other = manager.Begin(Request("survey-other", 20, SessionPurpose.Survey)); + + Assert.True(other.TryGetSession(out TunerSession? served)); + + served.Stop(); + served.WaitForEnd(Deadlock); } [Fact] - public void ASessionThatDeliveredResetsTheFailureStreakOfItsDevice() + public void ARefusalForDeliveringNothingNamesTheChannel() { - TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannel: 14)); + TunerSessionManager manager = Manager(new ThrowingDeviceFactory(() => + DvbFailure.LockedWithoutData("the frontend is still locked and nothing arrived."))); - for (int round = 1; round < TunerSessionManager.RepeatedTuneFailureCeiling; round++) - { - Assert.Equal( - SessionRefusal.NoLock, - manager.Begin(Request($"before-{round}", 14)).Refusal - ); + SessionStart start = manager.Begin(Request("survey-1", 14, SessionPurpose.Survey)); - clock.Advance(TimeSpan.FromSeconds(6)); - } + Assert.Equal(SessionRefusal.NoData, start.Refusal); + Assert.Contains("terrestrial channel 14", start.Detail, StringComparison.Ordinal); + } - SessionStart delivered = manager.Begin(Request("delivering", 20)); + [Fact] + public void FailingOnTwoDifferentChannelsInARowFaultsTheTuner() + { + TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannels: [14, 15])); - Assert.True(delivered.TryGetSession(out TunerSession? session)); + Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("survey-1", 14, SessionPurpose.Survey))); + Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("survey-2", 14, SessionPurpose.Survey))); - session.Stop(); - session.WaitForEnd(Deadlock); + Assert.False(manager.IsFaulted("adapter0", out _)); - Assert.Equal(SessionState.Stopped, session.State); + Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("survey-3", 15, SessionPurpose.Survey))); - for (int round = 1; round < TunerSessionManager.RepeatedTuneFailureCeiling; round++) - { - Assert.Equal( - SessionRefusal.NoLock, - manager.Begin(Request($"after-{round}", 14)).Refusal - ); + Assert.True(manager.IsFaulted("adapter0", out string? detail)); + Assert.Contains("terrestrial channel 14", detail, StringComparison.Ordinal); + Assert.Contains("terrestrial channel 15", detail, StringComparison.Ordinal); + Assert.Equal( + SessionRefusal.FaultedDevice, + manager.Begin(Request("survey-4", 20, SessionPurpose.Survey)).Refusal + ); + } - clock.Advance(TimeSpan.FromSeconds(6)); - } + [Fact] + public void ASessionThatDeliveredBetweenTwoFailingChannelsKeepsTheTunerInService() + { + TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannels: [14, 15, 16])); + Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("before", 14, SessionPurpose.Survey))); + + Assert.True(manager.Begin(Request("delivering", 20, SessionPurpose.Survey)).TryGetSession(out TunerSession? session)); + + session.Stop(); + session.WaitForEnd(Deadlock); + + Assert.Equal(SessionState.Stopped, session.State); + Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("after", 15, SessionPurpose.Survey))); Assert.False(manager.IsFaulted("adapter0", out _)); - Assert.Equal( - SessionRefusal.NoLock, - manager.Begin(Request("the-last-straw", 14)).Refusal - ); + Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("the-last-straw", 16, SessionPurpose.Survey))); Assert.True(manager.IsFaulted("adapter0", out _)); } [Fact] - public void AChannelThatKeepsGoingSilentAfterLockAlsoEndsInAFault() + public void TwoDifferentChannelsThatGoSilentAfterLockingInARowFaultTheTuner() { - TunerSessionManager manager = Manager(new QueuedDeviceFactory(new ConcurrentQueue( - [new SilentAfterLockDevice(), new SilentAfterLockDevice(), new SilentAfterLockDevice()] - ))); + TunerSessionManager manager = Manager(new SilentChannelDeviceFactory(silentChannels: [14, 15])); - for (int attempt = 1; attempt <= TunerSessionManager.RepeatedTuneFailureCeiling; attempt++) + foreach (int channel in new[] { 14, 15 }) { - SessionStart start = manager.Begin(Request($"scan-{attempt}", 14)); - - Assert.True(start.TryGetSession(out TunerSession? session)); + Assert.True( + manager.Begin(Request($"survey-{channel}", channel, SessionPurpose.Survey)) + .TryGetSession(out TunerSession? session) + ); session.WaitForEnd(Deadlock); Assert.Equal(SessionState.Failed, session.State); + Assert.NotEqual(SessionStopReason.DeviceFailed, session.StopReason); clock.Advance(TimeSpan.FromSeconds(6)); } Assert.True(manager.IsFaulted("adapter0", out string? detail)); - Assert.Contains("channel 14", detail, StringComparison.Ordinal); + Assert.Contains("terrestrial channel 15", detail, StringComparison.Ordinal); } [Fact] - public void ATunerFaultedForNotLockingSaysSoInItsHealth() + public void EmptyChannelsAScanMeetsNeitherCountTowardsAFaultNorExcuseTheOthers() + { + TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannels: [14, 15, 16])); + + Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("survey-14", 14, SessionPurpose.Survey))); + Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("scan-15", 15))); + + Assert.False(manager.IsFaulted("adapter0", out _)); + + Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("survey-16", 16, SessionPurpose.Survey))); + + Assert.True(manager.IsFaulted("adapter0", out _)); + } + + [Fact] + public void ASweepAcrossManyEmptyChannelsLeavesTheTunerInGoodStanding() { TunerSessionManager manager = Manager(new ThrowingDeviceFactory(() => DvbFailure.NoLock( "the frontend did not lock within 5 seconds." ))); - for (int attempt = 1; attempt <= TunerSessionManager.RepeatedTuneFailureCeiling; attempt++) + for (int channel = 13; channel <= 22; channel++) { - manager.Begin(Request($"scan-{attempt}", 14)); + SessionStart start = manager.Begin(Request($"scan-{channel}", channel)); + + Assert.Equal(SessionRefusal.NoLock, start.Refusal); clock.Advance(TimeSpan.FromSeconds(6)); } + Assert.False(manager.IsFaulted("adapter0", out _)); + } + + [Fact] + public void ATunerFaultedForNotLockingSaysSoInItsHealth() + { + TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannels: [14, 15])); + + Refusal(manager, Request("survey-14", 14, SessionPurpose.Survey)); + Refusal(manager, Request("survey-15", 15, SessionPurpose.Survey)); + TunerSnapshot tuner = Assert.Single(SessionViews.Tuners(Configuration, manager)); Assert.Equal(TunerState.Faulted, tuner.State); @@ -243,13 +297,14 @@ public void ATunerFaultedForNotLockingSaysSoInItsHealth() [Fact] public void ATunerFaultedForDeliveringNothingAfterLockingSaysSoInItsHealth() { - TunerSessionManager manager = Manager(new QueuedDeviceFactory(new ConcurrentQueue( - [new SilentAfterLockDevice(), new SilentAfterLockDevice(), new SilentAfterLockDevice()] - ))); + TunerSessionManager manager = Manager(new SilentChannelDeviceFactory(silentChannels: [14, 15])); - for (int attempt = 1; attempt <= TunerSessionManager.RepeatedTuneFailureCeiling; attempt++) + foreach (int channel in new[] { 14, 15 }) { - Assert.True(manager.Begin(Request($"scan-{attempt}", 14)).TryGetSession(out TunerSession? session)); + Assert.True( + manager.Begin(Request($"survey-{channel}", channel, SessionPurpose.Survey)) + .TryGetSession(out TunerSession? session) + ); session.WaitForEnd(Deadlock); @@ -299,9 +354,9 @@ public void ATunerThatHasNotFaultedNamesNoTuningFailureEvenAfterFailingToLock() [Fact] public void ATunerThatDeliversASessionThroughAfterFailingToTuneIsHealthyAgain() { - TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannel: 14)); + TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannels: [14])); - Assert.Equal(SessionRefusal.NoLock, manager.Begin(Request("scan-1", 14)).Refusal); + Assert.Equal(SessionRefusal.NoLock, Refusal(manager, Request("scan-1", 14))); clock.Advance(TimeSpan.FromSeconds(6)); Assert.Equal( @@ -322,16 +377,10 @@ public void ATunerThatDeliversASessionThroughAfterFailingToTuneIsHealthyAgain() [Fact] public void ATunerThatFaultedIsFaultedRatherThanDegraded() { - TunerSessionManager manager = Manager(new ThrowingDeviceFactory(() => DvbFailure.NoLock( - "the frontend did not lock within 5 seconds." - ))); - - for (int attempt = 1; attempt <= TunerSessionManager.RepeatedTuneFailureCeiling; attempt++) - { - manager.Begin(Request($"scan-{attempt}", 14)); + TunerSessionManager manager = Manager(new FailingChannelDeviceFactory(deadChannels: [14, 15])); - clock.Advance(TimeSpan.FromSeconds(6)); - } + Refusal(manager, Request("survey-14", 14, SessionPurpose.Survey)); + Refusal(manager, Request("survey-15", 15, SessionPurpose.Survey)); TunerSnapshot tuner = Assert.Single(SessionViews.Tuners(Configuration, manager)); @@ -339,14 +388,31 @@ public void ATunerThatFaultedIsFaultedRatherThanDegraded() Assert.Equal(TunerDegradedKind.Unspecified, tuner.Health?.DegradedKind); } - private sealed class FailingChannelDeviceFactory(int deadChannel) : ITunerDeviceFactory + private sealed class FailingChannelDeviceFactory(int[] deadChannels) : ITunerDeviceFactory { public ITunerDevice Create(DeviceSettings device, TuningRequest tuning, TuneParams? tune, HeadHold hold) => - tuning.PhysicalChannel == deadChannel + deadChannels.Contains(tuning.PhysicalChannel) ? throw DvbFailure.NoLock("the frontend did not lock within 5 seconds.") : new ScriptedTunerDevice(); } + private sealed class SilentChannelDeviceFactory(int[] silentChannels) : ITunerDeviceFactory + { + public ITunerDevice Create(DeviceSettings device, TuningRequest tuning, TuneParams? tune, HeadHold hold) => + silentChannels.Contains(tuning.PhysicalChannel) + ? new SilentAfterLockDevice() + : new ScriptedTunerDevice(); + } + + private SessionRefusal Refusal(TunerSessionManager manager, StartSessionRequest request) + { + SessionRefusal refusal = manager.Begin(request).Refusal; + + clock.Advance(TimeSpan.FromSeconds(6)); + + return refusal; + } + private TunerSessionManager Manager(ITunerDeviceFactory factory) => new( Configuration, @@ -355,11 +421,15 @@ private TunerSessionManager Manager(ITunerDeviceFactory factory) => NullLogger.Instance ); - private static StartSessionRequest Request(string sessionId, int channel) => + private static StartSessionRequest Request( + string sessionId, + int channel, + SessionPurpose purpose = SessionPurpose.Scan + ) => new() { SessionId = SessionId.Parse(sessionId), - Purpose = SessionPurpose.Scan, + Purpose = purpose, Tuning = new TuningRequest(TunerKind.Terrestrial, channel, 50001), EndsAt = Start.AddHours(1), }; From 0c58ba3051931835d0fb2b907e4a5b91199b894c Mon Sep 17 00:00:00 2001 From: fyui001 Date: Fri, 13 Nov 2026 22:13:04 +0900 Subject: [PATCH 2/2] =?UTF-8?q?=E6=88=BB=E3=81=97=E3=81=9F=E3=83=81?= =?UTF-8?q?=E3=83=A5=E3=83=BC=E3=83=8A=E3=83=BC=E3=81=8C=E3=81=BE=E3=81=9F?= =?UTF-8?q?=E6=AD=A2=E3=81=BE=E3=81=A3=E3=81=9F=E3=81=A8=E3=81=8D=E3=81=AE?= =?UTF-8?q?=E5=BE=85=E3=81=A1=E3=82=92=E8=A9=A6=E3=81=97=E7=9B=B4=E3=81=97?= =?UTF-8?q?=E3=81=AE=E5=A4=B1=E6=95=97=E3=81=AE=E5=9B=9E=E6=95=B0=E3=81=A7?= =?UTF-8?q?=E3=81=AF=E3=81=AA=E3=81=8F=E6=88=BB=E3=81=97=E3=81=9F=E7=9B=B4?= =?UTF-8?q?=E5=BE=8C=E3=81=AB=E6=AD=A2=E3=81=BE=E3=81=A3=E3=81=9F=E5=9B=9E?= =?UTF-8?q?=E6=95=B0=E3=81=A7=E6=B1=BA=E3=82=81=E3=80=81=E8=A9=A6=E3=81=97?= =?UTF-8?q?=E7=9B=B4=E3=81=95=E3=81=AA=E3=81=84=E3=81=A8=E3=81=8D=E3=81=AF?= =?UTF-8?q?=E3=81=9D=E3=81=AE=E7=90=86=E7=94=B1=E3=82=92=E3=83=AD=E3=82=B0?= =?UTF-8?q?=E3=81=AB=E5=87=BA=E3=81=99=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude --- .../Sessions/FaultedTunerRecheck.cs | 18 ++++---- .../Sessions/TunerSessionManager.cs | 26 +++++++++++- .../FaultedTunerRecheckTests.cs | 42 +++++++++++++------ .../Puppis.Driver.Tests/SessionTestSupport.cs | 9 ++++ .../TuneFailureClassificationTests.cs | 8 ---- 5 files changed, 74 insertions(+), 29 deletions(-) diff --git a/src/Puppis.Driver/Sessions/FaultedTunerRecheck.cs b/src/Puppis.Driver/Sessions/FaultedTunerRecheck.cs index 6c2cbfa..f897d23 100644 --- a/src/Puppis.Driver/Sessions/FaultedTunerRecheck.cs +++ b/src/Puppis.Driver/Sessions/FaultedTunerRecheck.cs @@ -49,19 +49,19 @@ ILogger logger return new RecheckPlan(waiting.NextTryAt, false); } - int tries = restored.TryGetValue(deviceId, out Restoration? last) && now - last.At <= RelapseWindow - ? Math.Min(last.Tries + 1, Intervals.Count - 1) + int relapse = restored.TryGetValue(deviceId, out Restoration? last) && now - last.At <= RelapseWindow + ? Math.Min(last.Relapse + 1, Intervals.Count - 1) : 0; - var entry = new Pending(device, restore, now + Intervals[tries]) { Tries = tries }; + Pending entry = new(device, restore, now + Intervals[relapse], relapse) { Tries = relapse }; entry.Timer = timeProvider.CreateTimer( _ => TryAgain(entry), null, - Intervals[tries], + Intervals[relapse], Timeout.InfiniteTimeSpan ); pending[deviceId] = entry; - return new RecheckPlan(entry.NextTryAt, tries > 0); + return new RecheckPlan(entry.NextTryAt, relapse > 0); } } @@ -138,7 +138,7 @@ private void TryAgain(Pending entry) } pending.Remove(deviceId); - restored[deviceId] = new Restoration(timeProvider.GetUtcNow(), entry.Tries); + restored[deviceId] = new Restoration(timeProvider.GetUtcNow(), entry.Relapse); } entry.Timer?.Dispose(); @@ -170,10 +170,12 @@ private void StillFaulted(Pending entry, Exception error) ); } - private sealed record Restoration(DateTimeOffset At, int Tries); + private sealed record Restoration(DateTimeOffset At, int Relapse); - private sealed class Pending(DeviceSettings device, Action restore, DateTimeOffset nextTryAt) + private sealed class Pending(DeviceSettings device, Action restore, DateTimeOffset nextTryAt, int relapse) { + public int Relapse { get; } = relapse; + public DeviceSettings Device { get; } = device; public Action Restore { get; } = restore; diff --git a/src/Puppis.Driver/Sessions/TunerSessionManager.cs b/src/Puppis.Driver/Sessions/TunerSessionManager.cs index 1c3df22..d5a250a 100644 --- a/src/Puppis.Driver/Sessions/TunerSessionManager.cs +++ b/src/Puppis.Driver/Sessions/TunerSessionManager.cs @@ -1399,6 +1399,13 @@ private void FaultAfterFailing(TunerSession session) if (FaultUntilItOpens(deviceId, new DeviceFault(fault, null, DeviceFaultKind.Failed)) is not { } plan) { + logger.LogError( + "The device {DeviceId} failed while serving {SessionId} and {NotTriedAgain}", + deviceId, + session.SessionId.Value, + NotTriedAgain() + ); + return; } @@ -1547,14 +1554,31 @@ SessionPurpose purpose new DeviceFault(fault, TuningFailureTitles.Of(cause.Failure), DeviceFaultKind.RepeatedTuneFailure) ); + if (plan is null) + { + logger.LogError( + "The device {DeviceId} failed to receive {Channels} one after another and {NotTriedAgain}", + deviceId, + string.Join(", ", channels), + NotTriedAgain() + ); + + return; + } + logger.LogWarning( "The device {DeviceId} failed to receive {Channels} one after another, so it is not handed out until it opens again; it is tried again at {NextTryAt}.", deviceId, string.Join(", ", channels), - plan?.NextTryAt + plan.NextTryAt ); } + private string NotTriedAgain() => + draining + ? "the driver is shutting down, so it is not tried again." + : "nothing is set to try it again, so it stays faulted until the driver restarts."; + private static bool OnTheSameChannel(TuningKey one, TuningKey other) => one.Kind == other.Kind && one.PhysicalChannel == other.PhysicalChannel; diff --git a/tests/Puppis.Driver.Tests/FaultedTunerRecheckTests.cs b/tests/Puppis.Driver.Tests/FaultedTunerRecheckTests.cs index 55cd3d9..e5cdb7c 100644 --- a/tests/Puppis.Driver.Tests/FaultedTunerRecheckTests.cs +++ b/tests/Puppis.Driver.Tests/FaultedTunerRecheckTests.cs @@ -4,11 +4,9 @@ using Puppis.Contracts; using Puppis.Driver.Configuration; -using Puppis.Driver.Descrambling; using Puppis.Driver.Events; using Puppis.Driver.Sessions; using Puppis.Driver.Tuning; -using Puppis.Driver.Tuning.Dvb; namespace Puppis.Driver.Tests; @@ -164,6 +162,29 @@ public void ATunerThatFailsAgainWithinHalfAnHourOfComingBackIsTriedAgainAfterALo Assert.False(manager.IsFaulted("adapter0", out _)); } + [Fact] + public void ATunerThatTookTwoFailedTriesToComeBackAndFailsAgainSoonWaitsFiveMinutes() + { + TunerSessionManager manager = Manager(); + check.RefuseTheNext(2); + + FailOn(manager, "s-1"); + Wait(TimeSpan.FromMinutes(1), 0, manager); + Wait(TimeSpan.FromMinutes(5), 1, manager); + Wait(TimeSpan.FromMinutes(15), 2, manager); + + Assert.False(manager.IsFaulted("adapter0", out _)); + + clock.Advance(TimeSpan.FromMinutes(10)); + FailOn(manager, "s-2"); + + Assert.Equal(TunerFaultKind.DeviceFailedAgain, manager.FaultKindOf("adapter0")?.Kind); + + Wait(TimeSpan.FromMinutes(5), 3, manager); + + Assert.False(manager.IsFaulted("adapter0", out _)); + } + [Fact] public void ATunerThatKeepsFailingRightAfterComingBackWaitsLongerEachTimeAndIsNeverGivenUpOn() { @@ -189,9 +210,9 @@ public void ATunerThatKeepsFailingRightAfterComingBackWaitsLongerEachTimeAndIsNe [Fact] public void ATunerFaultedForFailingOnTwoChannelsIsTriedAgainAndServesOnceItOpens() { - var manager = new TunerSessionManager( + TunerSessionManager manager = new( Configuration, - new DeadChannelsDeviceFactory(14, 15), + new FailingChannelDeviceFactory(deadChannels: [14, 15]), clock, NullLogger.Instance, events: hub, @@ -248,6 +269,11 @@ public void ATunerFaultedForDisagreeingWithTheLedgerIsNeverTriedAgain() Assert.Empty(check.Tried); Assert.True(manager.IsFaulted("adapter0", out _)); + + SessionStart refused = manager.Begin(Watching("s-1")); + + Assert.Equal(SessionRefusal.FaultedDevice, refused.Refusal); + Assert.Contains("until the driver restarts", refused.Detail, StringComparison.Ordinal); } [Fact] @@ -335,14 +361,6 @@ public void WithNothingToCheckADeviceWithAFailedTunerStaysFaultedUntilTheDriverR Assert.Contains("until the driver restarts", refused.Detail, StringComparison.Ordinal); } - private sealed class DeadChannelsDeviceFactory(params int[] deadChannels) : ITunerDeviceFactory - { - public ITunerDevice Create(DeviceSettings device, TuningRequest tuning, TuneParams? tune, HeadHold hold) => - deadChannels.Contains(tuning.PhysicalChannel) - ? throw DvbFailure.NoLock("the frontend did not lock within 5 seconds.") - : new ScriptedTunerDevice(); - } - private sealed class ScriptedTunerDeviceCheck : ITunerDeviceCheck { private readonly ConcurrentQueue tried = new(); diff --git a/tests/Puppis.Driver.Tests/SessionTestSupport.cs b/tests/Puppis.Driver.Tests/SessionTestSupport.cs index 60d05dc..fdab22d 100644 --- a/tests/Puppis.Driver.Tests/SessionTestSupport.cs +++ b/tests/Puppis.Driver.Tests/SessionTestSupport.cs @@ -7,6 +7,7 @@ using Puppis.Driver.Descrambling; using Puppis.Driver.Recording; using Puppis.Driver.Tuning; +using Puppis.Driver.Tuning.Dvb; namespace Puppis.Driver.Tests; @@ -193,6 +194,14 @@ public ITunerDevice Create(DeviceSettings device, TuningRequest tuning, TunePara : new ScriptedTunerDevice(); } +public sealed class FailingChannelDeviceFactory(int[] deadChannels) : ITunerDeviceFactory +{ + public ITunerDevice Create(DeviceSettings device, TuningRequest tuning, TuneParams? tune, HeadHold hold) => + deadChannels.Contains(tuning.PhysicalChannel) + ? throw DvbFailure.NoLock("the frontend did not lock within 5 seconds.") + : new ScriptedTunerDevice(); +} + public sealed class CountingRecordingWriter : IRecordingWriter { private long bytesWritten; diff --git a/tests/Puppis.Driver.Tests/TuneFailureClassificationTests.cs b/tests/Puppis.Driver.Tests/TuneFailureClassificationTests.cs index 2f6f918..58ca889 100644 --- a/tests/Puppis.Driver.Tests/TuneFailureClassificationTests.cs +++ b/tests/Puppis.Driver.Tests/TuneFailureClassificationTests.cs @@ -388,14 +388,6 @@ public void ATunerThatFaultedIsFaultedRatherThanDegraded() Assert.Equal(TunerDegradedKind.Unspecified, tuner.Health?.DegradedKind); } - private sealed class FailingChannelDeviceFactory(int[] deadChannels) : ITunerDeviceFactory - { - public ITunerDevice Create(DeviceSettings device, TuningRequest tuning, TuneParams? tune, HeadHold hold) => - deadChannels.Contains(tuning.PhysicalChannel) - ? throw DvbFailure.NoLock("the frontend did not lock within 5 seconds.") - : new ScriptedTunerDevice(); - } - private sealed class SilentChannelDeviceFactory(int[] silentChannels) : ITunerDeviceFactory { public ITunerDevice Create(DeviceSettings device, TuningRequest tuning, TuneParams? tune, HeadHold hold) =>