Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 20 additions & 13 deletions src/Puppis.Driver/Sessions/FaultedTunerRecheck.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,11 @@

namespace Puppis.Driver.Sessions;

/// <summary>
/// When a faulted device is next tried, and whether it failed again soon after it was last handed back out.
/// </summary>
public sealed record RecheckPlan(DateTimeOffset NextTryAt, bool Relapsed);

public sealed class FaultedTunerRecheck(
ITunerDeviceCheck check,
TimeProvider timeProvider,
Expand All @@ -23,11 +28,11 @@ ILogger logger

private readonly Lock gate = new();
private readonly Dictionary<string, Pending> pending = new(StringComparer.Ordinal);
private readonly Dictionary<string, DateTimeOffset> restoredAt = new(StringComparer.Ordinal);
private readonly Dictionary<string, Restoration> restored = new(StringComparer.Ordinal);

private bool disposed;

public DateTimeOffset? Schedule(DeviceSettings device, Action<string> restore)
public RecheckPlan? Schedule(DeviceSettings device, Action<string> restore)
{
string deviceId = device.Id!;
DateTimeOffset now = timeProvider.GetUtcNow();
Expand All @@ -39,26 +44,24 @@ ILogger logger
return null;
}

if (restoredAt.TryGetValue(deviceId, out DateTimeOffset restored) && now - restored <= RelapseWindow)
{
return null;
}

if (pending.TryGetValue(deviceId, out Pending? waiting))
{
return waiting.NextTryAt;
return new RecheckPlan(waiting.NextTryAt, false);
}

var entry = new Pending(device, restore, now + Intervals[0]);
int relapse = restored.TryGetValue(deviceId, out Restoration? last) && now - last.At <= RelapseWindow
? Math.Min(last.Relapse + 1, Intervals.Count - 1)
: 0;
Pending entry = new(device, restore, now + Intervals[relapse], relapse) { Tries = relapse };
entry.Timer = timeProvider.CreateTimer(
_ => TryAgain(entry),
null,
Intervals[0],
Intervals[relapse],
Timeout.InfiniteTimeSpan
);
pending[deviceId] = entry;

return entry.NextTryAt;
return new RecheckPlan(entry.NextTryAt, relapse > 0);
}
}

Expand Down Expand Up @@ -135,7 +138,7 @@ private void TryAgain(Pending entry)
}

pending.Remove(deviceId);
restoredAt[deviceId] = timeProvider.GetUtcNow();
restored[deviceId] = new Restoration(timeProvider.GetUtcNow(), entry.Relapse);
}

entry.Timer?.Dispose();
Expand Down Expand Up @@ -167,8 +170,12 @@ private void StillFaulted(Pending entry, Exception error)
);
}

private sealed class Pending(DeviceSettings device, Action<string> restore, DateTimeOffset nextTryAt)
private sealed record Restoration(DateTimeOffset At, int Relapse);

private sealed class Pending(DeviceSettings device, Action<string> restore, DateTimeOffset nextTryAt, int relapse)
{
public int Relapse { get; } = relapse;

public DeviceSettings Device { get; } = device;

public Action<string> Restore { get; } = restore;
Expand Down
150 changes: 105 additions & 45 deletions src/Puppis.Driver/Sessions/TunerSessionManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public sealed class TunerSessionManager(
{
public const int RetainedSessions = 64;

public const int RepeatedTuneFailureCeiling = 3;
public const int DifferentChannelsThatFaultTheDevice = 2;

public static readonly TimeSpan DefaultHardStopLimit = TimeSpan.FromSeconds(30);

Expand All @@ -57,9 +57,7 @@ public sealed class TunerSessionManager(
private readonly ConcurrentQueue<TunerSession> ended = new();
private readonly ConcurrentDictionary<SessionId, TuningKey> tunings = [];
private readonly ConcurrentDictionary<string, SessionId> writing = new(StringComparer.Ordinal);
private readonly Dictionary<string, Dictionary<TuningKey, int>> tuneFailureStreaks = new(
StringComparer.Ordinal
);
private readonly Dictionary<string, List<TuningKey>> tuneFailures = new(StringComparer.Ordinal);
private readonly Lock streakGate = new();
private readonly TimeSpan drainCap = TimeSpan.FromHours(
Math.Max(0, configuration.ShutdownGraceHours)
Expand Down Expand Up @@ -713,19 +711,19 @@ error is DvbDeviceException
} failed
)
{
RecordTuneFailure(deviceId, TuningKey.Of(request), failed);
RecordTuneFailure(deviceId, TuningKey.Of(request), failed, request.Purpose);
}

refusal = error switch
{
DvbDeviceException { Failure: TuningFailure.NoLock } => SessionStart.Refused(
SessionRefusal.NoLock,
$"The device '{deviceId}' opened but the frontend did not lock: {error.Message}"
$"The device '{deviceId}' opened but the frontend did not lock on {TuningKey.Of(request)}: {error.Message}"
),
DvbDeviceException { Failure: TuningFailure.LockedWithoutData } =>
SessionStart.Refused(
SessionRefusal.NoData,
$"The device '{deviceId}' locked but delivered nothing: {error.Message}"
$"The device '{deviceId}' locked on {TuningKey.Of(request)} but delivered nothing: {error.Message}"
),
_ => SessionStart.Refused(
SessionRefusal.DeviceUnavailable,
Expand Down Expand Up @@ -1382,7 +1380,7 @@ session.FailureCause is DvbDeviceException
} channel
)
{
RecordTuneFailure(session.DeviceId, tuning, channel);
RecordTuneFailure(session.DeviceId, tuning, channel, session.Purpose);
}
}

Expand All @@ -1399,46 +1397,78 @@ private void FaultAfterFailing(TunerSession session)
$"The device failed while serving '{session.SessionId}': "
+ (session.FailureCause?.Message ?? "no cause was recorded.");

faultedDevices[deviceId] = new DeviceFault(fault, null, DeviceFaultKind.Failed);
healthChangedAt[deviceId] = timeProvider.GetUtcNow();

pool.Discard(deviceId);

if (recheck is null || draining || DeviceNamed(deviceId) is not { } device)
if (FaultUntilItOpens(deviceId, new DeviceFault(fault, null, DeviceFaultKind.Failed)) is not { } plan)
{
logger.LogError(
"The device {DeviceId} failed while serving {SessionId} and {NotTriedAgain}",
deviceId,
session.SessionId.Value,
NotTriedAgain()
);

return;
}

if (recheck.Schedule(device, Restore) is { } nextTryAt)
if (!plan.Relapsed)
{
logger.LogWarning(
"The device {DeviceId} failed while serving {SessionId}, so it is not handed out until it opens again; it is tried again at {NextTryAt}.",
deviceId,
session.SessionId.Value,
nextTryAt
plan.NextTryAt
);

return;
}

faultedDevices[deviceId] = new DeviceFault(
fault
+ $" It had been handed back out less than {RelapseMinutes} minutes earlier, so it is not tried again.",
null,
DeviceFaultKind.FailedAgain);

logger.LogError(
"The device {DeviceId} failed again while serving {SessionId} within {RelapseMinutes} minutes of being handed back out, so it is not tried again and stays faulted until the driver restarts.",
"The device {DeviceId} failed again while serving {SessionId} within {RelapseMinutes} minutes of being handed back out, so it waits longer than last time and is tried again at {NextTryAt}.",
deviceId,
session.SessionId.Value,
RelapseMinutes
RelapseMinutes,
plan.NextTryAt
);
}

private RecheckPlan? FaultUntilItOpens(string deviceId, DeviceFault fault)
{
faultedDevices[deviceId] = fault;
healthChangedAt[deviceId] = timeProvider.GetUtcNow();

pool.Discard(deviceId);

events?.Signal(DriverEvents.TunerHealthChanged);
events?.Signal(DriverEvents.Tuners);

if (recheck is null || draining || DeviceNamed(deviceId) is not { } device)
{
return null;
}

RecheckPlan? plan = recheck.Schedule(device, Restore);

if (plan is { Relapsed: true } && fault.Kind.Kind is TunerFaultKind.DeviceFailed)
{
faultedDevices[deviceId] = fault with
{
Detail = fault.Detail
+ $" It had been handed back out less than {RelapseMinutes} minutes earlier, so it waits longer than last time before it is tried again.",
Kind = DeviceFaultKind.FailedAgain,
};
}

return plan;
}

private static double RelapseMinutes => FaultedTunerRecheck.RelapseWindow.TotalMinutes;

private void Restore(string deviceId)
{
lock (streakGate)
{
tuneFailures.Remove(deviceId);
}

faultedDevices.TryRemove(deviceId, out _);
healthChangedAt[deviceId] = timeProvider.GetUtcNow();

Expand Down Expand Up @@ -1481,25 +1511,32 @@ private void Announce()
events?.Signal(DriverEvents.Tuners);
}

private void RecordTuneFailure(string deviceId, TuningKey tuning, DvbDeviceException cause)
private void RecordTuneFailure(
string deviceId,
TuningKey tuning,
DvbDeviceException cause,
SessionPurpose purpose
)
{
int streak;
bool first;
TuningKey[] channels;

lock (streakGate)
{
if (!tuneFailureStreaks.TryGetValue(deviceId, out Dictionary<TuningKey, int>? perChannel))
first = !tuneFailures.TryGetValue(deviceId, out List<TuningKey>? counted);
counted ??= tuneFailures[deviceId] = [];

if (purpose is not SessionPurpose.Scan && !counted.Any(seen => OnTheSameChannel(seen, tuning)))
{
perChannel = tuneFailureStreaks[deviceId] = [];
counted.Add(tuning);
}

streak = perChannel[tuning] = perChannel.TryGetValue(tuning, out int seen)
? seen + 1
: 1;
channels = [.. counted];
}

if (streak < RepeatedTuneFailureCeiling)
if (channels.Length < DifferentChannelsThatFaultTheDevice)
{
if (streak is 1)
if (first)
{
healthChangedAt[deviceId] = timeProvider.GetUtcNow();
events?.Signal(DriverEvents.TunerHealthChanged);
Expand All @@ -1509,24 +1546,49 @@ private void RecordTuneFailure(string deviceId, TuningKey tuning, DvbDeviceExcep
return;
}

Fault(
string fault =
$"The device failed to receive {string.Join(", ", channels)} one after another without delivering"
+ $" anything in between; the last failure was: {cause.Message}";
RecheckPlan? plan = FaultUntilItOpens(
deviceId,
new DeviceFault(
$"The device failed to receive {tuning} {streak} times in a row without delivering"
+ $" anything in between; the last failure was: {cause.Message}",
TuningFailureTitles.Of(cause.Failure),
DeviceFaultKind.RepeatedTuneFailure
)
new DeviceFault(fault, TuningFailureTitles.Of(cause.Failure), DeviceFaultKind.RepeatedTuneFailure)
);

if (plan is null)
{
logger.LogError(
"The device {DeviceId} failed to receive {Channels} one after another and {NotTriedAgain}",
deviceId,
string.Join(", ", channels),
NotTriedAgain()
);

return;
}

logger.LogWarning(
"The device {DeviceId} failed to receive {Channels} one after another, so it is not handed out until it opens again; it is tried again at {NextTryAt}.",
deviceId,
string.Join(", ", channels),
plan.NextTryAt
);
}

private string NotTriedAgain() =>
draining
? "the driver is shutting down, so it is not tried again."
: "nothing is set to try it again, so it stays faulted until the driver restarts.";

private static bool OnTheSameChannel(TuningKey one, TuningKey other) =>
one.Kind == other.Kind && one.PhysicalChannel == other.PhysicalChannel;

private void ForgiveTuneFailures(string deviceId)
{
bool forgiven;

lock (streakGate)
{
forgiven = tuneFailureStreaks.Remove(deviceId);
forgiven = tuneFailures.Remove(deviceId);
}

if (forgiven && !faultedDevices.ContainsKey(deviceId))
Expand All @@ -1538,15 +1600,13 @@ private void ForgiveTuneFailures(string deviceId)
}

/// <summary>
/// Whether a channel has failed to tune on this device since it last delivered a session through to its end,
/// fewer times than it takes to fault the device.
/// Whether a channel has failed to tune on this device since it last delivered a session through to its end.
/// </summary>
public bool IsFailingToTune(string deviceId)
{
lock (streakGate)
{
return tuneFailureStreaks.TryGetValue(deviceId, out Dictionary<TuningKey, int>? perChannel)
&& perChannel.Values.Any(streak => streak > 0);
return tuneFailures.ContainsKey(deviceId);
}
}

Expand Down
Loading
Loading