From d5de792a65c6905c28762e99dde7888267e0444f Mon Sep 17 00:00:00 2001 From: Jumpstile Date: Tue, 17 Nov 2026 06:43:17 -0500 Subject: [PATCH] Stop the guide scorecard from reporting Pass on gates it cannot measure (#191) Running the scorecard on a real lineup showed three gates reading Pass without measuring anything. Accuracy is now NotMeasured unless guides share a channel or readiness reports a contradiction. Completeness takes an optional playlist guide id and goes to Review when playlist channels have no programmes. Deduplication is NotMeasured unless some programme carries a canonical event id, and filler titles are skipped. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NzHAcjThVoUGWWfXrYWsNh --- .../Get-ChannelForgeGuideQualityScorecard.ps1 | 46 +++++++++++++------ .../Get-ChannelForgeGuideQualityReport.ps1 | 8 ++-- tests/unit/GuideQualityScorecard.Tests.ps1 | 29 +++++++++++- 3 files changed, 64 insertions(+), 19 deletions(-) diff --git a/src/ChannelForge/Private/Get-ChannelForgeGuideQualityScorecard.ps1 b/src/ChannelForge/Private/Get-ChannelForgeGuideQualityScorecard.ps1 index 9643ca7..605c983 100644 --- a/src/ChannelForge/Private/Get-ChannelForgeGuideQualityScorecard.ps1 +++ b/src/ChannelForge/Private/Get-ChannelForgeGuideQualityScorecard.ps1 @@ -12,14 +12,17 @@ function Get-ChannelForgeGuideQualityScorecard { - Gates: Accuracy, Completeness, Freshness, Deduplication, Classification, Availability. Each gate has Name, Verdict, Counts (plain numbers) and Reason (one line). - Verdict rule: any Fail-level finding gives Fail; else any Review-level finding gives Review; else Pass. - A gate whose input was not supplied is NotMeasured, never Pass. No percentages. - Accuracy: Fail = readiness Contradiction. Review = xguide TitleDiffers or StartShift. - Completeness: Fail = a guide with no programmes. Review = xguide MissingChannel. + A gate whose input was not supplied, or that compared nothing, is NotMeasured, never Pass. No percentages. + PlaylistGuideId = the tvg-id of every playlist channel (Read-ChannelForgeCandidateChannels TvgId), optional. + Accuracy: NotMeasured unless guides share a channel or readiness found a Contradiction (a clean readiness alone compared nothing). Fail = readiness Contradiction. Review = xguide TitleDiffers or StartShift. + Completeness: Measured by shared guide channels, an empty guide, or PlaylistGuideId plus Programme. Fail = a guide with no programmes. + Review = xguide MissingChannel, or a playlist channel with no programmes. Freshness: Fail = a guide that ended before the build. Review = a guide more than 24 h behind, or readiness StaleSource. - Deduplication: Review = channels with the same title and start time and no CanonicalEventId. Count only, nothing is merged. + Deduplication: Measured only when programmes carry a CanonicalEventId; without it same title and start is a normal repeat + (syndication), so NotMeasured. Review = same title and start on different channels with no CanonicalEventId. + Filler titles (Test-ChannelForgeEventFillerKey) are never counted. Count only, nothing is merged. Classification: Review = any ambiguous (conflict) or unknown (no-rule) channel. An ambiguous category is never Pass. Availability: Fail = readiness SourceUnavailable. Review = readiness Unresolved. - - Overall = the worst gate verdict (Fail, then Review). NotMeasured counts as Review for Overall. - Totals carry the input channel counts so a caller can prove no channel was dropped. #> [CmdletBinding()] @@ -27,13 +30,14 @@ function Get-ChannelForgeGuideQualityScorecard { [AllowNull()][object]$XguideReport = $null, [AllowNull()][object]$CategoryReport = $null, [AllowNull()][object]$ReadinessResult = $null, - [AllowNull()][object[]]$Programme = $null + [AllowNull()][object[]]$Programme = $null, + [AllowNull()][string[]]$PlaylistGuideId = $null ) - function New-Gate([string]$Name, [bool]$Measured, [System.Collections.IDictionary]$Counts, [string[]]$Fail, [string[]]$Review) { + function New-Gate([string]$Name, [bool]$Measured, [System.Collections.IDictionary]$Counts, [string[]]$Fail, [string[]]$Review, [string]$NotMeasuredReason = 'No report was supplied for this gate, so nothing is claimed.') { $Fail = @($Fail | Where-Object { $_ }) $Review = @($Review | Where-Object { $_ }) - if (-not $Measured) { $verdict = 'NotMeasured'; $reason = 'No report was supplied for this gate, so nothing is claimed.' } + if (-not $Measured) { $verdict = 'NotMeasured'; $reason = $NotMeasuredReason } elseif ($Fail.Count -gt 0) { $verdict = 'Fail'; $reason = $Fail -join '; ' } elseif ($Review.Count -gt 0) { $verdict = 'Review'; $reason = $Review -join '; ' } else { $verdict = 'Pass'; $reason = 'No problems found in the supplied report.' } @@ -46,7 +50,9 @@ function Get-ChannelForgeGuideQualityScorecard { $channel = if ([string]::IsNullOrEmpty($p.RawChannelId)) { [string]$p.ChannelId } else { [string]$p.RawChannelId } $identity = $p.PSObject.Properties['CanonicalEventId'] if ($identity -and -not [string]::IsNullOrWhiteSpace([string]$identity.Value)) { continue } - $key = '{0}|{1}' -f ([regex]::Replace(([string]$p.Title).Trim().ToLowerInvariant(), '\s+', ' ')), $p.Start.UtcTicks + $titleKey = ConvertTo-ChannelForgeEventTitleKey -Text ([string]$p.Title) + if (Test-ChannelForgeEventFillerKey -Key $titleKey) { continue } + $key = '{0}|{1}' -f $titleKey, $p.Start.UtcTicks if (-not $groups.ContainsKey($key)) { $groups[$key] = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) } [void]$groups[$key].Add($channel) } @@ -64,25 +70,35 @@ function Get-ChannelForgeGuideQualityScorecard { $contradiction = Get-Number $rCounts 'Contradiction'; $stale = Get-Number $rCounts 'StaleSource' $unavailable = Get-Number $rCounts 'SourceUnavailable'; $unresolved = Get-Number $rCounts 'Unresolved' $missing = Get-Number $xCounts 'MissingChannel' + $shared = Get-Number $x 'SharedChannelCount' + $playlistMeasured = $null -ne $PlaylistGuideId -and $null -ne $Programme + $noProgramme = 0 + if ($playlistMeasured) { + $withProgrammes = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($p in $Programme) { [void]$withProgrammes.Add($(if ([string]::IsNullOrEmpty($p.RawChannelId)) { [string]$p.ChannelId } else { [string]$p.RawChannelId })) } + $noProgramme = @($PlaylistGuideId | Where-Object { [string]::IsNullOrEmpty($_) -or -not $withProgrammes.Contains($_) }).Count + } + $hasIdentity = $null -ne $Programme -and @($Programme | Where-Object { $_.PSObject.Properties['CanonicalEventId'] -and -not [string]::IsNullOrWhiteSpace([string]$_.CanonicalEventId) }).Count -gt 0 $rows = if ($CategoryReport) { @($CategoryReport.Rows) } else { @() } $ambiguous = @($rows | Where-Object { $_.Decision -ceq 'conflict' }).Count $unknown = @($rows | Where-Object { $_.Decision -ceq 'no-rule' }).Count $high = @($rows | Where-Object { $_.Confidence -ceq 'high' }).Count - $dupGroups = if ($null -ne $Programme) { Get-DuplicateGroupCount @($Programme) } else { 0 } + $dupGroups = if ($hasIdentity) { Get-DuplicateGroupCount @($Programme) } else { 0 } $gates = @( - New-Gate 'Accuracy' ($null -ne $x -or $null -ne $r) ([ordered]@{ TitleDiffers = $titleDiffers; StartShift = $startShift; Contradiction = $contradiction }) ` + New-Gate 'Accuracy' ($contradiction -gt 0 -or ($null -ne $x -and $shared -gt 0)) ([ordered]@{ TitleDiffers = $titleDiffers; StartShift = $startShift; Contradiction = $contradiction }) ` @(if ($contradiction) { "$contradiction readiness contradictions" }) ` @(if ($titleDiffers + $startShift) { "$titleDiffers title and $startShift start-time differences between guides" }) - New-Gate 'Completeness' ($null -ne $x) ([ordered]@{ MissingChannel = $missing; EmptyGuides = $empty }) ` + New-Gate 'Completeness' (($null -ne $x -and ($shared -gt 0 -or $empty -gt 0)) -or $playlistMeasured) ([ordered]@{ MissingChannel = $missing; EmptyGuides = $empty; PlaylistChannelsWithoutProgrammes = $noProgramme }) ` @(if ($empty) { "$empty guides have no programmes" }) ` - @(if ($missing) { "$missing channels are missing from some guides" }) + @($(if ($missing) { "$missing channels are missing from some guides" }), $(if ($noProgramme) { "$noProgramme playlist channels have no programmes in the guide" })) New-Gate 'Freshness' ($null -ne $x -or $null -ne $r) ([ordered]@{ EndedGuides = $ended; BehindGuides = $behind; StaleSource = $stale }) ` @(if ($ended) { "$ended guides ended before this build" }) ` @(if ($behind + $stale) { "$behind guides behind and $stale stale sources" }) - New-Gate 'Deduplication' ($null -ne $Programme) ([ordered]@{ DuplicateGroups = $dupGroups }) ` - @() @(if ($dupGroups) { "$dupGroups same-title same-time groups on different channels with no canonical identity" }) + New-Gate 'Deduplication' $hasIdentity ([ordered]@{ DuplicateGroups = $dupGroups }) ` + @() @(if ($dupGroups) { "$dupGroups same-title same-time groups on different channels with no canonical identity" }) ` + 'No programme has a canonical event identity, so a repeat of the same title and start cannot be told from a normal syndicated show.' New-Gate 'Classification' ($null -ne $CategoryReport) ([ordered]@{ High = $high; Ambiguous = $ambiguous; Unknown = $unknown }) ` @() @(if ($ambiguous + $unknown) { "$ambiguous ambiguous and $unknown unknown channels" }) New-Gate 'Availability' ($null -ne $r) ([ordered]@{ SourceUnavailable = $unavailable; Unresolved = $unresolved }) ` diff --git a/src/ChannelForge/Public/Get-ChannelForgeGuideQualityReport.ps1 b/src/ChannelForge/Public/Get-ChannelForgeGuideQualityReport.ps1 index 6003c35..ed51489 100644 --- a/src/ChannelForge/Public/Get-ChannelForgeGuideQualityReport.ps1 +++ b/src/ChannelForge/Public/Get-ChannelForgeGuideQualityReport.ps1 @@ -6,15 +6,17 @@ function Get-ChannelForgeGuideQualityReport { .DESCRIPTION Read-only report (contract guide-quality-scorecard/v1). Pass in the outputs of the existing reports: Get-ChannelForgeXguideReport, Get-ChannelForgeCategoryReport, Get-ChannelForgeGuideReadiness and the programme rows. - A gate with no input is NotMeasured, never Pass. It writes nothing and changes nothing. + Optional PlaylistGuideId = the tvg-id of each playlist channel; it lets Completeness count playlist channels with no programmes. + A gate with no input, or that compared nothing, is NotMeasured, never Pass. It writes nothing and changes nothing. #> [CmdletBinding()] param( [AllowNull()][object]$XguideReport = $null, [AllowNull()][object]$CategoryReport = $null, [AllowNull()][object]$ReadinessResult = $null, - [AllowNull()][object[]]$Programme = $null + [AllowNull()][object[]]$Programme = $null, + [AllowNull()][string[]]$PlaylistGuideId = $null ) - return Get-ChannelForgeGuideQualityScorecard -XguideReport $XguideReport -CategoryReport $CategoryReport -ReadinessResult $ReadinessResult -Programme $Programme + return Get-ChannelForgeGuideQualityScorecard -XguideReport $XguideReport -CategoryReport $CategoryReport -ReadinessResult $ReadinessResult -Programme $Programme -PlaylistGuideId $PlaylistGuideId } diff --git a/tests/unit/GuideQualityScorecard.Tests.ps1 b/tests/unit/GuideQualityScorecard.Tests.ps1 index fff486a..142a5d6 100644 --- a/tests/unit/GuideQualityScorecard.Tests.ps1 +++ b/tests/unit/GuideQualityScorecard.Tests.ps1 @@ -31,6 +31,7 @@ BeforeAll { @( New-ChannelForgeProgramme -ChannelId 'one' -Start $start -End $start.AddHours(1) -Title 'Show A' -SourceId 'p' New-ChannelForgeProgramme -ChannelId 'two' -Start $start -End $start.AddHours(1) -Title 'Show B' -SourceId 'p' + [pscustomobject]@{ ChannelId = 'three'; RawChannelId = 'three'; Start = $start; End = $start.AddHours(1); Title = 'Show C'; CanonicalEventId = 'evt-1' } ) } function Get-Card($Params) { InModuleScope ChannelForge -Parameters @{ P = $Params } { Get-ChannelForgeGuideQualityScorecard @P } } @@ -73,14 +74,40 @@ Describe 'Guide quality scorecard: what the owner sees' { $start = [datetimeoffset]::Parse('2026-10-01T12:00:00Z') $in = New-CleanInput $in.Programme = @( + $in.Programme[2] New-ChannelForgeProgramme -ChannelId 'one' -Start $start -End $start.AddHours(1) -Title 'Same Show' -SourceId 'p' New-ChannelForgeProgramme -ChannelId 'two' -Start $start -End $start.AddHours(1) -Title 'same show' -SourceId 'p' + New-ChannelForgeProgramme -ChannelId 'one' -Start $start -End $start.AddHours(1) -Title 'Sign Off' -SourceId 'p' + New-ChannelForgeProgramme -ChannelId 'two' -Start $start -End $start.AddHours(1) -Title 'Sign Off' -SourceId 'p' ) $gate = Get-Gate (Get-Card $in) 'Deduplication' $gate.Verdict | Should -Be 'Review' $gate.Counts.DuplicateGroups | Should -Be 1 } + It 'does not call syndicated repeats duplicates when no programme has an event identity (Deduplication NotMeasured)' { + $in = New-CleanInput; $in.Programme = @(New-Programmes)[0..1] + $gate = Get-Gate (Get-Card $in) 'Deduplication' + $gate.Verdict | Should -Be 'NotMeasured' + $gate.Reason | Should -BeLike '*syndicated*' + } + + It 'does not pass Accuracy or Completeness when one guide compared nothing' { + $in = New-CleanInput + $in.XguideReport = [pscustomobject]@{ Contract = 'xguide-compare/v1'; SharedChannelCount = 0; Guides = @([pscustomobject]@{ SourceId = 'a'; State = 'ok' }) + Counts = [pscustomobject]@{ MissingChannel = 0; StartShift = 0; TitleDiffers = 0; Unmatched = 0; GuideBehind = 0 } } + $card = Get-Card $in + (Get-Gate $card 'Accuracy').Verdict | Should -Be 'NotMeasured' + (Get-Gate $card 'Completeness').Verdict | Should -Be 'NotMeasured' + } + + It 'makes Completeness Review when playlist channels have no programmes' { + $in = New-CleanInput; $in.PlaylistGuideId = @('one', 'two', 'ghost', '') + $gate = Get-Gate (Get-Card $in) 'Completeness' + $gate.Verdict | Should -Be 'Review' + $gate.Counts.PlaylistChannelsWithoutProgrammes | Should -Be 2 + } + It 'reports NotMeasured, never Pass, for a missing input report' { $card = Get-Card @{ CategoryReport = (New-Categories) } foreach ($name in 'Accuracy', 'Completeness', 'Freshness', 'Deduplication', 'Availability') { @@ -98,7 +125,7 @@ Describe 'Guide quality scorecard: what the owner sees' { $card.Totals.CategoryChannelCount | Should -Be 3 $card.Totals.XguideSharedChannels | Should -Be 4 $card.Totals.ReadinessRecordCount | Should -Be 2 - $card.Totals.ProgrammeCount | Should -Be 2 + $card.Totals.ProgrammeCount | Should -Be 3 ($in | ConvertTo-Json -Depth 8) | Should -Be $before ($card | ConvertTo-Json -Depth 8) | Should -Be ($again | ConvertTo-Json -Depth 8) }