CMS: instant and on-page search, its analytics, and a configurable page-contents nav

src/DfE.CheckPerformanceData.Application/Analytics/ISearchAnalyticsQueryService.cs

@@ -318,4 +318,22 @@ Task<SearchAnalyticsSummaryDeltas> GetSummaryDeltasAsync(
         DateTime fromUtc,
         DateTime toUtc,
         CancellationToken cancellationToken = default);
+
+    // Pages that carry an on-page search widget, ranked by how much they were searched.
+    // Only instant-page rows have a host path, so the shape of the data does the filtering.
+    Task<(IReadOnlyList<OnPageSearchPageRow> Rows, int TotalCount)> GetOnPageSearchPagesAsync(
+        DateTime fromUtc,
+        DateTime toUtc,
+        int page,
+        int pageSize,
+        CancellationToken cancellationToken = default);
+
+    // What was searched for on one page.
+    Task<(IReadOnlyList<OnPageSearchTermRow> Rows, int TotalCount)> GetOnPageSearchTermsAsync(
+        string hostPath,
+        DateTime fromUtc,
+        DateTime toUtc,
+        int page,
+        int pageSize,
+        CancellationToken cancellationToken = default);
 }

src/DfE.CheckPerformanceData.Application/Analytics/ISearchSurfaceFilter.cs

@@ -0,0 +1,20 @@
+namespace DfE.CheckPerformanceData.Application.Analytics;
+
+// Which search surfaces the dashboard is currently looking at.
+//
+// Ambient rather than a parameter on all thirty-odd read methods, following the same shape as
+// ISearchDebugOptions: the value is a property of the request, every query wants it, and
+// threading it through each signature would add a parameter that no caller ever varies within
+// one request.
+//
+// The default is every surface. A reader that does not care must see everything rather than
+// silently see only site searches.
+public interface ISearchSurfaceFilter
+{
+    IReadOnlyList<string> Surfaces { get; }
+}
+
+public sealed class AllSearchSurfaces : ISearchSurfaceFilter
+{
+    public IReadOnlyList<string> Surfaces { get; } = SearchSurfaces.All;
+}

src/DfE.CheckPerformanceData.Application/Analytics/InstantSearchEventMapper.cs

@@ -0,0 +1,60 @@
+namespace DfE.CheckPerformanceData.Application.Analytics;
+
+// Transform from a browser-reported instant search to the same (SearchEventDto, result rows)
+// pair the site-search mapper produces, so both surfaces land in one table and the dashboard
+// reads them through one query layer.
+//
+// The counting rule is the only real decision here: a "section" hit increments ResultsSections
+// rather than ResultsPages. A section is a heading on the page the person was already reading,
+// not a document they could have found any other way — counting it as a page would inflate
+// every per-page figure on the dashboard and make an on-page search look like site traffic.
+public static class InstantSearchEventMapper
+{
+    public const string SectionKind = "section";
+    public const string PageKind = "page";
+
+    public static (SearchEventDto Event, IReadOnlyList<SearchEventResultDto> Results) From(
+        InstantSearchTelemetryEvent evt, string sessionId)
+    {
+        var results = new List<SearchEventResultDto>(evt.Shown.Count);
+        var pages = 0;
+        var sections = 0;
+
+        foreach (var hit in evt.Shown)
+        {
+            var kind = string.Equals(hit.Kind, SectionKind, StringComparison.Ordinal)
+                ? SectionKind
+                : PageKind;
+
+            if (kind == SectionKind) sections++;
+            else pages++;
+
+            results.Add(new SearchEventResultDto(
+                Position: hit.Position,
+                ResultKind: kind,
+                ResultKey: hit.Key,
+                // A typeahead has no ts_rank to report: the browser knows the order it showed
+                // but not the score behind it. Zero rather than a fabricated number, so a rank
+                // sort on the dashboard cannot mistake a suggestion for a scored search hit.
+                Rank: 0f));
+        }
+
+        var dto = new SearchEventDto(
+            OccurredAtUtc: evt.UtcTimestamp,
+            SessionId: sessionId,
+            QueryRaw: evt.QueryRaw,
+            QueryNormalised: evt.QueryNormalised,
+            Scope: evt.Scope,
+            ResultsPages: pages,
+            ResultsBlocks: 0,
+            LatencyMs: evt.LatencyMs,
+            Results: results,
+            ResultsSections: sections,
+            Surface: evt.Surface,
+            HostPath: evt.HostPath,
+            SelectedKey: evt.SelectedKey,
+            SelectedPosition: evt.SelectedPosition);
+
+        return (dto, results);
+    }
+}

src/DfE.CheckPerformanceData.Application/Analytics/InstantSearchTelemetryEvent.cs

@@ -0,0 +1,35 @@
+namespace DfE.CheckPerformanceData.Application.Analytics;
+
+// What a typeahead reports once the person has settled on a query.
+//
+// Deliberately a different record from SearchTelemetryEvent. That one describes a search the
+// server ran and carries per-field rank breakdowns and filter breadcrumbs it owns; this one
+// describes what a person did in a menu, and the only authority on what was actually shown to
+// them, and on whether they took any of it, is the browser. Keeping the two apart stops either
+// from growing fields that are meaningless on the other.
+public sealed record InstantSearchTelemetryEvent(
+    Guid SearchId,
+    DateTime UtcTimestamp,
+    string QueryRaw,
+    string QueryNormalised,
+    string? Scope,
+    // SearchSurfaces.Instant or SearchSurfaces.InstantPage.
+    string Surface,
+    // The page the widget sat on, for an on-page search; null otherwise.
+    string? HostPath,
+    // Time from the query being issued to the menu rendering, measured in the browser.
+    int LatencyMs,
+    IReadOnlyList<InstantSearchShownHit> Shown,
+    // Null when the menu was shown and nothing was taken from it.
+    string? SelectedKey,
+    int? SelectedPosition);
+
+// One row as it appeared in the menu. Position is one-indexed and is the order the person saw,
+// which is the only ordering that means anything when asking why they picked the third one.
+public sealed record InstantSearchShownHit(
+    int Position,
+    // "page" for a document, "section" for a heading on the current page.
+    string Kind,
+    // Canonical URL for a page, "#anchor" for a section.
+    string Key,
+    string Label);

src/DfE.CheckPerformanceData.Application/Analytics/SearchAnalyticsSummaries.cs

@@ -178,3 +178,22 @@ public sealed record ZeroResultJourney(
     IReadOnlyList<RefinementStep> Steps,
     bool EventuallyRecovered,
     bool SentFeedback);
+
+// One page that carries an on-page search widget, summarised over the window.
+//
+// SelectedCount is the number of searches where the person took one of the sections offered.
+// The gap between Searches and SelectedCount is the interesting number: searches where the
+// page was asked a question and the answer on offer was not taken.
+public sealed record OnPageSearchPageRow(
+    string HostPath,
+    int Searches,
+    int UniqueSessions,
+    int ZeroResultCount,
+    int SelectedCount);
+
+// One term searched for on a single page.
+public sealed record OnPageSearchTermRow(
+    string QueryNormalised,
+    int Searches,
+    int ZeroResultCount,
+    int SelectedCount);

src/DfE.CheckPerformanceData.Application/Analytics/SearchEventDto.cs

@@ -15,6 +15,17 @@ public sealed record SearchEventDto(
     int ResultsBlocks,
     int LatencyMs,
     IReadOnlyList<SearchEventResultDto> Results,
+    // Sections offered by an on-page instant search; zero for every other surface.
+    int ResultsSections = 0,
+    // Which surface produced the event. Defaulted so the existing site-search call sites
+    // need no change and cannot accidentally file themselves as something else.
+    string Surface = SearchSurfaces.Site,
+    // The page an on-page search ran on; null elsewhere.
+    string? HostPath = null,
+    // What was chosen from a suggestion menu, and where it sat. Null means nothing was
+    // taken — which for a typeahead is a result in its own right.
+    string? SelectedKey = null,
+    int? SelectedPosition = null,
     // Optional marker set by the sample-data seeder to true. Real events captured from
     // live user requests leave this at its default (false). The sink propagates the
     // flag onto both the parent SearchEvent row and its child SearchEventResult rows so

src/DfE.CheckPerformanceData.Application/Analytics/SearchSurfaces.cs

@@ -0,0 +1,27 @@
+namespace DfE.CheckPerformanceData.Application.Analytics;
+
+// The search surfaces a recorded event can come from. String constants rather than an enum
+// because the value is persisted as text, crosses an HTTP boundary from the browser, and is
+// read back in raw SQL — one spelling in one place keeps those three honest.
+public static class SearchSurfaces
+{
+    // A submitted search: /search, or a results widget on a content page.
+    public const string Site = "site";
+
+    // Typeahead over the whole site or a section of it. Choosing a suggestion opens a page.
+    public const string Instant = "instant";
+
+    // Typeahead over the sections of the page the widget sits on. Choosing a suggestion
+    // moves to a heading on that page rather than opening anything.
+    public const string InstantPage = "instant-page";
+
+    public static readonly IReadOnlyList<string> All = [Site, Instant, InstantPage];
+
+    public static bool IsKnown(string? surface) =>
+        surface is not null && All.Contains(surface, StringComparer.Ordinal);
+
+    // True for the two typeahead surfaces — the ones whose rows carry a selection.
+    public static bool IsInstant(string? surface) =>
+        string.Equals(surface, Instant, StringComparison.Ordinal)
+        || string.Equals(surface, InstantPage, StringComparison.Ordinal);
+}

src/DfE.CheckPerformanceData.Application/Analytics/SinkAndLogSearchTelemetry.cs

@@ -69,4 +69,23 @@ public void RecordSearch(SearchTelemetryEvent evt)
         // its summary log line.
         _inner.RecordSearch(evt);
     }
+
+    public void RecordInstantSearch(InstantSearchTelemetryEvent evt)
+    {
+        var sessionId = _sessionProvider.GetSessionId();
+
+        if (!string.IsNullOrEmpty(sessionId))
+        {
+            var (dto, _) = InstantSearchEventMapper.From(evt, sessionId);
+            if (!_writer.TryWrite(dto))
+            {
+                _droppedCounter.Increment();
+                _logger.LogWarning(
+                    "Search analytics event dropped: channel full (SearchId={SearchId})",
+                    evt.SearchId);
+            }
+        }
+
+        _inner.RecordInstantSearch(evt);
+    }
 }

src/DfE.CheckPerformanceData.Application/ContentPages/ContentNavBuilder.cs

@@ -1,32 +1,64 @@
 namespace DfE.CheckPerformanceData.Application.ContentPages;
 
-// Builds a page's left-hand nav by walking the content tree (depth-first, document order) for
-// Heading widgets. H2 → top-level item; H3 → nested under the most recent H2 (an H3 before any H2
-// falls back to top-level). Other widgets and other heading levels (H1, H4–H6) do not contribute.
+// Builds a page's contents nav by walking the content tree (depth-first, document order) for
+// Heading widgets.
+//
+// Which heading levels appear is the author's choice — any combination of H1 to H6. The chosen
+// levels form the hierarchy in the order they are chosen rather than by their absolute numbers,
+// so an author who picks H2 and H4 because the page does not use H3 gets H4 nested directly
+// under H2, and the H3s are ignored. That is the whole point of choosing: the nav should follow
+// the structure of this page, not the structure the levels imply in the abstract.
+//
+// Levels fall back rather than being dropped: a heading whose parent level has not appeared yet
+// attaches to the nearest chosen level above it that has, and one with nothing above it at all
+// becomes top-level. Authors skip levels, and dropping those headings would leave sections of
+// the page the nav cannot reach, which is the job it exists to do.
 public static class ContentNavBuilder
 {
-    public static IReadOnlyList<ContentNavItem> Build(IReadOnlyList<ContentNode> tree)
+    // What a page gets when nobody has chosen: the two levels a contents list is usually made of.
+    public static readonly IReadOnlyList<int> DefaultLevels = [2, 3];
+
+    public static IReadOnlyList<ContentNavItem> Build(
+        IReadOnlyList<ContentNode> tree,
+        IReadOnlyCollection<int>? levels = null)
     {
+        // Sorted and de-duplicated: the rank of a level is its position in this list, and that
+        // is what decides nesting.
+        var chosen = (levels ?? DefaultLevels)
+            .Where(level => level is >= 1 and <= 6)
+            .Distinct()
+            .OrderBy(level => level)
+            .ToList();
+
+        if (chosen.Count == 0) return [];
+
         var top = new List<MutableItem>();
-        MutableItem? currentH2 = null;
+
+        // The most recent item at each rank. A heading attaches to the nearest non-null entry
+        // above its own rank; everything below its rank is cleared, so a later sibling cannot
+        // collect the children of the branch that just ended.
+        var openAtRank = new MutableItem?[chosen.Count];
 
         foreach (var heading in Walk(tree))
         {
             var (level, text) = HeadingProps(heading);
-            if (text is null || heading.Anchor is null) continue;
+            if (text is null || heading.Anchor is null || level is null) continue;
+
+            var rank = chosen.IndexOf(level.Value);
+            if (rank < 0) continue;
 
             var item = new MutableItem(text, $"#{heading.Anchor}");
-            if (level == 2)
-            {
-                top.Add(item);
-                currentH2 = item;
-            }
-            else if (level == 3)
-            {
-                if (currentH2 is null) top.Add(item);
-                else currentH2.Children.Add(item);
-            }
-            // Other levels do not appear in the nav.
+
+            MutableItem? parent = null;
+            for (var above = rank - 1; above >= 0 && parent is null; above--)
+                parent = openAtRank[above];
+
+            if (parent is null) top.Add(item);
+            else parent.Children.Add(item);
+
+            openAtRank[rank] = item;
+            for (var below = rank + 1; below < openAtRank.Length; below++)
+                openAtRank[below] = null;
         }
 
         return top.Select(Freeze).ToList();

src/DfE.CheckPerformanceData.Application/ContentPages/NavLevelSelection.cs

@@ -0,0 +1,28 @@
+namespace DfE.CheckPerformanceData.Application.ContentPages;
+
+// Reads the page-nav widget's per-level tick boxes into the set of heading levels its contents
+// list should show.
+//
+// A widget placed before the tick boxes existed carries none of these props. That is not the
+// same as an author unticking everything: the first means "never asked", which is the default
+// pair, and the second means "asked for none", which is an empty list. Telling them apart is
+// why this checks for the props' presence rather than just reading each one as a boolean.
+public static class NavLevelSelection
+{
+    public static IReadOnlyCollection<int> For(WidgetNode widget)
+    {
+        var anyDeclared = false;
+        var levels = new List<int>();
+
+        for (var level = 1; level <= 6; level++)
+        {
+            var key = $"h{level}";
+            if (widget.Props?.ContainsKey(key) != true) continue;
+
+            anyDeclared = true;
+            if (widget.GetBool(key) == true) levels.Add(level);
+        }
+
+        return anyDeclared ? levels : ContentNavBuilder.DefaultLevels;
+    }
+}

src/DfE.CheckPerformanceData.Application/ContentPages/WidgetRegistry.cs

@@ -15,9 +15,9 @@ public static class WidgetRegistry
         new("card", "Card", ContributesToNav: false, """{"title":"","body":"","href":""}"""),
         new("summarylist", "Summary list", ContributesToNav: false, """{"rows":[]}"""),
         new("published", "Published callout", ContributesToNav: false, """{"text":""}"""),
-        new("search",    "Search",            ContributesToNav: false, """{"label":"Search","placeholder":"","action":"/search","buttonText":"Search","scope":""}"""),
+        new("search",    "Search",            ContributesToNav: false, """{"label":"Search","placeholder":"","action":"/search","buttonText":"Search","scope":"","searchIn":"site","instant":"false","noResultsText":"No results found"}"""),
         new("results",   "Search results",    ContributesToNav: false, """{"scope":"","emptyText":"No results found."}"""),
-        new("pagenav",   "Page navigation",   ContributesToNav: false, """{"mode":"headings","childrenParentPath":"","showSearch":false,"searchPath":"","searchLabel":"Search"}""")
+        new("pagenav",   "Page navigation",   ContributesToNav: false, """{"mode":"headings","childrenParentPath":"","showSearch":false,"searchPath":"","searchLabel":"Search","h1":"false","h2":"true","h3":"true","h4":"false","h5":"false","h6":"false"}""")
     ];
 
     private static readonly Dictionary<string, WidgetDefinition> ByType =

src/DfE.CheckPerformanceData.Application/Search/ISearchTelemetry.cs

@@ -1,3 +1,5 @@
+using DfE.CheckPerformanceData.Application.Analytics;
+
 namespace DfE.CheckPerformanceData.Application.Search;
 
 // Fire-and-forget emission surface for a completed search request. Callers assemble a
@@ -9,4 +11,10 @@ namespace DfE.CheckPerformanceData.Application.Search;
 public interface ISearchTelemetry
 {
     void RecordSearch(SearchTelemetryEvent evt);
+
+    // A typeahead reports once the person has settled on a query, not once per keystroke, and
+    // it reports from the browser — which is the only place that knows what was actually shown
+    // and whether any of it was taken. Separate from RecordSearch because the event shape is
+    // different and because the two must stay countable apart on the dashboard.
+    void RecordInstantSearch(InstantSearchTelemetryEvent evt);
 }

src/DfE.CheckPerformanceData.Application/Search/ISiteSearchService.cs

@@ -10,8 +10,22 @@ namespace DfE.CheckPerformanceData.Application.Search;
 public interface ISiteSearchService
 {
     Task<SiteSearchPagedResult> SearchAsync(SiteSearchQuery query);
+
+    // Typeahead for the instant-search widget: the same corpus, the same scope rules and the
+    // same silent filters as SearchAsync, reduced to the handful of rows a suggestion menu can
+    // show. Deliberately records no telemetry — see the implementation for why.
+    Task<IReadOnlyList<SiteSearchSuggestion>> SuggestAsync(SiteSearchSuggestQuery query);
 }
 
+// A typeahead request. Limit is the number of rows the caller can display; the service clamps
+// it so a hand-crafted request cannot ask for the whole corpus.
+public sealed record SiteSearchSuggestQuery(string? Query, string? ScopePath = null, int Limit = 10);
+
+// One row in a suggestion menu. Label is what the visitor reads, Url where choosing it takes
+// them — the menu navigates rather than filling a hidden field, which is why this carries a URL
+// and not the id/label shape the journey-domain suggestion endpoints use.
+public sealed record SiteSearchSuggestion(string Label, string Url);
+
 // Positional record: the trailing three ints carry the paging window.
 //   MaxPerType — per-corpus fetch window feeding the canonicaliser. Default 500 is the retired
 //                MergedFetchCap — represents the ceiling from which the in-memory pager slices.