Track eval scenario digests per size in the comparison

CHANGELOG.md

@@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [Unreleased]
 
+- The eval comparison tracks scenario digests per size, so a baseline that
+  holds several sizes no longer reads a smaller candidate run as a changed
+  scenario, and a changed workload is named as scenario/size.
 - `Compaction.new(fallback:)` names a second summarizer, a provider or a
   model id, that gets one try when the session's provider cannot write a
   usable summary, a refusal included; a fallback naming the primary's own

eval/compaction/report.rb

@@ -66,8 +66,8 @@ def compare(base, candidate)
       end
 
       changed = changed_scenarios(base, candidate)
-      base_cells = grouped(base).reject { |key, _| changed.include?(key[1]) }
-      candidate_cells = grouped(candidate).reject { |key, _| changed.include?(key[1]) }
+      base_cells = grouped(base).reject { |key, _| changed.include?(key[1..2]) }
+      candidate_cells = grouped(candidate).reject { |key, _| changed.include?(key[1..2]) }
       matched = base_cells.keys & candidate_cells.keys
       regressions = []
       per_model = per_model(base_cells.slice(*matched), candidate_cells.slice(*matched),
@@ -171,7 +171,8 @@ def combine(summaries, field)
     def coverage(base_keys, candidate_keys, changed)
       notes = []
       unless changed.empty?
-        notes << "Scenarios changed since the baseline and left out: #{changed.join(", ")}. " \
+        names = changed.map { |scenario, size| "#{scenario}/#{size}" }
+        notes << "Scenarios changed since the baseline and left out: #{names.join(", ")}. " \
                  "Rerun the baseline to compare them."
       end
       missing = base_keys - candidate_keys
@@ -188,12 +189,15 @@ def coverage(base_keys, candidate_keys, changed)
     end
 
     # A scenario edited since the baseline has no comparable rows; say so
-    # instead of comparing different questions.
+    # instead of comparing different questions. Each size builds its own
+    # workload, so a scenario is tracked per size.
     def changed_scenarios(base, candidate)
-      digests = ->(rows) { rows.to_h { |row| [row[:scenario].to_s, row[:scenario_digest]] } }
+      digests = lambda do |rows|
+        rows.to_h { |row| [[row[:scenario].to_s, row[:size].to_s], row[:scenario_digest]] }
+      end
       before = digests.call(base)
-      digests.call(candidate).filter_map do |scenario, digest|
-        scenario if before.key?(scenario) && before[scenario] != digest
+      digests.call(candidate).filter_map do |workload, digest|
+        workload if before.key?(workload) && before[workload] != digest
       end
     end
 

test/test_compaction_eval.rb

@@ -237,7 +237,12 @@ def test_compare_matches_cells_refuses_the_incomparable_and_flags_real_drops
 
     refute_predicate CompactionEval::Report.compare([base_s], old_grader), :passed
     assert_includes CompactionEval::Report.compare([base_s], [base_s.merge(scenario_digest: "d2")])
-                                          .markdown, "changed since the baseline"
+                                          .markdown, "changed since the baseline and left out: s/S."
+    per_size = CompactionEval::Report.compare([base_s, base_m.merge(scenario_digest: "dm")],
+                                              [base_s])
+
+    assert_predicate per_size, :passed, "digests are per size: M cannot change the S workload"
+    refute_includes per_size.markdown, "changed since the baseline"
   end
 
   def test_compare_weighs_matched_cells_equally_whatever_their_repetitions