diff --git a/CHANGELOG.md b/CHANGELOG.md index 23323ae..cc1aed7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +- The eval comparison tracks scenario digests per size, so a baseline that + holds several sizes no longer reads a smaller candidate run as a changed + scenario, and a changed workload is named as scenario/size. - `Compaction.new(fallback:)` names a second summarizer, a provider or a model id, that gets one try when the session's provider cannot write a usable summary, a refusal included; a fallback naming the primary's own diff --git a/eval/compaction/report.rb b/eval/compaction/report.rb index c3fd30d..95f5af1 100644 --- a/eval/compaction/report.rb +++ b/eval/compaction/report.rb @@ -66,8 +66,8 @@ def compare(base, candidate) end changed = changed_scenarios(base, candidate) - base_cells = grouped(base).reject { |key, _| changed.include?(key[1]) } - candidate_cells = grouped(candidate).reject { |key, _| changed.include?(key[1]) } + base_cells = grouped(base).reject { |key, _| changed.include?(key[1..2]) } + candidate_cells = grouped(candidate).reject { |key, _| changed.include?(key[1..2]) } matched = base_cells.keys & candidate_cells.keys regressions = [] per_model = per_model(base_cells.slice(*matched), candidate_cells.slice(*matched), @@ -171,7 +171,8 @@ def combine(summaries, field) def coverage(base_keys, candidate_keys, changed) notes = [] unless changed.empty? - notes << "Scenarios changed since the baseline and left out: #{changed.join(", ")}. " \ + names = changed.map { |scenario, size| "#{scenario}/#{size}" } + notes << "Scenarios changed since the baseline and left out: #{names.join(", ")}. " \ "Rerun the baseline to compare them." end missing = base_keys - candidate_keys @@ -188,12 +189,15 @@ def coverage(base_keys, candidate_keys, changed) end # A scenario edited since the baseline has no comparable rows; say so - # instead of comparing different questions. + # instead of comparing different questions. Each size builds its own + # workload, so a scenario is tracked per size. def changed_scenarios(base, candidate) - digests = ->(rows) { rows.to_h { |row| [row[:scenario].to_s, row[:scenario_digest]] } } + digests = lambda do |rows| + rows.to_h { |row| [[row[:scenario].to_s, row[:size].to_s], row[:scenario_digest]] } + end before = digests.call(base) - digests.call(candidate).filter_map do |scenario, digest| - scenario if before.key?(scenario) && before[scenario] != digest + digests.call(candidate).filter_map do |workload, digest| + workload if before.key?(workload) && before[workload] != digest end end diff --git a/test/test_compaction_eval.rb b/test/test_compaction_eval.rb index f3470f7..2909920 100644 --- a/test/test_compaction_eval.rb +++ b/test/test_compaction_eval.rb @@ -237,7 +237,12 @@ def test_compare_matches_cells_refuses_the_incomparable_and_flags_real_drops refute_predicate CompactionEval::Report.compare([base_s], old_grader), :passed assert_includes CompactionEval::Report.compare([base_s], [base_s.merge(scenario_digest: "d2")]) - .markdown, "changed since the baseline" + .markdown, "changed since the baseline and left out: s/S." + per_size = CompactionEval::Report.compare([base_s, base_m.merge(scenario_digest: "dm")], + [base_s]) + + assert_predicate per_size, :passed, "digests are per size: M cannot change the S workload" + refute_includes per_size.markdown, "changed since the baseline" end def test_compare_weighs_matched_cells_equally_whatever_their_repetitions