From 49036b92a3e6406d7d31018a16d8abb88e6d8a12 Mon Sep 17 00:00:00 2001 From: Patrick Linnane Date: Sat, 10 Oct 2026 11:23:23 -0700 Subject: [PATCH] advisory-match: emit split affected intervals Reconstruct every affected interval from complete formula history and union known intervals across macOS and Linux on ARM and Intel, so histories that leave and re-enter an affected range are no longer held as unrepresentable. Reconciliation still cannot fill reviewed gaps, expand coverage or reopen a terminal range. Signed-off-by: Patrick Linnane --- Library/Homebrew/dev-cmd/advisory-match.rb | 313 +++++++++----- .../test/dev-cmd/advisory-match_spec.rb | 386 ++++++++++++++++-- Library/Homebrew/test/vulns/match_spec.rb | 125 +++++- .../Homebrew/test/vulns/osv_export_spec.rb | 36 ++ Library/Homebrew/vulns/match.rb | 146 +++++-- Library/Homebrew/vulns/osv_export.rb | 31 ++ docs/Advisory-Matching.md | 31 +- 7 files changed, 879 insertions(+), 189 deletions(-) diff --git a/Library/Homebrew/dev-cmd/advisory-match.rb b/Library/Homebrew/dev-cmd/advisory-match.rb index e9dfd463b2678..ff6c22dfccf65 100644 --- a/Library/Homebrew/dev-cmd/advisory-match.rb +++ b/Library/Homebrew/dev-cmd/advisory-match.rb @@ -127,6 +127,7 @@ def run next end + new_histories = T.let({}, T::Hash[String, Homebrew::Vulns::Match::ReconciledHistory]) actionable.each do |hit, status| record_id = matcher.record_id(formula, hit) next if emitter.alias_protected?(record_id) @@ -158,6 +159,35 @@ def run end end + if initial_introduction + if new_histories.empty? + new_hits = actionable.filter_map do |candidate_hit, candidate_status| + id = matcher.record_id(formula, candidate_hit) + next if !candidate_status || emitter.alias_protected?(id) || emitter.reviewed_range_state(id) + + candidate_hit + end + new_histories = new_record_histories(matcher, formula, new_hits, emitter, latest_macos:) + end + history = new_histories.fetch(record_id) + next if history.state == :never_affected + + if history.state != :range + emitter.record_history_unavailable(formula.name) + reason = if history.reasons.include?(:history_unavailable) + "formula history is unavailable" + else + "complete history requires review" + end + opoo "#{record_id}: #{reason} (#{history.reasons.join(", ")}); skipping automatic update" + next + end + candidate = matcher.to_brew_record(formula, hit, history:) + revalidated = !history.fixed.nil? && !override&.fixed_in_overridden + emitter.emit(candidate, revalidated:, initial_introduction: true) + next + end + has_open_range = reviewed_state == :open has_terminal_range = reviewed_state == :terminal transition = (status&.fixed? && has_open_range) || @@ -199,20 +229,6 @@ def run end first_introduced = T.let(nil, T.nilable(String)) - if initial_introduction - emitter.record_history_walk - introduced = matcher.first_introduced_version(formula, hit, first_fixed: fixed_boundary) - case introduced - when String - first_introduced = introduced - when :history_unavailable - emitter.record_history_unavailable(formula.name) - opoo "#{record_id}: affected introduction cannot be established; skipping automatic update" - next - else - raise TypeError, "unexpected introduction-history result: #{introduced.inspect}" - end - end if status&.affected? && has_terminal_range emitter.record_history_walk reintroduced = matcher.first_reintroduced_version(formula, hit) @@ -252,6 +268,67 @@ def run end end + # Query each platform once per formula. Stop visiting platforms for a + # candidate as soon as its history or provenance requires review. + sig { + params(matcher: Homebrew::Vulns::Match, formula: Formula, hits: T::Array[Homebrew::Vulns::Match::Hit], + emitter: Emitter, latest_macos: Symbol) + .returns(T::Hash[String, Homebrew::Vulns::Match::ReconciledHistory]) + } + def new_record_histories(matcher, formula, hits, emitter, latest_macos:) + records = hits.to_h { |hit| [matcher.record_id(formula, hit), matcher.to_brew_record(formula, hit)] } + outcomes = T.let({}, T::Hash[String, T::Array[Homebrew::Vulns::Match::ReconciledHistory]]) + records.each_key { |id| outcomes[id] = [] } + [[latest_macos, :arm], [latest_macos, :intel], [:linux, :arm], [:linux, :intel]] + .each_with_index do |(os, arch), index| + pending = hits.select do |hit| + outcomes.fetch(matcher.record_id(formula, hit)) + .all? { |result| [:range, :never_affected].include?(result.state) } + end + break if pending.empty? + + Homebrew::SimulateSystem.with(os:, arch:) do + begin + view = index.zero? ? formula : Formulary.factory(formula.path) + platform_hits = index.zero? ? hits : matcher.dedup_by_aliases(matcher.advisories_for(view)) + rescue Homebrew::Vulns::OSV::Error + raise + rescue => e + pending.each do |hit| + outcomes.fetch(matcher.record_id(formula, hit)) << Homebrew::Vulns::Match::ReconciledHistory.new( + state: :unresolved, reasons: [:platform_unavailable], + ) + end + opoo "#{formula.name} (#{os}/#{arch}): #{e.message}" + next + end + pending.each do |hit| + id = matcher.record_id(formula, hit) + family = platform_hits.select { |other| other.identifiers.intersect?(hit.identifiers) } + # Missing query results alone never prove absence. A resource-only + # hit can reuse its upstream constraints when every subject is + # demonstrably absent; the full walk still checks older builds. + absent_resources = family.empty? && hit.evidence.present? && hit.evidence.all? do |evidence| + evidence.resource && !matcher.subject_version_at(view, evidence).first + end + if absent_resources + platform_hit = hit + elsif platform_provenance_changed?(matcher, emitter, records.fetch(id), view, family) + outcomes.fetch(id) << Homebrew::Vulns::Match::ReconciledHistory.new( + state: :unresolved, reasons: [:platform_provenance_changed], + ) + next + else + platform_hit = family.fetch(0) + end + emitter.record_history_walk + outcomes.fetch(id) << matcher.reconcile_history(view, platform_hit) + end + end + end + outcomes.transform_values { |results| Homebrew::Vulns::Match::ReconciledHistory.combine(results) } + end + sig { params(matcher: Homebrew::Vulns::Match).void } def report_history_load_failures(matcher) failures = matcher.history_load_failures @@ -334,7 +411,7 @@ def reconcile_formula(matcher, formula, hits, emitter, latest_macos:) rescue => e pending.each_key do |id| outcomes.fetch(id) << Homebrew::Vulns::Match::ReconciledHistory.new( - state: :unresolved, introduced: nil, fixed: nil, reasons: [:platform_unavailable], + state: :unresolved, reasons: [:platform_unavailable], ) end opoo "#{formula.name} (#{os}/#{arch}): #{e.message}" @@ -344,7 +421,7 @@ def reconcile_formula(matcher, formula, hits, emitter, latest_macos:) family = platform_hits.select { |hit| hit.identifiers.intersect?(record.fetch(:upstream)) } if platform_provenance_changed?(matcher, emitter, record, view, family) result = Homebrew::Vulns::Match::ReconciledHistory.new( - state: :unresolved, introduced: nil, fixed: nil, reasons: [:platform_provenance_changed], + state: :unresolved, reasons: [:platform_provenance_changed], ) else hit = family.fetch(0) @@ -363,14 +440,11 @@ def reconcile_formula(matcher, formula, hits, emitter, latest_macos:) results = outcomes.fetch(id) reasons = results.flat_map(&:reasons) reasons << :preserved if results.any? { |result| result.state == :preserved } - decisions = results.map { |result| [result.state, result.introduced, result.fixed] }.uniq - if reasons.empty? && (results.length != platforms.length || !decisions.one?) - reasons << :platform_disagreement - end + reasons << :platform_unavailable if results.length != platforms.length if reasons.any? emitter.skip_reconciliation(id, reasons.uniq) else - emitter.reconcile(record, results.fetch(0)) + emitter.reconcile(record, Homebrew::Vulns::Match::ReconciledHistory.combine(results)) end end rescue Homebrew::Vulns::OSV::Error => e @@ -381,7 +455,7 @@ def reconcile_formula(matcher, formula, hits, emitter, latest_macos:) # carrying its stored upstream family with the same matching provenance; # anything else means the stored ranges cannot be compared across views. sig { - params(matcher: Homebrew::Vulns::Match, emitter: DirEmitter, record: T::Hash[Symbol, T.untyped], + params(matcher: Homebrew::Vulns::Match, emitter: Emitter, record: T::Hash[Symbol, T.untyped], view: Formula, family: T::Array[Homebrew::Vulns::Match::Hit]).returns(T::Boolean) } def platform_provenance_changed?(matcher, emitter, record, view, family) @@ -481,6 +555,61 @@ def prepare_aliases(_formula_name, _groups) = [] sig { params(_record_id: String).returns(T::Boolean) } def alias_protected?(_record_id) = false + # Keep every strategy and runtime subject. Ignore version values and + # resource labels, but retain checkability and separate copies of the + # same resource package: either can change the aggregate history. + sig { params(record: T::Hash[T.untyped, T.untyped]).returns(T::Hash[String, T.untyped]) } + def range_basis(record) + record = JSON.parse(JSON.generate(record)) + affected = record["affected"] + return {} unless affected.is_a?(Array) + return {} unless affected.one? + + entry = affected.fetch(0) + return {} unless entry.is_a?(Hash) + + ecosystem_specific = entry["ecosystem_specific"] + ecosystem_specific = {} unless ecosystem_specific.is_a?(Hash) + + database_specific = record["database_specific"] + evidence = Array(database_specific["upstream_evidence"]) if database_specific.is_a?(Hash) + evidence = Array(evidence).grep(Hash) + subjects = evidence.group_by { |row| row["resource"] }.map do |resource, rows| + identities = rows.filter_map do |row| + identity = subject_identity(row) + next if identity.empty? + + [row["strategy"].to_s, identity, row["subject_version"].nil? ? "unknown" : "versioned"] + end + [resource ? "resource" : "primary", identities.uniq.sort] + end + basis = { "subjects" => subjects.sort } + resource_purl = ecosystem_specific["resource_purl"] + if resource_purl.is_a?(String) + resource = evidence.find { |row| row["resource"] && row["key"] == resource_purl } + basis["resource_purl"] = resource ? subject_identity(resource) : [unversioned_key(resource_purl)] + end + upstream_fixed_in = ecosystem_specific["upstream_fixed_in"] + basis["upstream_fixed_in"] = upstream_fixed_in if upstream_fixed_in.is_a?(String) + basis + end + + sig { params(evidence: T::Hash[String, T.untyped]).returns(T::Array[String]) } + def subject_identity(evidence) + ecosystem = evidence["ecosystem"] + name = evidence["name"] + return [ecosystem, name] if ecosystem.is_a?(String) && name.is_a?(String) + + key = evidence["key"] + key.is_a?(String) ? [unversioned_key(key)] : [] + end + + sig { params(key: String).returns(String) } + def unversioned_key(key) + key = key.delete_prefix("upstream:") + key.start_with?("pkg:") ? key.sub(%r{@[^/@]*\z}, "") : key + end + sig { params(_record: T::Hash[Symbol, T.untyped]).returns(T::Boolean) } def range_basis_changed?(_record) = false @@ -652,7 +781,7 @@ def reconciliation_resource_records(formula_name) record = alias_record(path) next unless record.is_a?(Hash) next unless reconcilable_record?(record) - next unless single_terminal_range?(record) + next unless terminal_intervals(record) next if affected_formula_names(record) != [formula_name] next if record["id"] != File.basename(path, ".json") @@ -705,7 +834,7 @@ def reconciliation_record(record) skip_reconciliation(id, [:unsupported_record]) return end - unless single_terminal_range?(existing) + unless terminal_intervals(existing) skip_reconciliation(id, [:unsupported_range]) return end @@ -726,23 +855,44 @@ def reconcilable_record?(existing) existing.dig("affected", 0, "ecosystem_specific", "fix") == "bump" end - # Exactly one ECOSYSTEM range holding an `introduced` and a `fixed` - # event, each a non-blank string. - sig { params(existing: T::Hash[String, T.untyped]).returns(T::Boolean) } - def single_terminal_range?(existing) + # Only complete introduced/fixed pairs can be automatically narrowed. + # Other event types and range metadata remain protected for review. + sig { + params(existing: T::Hash[String, T.untyped]) + .returns(T.nilable(T::Array[Homebrew::Vulns::Match::Interval])) + } + def terminal_intervals(existing) ranges = existing.dig("affected", 0, "ranges") - return false if !ranges.is_a?(Array) || !ranges.one? + return if !ranges.is_a?(Array) || ranges.empty? + + intervals = T.let([], T::Array[Homebrew::Vulns::Match::Interval]) + ranges.each do |range| + return nil if !range.is_a?(Hash) || range["type"] != "ECOSYSTEM" + return nil if (range.keys - %w[type events]).any? + + events = range["events"] + return nil if !events.is_a?(Array) || events.empty? || events.length.odd? - range = ranges.fetch(0) - return false if !range.is_a?(Hash) || range["type"] != "ECOSYSTEM" + previous_fixed = T.let(nil, T.nilable(String)) + events.each_slice(2) do |introduced, fixed| + return nil if !introduced.is_a?(Hash) || introduced.keys != ["introduced"] + return nil if !fixed.is_a?(Hash) || fixed.keys != ["fixed"] - events = range["events"] - return false if !events.is_a?(Array) || events.length != 2 + first = introduced["introduced"] + last = fixed["fixed"] + return nil if !first.is_a?(String) || !last.is_a?(String) - introduced, fixed = events - introduced.is_a?(Hash) && introduced.keys == ["introduced"] && introduced["introduced"].is_a?(String) && - introduced["introduced"].present? && - fixed.is_a?(Hash) && fixed.keys == ["fixed"] && fixed["fixed"].is_a?(String) && fixed["fixed"].present? + interval = Homebrew::Vulns::Match::Interval.new(introduced: first, fixed: last) + return nil unless interval.valid? + return nil if previous_fixed && PkgVersion.parse(first) < PkgVersion.parse(previous_fixed) + + intervals << interval + previous_fixed = last + end + end + Homebrew::Vulns::Match::Interval.union(intervals) + rescue ArgumentError + nil end # The stored record must still be reached through the same upstream @@ -771,29 +921,37 @@ def reconcile(record, result) id = record.fetch(:id) path = alias_target_paths(id).fetch(0) updated = existing.deep_dup - if result.state == :range && (introduced = result.introduced) && (fixed = result.fixed) - events = existing.fetch("affected").fetch(0).fetch("ranges").fetch(0).fetch("events") - old_introduced = PkgVersion.parse(events.fetch(0).fetch("introduced")) - old_fixed = PkgVersion.parse(events.fetch(1).fetch("fixed")) - if PkgVersion.parse(introduced) >= PkgVersion.parse(fixed) + if result.state == :range + if result.intervals.empty? || result.intervals.any? { |interval| !interval.valid? } skip_reconciliation(id, [:invalid_interval]) return end - if PkgVersion.parse(introduced) < old_introduced || PkgVersion.parse(fixed) > old_fixed + if result.intervals.any? { |interval| interval.fixed.nil? } + skip_reconciliation(id, [:reopened]) + return + end + reviewed = terminal_intervals(existing) + return unless reviewed + + if result.intervals.any? do |interval| + reviewed.none? do |old| + fixed = interval.fixed + old_fixed = old.fixed + fixed && old_fixed && PkgVersion.parse(interval.introduced) >= PkgVersion.parse(old.introduced) && + PkgVersion.parse(fixed) <= PkgVersion.parse(old_fixed) + end + end skip_reconciliation(id, [:range_expansion]) return end - updated.fetch("affected").fetch(0).fetch("ranges").fetch(0)["events"] = [ - { "introduced" => introduced }, { "fixed" => fixed } - ] - if updated == existing + if result.events == reviewed.flat_map(&:events) @unchanged += 1 return end + updated.fetch("affected").fetch(0)["ranges"] = [ + { "type" => "ECOSYSTEM", "events" => result.events }, + ] updated["modified"] = Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ") - elsif result.state == :range && result.fixed.nil? - skip_reconciliation(id, [:reopened]) - return elsif result.state != :never_affected skip_reconciliation(id, result.reasons.presence || [:unsupported_result]) return @@ -880,61 +1038,6 @@ def emit(record, revalidated: false, initial_introduction: false) end end - # Keep every strategy and runtime subject. Ignore version values and - # resource labels, but retain checkability and separate copies of the - # same resource package: either can change the aggregate history. - sig { params(record: T::Hash[T.untyped, T.untyped]).returns(T::Hash[String, T.untyped]) } - def range_basis(record) - record = JSON.parse(JSON.generate(record)) - affected = record["affected"] - return {} unless affected.is_a?(Array) - return {} unless affected.one? - - entry = affected.fetch(0) - return {} unless entry.is_a?(Hash) - - ecosystem_specific = entry["ecosystem_specific"] - ecosystem_specific = {} unless ecosystem_specific.is_a?(Hash) - - database_specific = record["database_specific"] - evidence = Array(database_specific["upstream_evidence"]) if database_specific.is_a?(Hash) - evidence = Array(evidence).grep(Hash) - subjects = evidence.group_by { |row| row["resource"] }.map do |resource, rows| - identities = rows.filter_map do |row| - identity = subject_identity(row) - next if identity.empty? - - [row["strategy"].to_s, identity, row["subject_version"].nil? ? "unknown" : "versioned"] - end - [resource ? "resource" : "primary", identities.uniq.sort] - end - basis = { "subjects" => subjects.sort } - resource_purl = ecosystem_specific["resource_purl"] - if resource_purl.is_a?(String) - resource = evidence.find { |row| row["resource"] && row["key"] == resource_purl } - basis["resource_purl"] = resource ? subject_identity(resource) : [unversioned_key(resource_purl)] - end - upstream_fixed_in = ecosystem_specific["upstream_fixed_in"] - basis["upstream_fixed_in"] = upstream_fixed_in if upstream_fixed_in.is_a?(String) - basis - end - - sig { params(evidence: T::Hash[String, T.untyped]).returns(T::Array[String]) } - def subject_identity(evidence) - ecosystem = evidence["ecosystem"] - name = evidence["name"] - return [ecosystem, name] if ecosystem.is_a?(String) && name.is_a?(String) - - key = evidence["key"] - key.is_a?(String) ? [unversioned_key(key)] : [] - end - - sig { params(key: String).returns(String) } - def unversioned_key(key) - key = key.delete_prefix("upstream:") - key.start_with?("pkg:") ? key.sub(%r{@[^/@]*\z}, "") : key - end - sig { override.params(record_id: String).returns(T::Boolean) } def history_required?(record_id) alias_target_paths(record_id).any? do |path| diff --git a/Library/Homebrew/test/dev-cmd/advisory-match_spec.rb b/Library/Homebrew/test/dev-cmd/advisory-match_spec.rb index 6a9045afee760..86b479fc5246a 100644 --- a/Library/Homebrew/test/dev-cmd/advisory-match_spec.rb +++ b/Library/Homebrew/test/dev-cmd/advisory-match_spec.rb @@ -24,6 +24,7 @@ before do allow(Formulary).to receive(:enable_factory_cache!) + allow(Formulary).to receive(:factory).with(requests.path).and_return(requests) allow(Homebrew::Vulns::Repology).to receive_messages( load: Homebrew::Vulns::Repology.new({ "meta" => {}, "formulae" => {} }), lookup: {}, @@ -53,10 +54,20 @@ def stub_osv_hit(cve, fixed:, aliases: []) ) end + def history_result(introduced: "2.27.0", fixed: "2.28.1", state: :range, reasons: []) + Homebrew::Vulns::Match::ReconciledHistory.new( + state:, reasons:, + intervals: (state == :range) ? [Homebrew::Vulns::Match::Interval.new(introduced:, fixed:)] : [] + ) + end + context "with --reconcile-history" do let(:matcher) { Homebrew::Vulns::Match.new(strict_upstream: true) } let(:result) do - Homebrew::Vulns::Match::ReconciledHistory.new(state: :range, introduced: "2.20.0", fixed: "2.28.1", reasons: []) + Homebrew::Vulns::Match::ReconciledHistory.new( + state: :range, + intervals: [Homebrew::Vulns::Match::Interval.new(introduced: "2.20.0", fixed: "2.28.1")], reasons: [] + ) end let(:stored) do hit = matcher.advisories_for(requests).fetch(0) @@ -367,6 +378,69 @@ class Requests < Formula .to output(/upstream_unavailable: 1/).to_stdout end + it "writes split history without covering the unaffected gap" do + intervals = [ + Homebrew::Vulns::Match::Interval.new(introduced: "2.20.0", fixed: "2.21.0"), + Homebrew::Vulns::Match::Interval.new(introduced: "2.27.0", fixed: "2.28.1"), + ] + allow(matcher).to receive(:reconcile_history).and_return(result.with(intervals: intervals.map(&:serialize))) + + reconcile_record do |path, _original| + expect(JSON.parse(File.read(path)).dig("affected", 0, "ranges", 0, "events")).to eq [ + { "introduced" => "2.20.0" }, { "fixed" => "2.21.0" }, + { "introduced" => "2.27.0" }, { "fixed" => "2.28.1" } + ] + end + end + + it "does not fill a reviewed unaffected gap during reconciliation" do + stored.fetch("affected").fetch(0).fetch("ranges").fetch(0)["events"] = [ + { "introduced" => "0" }, { "fixed" => "2.21.0" }, + { "introduced" => "2.27.0" }, { "fixed" => "2.31.0" } + ] + + expect { reconcile_record { |path, original| expect(File.read(path)).to eq original } } + .to output(/range_expansion: 1/).to_stdout + end + + it "preserves an identical reviewed split history byte for byte" do + intervals = [ + Homebrew::Vulns::Match::Interval.new(introduced: "2.20.0", fixed: "2.21.0"), + Homebrew::Vulns::Match::Interval.new(introduced: "2.27.0", fixed: "2.28.1"), + ] + stored.fetch("affected").fetch(0)["ranges"] = intervals.map do |interval| + { "type" => "ECOSYSTEM", "events" => JSON.parse(JSON.generate(interval.events)) } + end + allow(matcher).to receive(:reconcile_history).and_return(result.with(intervals: intervals.map(&:serialize))) + + reconcile_record { |path, original| expect(File.read(path)).to eq original } + end + + it "keeps unknown platforms held even when another platform is affected" do + allow(matcher).to receive(:reconcile_history) do + if Homebrew::SimulateSystem.simulating_or_running_on_linux? + history_result(state: :unresolved, reasons: [:history_unavailable]) + else + result + end + end + + expect { reconcile_record { |path, original| expect(File.read(path)).to eq original } } + .to output(/history_unavailable: 1/).to_stdout + end + + it "does not delete a record affected on only one platform" do + allow(matcher).to receive(:reconcile_history) do + Homebrew::SimulateSystem.simulating_or_running_on_linux? ? result : result.with(state: :never_affected) + end + + reconcile_record do |path, _original| + expect(JSON.parse(File.read(path)).dig("affected", 0, "ranges", 0, "events")).to eq [ + { "introduced" => "2.20.0" }, { "fixed" => "2.28.1" } + ] + end + end + it "shrinks a matched terminal range while preserving every other field" do reconcile_record do |path, _original| updated = JSON.parse(File.read(path)) @@ -456,18 +530,24 @@ class Requests < Formula end it "does not expand an existing affected interval" do - allow(matcher).to receive(:reconcile_history).and_return(result.with(fixed: "2.32.0")) + allow(matcher).to receive(:reconcile_history).and_return( + result.with(intervals: [result.intervals.fetch(0).with(fixed: "2.32.0").serialize]), + ) reconcile_record { |path, original| expect(File.read(path)).to eq original } end it "classifies an open reconciliation result as reopened" do - allow(matcher).to receive(:reconcile_history).and_return(result.with(fixed: nil)) + allow(matcher).to receive(:reconcile_history).and_return( + result.with(intervals: [result.intervals.fetch(0).with(fixed: nil).serialize]), + ) expect { reconcile_record { |path, original| expect(File.read(path)).to eq original } } .to output(/reopened: 1/).to_stdout end it "distinguishes an invalid interval from range expansion" do - allow(matcher).to receive(:reconcile_history).and_return(result.with(introduced: "2.28.1")) + allow(matcher).to receive(:reconcile_history).and_return( + result.with(intervals: [result.intervals.fetch(0).with(introduced: "2.28.1").serialize]), + ) expect { reconcile_record { |path, original| expect(File.read(path)).to eq original } } .to output(/invalid_interval: 1/).to_stdout end @@ -484,21 +564,55 @@ class Requests < Formula reconcile_record { |path, original| expect(File.read(path)).to eq original } end - it "holds differing platform histories" do + it "unions differing known platform histories" do allow(matcher).to receive(:reconcile_history) do Homebrew::Vulns::Match::ReconciledHistory.new( - state: :range, - introduced: "2.20.0", - fixed: - if Homebrew::SimulateSystem.simulating_or_running_on_linux? - "2.29.0" - else - "2.28.1" - end, - reasons: [], + state: :range, + intervals: [Homebrew::Vulns::Match::Interval.new( + introduced: "2.20.0", + fixed: Homebrew::SimulateSystem.simulating_or_running_on_linux? ? "2.29.0" : "2.28.1", + )], + reasons: [], ) end - reconcile_record { |path, original| expect(File.read(path)).to eq original } + reconcile_record do |path, _original| + expect(JSON.parse(File.read(path)).dig("affected", 0, "ranges", 0, "events")).to eq [ + { "introduced" => "2.20.0" }, { "fixed" => "2.29.0" } + ] + end + end + + it "unions disjoint platform intervals without filling their safe gap" do + allow(matcher).to receive(:reconcile_history) do + if Homebrew::SimulateSystem.simulating_or_running_on_linux? + history_result(introduced: "2.27.0", fixed: "2.28.1") + else + history_result(introduced: "2.20.0", fixed: "2.21.0") + end + end + + reconcile_record do |path, _original| + expect(JSON.parse(File.read(path)).dig("affected", 0, "ranges", 0, "events")).to eq [ + { "introduced" => "2.20.0" }, { "fixed" => "2.21.0" }, + { "introduced" => "2.27.0" }, { "fixed" => "2.28.1" } + ] + end + end + + it "unions overlapping platform intervals before narrowing the reviewed range" do + allow(matcher).to receive(:reconcile_history) do + if Homebrew::SimulateSystem.simulating_or_running_on_linux? + history_result(introduced: "2.26.0", fixed: "2.28.1") + else + history_result(introduced: "2.20.0", fixed: "2.27.0") + end + end + + reconcile_record do |path, _original| + expect(JSON.parse(File.read(path)).dig("affected", 0, "ranges", 0, "events")).to eq [ + { "introduced" => "2.20.0" }, { "fixed" => "2.28.1" } + ] + end end it "holds a match missing from an alternate platform" do @@ -956,7 +1070,7 @@ def basis_for(evidence, ecosystem_specific = {}) it "repairs legacy missing, empty or invalid ranges after verifying history" do matcher = Homebrew::Vulns::Match.new - allow(matcher).to receive_messages(first_fixed_version: "2.28.1", first_introduced_version: "2.27.0") + allow(matcher).to receive(:reconcile_history).and_return(history_result(introduced: "2.27.0")) allow(Homebrew::Vulns::Match).to receive(:new).and_return(matcher) cases = [nil, [], [{ type: "ECOSYSTEM", events: [] }]] outcomes = cases.map do |ranges| @@ -976,7 +1090,7 @@ def basis_for(evidence, ecosystem_specific = {}) it "derives introductions before repairing affected records with changed provenance" do stub_osv_hit("CVE-2024-1234", fixed: "2.32.0") matcher = Homebrew::Vulns::Match.new - allow(matcher).to receive(:first_introduced_version).and_return("2.30.0") + allow(matcher).to receive(:reconcile_history).and_return(history_result(introduced: "2.30.0", fixed: nil)) allow(Homebrew::Vulns::Match).to receive(:new).and_return(matcher) outcomes = [nil, [], [{ type: "ECOSYSTEM", events: [] }]].map do |ranges| record = existing.deep_dup @@ -990,7 +1104,7 @@ def basis_for(evidence, ecosystem_specific = {}) output = capture_stdout { cmd_for("requests", "--output", dir, "--new-history").run } refreshed = JSON.parse(File.read(path)) [refreshed.dig("affected", 0, "ranges", 0, "events"), - refreshed.dig("database_specific", "upstream_evidence"), output.include?("1 history walks")] + refreshed.dig("database_specific", "upstream_evidence"), output.include?("4 history walks")] end expect(outcomes).to all(eq([[{ "introduced" => "2.30.0" }], reviewed_git_evidence, true])) @@ -999,7 +1113,8 @@ def basis_for(evidence, ecosystem_specific = {}) it "preserves affected records with changed provenance when introduction history is unavailable" do stub_osv_hit("CVE-2024-1234", fixed: "2.32.0") matcher = Homebrew::Vulns::Match.new - allow(matcher).to receive(:first_introduced_version).and_return(:history_unavailable) + allow(matcher).to receive(:reconcile_history) + .and_return(history_result(state: :unresolved, reasons: [:history_unavailable])) allow(Homebrew::Vulns::Match).to receive(:new).and_return(matcher) outcomes = [nil, [], [{ type: "ECOSYSTEM", events: [] }]].map do |ranges| record = existing.deep_dup @@ -1319,17 +1434,208 @@ def basis_for(evidence, ecosystem_specific = {}) end end + it "queries each additional platform once for multiple new candidates" do + ids = %w[CVE-2024-1234 CVE-2024-5678] + ids.each { |id| stub_osv_hit(id, fixed: "2.28.1") } + allow(Homebrew::Vulns::OSV).to receive(:query_batch).and_return([ids.map { |id| { "id" => id } }, []]) + matcher = Homebrew::Vulns::Match.new + allow(matcher).to receive(:reconcile_history).and_return(history_result) + allow(Homebrew::Vulns::Match).to receive(:new).and_return(matcher) + expect(matcher).to receive(:advisories_for).exactly(3).times.and_call_original + + cmd_for("requests", "--json").run + end + + it "stops loading platforms once every new candidate is unresolved" do + stub_osv_hit("CVE-2024-1234", fixed: "2.28.1") + matcher = Homebrew::Vulns::Match.new + allow(Homebrew::Vulns::Match).to receive(:new).and_return(matcher) + expect(matcher).to receive(:reconcile_history).once + .and_return(history_result(state: :unresolved, + reasons: [:history_unavailable])) + expect(matcher).not_to receive(:advisories_for) + + cmd_for("requests", "--json").run + end + + context "with a resource absent from another platform's current formula" do + def resource_formula(version, resource_version, unknown: false) + formula("requests") do + T.bind(self, T.class_of(Formula)) + url "https://files.pythonhosted.org/packages/aa/bb/cc/requests-#{version}.tar.gz" + if resource_version + resource "certifi" do + url unknown ? "https://example.com/certifi-#{resource_version}.tar.gz" : "https://files.pythonhosted.org/packages/aa/bb/cc/certifi-#{resource_version}.tar.gz" + end + end + end + end + + let(:requests) { resource_formula("3.0", "1.2") } + let(:linux_resource) { nil } + let(:unknown_identity) { false } + let(:linux_history_unavailable) { false } + let(:formerly_on_linux) { false } + let(:history) { instance_double(Homebrew::Vulns::History) } + + before do + allow(Formulary).to receive(:factory).with(requests.path) do + if Homebrew::SimulateSystem.simulating_or_running_on_linux? + resource_formula("3.0", linux_resource, unknown: unknown_identity) + else + requests + end + end + allow(Homebrew::Vulns::OSV).to receive(:query_batch) do |packages| + packages.map do |package| + if package[:name] == "certifi" && !Homebrew::SimulateSystem.simulating_or_running_on_linux? + [{ "id" => "CVE-certifi" }] + else + [] + end + end + end + allow(Homebrew::Vulns::OSV).to receive(:vulnerability).with("CVE-certifi").and_return({ + "id" => "CVE-certifi", "affected" => [{ + "package" => { "ecosystem" => "PyPI", "name" => "certifi" }, + "ranges" => [{ "type" => "ECOSYSTEM", "events" => [{ "introduced" => "0" }, { "fixed" => "1.2" }] }], + }] + }) + allow(Homebrew::Vulns::History).to receive(:new).and_return(history) + allow(history).to receive(:walk).with(anything, complete: true) do |_, **_, &block| + linux = Homebrew::SimulateSystem.simulating_or_running_on_linux? + block.call(resource_formula("2.0", if linux + formerly_on_linux ? "1.0" : nil + else + "1.2" + end)) + block.call(resource_formula("1.0", linux ? nil : "1.0")) + :history_unavailable if linux && linux_history_unavailable + end + end + + it "emits a macOS-only resource after proving its absence throughout Linux history" do + records = JSON.parse(capture_stdout { cmd_for("requests", "--json").run }) + + expect(records.fetch(0).dig("affected", 0, "ranges", 0, "events")) + .to eq [{ "introduced" => "1.0" }, { "fixed" => "2.0" }] + end + + context "when Linux used to ship the resource" do + let(:formerly_on_linux) { true } + + it "includes affected Linux history even though the current resource is absent" do + records = JSON.parse(capture_stdout { cmd_for("requests", "--json").run }) + + expect(records.fetch(0).dig("affected", 0, "ranges", 0, "events")) + .to eq [{ "introduced" => "1.0" }, { "fixed" => "3.0" }] + end + end + + context "when Linux history is unavailable" do + let(:linux_history_unavailable) { true } + + it "does not infer historical absence from the current formula" do + expect(JSON.parse(capture_stdout { cmd_for("requests", "--json").run })).to eq [] + end + end + + context "when the resource is present but its upstream match is missing" do + let(:linux_resource) { "1.0" } + + it "holds the candidate instead of treating a missing query result as absence" do + expect(JSON.parse(capture_stdout { cmd_for("requests", "--json").run })).to eq [] + end + end + + context "when the resource's identity cannot be resolved" do + let(:linux_resource) { "1.0" } + let(:unknown_identity) { true } + + it "holds the candidate instead of treating an unknown identity as absence" do + expect(JSON.parse(capture_stdout { cmd_for("requests", "--json").run })).to eq [] + end + end + end + + context "with complete split history for a new record" do + let(:final_fixed) { "2.31.0" } + let(:history) { instance_double(Homebrew::Vulns::History) } + let(:old_formulae) do + %w[2.30.0 2.29.0 2.28.0 2.27.0].map do |version| + formula("requests") do + T.bind(self, T.class_of(Formula)) + url "https://files.pythonhosted.org/packages/aa/bb/cc/requests-#{version}.tar.gz" + head "https://github.com/psf/requests.git" + end + end + end + + before do + stub_osv_hit("CVE-2024-1234", fixed: final_fixed) + allow(Homebrew::Vulns::OSV).to receive(:vulnerability).with("CVE-2024-1234").and_return({ + "id" => "CVE-2024-1234", "affected" => [{ + "package" => { "ecosystem" => "GIT", "name" => "https://github.com/psf/requests" }, + "ranges" => [{ "type" => "ECOSYSTEM", "events" => [ + { "introduced" => "2.27.0" }, { "fixed" => "2.28.0" }, + { "introduced" => "2.30.0" }, { "fixed" => final_fixed } + ] }], + }] + }) + allow(Homebrew::Vulns::History).to receive(:new).and_return(history) + allow(history).to receive(:walk).with(anything, complete: true) do |_, **_, &block| + old_formulae.each { |old| block.call(old) } + nil + end + end + + it "emits and preserves both closed intervals on repeated daily runs" do + Dir.mktmpdir do |dir| + cmd_for("requests", "--output", dir, "--new-history").run + path = File.join(dir, "BREW-requests-CVE-2024-1234.json") + first = File.read(path) + cmd_for("requests", "--output", dir, "--new-history").run + + expect([JSON.parse(first).dig("affected", 0, "ranges", 0, "events"), File.read(path)]).to eq [ + [{ "introduced" => "2.27.0" }, { "fixed" => "2.28.0" }, + { "introduced" => "2.30.0" }, { "fixed" => "2.31.0" }], first + ] + end + end + + it "holds a new record when a later platform's complete walk is unavailable" do + allow(history).to receive(:walk).with(anything, complete: true) do |_, **_, &block| + old_formulae.each { |old| block.call(old) } + :history_unavailable if Homebrew::SimulateSystem.simulating_or_running_on_linux? + end + + expect(JSON.parse(capture_stdout { cmd_for("requests", "--json").run })).to eq [] + end + + context "when the final interval is open" do + let(:final_fixed) { "2.32.0" } + + it "emits all events and derives fix metadata from the final interval" do + record = JSON.parse(capture_stdout { cmd_for("requests", "--json").run }).fetch(0) + affected = record.fetch("affected").fetch(0) + + expect([affected.dig("ranges", 0, "events"), affected.fetch("ecosystem_specific")]).to eq [ + [{ "introduced" => "2.27.0" }, { "fixed" => "2.28.0" }, { "introduced" => "2.30.0" }], + { "fix" => nil, "range_state" => "affected", "upstream_fixed_in" => "2.32.0" }, + ] + end + end + end + it "uses both historical boundaries for a new record with --new-history" do stub_osv_hit("CVE-2024-1234", fixed: "2.28.1") matcher = Homebrew::Vulns::Match.new - expect(matcher).to receive(:first_fixed_version).and_return("2.28.1") - expect(matcher).to receive(:first_introduced_version).with(requests, anything, first_fixed: "2.28.1") - .and_return("2.27.0") + expect(matcher).to receive(:reconcile_history).exactly(4).times.and_return(history_result) allow(Homebrew::Vulns::Match).to receive(:new).and_return(matcher) Dir.mktmpdir do |dir| expect { cmd_for("requests", "--output", dir, "--new-history").run } - .to output(/2 history walks/).to_stdout + .to output(/4 history walks/).to_stdout path = File.join(dir, "BREW-requests-CVE-2024-1234.json") expect(JSON.parse(File.read(path)).dig("affected", 0, "ranges", 0, "events")) .to eq([{ "introduced" => "2.27.0" }, { "fixed" => "2.28.1" }]) @@ -1339,7 +1645,7 @@ def basis_for(evidence, ecosystem_specific = {}) it "repairs mixed ranges using an initial interval without reopening reviewed terminal ranges" do stub_osv_hit("CVE-2024-1234", fixed: "2.28.1") matcher = Homebrew::Vulns::Match.new - allow(matcher).to receive_messages(first_fixed_version: "2.28.1", first_introduced_version: "2.27.0") + allow(matcher).to receive(:reconcile_history).and_return(history_result(introduced: "2.27.0")) allow(Homebrew::Vulns::Match).to receive(:new).and_return(matcher) Dir.mktmpdir do |dir| @@ -1370,13 +1676,15 @@ def basis_for(evidence, ecosystem_specific = {}) it "skips a new record when formula history is unavailable" do stub_osv_hit("CVE-2024-1234", fixed: "2.28.1") matcher = Homebrew::Vulns::Match.new - expect(matcher).to receive(:first_fixed_version).and_return(:history_unavailable) + expect(matcher).to receive(:reconcile_history).once + .and_return(history_result(state: :unresolved, + reasons: [:history_unavailable])) allow(Homebrew::Vulns::Match).to receive(:new).and_return(matcher) Dir.mktmpdir do |dir| expect do expect { cmd_for("requests", "--output", dir, "--new-history").run } - .to output(/formula history is unavailable; skipping automatic update/).to_stderr + .to output(/formula history is unavailable.*skipping automatic update/).to_stderr end.to output(/1 history-unavailable skips.*Unavailable history by formula:\n requests: 1/m).to_stdout expect(Dir.glob(File.join(dir, "*.json"))).to be_empty end @@ -1385,14 +1693,15 @@ def basis_for(evidence, ecosystem_specific = {}) it "does not emit a fixed record when its introduction cannot be established" do stub_osv_hit("CVE-2024-1234", fixed: "2.28.1") matcher = Homebrew::Vulns::Match.new - allow(matcher).to receive_messages(first_fixed_version: "2.28.1", first_introduced_version: :history_unavailable) + allow(matcher).to receive(:reconcile_history) + .and_return(history_result(state: :unresolved, reasons: [:history_unavailable])) allow(Homebrew::Vulns::Match).to receive(:new).and_return(matcher) Dir.mktmpdir do |dir| expect do expect { cmd_for("requests", "--output", dir, "--new-history").run } - .to output(/affected introduction cannot be established; skipping automatic update/).to_stderr - end.to output(/2 history walks, 1 history-unavailable skips/).to_stdout + .to output(/formula history is unavailable.*history_unavailable.*skipping automatic update/).to_stderr + end.to output(/1 history walks, 1 history-unavailable skips/).to_stdout expect(Dir.glob(File.join(dir, "*.json"))).to be_empty end end @@ -1400,7 +1709,8 @@ def basis_for(evidence, ecosystem_specific = {}) it "does not emit an affected JSON record when its introduction cannot be established" do stub_osv_hit("CVE-2024-1234", fixed: "2.32.0") matcher = Homebrew::Vulns::Match.new - allow(matcher).to receive(:first_introduced_version).and_return(:history_unavailable) + allow(matcher).to receive(:reconcile_history) + .and_return(history_result(state: :unresolved, reasons: [:history_unavailable])) allow(Homebrew::Vulns::Match).to receive(:new).and_return(matcher) expect(JSON.parse(capture_stdout { cmd_for("requests", "--json").run })).to eq [] @@ -1499,6 +1809,11 @@ def basis_for(evidence, ecosystem_specific = {}) allow(Homebrew::Vulns::Match).to receive(:new).and_return(matcher) Dir.mktmpdir do |dir| + File.write(File.join(dir, "BREW-requests-CVE-2024-1234.json"), JSON.generate({ + "id" => "BREW-requests-CVE-2024-1234", "database_specific" => { "source" => "matched" }, + "affected" => [{ "package" => { "ecosystem" => "Homebrew", "name" => "requests" }, + "ranges" => [{ "type" => "ECOSYSTEM", "events" => [{ "introduced" => "0" }] }] }] + })) expect { cmd_for("requests", "--output", dir, "--new-history").run } .to raise_error(TypeError, /unexpected fixed-history result: :future_result/) end @@ -1507,8 +1822,7 @@ def basis_for(evidence, ecosystem_specific = {}) it "walks history when an existing matched record has no ranges" do stub_osv_hit("CVE-2024-1234", fixed: "2.28.1") matcher = Homebrew::Vulns::Match.new - expect(matcher).to receive(:first_fixed_version).and_return("2.28.1") - expect(matcher).to receive(:first_introduced_version).and_return("2.27.0") + expect(matcher).to receive(:reconcile_history).exactly(4).times.and_return(history_result) allow(Homebrew::Vulns::Match).to receive(:new).and_return(matcher) Dir.mktmpdir do |dir| @@ -1757,13 +2071,13 @@ def basis_for(evidence, ecosystem_specific = {}) stub_osv_hit("CVE-2024-1234", fixed: "2.32.0") matcher = Homebrew::Vulns::Match.new expect(matcher).not_to receive(:first_fixed_version) - expect(matcher).to receive(:first_introduced_version).with(requests, anything, first_fixed: nil) - .and_return("2.30.0") + expect(matcher).to receive(:reconcile_history).exactly(4).times + .and_return(history_result(introduced: "2.30.0", fixed: nil)) allow(Homebrew::Vulns::Match).to receive(:new).and_return(matcher) Dir.mktmpdir do |dir| expect { cmd_for("requests", "--output", dir, "--new-history").run } - .to output(/1 history walks/).to_stdout + .to output(/4 history walks/).to_stdout path = File.join(dir, "BREW-requests-CVE-2024-1234.json") expect(JSON.parse(File.read(path)).dig("affected", 0, "ranges", 0, "events")) .to eq([{ "introduced" => "2.30.0" }]) @@ -2168,7 +2482,7 @@ def run_formula_list(contents, *extra_args, mode: "--reconcile-history") it "writes only the selected formula without overrides" do stub_osv_hit("CVE-2024-1234", fixed: "2.28.1") matcher = Homebrew::Vulns::Match.new(bulk: true) - allow(matcher).to receive_messages(first_fixed_version: "2.28.1", first_introduced_version: "2.20.0") + allow(matcher).to receive(:reconcile_history).and_return(history_result(introduced: "2.20.0")) expect(Homebrew::Vulns::Match).to receive(:new) .with(repology: nil, overrides: nil, bulk: true, strict_upstream: false).and_return(matcher) expect(Formulary).to receive(:factory).with("requests").once.and_return(requests) diff --git a/Library/Homebrew/test/vulns/match_spec.rb b/Library/Homebrew/test/vulns/match_spec.rb index d52041c04be0b..8f2ae9abef6de 100644 --- a/Library/Homebrew/test/vulns/match_spec.rb +++ b/Library/Homebrew/test/vulns/match_spec.rb @@ -912,6 +912,16 @@ def registry_hit(affected_events:, subject_version: "2.31.0", resource: nil, nam expect(record.dig(:database_specific, :confidence)).to eq "high" end + it "does not mark a shared current label fixed when its complete history is open" do + hit = registry_hit(affected_events: [{ "introduced" => "0" }, { "fixed" => "2.28.1" }]) + history = Homebrew::Vulns::Match::ReconciledHistory.new( + state: :range, reasons: [], intervals: [Homebrew::Vulns::Match::Interval.new(introduced: "2.31.0")], + ) + + expect(matcher.to_brew_record(requests, hit, history:, now:).dig(:affected, 0, :ecosystem_specific)) + .to eq(fix: nil, range_state: "affected", upstream_fixed_in: "2.28.1") + end + it "deduplicates exported evidence after removing internal provenance" do records = ["CVE-2024-1234", "GHSA-abcd"].map do |id| vulnerability = vuln( @@ -1420,18 +1430,100 @@ def resource_hit expect(results).to eq histories.transform_values { [:unresolved, nil, nil, true] } end - it "rejects a range covering an unaffected gap" do + it "reconstructs every closed affected interval" do stub_history([["2.31.0", "1.2"], ["2.30.0", "1.0"], ["2.29.0", "1.2"], ["2.28.0", "1.0"]]) - expect(matcher.reconcile_history(requests, resource_hit)) - .to have_attributes(state: :unresolved, reasons: [:unrepresentable_interval]) + expect(matcher.reconcile_history(requests, resource_hit).events).to eq [ + { introduced: "2.28.0" }, { fixed: "2.29.0" }, + { introduced: "2.30.0" }, { fixed: "2.31.0" } + ] end - it "rejects affected and unaffected builds sharing a package version" do + it "leaves the final interval open after a resource returns" do + formulae = stub_history([["2.31.0", "1.0"], ["2.30.0", nil], ["2.29.0", "1.0"]]) + + expect(matcher.reconcile_history(formulae.fetch(0), resource_hit).events).to eq [ + { introduced: "2.29.0" }, { fixed: "2.30.0" }, { introduced: "2.31.0" } + ] + end + + it "keeps a shared current label affected when an earlier bottle was affected" do + stub_history([["2.31.0", "1.2"], ["2.31.0", "1.0"], ["2.30.0", nil]]) + + expect(matcher.reconcile_history(requests, resource_hit).events).to eq [{ introduced: "2.31.0" }] + end + + it "unions equivalent version spellings before choosing boundaries" do + stub_history([["2.31.0", "1.2"], ["2.30.0", "1.0"], ["2.30", "1.2"]]) + + expect(matcher.reconcile_history(requests, resource_hit).events).to eq [ + { introduced: "2.30.0" }, { fixed: "2.31.0" } + ] + end + + it "keeps revisions in each interval boundary" do + formulae = stub_history([["2.31.0", "1.2"], ["2.30.0", "1.0"], + ["2.30.0", "1.2"], ["2.30.0", "1.0"]]) + allow(formulae.fetch(1)).to receive(:pkg_version).and_return(PkgVersion.parse("2.30.0_2")) + allow(formulae.fetch(2)).to receive(:pkg_version).and_return(PkgVersion.parse("2.30.0_1")) + + expect(matcher.reconcile_history(requests, resource_hit).events).to eq [ + { introduced: "2.30.0" }, { fixed: "2.30.0_1" }, + { introduced: "2.30.0_2" }, { fixed: "2.31.0" } + ] + end + + it "counts a shared historical label as affected until the next safe label" do stub_history([["2.31.0", "1.2"], ["2.30.0", "1.2"], ["2.30.0", "1.0"]]) - expect(matcher.reconcile_history(requests, resource_hit)) - .to have_attributes(state: :unresolved, reasons: [:unrepresentable_interval]) + expect(matcher.reconcile_history(requests, resource_hit).events).to eq [ + { introduced: "2.30.0" }, { fixed: "2.31.0" } + ] + end + + it "holds unknown observations around every boundary of split history" do + known = [["2.32.0", "1.2"], ["2.31.0", "1.0"], ["2.30.0", "1.2"], ["2.29.0", "1.0"]] + results = (0..known.length).map do |position| + stub_history([*known.take(position), nil, *known.drop(position)]) + described_class.new(repology:, cpan_sec:).reconcile_history(requests, resource_hit) + end + + expect(results).to all(have_attributes(state: :unresolved, intervals: [], reasons: [:history_unavailable])) + end + + it "holds a rollback below an unclosed later affected interval" do + current = stub_history([["2.31.0", "1.2"], ["3.0.0", "1.0"], + ["2.30.0", "1.2"], ["2.29.0", "1.0"]]).fetch(0) + + expect(matcher.reconcile_history(current, resource_hit)) + .to have_attributes(state: :unresolved, intervals: [], reasons: [:unrepresentable_interval]) + end + + it "holds an upstream alias conflict in the second affected interval" do + stub_history([["2.32.0", "1.2"], ["2.31.0", "1.1"], ["2.30.0", "1.2"], ["2.29.0", "1.0"]]) + first = resource_hit + other = vuln("id" => "GHSA-1", "aliases" => ["CVE-1"], "affected" => [ + { "package" => { "ecosystem" => "PyPI", "name" => "certifi" }, + "ranges" => [{ "type" => "ECOSYSTEM", "events" => [ + { "introduced" => "0" }, { "fixed" => "1.1" } + ] }] }, + ]) + hit = make_hit(first.vulnerability, *first.evidence, + ev(:registry, ecosystem: "PyPI", name: "certifi", subject_version: "1.2", resource: "certifi") + .with_source(other)) + + expect(matcher.reconcile_history(requests, hit)) + .to have_attributes(state: :unresolved, intervals: [], reasons: [:conflicting_upstream]) + end + + it "holds split history when an upstream override needs review" do + stub_history([["2.32.0", "1.2"], ["2.31.0", "1.0"], ["2.30.0", "1.2"], ["2.29.0", "1.0"]]) + overrides = Homebrew::Vulns::AdvisoryOverrides.new({ + "requests" => { "advisories" => { "CVE-1" => { "upstream_fixed_in" => "1.1" } } }, + }) + + expect(described_class.new(repology:, cpan_sec:, overrides:).reconcile_history(requests, resource_hit)) + .to have_attributes(state: :unresolved, intervals: [], reasons: [:upstream_override_requires_review]) end it "does not hide disagreement between upstream aliases" do @@ -1471,6 +1563,19 @@ def resource_hit .to have_attributes(state: :unresolved, reasons: [:advisory_patch_requires_review]) end + it "holds split history for an advisory patch at either fix boundary" do + results = [0, 2].map do |boundary| + formulae = stub_history([["2.31.0", "1.2"], ["2.30.0", "1.0"], + ["2.29.0", "1.2"], ["2.28.0", "1.0"]]) + allow(formulae.fetch(boundary)).to receive(:serialized_patches) + .and_return([{ "resolves" => [{ "type" => "security", "id" => "CVE-1" }] }]) + + described_class.new(repology:, cpan_sec:).reconcile_history(requests, resource_hit) + end + + expect(results).to all(have_attributes(state: :unresolved, reasons: [:advisory_patch_requires_review])) + end + it "does not block on patches after the affected interval and fix boundary" do formulae = stub_history(["2.31.0", "2.28.1", "2.28.0"]) allow(formulae.fetch(0)).to receive(:serialized_patches) @@ -1500,13 +1605,13 @@ def resource_hit .to have_attributes(state: :unresolved, reasons: [:advisory_patch_requires_review]) end - it "collects patch and interval problems independently" do + it "keeps patch review required when a shared label is affected" do formulae = stub_history([["2.31.0", "1.2"], ["2.30.0", "1.2"], ["2.30.0", "1.0"]]) allow(formulae.fetch(2)).to receive(:serialized_patches) .and_return([{ "resolves" => [{ "type" => "security", "id" => "CVE-1" }] }]) expect(matcher.reconcile_history(requests, resource_hit).reasons) - .to contain_exactly(:unrepresentable_interval, :advisory_patch_requires_review) + .to contain_exactly(:advisory_patch_requires_review) end it "preserves a reviewed range without walking history" do @@ -1643,14 +1748,14 @@ def install .to have_attributes(state: :never_affected, reasons: []) end - it "does not add an irrelevant patch reason to an unrepresentable interval" do + it "does not hold split intervals for a patch after their final fix" do formulae = stub_history([ ["2.31.0", "1.2"], ["2.30.0", "1.2"], ["2.29.0", "1.0"], ["2.28.0", "1.2"], ["2.27.0", "1.0"] ]) allow(formulae.fetch(0)).to receive(:serialized_patches) .and_return([{ "resolves" => [{ "type" => "security", "id" => "CVE-1" }] }]) - expect(matcher.reconcile_history(requests, resource_hit).reasons).to eq [:unrepresentable_interval] + expect(matcher.reconcile_history(requests, resource_hit).reasons).to eq [] end end diff --git a/Library/Homebrew/test/vulns/osv_export_spec.rb b/Library/Homebrew/test/vulns/osv_export_spec.rb index 8bab589480442..e044804b49b31 100644 --- a/Library/Homebrew/test/vulns/osv_export_spec.rb +++ b/Library/Homebrew/test/vulns/osv_export_spec.rb @@ -55,6 +55,42 @@ end describe ".merge_existing" do + it "repairs split initial history without truncating it or closing reviewed open ranges" do + Dir.mktmpdir do |dir| + path = File.join(dir, "BREW-x-CVE-1.json") + reviewed = [{ "introduced" => "2.5" }] + File.write(path, JSON.generate({ + "id" => "BREW-x-CVE-1", "affected" => [{ "ranges" => [ + { "type" => "ECOSYSTEM", "events" => reviewed }, + { "type" => "ECOSYSTEM", "events" => [] }, + ] }] + })) + events = [{ introduced: "1.0" }, { fixed: "2.0" }, { introduced: "3.0" }] + incoming = { id: "BREW-x-CVE-1", affected: [{ ranges: [{ type: "ECOSYSTEM", events: }] }] } + merged = described_class.merge_existing(path, incoming, close_open_ranges: true, initial_introduction: true) + + expect(JSON.parse(JSON.generate(merged))&.dig("affected", 0, "ranges")).to eq [ + { "type" => "ECOSYSTEM", "events" => reviewed }, + { "type" => "ECOSYSTEM", "events" => JSON.parse(JSON.generate(events)) }, + ] + end + end + + it "does not reduce a complete incoming history to one reviewed-range transition" do + Dir.mktmpdir do |dir| + path = File.join(dir, "BREW-x-CVE-1.json") + File.write(path, JSON.generate({ + "id" => "BREW-x-CVE-1", "affected" => [{ "ranges" => [{ "type" => "ECOSYSTEM", "events" => [ + { "introduced" => "0" }, { "fixed" => "1.0" } + ] }] }] + })) + events = [{ introduced: "1.5" }, { fixed: "2.0" }, { introduced: "3.0" }] + incoming = { id: "BREW-x-CVE-1", affected: [{ ranges: [{ type: "ECOSYSTEM", events: }] }] } + + expect(described_class.merge_existing(path, incoming, close_open_ranges: true)).to be_nil + end + end + it "repairs an unreviewed range with a nonzero initial introduction" do Dir.mktmpdir do |dir| path = File.join(dir, "BREW-x-CVE-1.json") diff --git a/Library/Homebrew/vulns/match.rb b/Library/Homebrew/vulns/match.rb index a4c936f50e190..d3f0c08c88de9 100644 --- a/Library/Homebrew/vulns/match.rb +++ b/Library/Homebrew/vulns/match.rb @@ -44,13 +44,88 @@ class Match T::Hash[Symbol, String], ) - # A single-platform history decision, not permission to replace a stored - # record. Callers must also validate its provenance and other platforms. + # An affected Homebrew interval. A missing fix leaves the interval open. + class Interval < T::Struct + const :introduced, String + const :fixed, T.nilable(String) + + sig { returns(T::Boolean) } + def valid? + return false if introduced.blank? + + introduced_version = PkgVersion.parse(introduced) + fixed = self.fixed + return true if fixed.nil? + + fixed.present? && introduced_version < PkgVersion.parse(fixed) + rescue ArgumentError + false + end + + sig { returns(T::Array[T::Hash[Symbol, String]]) } + def events + events = T.let([{ introduced: }], T::Array[T::Hash[Symbol, String]]) + fixed = self.fixed + events << { fixed: } if fixed + events + end + + # Union copies and platforms before choosing Homebrew label boundaries. + sig { params(intervals: T::Array[Interval]).returns(T::Array[Interval]) } + def self.union(intervals) + merged = T.let([], T::Array[Interval]) + intervals.sort_by { |interval| PkgVersion.parse(interval.introduced) } + .each_with_object(merged) do |interval, merged| + previous = merged.last + if previous.nil? + merged << interval + elsif (fixed = previous.fixed) + if PkgVersion.parse(interval.introduced) > PkgVersion.parse(fixed) + merged << interval + else + last_fixed = interval.fixed + last_fixed = fixed if last_fixed && PkgVersion.parse(last_fixed) < PkgVersion.parse(fixed) + merged[-1] = previous.with(fixed: last_fixed) + end + end + end + end + end + + # A history decision, not permission to replace a stored record. Callers + # must also validate its provenance and observe every required platform. class ReconciledHistory < T::Struct const :state, Symbol - const :introduced, T.nilable(String) - const :fixed, T.nilable(String) + const :intervals, T::Array[Interval], default: [] const :reasons, T::Array[Symbol] + + sig { returns(T.nilable(String)) } + def introduced = intervals.first&.introduced + + sig { returns(T.nilable(String)) } + def fixed = intervals.last&.fixed + + sig { returns(T::Array[T::Hash[Symbol, String]]) } + def events = intervals.flat_map(&:events) + + sig { params(results: T::Array[ReconciledHistory]).returns(ReconciledHistory) } + def self.combine(results) + reasons = results.flat_map(&:reasons) + reasons << :preserved if results.any? { |result| result.state == :preserved } + if (results.empty? || results.any? do |result| + [:range, :never_affected].exclude?(result.state) + end) && reasons.empty? + reasons << :unsupported_result + end + ranges = results.select { |result| result.state == :range } + if ranges.any? { |result| result.intervals.empty? || result.intervals.any? { |interval| !interval.valid? } } + reasons << :invalid_interval + end + return new(state: :unresolved, reasons: reasons.uniq.sort) if reasons.any? + + intervals = Interval.union(ranges.flat_map(&:intervals)) + new(state: intervals.empty? ? :never_affected : :range, intervals:, reasons: []) + end end class HistoricalObservation < T::Struct @@ -705,21 +780,28 @@ def evidence_range_status(evidence, subject_version) # `affected? == true` (or no comparable range) emits no `fixed` event and # `fix: null`. As with {OsvExport.record_for}, {OsvExport.merge_existing} # preserves on-disk `ranges` on rewrite so a hand-corrected boundary - # sticks. + # sticks. With `history`, emit all reconstructed intervals and derive + # fix metadata from the last interval, including a final open interval. sig { params(formula: Formula, hit: Hit, first_fixed: T.nilable(String), - first_introduced: T.nilable(String), now: Time) + first_introduced: T.nilable(String), history: T.nilable(ReconciledHistory), now: Time) .returns(T::Hash[Symbol, T.untyped]) } - def to_brew_record(formula, hit, first_fixed: nil, first_introduced: nil, now: Time.now.utc) + def to_brew_record(formula, hit, first_fixed: nil, first_introduced: nil, history: nil, now: Time.now.utc) vuln = hit.vulnerability timestamp = now.strftime("%Y-%m-%dT%H:%M:%SZ") status, status_evidence = range_status(hit, formula_name: formula.name) - fixed = first_fixed - fixed ||= formula.pkg_version.to_s if status&.fixed? - events = T.let([{ introduced: first_introduced || "0" }], T::Array[T::Hash[Symbol, String]]) - events << { fixed: } if fixed + if history && (history.state != :range || history.intervals.empty? || + history.intervals.any? { |interval| !interval.valid? }) + raise ArgumentError, "Candidate history must contain valid affected intervals" + end + + fixed = history ? history.fixed : first_fixed + fixed ||= formula.pkg_version.to_s if !history && status&.fixed? + events = T.let(history ? history.events : [{ introduced: first_introduced || "0" }], + T::Array[T::Hash[Symbol, String]]) + events << { fixed: } if !history && fixed record = T.let({ schema_version: OsvExport::SCHEMA_VERSION, @@ -736,6 +818,10 @@ def to_brew_record(formula, hit, first_fixed: nil, first_introduced: nil, now: T }, }, T::Hash[Symbol, T.untyped]) + if history + record.fetch(:affected).fetch(0).fetch(:ecosystem_specific)[:range_state] = fixed ? "fixed" : "affected" + end + if status.nil? && current_prerelease_boundary?(hit) record[:database_specific][:review_reason] = "prerelease_boundary" end @@ -855,9 +941,9 @@ def first_fixed_version(formula, hit) result || :never_affected end - # Reconstruct one representable affected interval from complete history. - # Callers may only reconcile matched records; generated patch-fix records - # must retain their annotation-based ranges. + # Reconstruct every representable affected interval from complete history. + # Callers can emit new candidates or reconcile matched records; generated + # patch-fix records must retain their annotation-based ranges. # Unlike first_fixed_version, this must visit builds before the first # transition, including below-introduced versions and removed resources. # An unresolved or protected result never supplies replacement boundaries. @@ -889,30 +975,34 @@ def reconcile_history(formula, hit, require_resource_presence: false) reasons << :upstream_override_requires_review end - affected = observations.select { |row| row.state == :affected }.map(&:version) - unaffected = observations.select { |row| [:fixed, :not_applicable].include?(row.state) }.map(&:version) - introduced = affected.min - last_affected = affected.max - fixed = T.let(nil, T.nilable(PkgVersion)) - if introduced && last_affected - fixed = unaffected.select { |version| version > last_affected }.min - if unaffected.any? { |version| version.between?(introduced, last_affected) } - reasons << :unrepresentable_interval + intervals = T.let([], T::Array[Interval]) + observations.sort_by(&:version).chunk_while { |left, right| left.version == right.version }.each do |builds| + version = builds.fetch(0).version + previous = intervals.last + if builds.any? { |build| build.state == :affected } + intervals << Interval.new(introduced: version.to_s) if previous.nil? || previous.fixed + elsif previous && previous.fixed.nil? + intervals[-1] = previous.with(fixed: version.to_s) end - reasons << :unrepresentable_interval if fixed.nil? && observations.fetch(0).state != :affected + end + if intervals.last&.fixed.nil? && intervals.any? && + observations.none? { |row| row.version == formula.pkg_version && row.state == :affected } + reasons << :unrepresentable_interval end if observations.any? do |row| - row.patched && (!introduced || - (row.version >= introduced && (!fixed || row.version <= fixed))) + row.patched && (intervals.empty? || intervals.any? do |interval| + fixed = interval.fixed + row.version >= PkgVersion.parse(interval.introduced) && (!fixed || row.version <= PkgVersion.parse(fixed)) + end) end reasons << :advisory_patch_requires_review end return ReconciledHistory.new(state: :unresolved, reasons: reasons.uniq.sort) if reasons.any? - return ReconciledHistory.new(state: :never_affected, reasons: []) unless introduced + return ReconciledHistory.new(state: :never_affected, reasons: []) if intervals.empty? - ReconciledHistory.new(state: :range, introduced: introduced.to_s, fixed: fixed&.to_s, reasons: []) + ReconciledHistory.new(state: :range, intervals:, reasons: []) end # Preserve disagreements within an upstream alias family and unknown diff --git a/Library/Homebrew/vulns/osv_export.rb b/Library/Homebrew/vulns/osv_export.rb index 63503abec440a..070b2e59d7d1e 100644 --- a/Library/Homebrew/vulns/osv_export.rb +++ b/Library/Homebrew/vulns/osv_export.rb @@ -120,9 +120,17 @@ def self.merge_existing(path, record, close_open_ranges: false, initial_introduc record[:published] = existing_published end affected_records = Array(record[:affected]) + # Complete histories are not single boundary transitions. Only an + # initial repair may copy their events into unreviewed ranges. + if close_open_ranges && !initial_introduction && + affected_records.any? { |affected| multiple_intervals?(affected[:ranges]) } + return + end + invalid_transition = affected_records.each_with_index.any? do |affected, index| existing_ranges = existing.dig("affected", index, "ranges") next false unless existing_ranges + next false if initial_introduction && multiple_intervals?(affected[:ranges]) close_open_ranges && ( ((fixed = explicit_fixed(affected[:ranges])) && !fixed_follows?(existing_ranges, fixed)) || @@ -244,6 +252,17 @@ def self.merge_range_transitions(existing_ranges, incoming_ranges, initial_intro return incoming_ranges if !existing_ranges.is_a?(Array) || existing_ranges.empty? incoming_ranges = Array(incoming_ranges) + if initial_introduction && multiple_intervals?(incoming_ranges) + return existing_ranges.map.with_index do |range, index| + next range if range_state(range) != :invalid + + replacement = incoming_ranges[index] || incoming_ranges.first + next range unless replacement + next replacement unless range.is_a?(Hash) + + range.merge("events" => replacement[:events] || replacement["events"]) + end + end fixed = incoming_ranges.filter_map do |range| Array(range[:events] || range["events"]).rfind do |event| event.is_a?(Hash) && (event.key?(:fixed) || event.key?("fixed")) @@ -306,6 +325,18 @@ def self.merge_range_transitions(existing_ranges, incoming_ranges, initial_intro end private_class_method :merge_range_transitions + sig { params(ranges: T.untyped).returns(T::Boolean) } + def self.multiple_intervals?(ranges) + Array(ranges).sum do |range| + next 0 unless range.is_a?(Hash) + + Array(range[:events] || range["events"]).count do |event| + event.is_a?(Hash) && (event.key?(:introduced) || event.key?("introduced")) + end + end > 1 + end + private_class_method :multiple_intervals? + sig { params(range: T.untyped).returns(Symbol) } def self.range_state(range) return :invalid unless range.is_a?(Hash) diff --git a/docs/Advisory-Matching.md b/docs/Advisory-Matching.md index 70e17a91e28dd..8c882b64726a5 100644 --- a/docs/Advisory-Matching.md +++ b/docs/Advisory-Matching.md @@ -1,5 +1,5 @@ --- -last_review_date: "2026-09-16" +last_review_date: "2026-10-09" --- # Advisory Matching @@ -10,12 +10,17 @@ Candidates still need [human review](https://github.com/Homebrew/advisory-databa ## Range accuracy and coverage -With history enabled, a candidate with a comparable current state and no reviewed range must have an affected interval that can be established from formula history. +With history enabled, a candidate with a comparable current state and no reviewed range must have its complete affected intervals established from formula history on the latest supported macOS and Linux for both ARM and Intel. This applies even when the current formula is known to be affected: knowing today's state does not establish when the affected range began. An `introduced: "0"` boundary asserts that every earlier version is affected; it is not a marker for an unknown introduction. Automatic matching deliberately favours range accuracy over coverage. -Unreadable or uncomparable history and disjoint affected intervals can prevent a new record from being emitted. +Unreadable or uncomparable history can prevent a new record from being emitted. +Disjoint affected runs produce separate introduced/fixed pairs, including an open final interval when the package is still affected. +Known intervals are unioned across platforms; a history or provenance failure on any platform holds the whole candidate. +Additional platform views are queried once per formula for its new candidates, and a held candidate stops visiting further platforms. +For a resource-only hit absent from another platform's current formula, the original upstream constraints are checked against that platform's complete history. +Absence today alone does not establish historical absence; unreadable history, unknown resource identities and missing query results for present resources still require review. The command warns and counts these as history-unavailable skips instead of inventing a boundary. This includes a historical subject whose prerelease suffix changes its `SEMVER` range state compared with its release version. A skip does not mean the formula is unaffected, so an ingest run can omit a currently affected formula and is not evidence of complete vulnerability coverage. @@ -24,8 +29,10 @@ Reviewers must establish the ranges and matching provenance together from upstre For a new record, a `pkg_version` shared by affected and unaffected builds is conservatively included in the affected interval, because the advisory cannot distinguish those builds by version. This can happen when a resource changes without a formula revision bump. For the same reason, a fix that shares its `pkg_version` with affected builds is recorded at the next version with only fixed builds, and a range stays unchanged while the current version also has an affected build. -A distinct unaffected version inside the interval still requires review. -Existing-range reconciliation and reintroduction checks retain their stricter rules for mixed builds, so reconciliation reports a record whose `introduced` version has mixed builds as an unrepresentable interval and leaves it unchanged. +A distinct unaffected label closes the preceding interval; a later affected label opens another. +Complete-history reconciliation applies the same any-affected-build rule. +Unknown builds, conflicting upstream constraints and rollbacks with no representable fix still require review. +Existing daily boundary-transition checks retain their conservative guards. Current-version prerelease ambiguity produces an uncomparable review lead with no `range_state`, reduced confidence and `database_specific.review_reason: "prerelease_boundary"`. Ingest drops these leads and lists their IDs by reason in its run summary. @@ -45,11 +52,15 @@ It is an unchecked authoring mode, not a way to validate a skipped candidate. ## Reconciling existing ranges -`--reconcile-history --output advisories --overrides data/overrides.yml` explicitly revisits existing `source: matched`, bump-fixed records with one terminal interval. -It requires complete formula history and matching provenance, with the same result on the latest supported macOS and Linux for both ARM and Intel. +`--reconcile-history --output advisories --overrides data/overrides.yml` explicitly revisits existing `source: matched`, bump-fixed records with terminal introduced/fixed intervals. +It requires complete formula history and matching provenance, on the latest supported macOS and Linux for both ARM and Intel. +Known affected intervals are unioned across those platforms; an unaffected platform cannot cancel another platform’s affected builds. These simulations check declared formula and resource versions; they do not rebuild historical bottles or verify installation steps. -It can narrow an affected interval or delete a record proven never affected; it preserves other fields and updates `modified` only for a range change. -Generated records, patch fixes and open or multiple intervals are left unchanged. +It can narrow or split affected intervals or delete a record proven never affected on every platform; it preserves other fields and updates `modified` only for a range change. +Generated records, patch fixes, open intervals and unsupported range shapes are left unchanged. +Reviewed closed multi-interval records can be narrowed, but a proposed interval must remain entirely inside the reviewed affected set. +Reconciliation cannot fill reviewed safe gaps, expand coverage or reopen a terminal range. +Initial repairs preserve complete split histories in unreviewed ranges without replacing reviewed intervals. For records missed by current matching, complete registry-resource evidence can supply versionless upstream queries even after a resource has been removed. The fresh results must reproduce the stored upstream family and range basis, and every resource identity must occur in formula history on every platform. Stored subject versions only reproduce the range basis; historical decisions use versions from the formula sources. @@ -61,7 +72,7 @@ The command reports skip reasons and the number of matched records it did not re Before running this mode, protect hand-reviewed Homebrew boundaries with `preserve_homebrew_ranges: true` under the formula and advisory in `overrides.yml`. The pin applies across advisory aliases and is separate from `upstream_fixed_in`, which describes the upstream fix. -Unavailable history or upstream records, changed subjects, conflicting platform results and advisory-specific patches require review. +Unavailable history or upstream records, changed subjects, unresolved platform results and advisory-specific patches require review. A failed upstream lookup holds every record for that formula with `upstream_unavailable`; a failed batch query holds its entire formula batch. A confirmed HTTP 404 while following an upstream link is cached as a missing target; a record with no resolved targets keeps its original evidence. A 404 for a supplemental CPANSA CVE lookup uses the CPANSA evidence for that CVE, including on subsequent platform passes.