Skip to content

Share analytics queries and package downloads #24350

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MikeMcQuaid merged 1 commit into from
Oct 24, 2026
Merged

Share analytics queries and package downloads #24350

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
393 changes: 7 additions & 386 deletions Library/Homebrew/dev-cmd/formula-analytics.rb
View file
Open in desktop

Large diffs are not rendered by default.

50 changes: 50 additions & 0 deletions Library/Homebrew/github_packages.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,56 @@ class GitHubPackages
private_constant :URL_PREFIX
private_constant :DOCKER_PREFIX

class Downloads < T::Struct
const :source, String
const :all_time, Integer
const :last_30_days, Integer
const :start_date, String
const :end_date, String
const :fetched_at, String
end

# GitHub's package overview includes exact totals and daily chart values.
# Cache successful snapshots per UTC day for CLI and API generation callers.
sig { params(formula_name: String).returns(T.nilable(Downloads)) }
def self.downloads(formula_name)
cache = HOMEBREW_CACHE/"github-packages-downloads/#{formula_name}.json"
if cache.exist?
begin
cached = Downloads.from_hash(JSON.parse(cache.read))
return cached if cached.fetched_at.start_with?(Time.now.utc.strftime("%F"))
rescue JSON::ParserError, TypeError, ArgumentError
nil
end
end

url = "https://github.com/Homebrew/homebrew-core/pkgs/container/core%2F" \
"#{image_formula_name(formula_name).gsub("/", "%2F")}"
output = Utils::Curl.curl_output("--fail", "--compressed", url)
return unless output.success?

total = output.stdout[%r{Total downloads</span>\s*<h3[^>]*title="(\d+)"}, 1]
chart = output.stdout[%r{aria-label="Downloads for the last 30 days">.*?</svg>}m]
return if total.nil? || chart.nil?

bars = chart.scan(/<rect\b[^>]*>/m).flatten
daily = bars.filter_map do |bar|
date = bar[/\bdata-date="(\d{4}-\d{2}-\d{2})"/, 1]
count = bar[/\bdata-merge-count="(\d+)"/, 1]
next if date.nil? || count.nil?

[date, count.to_i]
end.to_h
return if daily.empty? || daily.length != bars.length

dates = daily.keys.sort
downloads = Downloads.new(source: url, all_time: total.to_i, last_30_days: daily.values.sum,
start_date: dates.fetch(0), end_date: dates.fetch(-1), fetched_at: Time.now.utc.iso8601)
cache.dirname.mkpath
cache.atomic_write(JSON.generate(downloads.serialize))
downloads
end

URL_REGEX = %r{(?:#{Regexp.escape(URL_PREFIX)}|#{Regexp.escape(DOCKER_PREFIX)})([\w-]+)/([\w-]+)}

# Valid OCI tag characters
Expand Down
152 changes: 0 additions & 152 deletions Library/Homebrew/test/dev-cmd/formula-analytics_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,159 +3,7 @@

require "cmd/shared_examples/args_parse"
require "dev-cmd/formula-analytics"
require "json"
require "utils/analytics"

RSpec.describe Homebrew::DevCmd::FormulaAnalytics do
it_behaves_like "parseable arguments"

describe "#format_os_version_dimension" do
it "formats retired macOS versions" do
expect(described_class.new([]).format_os_version_dimension("10.15.7"))
.to eq("macOS Catalina (10.15)")
end

it "normalizes Big Sur compatibility versions" do
expect(described_class.new([]).format_os_version_dimension("10.16.0"))
.to eq("macOS Big Sur (11)")
end

it "formats retired OS X versions" do
expect(described_class.new([]).format_os_version_dimension("10.11.6"))
.to eq("OS X El Capitan (10.11)")
end

it "formats unknown legacy macOS versions" do
expect(described_class.new([]).format_os_version_dimension("10.10.5")).to eq("OS X 10.10")
end

it "formats unknown future macOS versions" do
expect(described_class.new([]).format_os_version_dimension("28.0")).to eq("macOS 28")
end

it "does not format malformed macOS versions" do
expect(described_class.new([]).format_os_version_dimension("10.15.invalid")).to eq("10.15.invalid")
end

it "formats Debian point releases as major releases" do
expect(described_class.new([]).format_os_version_dimension("Debian GNU/Linux 12.11")).to eq("Debian 12")
end

it "formats Debian major releases" do
expect(described_class.new([]).format_os_version_dimension("Debian GNU/Linux 12")).to eq("Debian 12")
end

it "preserves WSL in formatted Linux versions" do
expect(described_class.new([]).format_os_version_dimension(
"Ubuntu 24.04.3 LTS#{Utils::Analytics::WSL_SUFFIX}",
)).to eq("Ubuntu 24.04 LTS#{Utils::Analytics::WSL_SUFFIX}")
end
end

describe "#influx_analytics" do
it "ranks sampled environment configurations by non-default use" do
ENV.delete("HOMEBREW_NO_ANALYTICS")
ENV["HOMEBREW_INFLUXDB_TOKEN"] = "token"
records = [
{ "env_config" => "HOMEBREW_BAT", "env_config_state" => "non_default", "count" => 2 },
{ "env_config" => "HOMEBREW_BAT", "env_config_state" => "default", "count" => 3 },
{ "env_config" => "HOMEBREW_BAT", "env_config_state" => "unset", "count" => 5 },
{ "env_config" => "HOMEBREW_NO_AUTO_UPDATE", "env_config_state" => "non_default", "count" => 1 },
{ "env_config" => "HOMEBREW_NO_AUTO_UPDATE", "env_config_state" => "unset", "count" => 1 },
{ "env_config" => "HOMEBREW_MAKE_JOBS", "env_config_state" => "default", "count" => 4 },
{ "env_config" => "HOMEBREW_TOTALLY_MADE_UP", "env_config_state" => "non_default", "count" => 100 },
{ "env_config" => "HOMEBREW_BAT", "env_config_state" => "borked", "count" => 50 },
]
queries = []
command = described_class.new(["--homebrew-env-config", "--json"])
allow(command).to receive(:each_influx_record) do |query, &block|
queries << query
records.each(&block)
end
expected = {
category: :homebrew_env_config,
total_items: 3,
start_date: Date.today - 30,
end_date: Date.today,
total_count: 16,
items: [
{
number: 1, env_config: "HOMEBREW_NO_AUTO_UPDATE", count: "2", non_default_count: "1",
set_default_count: "0", unset_count: "1", percent: "50", default_value: nil
},
{
number: 2, env_config: "HOMEBREW_BAT", count: "10", non_default_count: "2",
set_default_count: "3", unset_count: "5", percent: "20", default_value: nil
},
{
number: 3, env_config: "HOMEBREW_MAKE_JOBS", count: "4", non_default_count: "0",
set_default_count: "4", unset_count: "0", percent: "0",
default_value: "The number of available CPU cores."
},
],
}

expect { command.influx_analytics(command.args) }
.to output("#{JSON.pretty_generate(expected)}\n").to_stdout
expect(queries).to contain_exactly(
match(/FROM "command_run".*env_config_state IS NOT NULL GROUP BY/).and(
include('GROUP BY "env_config","env_config_state"'),
),
)
end
end

describe "#each_influx_record" do
it "streams the JSON request to the bridge script and parses JSON lines" do
command = described_class.new([])
bridge = mktmpdir/"fake-python"
bridge.write "#!/bin/sh\ncat\n"
bridge.chmod 0755
allow(command).to receive(:venv_python).and_return(bridge)
records = []

command.each_influx_record("SELECT 1") { |record| records << record }

expect(records).to eq [{
"host" => "analytics.brew.sh",
"org" => Utils::Analytics::INFLUX_ORG,
"database" => Utils::Analytics::INFLUX_BUCKET,
"query" => "SELECT 1",
}]
end

it "reports unauthenticated bridge errors as a token problem" do
command = described_class.new([])
bridge = mktmpdir/"fake-python"
bridge.write <<~SH
#!/bin/sh
cat >/dev/null
echo "pyarrow.flight.FlightUnauthenticatedError: message: unauthenticated" >&2
exit 1
SH
bridge.chmod 0755
allow(command).to receive(:venv_python).and_return(bridge)

expect { command.each_influx_record("SELECT 1") { nil } }
.to raise_error(SystemExit)
.and output(/Could not authenticate with InfluxDB/).to_stderr
end

it "reports other bridge failures with their standard error output" do
command = described_class.new([])
bridge = mktmpdir/"fake-python"
bridge.write <<~SH
#!/bin/sh
cat >/dev/null
echo "Traceback: boom" >&2
exit 1
SH
bridge.chmod 0755
allow(command).to receive(:venv_python).and_return(bridge)

expect { command.each_influx_record("SELECT 1") { nil } }
.to raise_error(SystemExit)
.and output(/InfluxDB query failed:\nTraceback: boom/).to_stderr
end
end
end
46 changes: 46 additions & 0 deletions Library/Homebrew/test/github_packages_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,52 @@
require "github_packages"

RSpec.describe GitHubPackages do
describe ".downloads" do
before do
allow(Time).to receive(:now).and_return(Time.utc(2026, 10, 23, 12))
allow(Utils::Curl).to receive(:curl_output).and_return(
instance_double(SystemCommand::Result, success?: true, stdout: <<~HTML),
<span>Total downloads</span><h3 title="22776486">22.8M</h3>
<div aria-label="Downloads for the last 30 days"><svg>
<rect data-merge-count="1234" data-date="2026-10-23"></rect>
<rect data-date="2026-10-22" data-merge-count="2345"></rect>
</svg></div>
HTML
)
end

it "reads exact package totals instead of rounded totals or recent tags" do
expect(described_class.downloads("wget")&.serialize).to include(
"all_time" => 22_776_486, "last_30_days" => 3579,
"start_date" => "2026-10-22", "end_date" => "2026-10-23",
"fetched_at" => "2026-10-23T12:00:00Z"
)
end

it "reuses a successful snapshot on the same UTC day" do
described_class.downloads("wget")
expect(Utils::Curl).not_to receive(:curl_output)

described_class.downloads("wget")
end

it "refreshes a snapshot on the next UTC day" do
described_class.downloads("wget")
allow(Time).to receive(:now).and_return(Time.utc(2026, 10, 24))
expect(Utils::Curl).to receive(:curl_output).once

described_class.downloads("wget")
end

it "does not interpret missing chart data as zero downloads" do
allow(Utils::Curl).to receive(:curl_output).and_return(
instance_double(SystemCommand::Result, success?: true, stdout: '<h3 title="100">100</h3>'),
)

expect(described_class.downloads("wget")).to be_nil
end
end

describe "#upload_bottles" do
it "reports progress when uploading many bottles" do
github_packages = described_class.new
Expand Down
Loading
Loading