diff --git a/Library/Homebrew/cmd/update.sh b/Library/Homebrew/cmd/update.sh index 5127022b42f2e..f9c3d2463b70f 100644 --- a/Library/Homebrew/cmd/update.sh +++ b/Library/Homebrew/cmd/update.sh @@ -443,7 +443,16 @@ fetch_api_file() { echo "Checking if we need to fetch ${filename}..." fi - local arg curl_exit_code json_url last_json_url + local etag_path new_etag_path + local -a etag_args=() + if api_curl_supports_etag + then + etag_path="${cache_path}.etag" + new_etag_path="${cache_path}.etag.new" + etag_args=(--etag-save "${new_etag_path}" --write-out "%{http_code}") + fi + + local arg curl_exit_code http_code json_url last_json_url local -a time_cond while read -r json_url do @@ -451,20 +460,33 @@ fetch_api_file() { while read -r arg do time_cond+=("${arg}") - done < <(api_time_cond_args "${cache_path}") - api_curl_download "${json_url}" "${cache_path}" "${time_cond[@]}" + done < <(api_time_cond_args "${cache_path}" "${etag_path}") + [[ -n "${new_etag_path}" ]] && rm -f "${new_etag_path}" + http_code="$(api_curl_download "${json_url}" "${cache_path}" "${etag_args[@]}" "${time_cond[@]}")" curl_exit_code=$? # A conditional request can fail with a receive error (curl exit code 56) when # an unconditional request for the same URL succeeds, so retry exactly once. if [[ ${curl_exit_code} -eq 56 ]] && [[ ${#time_cond[@]} -gt 0 ]] then - api_curl_download "${json_url}" "${cache_path}" + http_code="$(api_curl_download "${json_url}" "${cache_path}" "${etag_args[@]}")" curl_exit_code=$? fi last_json_url="${json_url}" [[ ${curl_exit_code} -eq 0 ]] && break done < <(api_urls "${filename}") + if [[ -n "${new_etag_path}" ]] + then + if [[ ${curl_exit_code} -eq 0 && -s "${new_etag_path}" ]] + then + mv -f "${new_etag_path}" "${etag_path}" + elif [[ ${curl_exit_code} -eq 0 && "${http_code}" != "304" ]] + then + rm -f "${etag_path}" + fi + rm -f "${new_etag_path}" + fi + if [[ -n ${is_formula_file} ]] && [[ -f "${api_cache}/formula_names.txt" ]] then mv -f "${api_cache}/formula_names.txt" "${api_cache}/formula_names.before.txt" @@ -1078,6 +1100,7 @@ EOS "${HOMEBREW_CACHE}/api/internal/packages.$(bottle_tag).jws.json") ;; "${HOMEBREW_CACHE}/api/internal/packages.$(bottle_tag).jws.json.payload") ;; "${HOMEBREW_CACHE}/api/internal/packages.$(bottle_tag).jws.json.payload.index") ;; + "${HOMEBREW_CACHE}/api/internal/packages.$(bottle_tag).jws.json.etag") ;; *) rm -f "${f}" ;; esac done diff --git a/Library/Homebrew/test/cmd/update_spec.rb b/Library/Homebrew/test/cmd/update_spec.rb index a0222953fa43f..dbdcd1cfd2e04 100644 --- a/Library/Homebrew/test/cmd/update_spec.rb +++ b/Library/Homebrew/test/cmd/update_spec.rb @@ -130,6 +130,7 @@ def setup_update_utils "HOMEBREW_CURL_SPEED_TIME" => "5", "HOMEBREW_LIBRARY" => (test_root/"Library").to_s, "HOMEBREW_USER_AGENT_CURL" => "Homebrew/test", + "API_CURL_SUPPORTS_ETAG" => "0", }, ) @@ -138,6 +139,141 @@ def setup_update_utils ) end + describe "API ETag revalidation" do + let(:cache_path) { test_root/"cache/api/formula.jws.json" } + let(:etag_path) { test_root/"cache/api/formula.jws.json.etag" } + let(:new_etag_path) { test_root/"cache/api/formula.jws.json.etag.new" } + let(:args_file) { test_root/"curl-args.txt" } + let(:update_failed_file) { test_root/"update_failed.txt" } + + def fetch_with_curl_response(status_code:, body: "", etag: "", saved_etag: '"old"', curl_supports_etag: "1") + setup_update_utils + cache_path.dirname.mkpath + cache_path.write "cached" + etag_path.write saved_etag if saved_etag + + _stdout, stderr, status = run_update_shell( + <<~SH, + source "#{test_root}/Library/Homebrew/utils.sh" + source "#{update_script}" + curl() { + if [[ "$1" == "--version" ]] + then + echo "curl 7.67.0 (x86_64-apple-darwin) libcurl/7.67.0" + return + fi + printf '%s\\n' "$@" >> "#{args_file}" + local output etag_save + while [[ $# -gt 0 ]] + do + case "$1" in + --output) output="$2"; shift ;; + --etag-save) etag_save="$2"; shift ;; + esac + shift + done + [[ -n "${etag_save}" ]] && printf '%s' '#{etag}' > "${etag_save}" + [[ -n '#{body}' ]] && printf '%s' '#{body}' > "${output}" + [[ -n "${etag_save}" ]] && printf '%s' '#{status_code}' + return 0 + } + fetch_api_file formula.jws.json "#{update_failed_file}" + SH + { + "HOMEBREW_API_DEFAULT_DOMAIN" => "https://formulae.example/api", + "HOMEBREW_API_DOMAIN" => nil, + "HOMEBREW_CACHE" => (test_root/"cache").to_s, + "HOMEBREW_CURL_SPEED_LIMIT" => "100", + "HOMEBREW_CURL_SPEED_TIME" => "5", + "HOMEBREW_LIBRARY" => (test_root/"Library").to_s, + "HOMEBREW_USER_AGENT_CURL" => "Homebrew/test", + "API_CURL_SUPPORTS_ETAG" => curl_supports_etag, + }, + ) + + conditional_flags = %w[--etag-compare --etag-save --time-cond] + { + success: status.success? && stderr.empty? && !update_failed_file.exist?, + curl_args: args_file.read.lines(chomp: true).each_cons(2).filter_map do |flag, value| + [flag, value] if conditional_flags.include?(flag) + end, + body: cache_path.read, + etag: (etag_path.read if etag_path.exist?), + temp_etag: new_etag_path.exist?, + } + end + + it "revalidates with the saved ETag and saves the new one with the new body" do + expect(fetch_with_curl_response(status_code: 200, body: "fresh", etag: '"new"')).to eq( + success: true, + curl_args: [ + ["--etag-save", new_etag_path.to_s], + ["--etag-compare", etag_path.to_s], + ], + body: "fresh", + etag: '"new"', + temp_etag: false, + ) + end + + it "keeps the saved ETag when the server answers 304" do + expect(fetch_with_curl_response(status_code: 304).except(:curl_args)).to eq( + success: true, body: "cached", etag: '"old"', temp_etag: false, + ) + end + + it "drops the saved ETag after a 200 that carries none" do + expect(fetch_with_curl_response(status_code: 200, body: "fresh").except(:curl_args)).to eq( + success: true, body: "fresh", etag: nil, temp_etag: false, + ) + end + + it "falls back to the time condition without a saved ETag" do + expect(fetch_with_curl_response(status_code: 200, body: "fresh", etag: '"new"', saved_etag: nil)).to eq( + success: true, + curl_args: [ + ["--etag-save", new_etag_path.to_s], + ["--time-cond", cache_path.to_s], + ], + body: "fresh", + etag: '"new"', + temp_etag: false, + ) + end + + it "uses only the time condition when curl is older than 7.68.0" do + expect(fetch_with_curl_response(status_code: 200, body: "fresh", curl_supports_etag: nil)).to eq( + success: true, + curl_args: [["--time-cond", cache_path.to_s]], + body: "fresh", + etag: '"old"', + temp_etag: false, + ) + end + end + + it "detects whether curl supports ETag revalidation" do + setup_update_utils + + stdout, stderr, status = run_update_shell( + <<~SH, + source "#{test_root}/Library/Homebrew/utils.sh" + source "#{update_script}" + for version in 7.41.0 7.67.0 7.68.0 8.7.1 + do + unset API_CURL_SUPPORTS_ETAG + curl() { echo "curl ${version} (x86_64-apple-darwin) libcurl/${version}"; } + api_curl_supports_etag && echo "${version}: yes" || echo "${version}: no" + done + SH + { "HOMEBREW_LIBRARY" => (test_root/"Library").to_s }, + ) + + expect([status.success?, stderr, stdout]).to eq( + [true, "", "7.41.0: no\n7.67.0: no\n7.68.0: yes\n8.7.1: yes\n"], + ) + end + it "passes all arguments through to delegated upgrades" do args_file = test_root/"brew-args.txt" brew_wrapper = test_root/"brew-wrapper" @@ -226,6 +362,56 @@ def setup_update_utils expect(args_file.read).to eq("update-report\n--auto-update\n") end + it "keeps the current API file's saved ETag when removing other OS versions' API files" do + api_internal = test_root/"cache/api/internal" + setup_update_utils + api_internal.mkpath + (test_root/"repository").mkpath + %w[ + packages.arm64_tahoe.jws.json + packages.arm64_tahoe.jws.json.etag + packages.arm64_sequoia.jws.json + packages.arm64_sequoia.jws.json.etag + ].each { |name| (api_internal/name).write "x" } + + _stdout, stderr, status = run_update_shell( + <<~SH, + source "#{update_script}" + bottle_tag() { echo arm64_tahoe; } + brew() { :; } + fetch_api_file() { :; } + git_init_if_necessary() { :; } + git() { + [[ "$1" == "--version" ]] && return 0 + return 1 + } + lock() { :; } + odie() { echo "Error: $*" >&2; exit 1; } + ohai() { :; } + onoe() { echo "Error: $*" >&2; } + safe_cd() { cd "$1" >/dev/null || exit 1; } + setup_ca_certificates() { :; } + setup_curl() { :; } + setup_git() { :; } + homebrew-update --auto-update + SH + { + "HOMEBREW_BREW_GIT_REMOTE" => "https://github.com/Homebrew/brew", + "HOMEBREW_CACHE" => (test_root/"cache").to_s, + "HOMEBREW_CELLAR" => (test_root/"cellar").to_s, + "HOMEBREW_LIBRARY" => (test_root/"Library").to_s, + "HOMEBREW_NO_INSTALL_FROM_API" => nil, + "HOMEBREW_PREFIX" => (test_root/"prefix").to_s, + "HOMEBREW_REPOSITORY" => (test_root/"repository").to_s, + }, + ) + + expect([status.success?, stderr]).to eq([true, ""]) + expect(api_internal.children.map { |path| path.basename.to_s }.sort).to eq( + %w[packages.arm64_tahoe.jws.json packages.arm64_tahoe.jws.json.etag], + ) + end + it "does not query redirected remote metadata for no-op tap updates" do args_file = test_root/"brew-args.txt" fetches_file = test_root/"fetches.txt" diff --git a/Library/Homebrew/utils/api.sh b/Library/Homebrew/utils/api.sh index f84640bbced92..c46e5333ff79f 100644 --- a/Library/Homebrew/utils/api.sh +++ b/Library/Homebrew/utils/api.sh @@ -25,12 +25,36 @@ api_curlrc_args() { fi } +api_curl_supports_etag() { + if [[ -z "${API_CURL_SUPPORTS_ETAG:-}" ]] + then + local curl_version_output curl_name_and_version + curl_version_output="$(curl --version 2>/dev/null)" + curl_name_and_version="${curl_version_output%% (*}" + if [[ "$(numeric "${curl_name_and_version##* }")" -ge "$(numeric "7.68.0")" ]] + then + API_CURL_SUPPORTS_ETAG=1 + else + API_CURL_SUPPORTS_ETAG=0 + fi + fi + + [[ "${API_CURL_SUPPORTS_ETAG}" == "1" ]] +} + api_time_cond_args() { local cache_path="$1" + local etag_path="$2" if [[ -s "${cache_path}" ]] then - echo "--time-cond" - echo "${cache_path}" + if [[ -n "${etag_path}" && -s "${etag_path}" ]] + then + echo "--etag-compare" + echo "${etag_path}" + else + echo "--time-cond" + echo "${cache_path}" + fi fi }