update.sh: revalidate API files with the ETag

Library/Homebrew/cmd/update.sh

@@ -443,28 +443,50 @@ fetch_api_file() {
     echo "Checking if we need to fetch ${filename}..."
   fi
 
-  local arg curl_exit_code json_url last_json_url
+  local etag_path new_etag_path
+  local -a etag_args=()
+  if api_curl_supports_etag
+  then
+    etag_path="${cache_path}.etag"
+    new_etag_path="${cache_path}.etag.new"
+    etag_args=(--etag-save "${new_etag_path}" --write-out "%{http_code}")
+  fi
+
+  local arg curl_exit_code http_code json_url last_json_url
   local -a time_cond
   while read -r json_url
   do
     time_cond=()
     while read -r arg
     do
       time_cond+=("${arg}")
-    done < <(api_time_cond_args "${cache_path}")
-    api_curl_download "${json_url}" "${cache_path}" "${time_cond[@]}"
+    done < <(api_time_cond_args "${cache_path}" "${etag_path}")
+    [[ -n "${new_etag_path}" ]] && rm -f "${new_etag_path}"
+    http_code="$(api_curl_download "${json_url}" "${cache_path}" "${etag_args[@]}" "${time_cond[@]}")"
     curl_exit_code=$?
     # A conditional request can fail with a receive error (curl exit code 56) when
     # an unconditional request for the same URL succeeds, so retry exactly once.
     if [[ ${curl_exit_code} -eq 56 ]] && [[ ${#time_cond[@]} -gt 0 ]]
     then
-      api_curl_download "${json_url}" "${cache_path}"
+      http_code="$(api_curl_download "${json_url}" "${cache_path}" "${etag_args[@]}")"
       curl_exit_code=$?
     fi
     last_json_url="${json_url}"
     [[ ${curl_exit_code} -eq 0 ]] && break
   done < <(api_urls "${filename}")
 
+  if [[ -n "${new_etag_path}" ]]
+  then
+    if [[ ${curl_exit_code} -eq 0 && -s "${new_etag_path}" ]]
+    then
+      mv -f "${new_etag_path}" "${etag_path}"
+    elif [[ ${curl_exit_code} -eq 0 && "${http_code}" != "304" ]]
+    then
+      rm -f "${etag_path}"
+    fi
+    rm -f "${new_etag_path}"
+  fi
+
   if [[ -n ${is_formula_file} ]] && [[ -f "${api_cache}/formula_names.txt" ]]
   then
     mv -f "${api_cache}/formula_names.txt" "${api_cache}/formula_names.before.txt"
@@ -1078,6 +1100,7 @@ EOS
         "${HOMEBREW_CACHE}/api/internal/packages.$(bottle_tag).jws.json") ;;
         "${HOMEBREW_CACHE}/api/internal/packages.$(bottle_tag).jws.json.payload") ;;
         "${HOMEBREW_CACHE}/api/internal/packages.$(bottle_tag).jws.json.payload.index") ;;
+        "${HOMEBREW_CACHE}/api/internal/packages.$(bottle_tag).jws.json.etag") ;;
         *) rm -f "${f}" ;;
       esac
     done

Library/Homebrew/test/cmd/update_spec.rb

@@ -130,6 +130,7 @@ def setup_update_utils
         "HOMEBREW_CURL_SPEED_TIME"    => "5",
         "HOMEBREW_LIBRARY"            => (test_root/"Library").to_s,
         "HOMEBREW_USER_AGENT_CURL"    => "Homebrew/test",
+        "API_CURL_SUPPORTS_ETAG"      => "0",
       },
     )
 
@@ -138,6 +139,141 @@ def setup_update_utils
     )
   end
 
+  describe "API ETag revalidation" do
+    let(:cache_path) { test_root/"cache/api/formula.jws.json" }
+    let(:etag_path) { test_root/"cache/api/formula.jws.json.etag" }
+    let(:new_etag_path) { test_root/"cache/api/formula.jws.json.etag.new" }
+    let(:args_file) { test_root/"curl-args.txt" }
+    let(:update_failed_file) { test_root/"update_failed.txt" }
+
+    def fetch_with_curl_response(status_code:, body: "", etag: "", saved_etag: '"old"', curl_supports_etag: "1")
+      setup_update_utils
+      cache_path.dirname.mkpath
+      cache_path.write "cached"
+      etag_path.write saved_etag if saved_etag
+
+      _stdout, stderr, status = run_update_shell(
+        <<~SH,
+          source "#{test_root}/Library/Homebrew/utils.sh"
+          source "#{update_script}"
+          curl() {
+            if [[ "$1" == "--version" ]]
+            then
+              echo "curl 7.67.0 (x86_64-apple-darwin) libcurl/7.67.0"
+              return
+            fi
+            printf '%s\\n' "$@" >> "#{args_file}"
+            local output etag_save
+            while [[ $# -gt 0 ]]
+            do
+              case "$1" in
+                --output) output="$2"; shift ;;
+                --etag-save) etag_save="$2"; shift ;;
+              esac
+              shift
+            done
+            [[ -n "${etag_save}" ]] && printf '%s' '#{etag}' > "${etag_save}"
+            [[ -n '#{body}' ]] && printf '%s' '#{body}' > "${output}"
+            [[ -n "${etag_save}" ]] && printf '%s' '#{status_code}'
+            return 0
+          }
+          fetch_api_file formula.jws.json "#{update_failed_file}"
+        SH
+        {
+          "HOMEBREW_API_DEFAULT_DOMAIN" => "https://formulae.example/api",
+          "HOMEBREW_API_DOMAIN"         => nil,
+          "HOMEBREW_CACHE"              => (test_root/"cache").to_s,
+          "HOMEBREW_CURL_SPEED_LIMIT"   => "100",
+          "HOMEBREW_CURL_SPEED_TIME"    => "5",
+          "HOMEBREW_LIBRARY"            => (test_root/"Library").to_s,
+          "HOMEBREW_USER_AGENT_CURL"    => "Homebrew/test",
+          "API_CURL_SUPPORTS_ETAG"      => curl_supports_etag,
+        },
+      )
+
+      conditional_flags = %w[--etag-compare --etag-save --time-cond]
+      {
+        success:   status.success? && stderr.empty? && !update_failed_file.exist?,
+        curl_args: args_file.read.lines(chomp: true).each_cons(2).filter_map do |flag, value|
+          [flag, value] if conditional_flags.include?(flag)
+        end,
+        body:      cache_path.read,
+        etag:      (etag_path.read if etag_path.exist?),
+        temp_etag: new_etag_path.exist?,
+      }
+    end
+
+    it "revalidates with the saved ETag and saves the new one with the new body" do
+      expect(fetch_with_curl_response(status_code: 200, body: "fresh", etag: '"new"')).to eq(
+        success:   true,
+        curl_args: [
+          ["--etag-save", new_etag_path.to_s],
+          ["--etag-compare", etag_path.to_s],
+        ],
+        body:      "fresh",
+        etag:      '"new"',
+        temp_etag: false,
+      )
+    end
+
+    it "keeps the saved ETag when the server answers 304" do
+      expect(fetch_with_curl_response(status_code: 304).except(:curl_args)).to eq(
+        success: true, body: "cached", etag: '"old"', temp_etag: false,
+      )
+    end
+
+    it "drops the saved ETag after a 200 that carries none" do
+      expect(fetch_with_curl_response(status_code: 200, body: "fresh").except(:curl_args)).to eq(
+        success: true, body: "fresh", etag: nil, temp_etag: false,
+      )
+    end
+
+    it "falls back to the time condition without a saved ETag" do
+      expect(fetch_with_curl_response(status_code: 200, body: "fresh", etag: '"new"', saved_etag: nil)).to eq(
+        success:   true,
+        curl_args: [
+          ["--etag-save", new_etag_path.to_s],
+          ["--time-cond", cache_path.to_s],
+        ],
+        body:      "fresh",
+        etag:      '"new"',
+        temp_etag: false,
+      )
+    end
+
+    it "uses only the time condition when curl is older than 7.68.0" do
+      expect(fetch_with_curl_response(status_code: 200, body: "fresh", curl_supports_etag: nil)).to eq(
+        success:   true,
+        curl_args: [["--time-cond", cache_path.to_s]],
+        body:      "fresh",
+        etag:      '"old"',
+        temp_etag: false,
+      )
+    end
+  end
+
+  it "detects whether curl supports ETag revalidation" do
+    setup_update_utils
+
+    stdout, stderr, status = run_update_shell(
+      <<~SH,
+        source "#{test_root}/Library/Homebrew/utils.sh"
+        source "#{update_script}"
+        for version in 7.41.0 7.67.0 7.68.0 8.7.1
+        do
+          unset API_CURL_SUPPORTS_ETAG
+          curl() { echo "curl ${version} (x86_64-apple-darwin) libcurl/${version}"; }
+          api_curl_supports_etag && echo "${version}: yes" || echo "${version}: no"
+        done
+      SH
+      { "HOMEBREW_LIBRARY" => (test_root/"Library").to_s },
+    )
+
+    expect([status.success?, stderr, stdout]).to eq(
+      [true, "", "7.41.0: no\n7.67.0: no\n7.68.0: yes\n8.7.1: yes\n"],
+    )
+  end
+
   it "passes all arguments through to delegated upgrades" do
     args_file = test_root/"brew-args.txt"
     brew_wrapper = test_root/"brew-wrapper"
@@ -226,6 +362,56 @@ def setup_update_utils
     expect(args_file.read).to eq("update-report\n--auto-update\n")
   end
 
+  it "keeps the current API file's saved ETag when removing other OS versions' API files" do
+    api_internal = test_root/"cache/api/internal"
+    setup_update_utils
+    api_internal.mkpath
+    (test_root/"repository").mkpath
+    %w[
+      packages.arm64_tahoe.jws.json
+      packages.arm64_tahoe.jws.json.etag
+      packages.arm64_sequoia.jws.json
+      packages.arm64_sequoia.jws.json.etag
+    ].each { |name| (api_internal/name).write "x" }
+
+    _stdout, stderr, status = run_update_shell(
+      <<~SH,
+        source "#{update_script}"
+        bottle_tag() { echo arm64_tahoe; }
+        brew() { :; }
+        fetch_api_file() { :; }
+        git_init_if_necessary() { :; }
+        git() {
+          [[ "$1" == "--version" ]] && return 0
+          return 1
+        }
+        lock() { :; }
+        odie() { echo "Error: $*" >&2; exit 1; }
+        ohai() { :; }
+        onoe() { echo "Error: $*" >&2; }
+        safe_cd() { cd "$1" >/dev/null || exit 1; }
+        setup_ca_certificates() { :; }
+        setup_curl() { :; }
+        setup_git() { :; }
+        homebrew-update --auto-update
+      SH
+      {
+        "HOMEBREW_BREW_GIT_REMOTE"     => "https://github.com/Homebrew/brew",
+        "HOMEBREW_CACHE"               => (test_root/"cache").to_s,
+        "HOMEBREW_CELLAR"              => (test_root/"cellar").to_s,
+        "HOMEBREW_LIBRARY"             => (test_root/"Library").to_s,
+        "HOMEBREW_NO_INSTALL_FROM_API" => nil,
+        "HOMEBREW_PREFIX"              => (test_root/"prefix").to_s,
+        "HOMEBREW_REPOSITORY"          => (test_root/"repository").to_s,
+      },
+    )
+
+    expect([status.success?, stderr]).to eq([true, ""])
+    expect(api_internal.children.map { |path| path.basename.to_s }.sort).to eq(
+      %w[packages.arm64_tahoe.jws.json packages.arm64_tahoe.jws.json.etag],
+    )
+  end
+
   it "does not query redirected remote metadata for no-op tap updates" do
     args_file = test_root/"brew-args.txt"
     fetches_file = test_root/"fetches.txt"

Library/Homebrew/utils/api.sh

@@ -25,12 +25,36 @@ api_curlrc_args() {
   fi
 }
 
+api_curl_supports_etag() {
+  if [[ -z "${API_CURL_SUPPORTS_ETAG:-}" ]]
+  then
+    local curl_version_output curl_name_and_version
+    curl_version_output="$(curl --version 2>/dev/null)"
+    curl_name_and_version="${curl_version_output%% (*}"
+    if [[ "$(numeric "${curl_name_and_version##* }")" -ge "$(numeric "7.68.0")" ]]
+    then
+      API_CURL_SUPPORTS_ETAG=1
+    else
+      API_CURL_SUPPORTS_ETAG=0
+    fi
+  fi
+
+  [[ "${API_CURL_SUPPORTS_ETAG}" == "1" ]]
+}
+
 api_time_cond_args() {
   local cache_path="$1"
+  local etag_path="$2"
 
   if [[ -s "${cache_path}" ]]
   then
-    echo "--time-cond"
-    echo "${cache_path}"
+    if [[ -n "${etag_path}" && -s "${etag_path}" ]]
+    then
+      echo "--etag-compare"
+      echo "${etag_path}"
+    else
+      echo "--time-cond"
+      echo "${cache_path}"
+    fi
   fi
 }