diff --git a/Library/Homebrew/test/utils/curl_spec.rb b/Library/Homebrew/test/utils/curl_spec.rb index bb2696b19b855..20d7fc4578639 100644 --- a/Library/Homebrew/test/utils/curl_spec.rb +++ b/Library/Homebrew/test/utils/curl_spec.rb @@ -704,6 +704,56 @@ def curl_args_for(**options) expect(response[:etag]).to eq("bbb") expect(response[:final_url]).to eq("https://brew.sh/b") end + + describe "body encoding" do + def body_encoding_for(body, content_type: "text/html") + allow(self).to receive(:curl_output) do |*arguments, **_options| + File.binwrite(arguments.drop_while { |argument| argument != "--output" }.fetch(1), body) + ["HTTP/1.1 200 OK\r\nContent-Type: #{content_type}\r\n\r\n", "", + instance_double(Process::Status, success?: true, exitstatus: 0)] + end + curl_http_content_headers_and_checksum("https://brew.sh/")[:file].encoding + end + + it "uses the charset from the `Content-Type` header" do + expect(body_encoding_for("", content_type: "text/html; charset=Shift_JIS")).to eq(Encoding::Shift_JIS) + end + + it "parses a quoted `Content-Type` header charset case-insensitively" do + expect(body_encoding_for("", content_type: 'text/html; Charset="Shift_JIS"')).to eq(Encoding::Shift_JIS) + end + + it "guesses the charset from a `` tag" do + body = '' + expect(body_encoding_for(body)).to eq(Encoding::ISO_8859_1) + end + + it "guesses the charset from a `` tag" do + expect(body_encoding_for('')).to eq(Encoding::Shift_JIS) + end + + it "ignores attributes that only end in `charset`" do + expect(body_encoding_for('')).to eq(Encoding::UTF_8) + end + + it "ignores `` tags in HTML comments" do + expect(body_encoding_for('')).to eq(Encoding::UTF_8) + end + + it "prefers the `Content-Type` header charset to a `` tag" do + body = '' + expect(body_encoding_for(body, content_type: "text/html; charset=ISO-8859-1")).to eq(Encoding::ISO_8859_1) + end + + it "ignores a `` tag after the first 1024 bytes" do + body = "#{" " * 1024}" + expect(body_encoding_for(body)).to eq(Encoding.default_external) + end + + it "ignores charsets that are not ASCII-compatible" do + expect(body_encoding_for('')).to eq(Encoding.default_external) + end + end end describe "::curl_check_http_content" do @@ -779,6 +829,28 @@ def record_head_only(*responses) expect(curl_check_http_content("https://brew.sh/", "homepage URL")) .to include("is not reachable") end + + it "matches the same content declared in different charsets" do + body = "

café

" + allow(self).to receive(:curl_http_content_headers_and_checksum) do |url, **_options| + if url.start_with?("https:") + response.merge(final_url: url, file: body, file_hash: "https") + else + response.merge(file: body.encode(Encoding::ISO_8859_1), file_hash: "http") + end + end + expect(curl_check_http_content("http://brew.sh/", "homepage URL", check_content: true)) + .to eq("The homepage URL http://brew.sh/ should use HTTPS rather than HTTP") + end + + it "compares content in a charset that cannot be converted to UTF-8" do + body = "

caf\xE9

".b.force_encoding("Windows-1258") + allow(self).to receive(:curl_http_content_headers_and_checksum) do |url, **_options| + response.merge(final_url: url, file: body, file_hash: url) + end + expect { curl_check_http_content("http://brew.sh/", "homepage URL", check_content: true) } + .not_to raise_error + end end describe "curl request deadlines" do diff --git a/Library/Homebrew/utils/curl.rb b/Library/Homebrew/utils/curl.rb index 8c24d6cdef59b..b67712d1225f5 100644 --- a/Library/Homebrew/utils/curl.rb +++ b/Library/Homebrew/utils/curl.rb @@ -583,8 +583,14 @@ def curl_check_http_content(url, url_type, specs: {}, user_agents: [:default], r return unless check_content no_protocol_file_contents = %r{https?:\\?/\\?/} - http_content = details[:file]&.scrub&.gsub(no_protocol_file_contents, "/") - https_content = secure_details[:file]&.scrub&.gsub(no_protocol_file_contents, "/") + http_content, https_content = [details, secure_details].map do |response| + content = response[:file]&.scrub&.gsub(no_protocol_file_contents, "/") + # Each side may declare a different charset, so compare both as UTF-8. + content&.encode("UTF-8", invalid: :replace, undef: :replace) + rescue Encoding::ConverterNotFoundError + # e.g. Windows-1258 has no converter, so compare it as it is. + content + end # Check for the same content after removing all protocols if http_content && https_content && (http_content == https_content) && http_with_https_available @@ -673,15 +679,18 @@ def curl_http_content_headers_and_checksum( open_args = {} content_type = Array(headers["content-type"]).last - # Try to get encoding from Content-Type header - # TODO: add guessing encoding by tag - if content_type && - (match = content_type.match(/;\s*charset\s*=\s*([^\s]+)/)) && - (charset = match[1]) + # Try to get encoding from Content-Type header, then from a `` tag + # outside comments in the first 1024 bytes as browsers do. + charset = content_type[/;\s*charset\s*=\s*"?([^\s";]+)/i, 1] if content_type + charset ||= File.binread(file_path, 1024).to_s + .gsub(/|\z)/m, "")[/]*[\s;]charset\s*=\s*["']?([\w-]+)/i, 1] + if charset begin - open_args[:encoding] = Encoding.find(charset) + encoding = Encoding.find(charset) + # Content is compared with ASCII regexes, which raise on e.g. UTF-16. + open_args[:encoding] = encoding if encoding.ascii_compatible? rescue ArgumentError - # Unknown charset in Content-Type header + # Unknown charset end end