utils/curl: guess charset from a <meta> tag

Library/Homebrew/test/utils/curl_spec.rb

@@ -704,6 +704,56 @@ def curl_args_for(**options)
       expect(response[:etag]).to eq("bbb")
       expect(response[:final_url]).to eq("https://brew.sh/b")
     end
+
+    describe "body encoding" do
+      def body_encoding_for(body, content_type: "text/html")
+        allow(self).to receive(:curl_output) do |*arguments, **_options|
+          File.binwrite(arguments.drop_while { |argument| argument != "--output" }.fetch(1), body)
+          ["HTTP/1.1 200 OK\r\nContent-Type: #{content_type}\r\n\r\n", "",
+           instance_double(Process::Status, success?: true, exitstatus: 0)]
+        end
+        curl_http_content_headers_and_checksum("https://brew.sh/")[:file].encoding
+      end
+
+      it "uses the charset from the `Content-Type` header" do
+        expect(body_encoding_for("", content_type: "text/html; charset=Shift_JIS")).to eq(Encoding::Shift_JIS)
+      end
+
+      it "parses a quoted `Content-Type` header charset case-insensitively" do
+        expect(body_encoding_for("", content_type: 'text/html; Charset="Shift_JIS"')).to eq(Encoding::Shift_JIS)
+      end
+
+      it "guesses the charset from a `<meta http-equiv>` tag" do
+        body = '<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">'
+        expect(body_encoding_for(body)).to eq(Encoding::ISO_8859_1)
+      end
+
+      it "guesses the charset from a `<meta charset>` tag" do
+        expect(body_encoding_for('<meta charset="Shift_JIS">')).to eq(Encoding::Shift_JIS)
+      end
+
+      it "ignores attributes that only end in `charset`" do
+        expect(body_encoding_for('<meta data-charset="Shift_JIS"><meta charset="UTF-8">')).to eq(Encoding::UTF_8)
+      end
+
+      it "ignores `<meta>` tags in HTML comments" do
+        expect(body_encoding_for('<!-- <meta charset="Shift_JIS"> --><meta charset="UTF-8">')).to eq(Encoding::UTF_8)
+      end
+
+      it "prefers the `Content-Type` header charset to a `<meta>` tag" do
+        body = '<meta charset="Shift_JIS">'
+        expect(body_encoding_for(body, content_type: "text/html; charset=ISO-8859-1")).to eq(Encoding::ISO_8859_1)
+      end
+
+      it "ignores a `<meta>` tag after the first 1024 bytes" do
+        body = "#{" " * 1024}<meta charset=\"Shift_JIS\">"
+        expect(body_encoding_for(body)).to eq(Encoding.default_external)
+      end
+
+      it "ignores charsets that are not ASCII-compatible" do
+        expect(body_encoding_for('<meta charset="UTF-16">')).to eq(Encoding.default_external)
+      end
+    end
   end
 
   describe "::curl_check_http_content" do
@@ -779,6 +829,28 @@ def record_head_only(*responses)
       expect(curl_check_http_content("https://brew.sh/", "homepage URL"))
         .to include("is not reachable")
     end
+
+    it "matches the same content declared in different charsets" do
+      body = "<p>café</p>"
+      allow(self).to receive(:curl_http_content_headers_and_checksum) do |url, **_options|
+        if url.start_with?("https:")
+          response.merge(final_url: url, file: body, file_hash: "https")
+        else
+          response.merge(file: body.encode(Encoding::ISO_8859_1), file_hash: "http")
+        end
+      end
+      expect(curl_check_http_content("http://brew.sh/", "homepage URL", check_content: true))
+        .to eq("The homepage URL http://brew.sh/ should use HTTPS rather than HTTP")
+    end
+
+    it "compares content in a charset that cannot be converted to UTF-8" do
+      body = "<p>caf\xE9</p>".b.force_encoding("Windows-1258")
+      allow(self).to receive(:curl_http_content_headers_and_checksum) do |url, **_options|
+        response.merge(final_url: url, file: body, file_hash: url)
+      end
+      expect { curl_check_http_content("http://brew.sh/", "homepage URL", check_content: true) }
+        .not_to raise_error
+    end
   end
 
   describe "curl request deadlines" do

Library/Homebrew/utils/curl.rb

@@ -583,8 +583,14 @@ def curl_check_http_content(url, url_type, specs: {}, user_agents: [:default], r
       return unless check_content
 
       no_protocol_file_contents = %r{https?:\\?/\\?/}
-      http_content = details[:file]&.scrub&.gsub(no_protocol_file_contents, "/")
-      https_content = secure_details[:file]&.scrub&.gsub(no_protocol_file_contents, "/")
+      http_content, https_content = [details, secure_details].map do |response|
+        content = response[:file]&.scrub&.gsub(no_protocol_file_contents, "/")
+        # Each side may declare a different charset, so compare both as UTF-8.
+        content&.encode("UTF-8", invalid: :replace, undef: :replace)
+      rescue Encoding::ConverterNotFoundError
+        # e.g. Windows-1258 has no converter, so compare it as it is.
+        content
+      end
 
       # Check for the same content after removing all protocols
       if http_content && https_content && (http_content == https_content) && http_with_https_available
@@ -673,15 +679,18 @@ def curl_http_content_headers_and_checksum(
           open_args = {}
           content_type = Array(headers["content-type"]).last
 
-          # Try to get encoding from Content-Type header
-          # TODO: add guessing encoding by <meta http-equiv="Content-Type" ...> tag
-          if content_type &&
-             (match = content_type.match(/;\s*charset\s*=\s*([^\s]+)/)) &&
-             (charset = match[1])
+          # Try to get encoding from Content-Type header, then from a `<meta>` tag
+          # outside comments in the first 1024 bytes as browsers do.
+          charset = content_type[/;\s*charset\s*=\s*"?([^\s";]+)/i, 1] if content_type
+          charset ||= File.binread(file_path, 1024).to_s
+                          .gsub(/<!--.*?(?:-->|\z)/m, "")[/<meta[^>]*[\s;]charset\s*=\s*["']?([\w-]+)/i, 1]
+          if charset
             begin
-              open_args[:encoding] = Encoding.find(charset)
+              encoding = Encoding.find(charset)
+              # Content is compared with ASCII regexes, which raise on e.g. UTF-16.
+              open_args[:encoding] = encoding if encoding.ascii_compatible?
             rescue ArgumentError
-              # Unknown charset in Content-Type header
+              # Unknown charset
             end
           end