Merge remote-tracking branch 'parent/main' into upstream-20240716

app/lib/link_details_extractor.rb

@@ -62,7 +62,8 @@ class LinkDetailsExtractor
     end
 
     def author_name
-      author['name']
+      name = author['name']
+      name.is_a?(Array) ? name.join(', ') : name
     end
 
     def author_url
@@ -156,11 +157,11 @@ class LinkDetailsExtractor
   end
 
   def title
-    html_entities.decode(structured_data&.headline || opengraph_tag('og:title') || document.xpath('//title').map(&:content).first).strip
+    html_entities_decode(structured_data&.headline || opengraph_tag('og:title') || document.xpath('//title').map(&:content).first)&.strip
   end
 
   def description
-    html_entities.decode(structured_data&.description || opengraph_tag('og:description') || meta_tag('description'))
+    html_entities_decode(structured_data&.description || opengraph_tag('og:description') || meta_tag('description'))
   end
 
   def published_at
@@ -180,7 +181,7 @@ class LinkDetailsExtractor
   end
 
   def provider_name
-    html_entities.decode(structured_data&.publisher_name || opengraph_tag('og:site_name'))
+    html_entities_decode(structured_data&.publisher_name || opengraph_tag('og:site_name'))
   end
 
   def provider_url
@@ -188,7 +189,7 @@ class LinkDetailsExtractor
   end
 
   def author_name
-    html_entities.decode(structured_data&.author_name || opengraph_tag('og:author') || opengraph_tag('og:author:username'))
+    html_entities_decode(structured_data&.author_name || opengraph_tag('og:author') || opengraph_tag('og:author:username'))
   end
 
   def author_url
@@ -257,7 +258,7 @@ class LinkDetailsExtractor
 
       next if json_ld.blank?
 
-      structured_data = StructuredData.new(html_entities.decode(json_ld))
+      structured_data = StructuredData.new(html_entities_decode(json_ld))
 
       next unless structured_data.valid?
 
@@ -273,10 +274,11 @@ class LinkDetailsExtractor
   end
 
   def detect_encoding_and_parse_document
-    [detect_encoding, nil, @html_charset, 'UTF-8'].uniq.each do |encoding|
+    [detect_encoding, nil, header_encoding].uniq.each do |encoding|
       document = Nokogiri::HTML(@html, nil, encoding)
       return document if document.to_s.valid_encoding?
     end
+    Nokogiri::HTML(@html, nil, 'UTF-8')
   end
 
   def detect_encoding
@@ -284,12 +286,28 @@ class LinkDetailsExtractor
     guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
   end
 
+  def header_encoding
+    Encoding.find(@html_charset).name if @html_charset
+  rescue ArgumentError
+    # Encoding from HTTP header is not recognized by ruby
+    nil
+  end
+
   def detector
     @detector ||= CharlockHolmes::EncodingDetector.new.tap do |detector|
       detector.strip_tags = true
     end
   end
 
+  def html_entities_decode(string)
+    return if string.nil?
+
+    unicode_string = string.to_s.encode('UTF-8')
+    raise EncodingError, 'cannot convert string to valid UTF-8' unless unicode_string.valid_encoding?
+
+    html_entities.decode(unicode_string)
+  end
+
   def html_entities
     @html_entities ||= HTMLEntities.new(:expanded)
   end