gitea/nas
1
0
Fork 2
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

Improve encoding detection for link cards (#30780)

Browse source
This commit is contained in:
David Roetzel 2024-06-21 14:51:10 +02:00 • committed by Claire
parent 88b2d6eca5
commit 9b6219c48f
3 changed files with 36 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

15
app/lib/link_details_extractor.rb
View file

@ -265,16 +265,21 @@ class LinkDetailsExtractor
end
def document
@document ||= Nokogiri::HTML(@html, nil, encoding)
@document ||= detect_encoding_and_parse_document
end
def encoding
@encoding ||= begin
guess = detector.detect(@html, @html_charset)
guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
def detect_encoding_and_parse_document
[detect_encoding, nil, @html_charset, 'UTF-8'].uniq.each do |encoding|
document = Nokogiri::HTML(@html, nil, encoding)
return document if document.to_s.valid_encoding?
end
end
def detect_encoding
guess = detector.detect(@html, @html_charset)
guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
end
def detector
@detector ||= CharlockHolmes::EncodingDetector.new.tap do |detector|
detector.strip_tags = true