Improve fetch_link_card_service to exclude status references url

app/services/fetch_link_card_service.rb

@@ -3,6 +3,7 @@
 class FetchLinkCardService < BaseService
   include Redisable
   include Lockable
+  include FormattingHelper
 
   URL_PATTERN = %r{
     (#{Twitter::TwitterText::Regex[:valid_url_preceding_chars]})                                                                #   $1 preceding chars
@@ -80,9 +81,25 @@ class FetchLinkCardService < BaseService
              links.filter_map { |a| Addressable::URI.parse(a['href']) unless skip_link?(a) }.filter_map(&:normalize)
            end
 
+    exclude_urls = referenced_urls
+    urls = urls.filter { |url| exclude_urls.exclude?(url.to_s) }
+
     urls.reject { |uri| bad_url?(uri) }.first
   end
 
+  def referenced_urls
+    unless @status.local?
+      document = Nokogiri::HTML(@status.text)
+      document.search('a[href^="http://"]', 'a[href^="https://"]').each do |link|
+        link.replace(link['href']) if link['href']
+      end
+
+      return PlainTextFormatter.new(document.to_s, false).to_s.scan(ProcessReferencesService::REFURL_EXP).pluck(3).uniq
+    end
+
+    extract_status_plain_text(@status).scan(ProcessReferencesService::REFURL_EXP).pluck(3).uniq
+  end
+
   def bad_url?(uri)
     # Avoid local instance URLs and invalid URLs
     uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme)

spec/services/fetch_link_card_service_spec.rb

@@ -10,6 +10,7 @@ RSpec.describe FetchLinkCardService, type: :service do
 
   before do
     stub_request(:get, 'http://example.com/html').to_return(headers: { 'Content-Type' => 'text/html' }, body: html)
+    stub_request(:get, 'http://example.com/html_sub').to_return(headers: { 'Content-Type' => 'text/html' }, body: html)
     stub_request(:get, 'http://example.com/not-found').to_return(status: 404, headers: { 'Content-Type' => 'text/html' }, body: html)
     stub_request(:get, 'http://example.com/text').to_return(status: 404, headers: { 'Content-Type' => 'text/plain' }, body: 'Hello')
     stub_request(:get, 'http://example.com/redirect').to_return(status: 302, headers: { 'Location' => 'http://example.com/html' })
@@ -234,6 +235,24 @@ RSpec.describe FetchLinkCardService, type: :service do
         end
       end
     end
+
+    context 'with URL of reference' do
+      let(:status) { Fabricate(:status, text: 'RT http://example.com/html') }
+
+      it 'creates preview card' do
+        expect(status.preview_card).to be_nil
+      end
+    end
+
+    context 'with URL of reference and normal page' do
+      let(:status) { Fabricate(:status, text: 'RT http://example.com/text http://example.com/html') }
+
+      it 'creates preview card' do
+        expect(status.preview_card).to_not be_nil
+        expect(status.preview_card.url).to eq 'http://example.com/html'
+        expect(status.preview_card.title).to eq 'Hello world'
+      end
+    end
   end
 
   context 'with a remote status' do
@@ -254,4 +273,31 @@ RSpec.describe FetchLinkCardService, type: :service do
       expect(a_request(:get, 'https://quitter.se/tag/wannacry')).to_not have_been_made
     end
   end
+
+  context 'with a remote status of reference' do
+    let(:status) do
+      Fabricate(:status, account: Fabricate(:account, domain: 'example.com'), text: <<-TEXT)
+      RT <a href="http://example.com/html" target="_blank" rel="noopener noreferrer" title="http://example.com/html">Hello</a>&nbsp;
+      TEXT
+    end
+
+    it 'creates preview card' do
+      expect(status.preview_card).to be_nil
+    end
+  end
+
+  context 'with a remote status of reference and normal link' do
+    let(:status) do
+      Fabricate(:status, account: Fabricate(:account, domain: 'example.com'), text: <<-TEXT)
+      RT <a href="http://example.com/html" target="_blank" rel="noopener noreferrer" title="http://example.com/html">Hello</a>&nbsp;
+      <a href="http://example.com/html_sub" target="_blank" rel="noopener noreferrer" title="http://example.com/html_sub">Hello</a>&nbsp;
+      TEXT
+    end
+
+    it 'creates preview card' do
+      expect(status.preview_card).to_not be_nil
+      expect(status.preview_card.url).to eq 'http://example.com/html_sub'
+      expect(status.preview_card.title).to eq 'Hello world'
+    end
+  end
 end