Slightly reduce RAM usage (#7301)

* No need to re-require sidekiq plugins, they are required via Gemfile

* Add derailed_benchmarks tool, no need to require TTY gems in Gemfile

* Replace ruby-oembed with FetchOEmbedService

Reduce startup by 45382 allocated objects

* Remove preloaded JSON-LD in favour of caching HTTP responses

Reduce boot RAM by about 6 MiB

* Fix tests

* Fix test suite by stubbing out JSON-LD contexts

app/services/fan_out_on_write_service.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'sidekiq-bulk'
-
 class FanOutOnWriteService < BaseService
   # Push a status into home and mentions feeds
   # @param [Status] status

app/services/fetch_link_card_service.rb

@@ -85,42 +85,40 @@ class FetchLinkCardService < BaseService
   end
 
   def attempt_oembed
-    embed = OEmbed::Providers.get(@url, html: @html)
+    embed = FetchOEmbedService.new.call(@url, html: @html)
 
-    return false unless embed.respond_to?(:type)
+    return false if embed.nil?
 
-    @card.type          = embed.type
-    @card.title         = embed.respond_to?(:title)         ? embed.title         : ''
-    @card.author_name   = embed.respond_to?(:author_name)   ? embed.author_name   : ''
-    @card.author_url    = embed.respond_to?(:author_url)    ? embed.author_url    : ''
-    @card.provider_name = embed.respond_to?(:provider_name) ? embed.provider_name : ''
-    @card.provider_url  = embed.respond_to?(:provider_url)  ? embed.provider_url  : ''
+    @card.type          = embed[:type]
+    @card.title         = embed[:title]         || ''
+    @card.author_name   = embed[:author_name]   || ''
+    @card.author_url    = embed[:author_url]    || ''
+    @card.provider_name = embed[:provider_name] || ''
+    @card.provider_url  = embed[:provider_url]  || ''
     @card.width         = 0
     @card.height        = 0
 
     case @card.type
     when 'link'
-      @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url)
+      @card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present?
     when 'photo'
-      return false unless embed.respond_to?(:url)
+      return false if embed[:url].blank?
 
-      @card.embed_url        = embed.url
-      @card.image_remote_url = embed.url
-      @card.width            = embed.width.presence  || 0
-      @card.height           = embed.height.presence || 0
+      @card.embed_url        = embed[:url]
+      @card.image_remote_url = embed[:url]
+      @card.width            = embed[:width].presence  || 0
+      @card.height           = embed[:height].presence || 0
     when 'video'
-      @card.width            = embed.width.presence  || 0
-      @card.height           = embed.height.presence || 0
-      @card.html             = Formatter.instance.sanitize(embed.html, Sanitize::Config::MASTODON_OEMBED)
-      @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url)
+      @card.width            = embed[:width].presence  || 0
+      @card.height           = embed[:height].presence || 0
+      @card.html             = Formatter.instance.sanitize(embed[:html], Sanitize::Config::MASTODON_OEMBED)
+      @card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present?
     when 'rich'
       # Most providers rely on <script> tags, which is a no-no
       return false
     end
 
     @card.save_with_optional_image!
-  rescue OEmbed::NotFound
-    false
   end
 
   def attempt_opengraph

app/services/fetch_oembed_service.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+class FetchOEmbedService
+  attr_reader :url, :options, :format, :endpoint_url
+
+  def call(url, options = {})
+    @url     = url
+    @options = options
+
+    discover_endpoint!
+    fetch!
+  end
+
+  private
+
+  def discover_endpoint!
+    return if html.nil?
+
+    @format = @options[:format]
+    page    = Nokogiri::HTML(html)
+
+    if @format.nil? || @format == :json
+      @endpoint_url ||= page.at_xpath('//link[@type="application/json+oembed"]')&.attribute('href')&.value
+      @format       ||= :json if @endpoint_url
+    end
+
+    if @format.nil? || @format == :xml
+      @endpoint_url ||= page.at_xpath('//link[@type="text/xml+oembed"]')&.attribute('href')&.value
+      @format       ||= :xml if @endpoint_url
+    end
+
+    return if @endpoint_url.blank?
+
+    @endpoint_url = Addressable::URI.parse(@endpoint_url).to_s
+  rescue Addressable::URI::InvalidURIError
+    @endpoint_url = nil
+  end
+
+  def fetch!
+    return if @endpoint_url.blank?
+
+    body = Request.new(:get, @endpoint_url).perform do |res|
+      res.code != 200 ? nil : res.body_with_limit
+    end
+
+    validate(parse_for_format(body)) unless body.nil?
+  rescue Oj::ParseError, Ox::ParseError
+    nil
+  end
+
+  def parse_for_format(body)
+    case @format
+    when :json
+      Oj.load(body, mode: :strict)&.with_indifferent_access
+    when :xml
+      Ox.load(body, mode: :hash_no_attrs)&.with_indifferent_access&.dig(:oembed)
+    end
+  end
+
+  def validate(oembed)
+    oembed if oembed[:version] == '1.0' && oembed[:type].present?
+  end
+
+  def html
+    return @html if defined?(@html)
+
+    @html = @options[:html] || Request.new(:get, @url).perform do |res|
+      res.code != 200 || res.mime_type != 'text/html' ? nil : res.body_with_limit
+    end
+  end
+end