Merge remote-tracking branch 'parent/stable-4.2' into kb-draft-5.19-lts

lib/action_dispatch/remote_ip_extensions.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# Mastodon is not made to be directly accessed without a reverse proxy.
+# This monkey-patch prevents remote IP address spoofing when being accessed
+# directly.
+#
+# See PR: https://github.com/rails/rails/pull/51610
+
+# In addition to the PR above, it also raises an error if a request with
+# `X-Forwarded-For` or `Client-Ip` comes directly from a client without
+# going through a trusted proxy.
+
+# rubocop:disable all -- This is a mostly vendored file
+
+module ActionDispatch
+  class RemoteIp
+    module GetIpExtensions
+      def calculate_ip
+        # Set by the Rack web server, this is a single value.
+        remote_addr = ips_from(@req.remote_addr).last
+
+        # Could be a CSV list and/or repeated headers that were concatenated.
+        client_ips    = ips_from(@req.client_ip).reverse!
+        forwarded_ips = ips_from(@req.x_forwarded_for).reverse!
+
+        # `Client-Ip` and `X-Forwarded-For` should not, generally, both be set. If they
+        # are both set, it means that either:
+        #
+        # 1) This request passed through two proxies with incompatible IP header
+        #     conventions.
+        #
+        # 2) The client passed one of `Client-Ip` or `X-Forwarded-For`
+        #     (whichever the proxy servers weren't using) themselves.
+        #
+        # Either way, there is no way for us to determine which header is the right one
+        # after the fact. Since we have no idea, if we are concerned about IP spoofing
+        # we need to give up and explode. (If you're not concerned about IP spoofing you
+        # can turn the `ip_spoofing_check` option off.)
+        should_check_ip = @check_ip && client_ips.last && forwarded_ips.last
+        if should_check_ip && !forwarded_ips.include?(client_ips.last)
+          # We don't know which came from the proxy, and which from the user
+          raise IpSpoofAttackError, "IP spoofing attack?! " \
+            "HTTP_CLIENT_IP=#{@req.client_ip.inspect} " \
+            "HTTP_X_FORWARDED_FOR=#{@req.x_forwarded_for.inspect}"
+        end
+
+        # NOTE: Mastodon addition to make sure we don't get requests from a non-trusted client
+        if @check_ip && (forwarded_ips.last || client_ips.last) && !@proxies.any? { |proxy| proxy === remote_addr }
+          raise IpSpoofAttackError, "IP spoofing attack?! client #{remote_addr} is not a trusted proxy " \
+            "HTTP_CLIENT_IP=#{@req.client_ip.inspect} " \
+            "HTTP_X_FORWARDED_FOR=#{@req.x_forwarded_for.inspect}"
+        end
+
+        # We assume these things about the IP headers:
+        #
+        #     - X-Forwarded-For will be a list of IPs, one per proxy, or blank
+        #     - Client-Ip is propagated from the outermost proxy, or is blank
+        #     - REMOTE_ADDR will be the IP that made the request to Rack
+        ips = forwarded_ips + client_ips
+        ips.compact!
+
+        # If every single IP option is in the trusted list, return the IP that's
+        # furthest away
+        filter_proxies([remote_addr] + ips).first || ips.last || remote_addr
+      end
+    end
+  end
+end
+
+ActionDispatch::RemoteIp::GetIp.prepend(ActionDispatch::RemoteIp::GetIpExtensions)
+
+# rubocop:enable all

lib/mastodon/cli/maintenance.rb

@@ -224,7 +224,7 @@ module Mastodon::CLI
         users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse
         ref_user = users.shift
         say "Multiple users registered with e-mail address #{ref_user.email}.", :yellow
-        say "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}", :yellow
+        say "e-mail will be disabled for the following accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow
         say 'Please reach out to them and set another address with `tootctl account modify` or delete them.', :yellow
 
         users.each_with_index do |user, index|

lib/mastodon/cli/media.rb

@@ -134,7 +134,7 @@ module Mastodon::CLI
 
             model_name      = path_segments.first.classify
             attachment_name = path_segments[1].singularize
-            record_id       = path_segments[2..-2].join.to_i
+            record_id       = path_segments[2...-2].join.to_i
             file_name       = path_segments.last
             record          = record_map.dig(model_name, record_id)
             attachment      = record&.public_send(attachment_name)
@@ -180,7 +180,7 @@ module Mastodon::CLI
           end
 
           model_name      = path_segments.first.classify
-          record_id       = path_segments[2..-2].join.to_i
+          record_id       = path_segments[2...-2].join.to_i
           attachment_name = path_segments[1].singularize
           file_name       = path_segments.last
 
@@ -311,7 +311,7 @@ module Mastodon::CLI
       end
 
       model_name = path_segments.first.classify
-      record_id  = path_segments[2..-2].join.to_i
+      record_id  = path_segments[2...-2].join.to_i
 
       unless PRELOAD_MODEL_WHITELIST.include?(model_name)
         say("Cannot find corresponding model: #{model_name}", :red)
@@ -361,7 +361,7 @@ module Mastodon::CLI
         next unless VALID_PATH_SEGMENTS_SIZE.include?(segments.size)
 
         model_name = segments.first.classify
-        record_id  = segments[2..-2].join.to_i
+        record_id  = segments[2...-2].join.to_i
 
         next unless PRELOAD_MODEL_WHITELIST.include?(model_name)
 

lib/mastodon/sidekiq_middleware.rb

@@ -8,6 +8,7 @@ class Mastodon::SidekiqMiddleware
   rescue Mastodon::HostValidationError
     # Do not retry
   rescue => e
+    clean_up_elasticsearch_connections!
     limit_backtrace_and_raise(e)
   ensure
     clean_up_sockets!
@@ -25,6 +26,32 @@ class Mastodon::SidekiqMiddleware
     clean_up_statsd_socket!
   end
 
+  # This is a hack to immediately free up unused Elasticsearch connections.
+  #
+  # Indeed, Chewy creates one `Elasticsearch::Client` instance per thread,
+  # and each such client manages its long-lasting connection to
+  # Elasticsearch.
+  #
+  # As far as I know, neither `chewy`,  `elasticsearch-transport` or even
+  # `faraday` provide a reliable way to immediately close a connection, and
+  # rely on the underlying object to be garbage-collected instead.
+  #
+  # Furthermore, `sidekiq` creates a new thread each time a job throws an
+  # exception, meaning that each failure will create a new connection, and
+  # the old one will only be closed on full garbage collection.
+  def clean_up_elasticsearch_connections!
+    return unless Chewy.enabled? && Chewy.current[:chewy_client].present?
+
+    Chewy.client.transport.connections.each do |connection|
+      # NOTE: This bit of code is tailored for the HTTPClient Faraday adapter
+      connection.connection.app.instance_variable_get(:@client)&.reset_all
+    end
+
+    Chewy.current.delete(:chewy_client)
+  rescue
+    nil
+  end
+
   def clean_up_redis_socket!
     RedisConfiguration.pool.checkin if Thread.current[:redis]
     Thread.current[:redis] = nil

lib/mastodon/version.rb

@@ -25,7 +25,7 @@ module Mastodon
     end
 
     def patch
-      8
+      9
     end
 
     def default_prerelease

lib/paperclip/image_extractor.rb

@@ -35,7 +35,7 @@ module Paperclip
       dst.binmode
 
       begin
-        command = Terrapin::CommandLine.new('ffmpeg', '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger)
+        command = Terrapin::CommandLine.new(Rails.configuration.x.ffmpeg_binary, '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger)
         command.run(source: @file.path, destination: dst.path, loglevel: 'fatal')
       rescue Terrapin::ExitStatusError
         dst.close(true)

lib/paperclip/transcoder.rb

@@ -61,7 +61,7 @@ module Paperclip
       command_arguments, interpolations = prepare_command(destination)
 
       begin
-        command = Terrapin::CommandLine.new('ffmpeg', command_arguments.join(' '), logger: Paperclip.logger)
+        command = Terrapin::CommandLine.new(Rails.configuration.x.ffmpeg_binary, command_arguments.join(' '), logger: Paperclip.logger)
         command.run(interpolations)
       rescue Terrapin::ExitStatusError => e
         raise Paperclip::Error, "Error while transcoding #{@basename}: #{e}"

lib/tasks/mastodon.rake

@@ -515,6 +515,7 @@ namespace :mastodon do
           owner_role = UserRole.find_by(name: 'Owner')
           user = User.new(email: email, password: password, confirmed_at: Time.now.utc, account_attributes: { username: username }, bypass_invite_request_check: true, role: owner_role)
           user.save(validate: false)
+          user.approve!
 
           Setting.site_contact_username = username