Merge remote-tracking branch 'parent/main' into upstream-20240906

lib/chewy/strategy/mastodon.rb

@@ -14,7 +14,7 @@ module Chewy
       end
 
       def leave
-        RedisConfiguration.with do |redis|
+        RedisConnection.with do |redis|
           redis.pipelined do |pipeline|
             @stash.each do |type, ids|
               pipeline.sadd("chewy:queue:#{type.name}", ids)

lib/mastodon/cli/accounts.rb

@@ -252,7 +252,7 @@ module Mastodon::CLI
       domain configuration.
     LONG_DESC
     def fix_duplicates
-      Account.remote.select(:uri, 'count(*)').group(:uri).having('count(*) > 1').pluck(:uri).each do |uri|
+      Account.remote.duplicate_uris.pluck(:uri).each do |uri|
         say("Duplicates found for #{uri}")
         begin
           ActivityPub::FetchRemoteAccountService.new.call(uri) unless dry_run?

lib/mastodon/cli/base.rb

@@ -40,7 +40,7 @@ module Mastodon
             .dup
             .tap { |config| config['pool'] = options[:concurrency] + 1 }
         )
-        RedisConfiguration.establish_pool(options[:concurrency])
+        RedisConnection.establish_pool(options[:concurrency])
       end
     end
   end

lib/mastodon/cli/media.rb

@@ -157,7 +157,7 @@ module Mastodon::CLI
       when :filesystem
         require 'find'
 
-        root_path = ENV.fetch('PAPERCLIP_ROOT_PATH', File.join(':rails_root', 'public', 'system')).gsub(':rails_root', Rails.root.to_s)
+        root_path = Rails.configuration.x.file_storage_root_path.gsub(':rails_root', Rails.root.to_s)
 
         Find.find(File.join(*[root_path, prefix].compact)) do |path|
           next if File.directory?(path)

lib/mastodon/cli/progress_helper.rb

@@ -51,7 +51,7 @@ module Mastodon::CLI
               result = ActiveRecord::Base.connection_pool.with_connection do
                 yield(item)
               ensure
-                RedisConfiguration.pool.checkin if Thread.current[:redis]
+                RedisConnection.pool.checkin if Thread.current[:redis]
                 Thread.current[:redis] = nil
               end
 

lib/mastodon/rack_middleware.rb

@@ -19,7 +19,7 @@ class Mastodon::RackMiddleware
   end
 
   def clean_up_redis_socket!
-    RedisConfiguration.pool.checkin if Thread.current[:redis]
+    RedisConnection.pool.checkin if Thread.current[:redis]
     Thread.current[:redis] = nil
   end
 

lib/mastodon/redis_config.rb

@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-
-def setup_redis_env_url(prefix = nil, defaults = true)
-  prefix = "#{prefix.to_s.upcase}_" unless prefix.nil?
-  prefix = '' if prefix.nil?
-
-  return if ENV["#{prefix}REDIS_URL"].present?
-
-  password = ENV.fetch("#{prefix}REDIS_PASSWORD") { '' if defaults }
-  host     = ENV.fetch("#{prefix}REDIS_HOST") { 'localhost' if defaults }
-  port     = ENV.fetch("#{prefix}REDIS_PORT") { 6379 if defaults }
-  db       = ENV.fetch("#{prefix}REDIS_DB") { 0 if defaults }
-
-  ENV["#{prefix}REDIS_URL"] = begin
-    if [password, host, port, db].all?(&:nil?)
-      ENV['REDIS_URL']
-    else
-      Addressable::URI.parse("redis://#{host}:#{port}/#{db}").tap do |uri|
-        uri.password = password if password.present?
-      end.normalize.to_str
-    end
-  end
-end
-
-setup_redis_env_url
-setup_redis_env_url(:cache, false)
-setup_redis_env_url(:sidekiq, false)
-
-namespace         = ENV.fetch('REDIS_NAMESPACE', nil)
-cache_namespace   = namespace ? "#{namespace}_cache" : 'cache'
-sidekiq_namespace = namespace
-
-redis_driver = ENV.fetch('REDIS_DRIVER', 'hiredis') == 'ruby' ? :ruby : :hiredis
-
-REDIS_CACHE_PARAMS = {
-  driver: redis_driver,
-  url: ENV['CACHE_REDIS_URL'],
-  expires_in: 10.minutes,
-  namespace: "#{cache_namespace}:7.1",
-  connect_timeout: 5,
-  pool: {
-    size: Sidekiq.server? ? Sidekiq[:concurrency] : Integer(ENV['MAX_THREADS'] || 5),
-    timeout: 5,
-  },
-}.freeze
-
-REDIS_SIDEKIQ_PARAMS = {
-  driver: redis_driver,
-  url: ENV['SIDEKIQ_REDIS_URL'],
-  namespace: sidekiq_namespace,
-}.freeze
-
-ENV['REDIS_NAMESPACE'] = "mastodon_test#{ENV['TEST_ENV_NUMBER']}" if Rails.env.test?

lib/mastodon/redis_configuration.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+class Mastodon::RedisConfiguration
+  DEFAULTS = {
+    host: 'localhost',
+    port: 6379,
+    db: 0,
+  }.freeze
+
+  def base
+    @base ||= setup_config(prefix: nil, defaults: DEFAULTS)
+              .merge(namespace: base_namespace)
+  end
+
+  def sidekiq
+    @sidekiq ||= setup_config(prefix: 'SIDEKIQ_')
+                 .merge(namespace: sidekiq_namespace)
+  end
+
+  def cache
+    @cache ||= setup_config(prefix: 'CACHE_')
+               .merge({
+                 namespace: cache_namespace,
+                 expires_in: 10.minutes,
+                 connect_timeout: 5,
+                 pool: {
+                   size: Sidekiq.server? ? Sidekiq[:concurrency] : Integer(ENV['MAX_THREADS'] || 5),
+                   timeout: 5,
+                 },
+               })
+  end
+
+  private
+
+  def driver
+    ENV['REDIS_DRIVER'] == 'ruby' ? :ruby : :hiredis
+  end
+
+  def namespace
+    @namespace ||= ENV.fetch('REDIS_NAMESPACE', nil)
+  end
+
+  def base_namespace
+    return "mastodon_test#{ENV.fetch('TEST_ENV_NUMBER', nil)}" if Rails.env.test?
+
+    namespace
+  end
+
+  def sidekiq_namespace
+    namespace
+  end
+
+  def cache_namespace
+    namespace ? "#{namespace}_cache" : 'cache'
+  end
+
+  def setup_config(prefix: nil, defaults: {})
+    prefix = "#{prefix}REDIS_"
+
+    url      = ENV.fetch("#{prefix}URL", nil)
+    user     = ENV.fetch("#{prefix}USER", nil)
+    password = ENV.fetch("#{prefix}PASSWORD", nil)
+    host     = ENV.fetch("#{prefix}HOST", defaults[:host])
+    port     = ENV.fetch("#{prefix}PORT", defaults[:port])
+    db       = ENV.fetch("#{prefix}DB", defaults[:db])
+
+    return { url:, driver: } if url
+
+    sentinel_options = setup_sentinels(prefix, default_user: user, default_password: password)
+
+    if sentinel_options.present?
+      host = sentinel_options[:name]
+      port = nil
+      db ||= 0
+    end
+
+    url = construct_uri(host, port, db, user, password)
+
+    if url.present?
+      { url:, driver: }.merge(sentinel_options)
+    else
+      # Fall back to base config, which has defaults for the URL
+      # so this cannot lead to endless recursion.
+      base
+    end
+  end
+
+  def setup_sentinels(prefix, default_user: nil, default_password: nil)
+    name              = ENV.fetch("#{prefix}SENTINEL_MASTER", nil)
+    sentinel_port     = ENV.fetch("#{prefix}SENTINEL_PORT", 26_379)
+    sentinel_list     = ENV.fetch("#{prefix}SENTINELS", nil)
+    sentinel_username = ENV.fetch("#{prefix}SENTINEL_USERNAME", default_user)
+    sentinel_password = ENV.fetch("#{prefix}SENTINEL_PASSWORD", default_password)
+
+    sentinels = parse_sentinels(sentinel_list, default_port: sentinel_port)
+
+    if name.present? && sentinels.present?
+      { name:, sentinels:, sentinel_username:, sentinel_password: }
+    else
+      {}
+    end
+  end
+
+  def construct_uri(host, port, db, user, password)
+    return nil if host.blank?
+
+    Addressable::URI.parse("redis://#{host}:#{port}/#{db}").tap do |uri|
+      uri.user = user if user.present?
+      uri.password = password if password.present?
+    end.normalize.to_str
+  end
+
+  def parse_sentinels(sentinels_string, default_port: 26_379)
+    (sentinels_string || '').split(',').map do |sentinel|
+      host, port = sentinel.split(':')
+      port = (port || default_port).to_i
+      { host: host, port: port }
+    end.presence
+  end
+end

lib/mastodon/sidekiq_middleware.rb

@@ -53,7 +53,7 @@ class Mastodon::SidekiqMiddleware
   end
 
   def clean_up_redis_socket!
-    RedisConfiguration.pool.checkin if Thread.current[:redis]
+    RedisConnection.pool.checkin if Thread.current[:redis]
     Thread.current[:redis] = nil
   end