Merge remote-tracking branch 'parent/stable-4.2' into kb-draft-5.19-lts

app/controllers/admin/domain_allows_controller.rb

@@ -25,6 +25,8 @@ class Admin::DomainAllowsController < Admin::BaseController
   def destroy
     authorize @domain_allow, :destroy?
     UnallowDomainService.new.call(@domain_allow)
+    log_action :destroy, @domain_allow
+
     redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.destroyed_msg')
   end
 

app/controllers/api/v1/admin/domain_blocks_controller.rb

@@ -29,10 +29,11 @@ class Api::V1::Admin::DomainBlocksController < Api::BaseController
   def create
     authorize :domain_block, :create?
 
+    @domain_block = DomainBlock.new(resource_params)
     existing_domain_block = resource_params[:domain].present? ? DomainBlock.rule_for(resource_params[:domain]) : nil
-    return render json: existing_domain_block, serializer: REST::Admin::ExistingDomainBlockErrorSerializer, status: 422 if existing_domain_block.present?
+    return render json: existing_domain_block, serializer: REST::Admin::ExistingDomainBlockErrorSerializer, status: 422 if conflicts_with_existing_block?(@domain_block, existing_domain_block)
 
-    @domain_block = DomainBlock.create!(resource_params)
+    @domain_block.save!
     DomainBlockWorker.perform_async(@domain_block.id)
     log_action :create, @domain_block
     render json: @domain_block, serializer: REST::Admin::DomainBlockSerializer
@@ -55,6 +56,10 @@ class Api::V1::Admin::DomainBlocksController < Api::BaseController
 
   private
 
+  def conflicts_with_existing_block?(domain_block, existing_domain_block)
+    existing_domain_block.present? && (existing_domain_block.domain == TagManager.instance.normalize_domain(domain_block.domain) || !domain_block.stricter_than?(existing_domain_block))
+  end
+
   def set_domain_blocks
     @domain_blocks = filtered_domain_blocks.order(id: :desc).to_a_paginated_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id))
   end

app/controllers/api/v1/featured_tags/suggestions_controller.rb

@@ -12,6 +12,10 @@ class Api::V1::FeaturedTags::SuggestionsController < Api::BaseController
   private
 
   def set_recently_used_tags
-    @recently_used_tags = Tag.recently_used(current_account).where.not(id: current_account.featured_tags).limit(10)
+    @recently_used_tags = Tag.recently_used(current_account).where.not(id: featured_tag_ids).limit(10)
+  end
+
+  def featured_tag_ids
+    current_account.featured_tags.pluck(:tag_id)
   end
 end

app/controllers/concerns/cache_concern.rb

@@ -208,34 +208,19 @@ module CacheConcern
     end
   end
 
+  # TODO: Rename this method, as it does not perform any caching anymore.
   def cache_collection(raw, klass)
-    return raw unless klass.respond_to?(:with_includes)
+    return raw unless klass.respond_to?(:preload_cacheable_associations)
 
-    raw = raw.cache_ids.to_a if raw.is_a?(ActiveRecord::Relation)
-    return [] if raw.empty?
+    records = raw.to_a
 
-    cached_keys_with_value = begin
-      Rails.cache.read_multi(*raw).transform_keys(&:id).transform_values { |r| ActiveRecordCoder.load(r) }
-    rescue ActiveRecordCoder::Error
-      {} # The serialization format may have changed, let's pretend it's a cache miss.
-    end
+    klass.preload_cacheable_associations(records)
 
-    uncached_ids = raw.map(&:id) - cached_keys_with_value.keys
-
-    klass.reload_stale_associations!(cached_keys_with_value.values) if klass.respond_to?(:reload_stale_associations!)
-
-    unless uncached_ids.empty?
-      uncached = klass.where(id: uncached_ids).with_includes.index_by(&:id)
-
-      uncached.each_value do |item|
-        Rails.cache.write(item, ActiveRecordCoder.dump(item))
-      end
-    end
-
-    raw.filter_map { |item| cached_keys_with_value[item.id] || uncached[item.id] }
+    records
   end
 
+  # TODO: Rename this method, as it does not perform any caching anymore.
   def cache_collection_paginated_by_id(raw, klass, limit, options)
-    cache_collection raw.cache_ids.to_a_paginated_by_id(limit, options), klass
+    cache_collection raw.to_a_paginated_by_id(limit, options), klass
   end
 end

app/controllers/well_known/webfinger_controller.rb

@@ -21,7 +21,7 @@ module WellKnown
       username = username_from_resource
 
       @account = begin
-        if username == Rails.configuration.x.local_domain
+        if username == Rails.configuration.x.local_domain || username == Rails.configuration.x.web_domain
           Account.representative
         else
           Account.find_local!(username)

app/lib/activitypub/parser/status_parser.rb

@@ -3,6 +3,8 @@
 class ActivityPub::Parser::StatusParser
   include JsonLdHelper
 
+  NORMALIZED_LOCALE_NAMES = LanguagesHelper::SUPPORTED_LOCALES.keys.index_by(&:downcase).freeze
+
   # @param [Hash] json
   # @param [Hash] magic_values
   # @option magic_values [String] :followers_collection
@@ -99,6 +101,13 @@ class ActivityPub::Parser::StatusParser
   end
 
   def language
+    lang = raw_language_code
+    lang.presence && NORMALIZED_LOCALE_NAMES.fetch(lang.downcase.to_sym, lang)
+  end
+
+  private
+
+  def raw_language_code
     if content_language_map?
       @object['contentMap'].keys.first
     elsif name_language_map?
@@ -108,8 +117,6 @@ class ActivityPub::Parser::StatusParser
     end
   end
 
-  private
-
   def audience_to
     as_array(@object['to'] || @json['to']).map { |x| value_or_id(x) }
   end

app/lib/vacuum/applications_vacuum.rb

@@ -1,10 +0,0 @@
-# frozen_string_literal: true
-
-class Vacuum::ApplicationsVacuum
-  def perform
-    Doorkeeper::Application.where(owner_id: nil)
-                           .where.missing(:created_users, :access_tokens, :access_grants)
-                           .where(created_at: ...1.day.ago)
-                           .in_batches.delete_all
-  end
-end

app/lib/video_metadata_extractor.rb

@@ -22,7 +22,7 @@ class VideoMetadataExtractor
   private
 
   def ffmpeg_command_output
-    command = Terrapin::CommandLine.new('ffprobe', '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel')
+    command = Terrapin::CommandLine.new(Rails.configuration.x.ffprobe_binary, '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel')
     command.run(path: @path, format: 'json', loglevel: 'fatal')
   end
 

app/models/concerns/account_interactions.rb

@@ -185,7 +185,7 @@ module AccountInteractions
   end
 
   def unblock_domain!(other_domain)
-    block = domain_blocks.find_by(domain: other_domain)
+    block = domain_blocks.find_by(domain: normalized_domain(other_domain))
     block&.destroy
   end
 
@@ -333,4 +333,8 @@ module AccountInteractions
   def remove_potential_friendship(other_account)
     PotentialFriendshipTracker.remove(id, other_account.id)
   end
+
+  def normalized_domain(domain)
+    TagManager.instance.normalize_domain(domain)
+  end
 end

app/models/concerns/cacheable.rb

@@ -14,6 +14,10 @@ module Cacheable
       includes(@cache_associated)
     end
 
+    def preload_cacheable_associations(records)
+      ActiveRecord::Associations::Preloader.new(records: records, associations: @cache_associated).call
+    end
+
     def cache_ids
       select(:id, :updated_at)
     end

app/models/feed.rb

@@ -28,7 +28,7 @@ class Feed
       unhydrated = redis.zrangebyscore(key, "(#{min_id}", "(#{max_id}", limit: [0, limit], with_scores: true).map(&:first).map(&:to_i)
     end
 
-    Status.where(id: unhydrated).cache_ids
+    Status.where(id: unhydrated)
   end
 
   def key

app/models/public_feed.rb

@@ -31,7 +31,7 @@ class PublicFeed
     # scope.merge!(anonymous_scope) unless account?
     scope = to_anonymous_scope(scope) unless account?
 
-    scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
+    scope.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
   end
 
   private

app/models/status.rb

@@ -490,38 +490,6 @@ class Status < ApplicationRecord
       Status.where(id: status_ids).pluck(:account_id).uniq.index_with { |a| Account.find_by(id: a).show_emoji_reaction?(my_account) }
     end
 
-    def reload_stale_associations!(cached_items)
-      account_ids = []
-
-      cached_items.each do |item|
-        account_ids << item.account_id
-        account_ids << item.reblog.account_id if item.reblog?
-      end
-
-      account_ids.uniq!
-
-      status_ids = cached_items.map { |item| item.reblog? ? item.reblog_of_id : item.id }.uniq
-
-      return if account_ids.empty?
-
-      accounts = Account.where(id: account_ids).includes(:account_stat, :user).index_by(&:id)
-
-      status_stats = StatusStat.where(status_id: status_ids).index_by(&:status_id)
-
-      cached_items.each do |item|
-        item.account = accounts[item.account_id]
-        item.reblog.account = accounts[item.reblog.account_id] if item.reblog?
-
-        if item.reblog?
-          status_stat = status_stats[item.reblog.id]
-          item.reblog.status_stat = status_stat if status_stat.present?
-        else
-          status_stat = status_stats[item.id]
-          item.status_stat = status_stat if status_stat.present?
-        end
-      end
-    end
-
     def from_text(text)
       return [] if text.blank?
 

app/models/tag_feed.rb

@@ -35,7 +35,7 @@ class TagFeed < PublicFeed
     # scope.merge!(anonymous_scope) unless account?
     scope = to_anonymous_scope(scope) unless account?
 
-    scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
+    scope.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
   end
 
   private

app/models/user.rb

@@ -98,6 +98,8 @@ class User < ApplicationRecord
   accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? && !Setting.require_invite_text }
   validates :invite_request, presence: true, on: :create, if: :invite_text_required?
 
+  validates :email, presence: true, email_address: true
+
   validates_with BlacklistedEmailValidator, if: -> { ENV['EMAIL_DOMAIN_LISTS_APPLY_AFTER_CONFIRMATION'] == 'true' || !confirmed? }
   validates_with EmailMxValidator, if: :validate_email_dns?
   validates :agreement, acceptance: { allow_nil: false, accept: [true, 'true', '1'] }, on: :create

app/serializers/rest/admin/domain_block_serializer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class REST::Admin::DomainBlockSerializer < ActiveModel::Serializer
-  attributes :id, :domain, :created_at, :severity,
+  attributes :id, :domain, :digest, :created_at, :severity,
              :reject_media, :reject_favourite, :reject_reply, :reject_reports,
              :reject_reply_exclude_followers,
              :reject_send_sensitive,
@@ -11,4 +11,8 @@ class REST::Admin::DomainBlockSerializer < ActiveModel::Serializer
   def id
     object.id.to_s
   end
+
+  def digest
+    object.domain_digest
+  end
 end

app/services/notify_service.rb

@@ -73,16 +73,17 @@ class NotifyService < BaseService
           LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
           WHERE s.id = :id
         UNION ALL
-          SELECT s.id, s.in_reply_to_id, m.id, st.path || s.id, st.depth + 1
-          FROM ancestors st
-          JOIN statuses s ON s.id = st.in_reply_to_id
-          LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
-          WHERE st.mention_id IS NULL AND NOT s.id = ANY(path) AND st.depth < :depth_limit
+          SELECT s.id, s.in_reply_to_id, m.id, ancestors.path || s.id, ancestors.depth + 1
+          FROM ancestors
+          JOIN statuses s ON s.id = ancestors.in_reply_to_id
+          /* early exit if we already have a mention matching our requirements */
+          LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id AND s.account_id = :recipient_id
+          WHERE ancestors.mention_id IS NULL AND NOT s.id = ANY(path) AND ancestors.depth < :depth_limit
       )
       SELECT COUNT(*)
-      FROM ancestors st
-      JOIN statuses s ON s.id = st.id
-      WHERE st.mention_id IS NOT NULL AND s.visibility = 3
+      FROM ancestors
+      JOIN statuses s ON s.id = ancestors.id
+      WHERE ancestors.mention_id IS NOT NULL AND s.account_id = :recipient_id AND s.visibility = 3
     SQL
   end
 

app/services/post_status_service.rb

@@ -240,7 +240,7 @@ class PostStatusService < BaseService
 
   def idempotency_duplicate
     if scheduled?
-      @account.schedule_statuses.find(@idempotency_duplicate)
+      @account.scheduled_statuses.find(@idempotency_duplicate)
     else
       @account.statuses.find(@idempotency_duplicate)
     end
@@ -298,7 +298,7 @@ class PostStatusService < BaseService
   end
 
   def scheduled_options
-    @options.tap do |options_hash|
+    @options.dup.tap do |options_hash|
       options_hash[:in_reply_to_id]  = options_hash.delete(:thread)&.id
       options_hash[:application_id]  = options_hash.delete(:application)&.id
       options_hash[:scheduled_at]    = nil

app/validators/email_address_validator.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# NOTE: I initially wrote this as `EmailValidator` but it ended up clashing
+# with an indirect dependency of ours, `validate_email`, which, turns out,
+# has the same approach as we do, but with an extra check disallowing
+# single-label domains. Decided to not switch to `validate_email` because
+# we do want to allow at least `localhost`.
+
+class EmailAddressValidator < ActiveModel::EachValidator
+  def validate_each(record, attribute, value)
+    value = value.strip
+
+    address = Mail::Address.new(value)
+    record.errors.add(attribute, :invalid) if address.address != value
+  rescue Mail::Field::FieldError
+    record.errors.add(attribute, :invalid)
+  end
+end

app/workers/scheduler/vacuum_scheduler.rb

@@ -22,7 +22,6 @@ class Scheduler::VacuumScheduler
       preview_cards_vacuum,
       backups_vacuum,
       access_tokens_vacuum,
-      applications_vacuum,
       feeds_vacuum,
       imports_vacuum,
     ]
@@ -56,10 +55,6 @@ class Scheduler::VacuumScheduler
     Vacuum::ImportsVacuum.new
   end
 
-  def applications_vacuum
-    Vacuum::ApplicationsVacuum.new
-  end
-
   def content_retention_policy
     ContentRetentionPolicy.current
   end