Merge remote-tracking branch 'parent/main' into upstream-20241126

app/lib/annual_report/commonly_interacted_with_accounts.rb

@@ -17,6 +17,6 @@ class AnnualReport::CommonlyInteractedWithAccounts < AnnualReport::Source
   private
 
   def commonly_interacted_with_accounts
-    report_statuses.where.not(in_reply_to_account_id: @account.id).group(:in_reply_to_account_id).having('count(*) > 1').order(total: :desc).limit(SET_SIZE).pluck(Arel.sql('in_reply_to_account_id, count(*) AS total'))
+    report_statuses.where.not(in_reply_to_account_id: @account.id).group(:in_reply_to_account_id).having('count(*) > 1').order(count_all: :desc).limit(SET_SIZE).count
   end
 end

app/lib/annual_report/most_reblogged_accounts.rb

@@ -17,6 +17,6 @@ class AnnualReport::MostRebloggedAccounts < AnnualReport::Source
   private
 
   def most_reblogged_accounts
-    report_statuses.where.not(reblog_of_id: nil).joins(reblog: :account).group('accounts.id').having('count(*) > 1').order(total: :desc).limit(SET_SIZE).pluck(Arel.sql('accounts.id, count(*) as total'))
+    report_statuses.where.not(reblog_of_id: nil).joins(reblog: :account).group(accounts: [:id]).having('count(*) > 1').order(count_all: :desc).limit(SET_SIZE).count
   end
 end

app/lib/annual_report/most_used_apps.rb

@@ -17,6 +17,6 @@ class AnnualReport::MostUsedApps < AnnualReport::Source
   private
 
   def most_used_apps
-    report_statuses.joins(:application).group('oauth_applications.name').order(total: :desc).limit(SET_SIZE).pluck(Arel.sql('oauth_applications.name, count(*) as total'))
+    report_statuses.joins(:application).group(oauth_applications: [:name]).order(count_all: :desc).limit(SET_SIZE).count
   end
 end

app/lib/annual_report/top_hashtags.rb

@@ -17,6 +17,12 @@ class AnnualReport::TopHashtags < AnnualReport::Source
   private
 
   def top_hashtags
-    Tag.joins(:statuses).where(statuses: { id: report_statuses.select(:id) }).group(:id).having('count(*) > 1').order(total: :desc).limit(SET_SIZE).pluck(Arel.sql('COALESCE(tags.display_name, tags.name), count(*) AS total'))
+    Tag.joins(:statuses).where(statuses: { id: report_statuses.select(:id) }).group(coalesced_tag_names).having('count(*) > 1').order(count_all: :desc).limit(SET_SIZE).count
+  end
+
+  def coalesced_tag_names
+    Arel.sql(<<~SQL.squish)
+      COALESCE(tags.display_name, tags.name)
+    SQL
   end
 end

app/lib/antispam.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+class Antispam
+  include Redisable
+
+  ACCOUNT_AGE_EXEMPTION = 1.week.freeze
+
+  class DummyStatus < SimpleDelegator
+    def self.model_name
+      Mention.model_name
+    end
+
+    def active_mentions
+      # Don't use the scope but the in-memory array
+      mentions.filter { |mention| !mention.silent? }
+    end
+  end
+
+  class SilentlyDrop < StandardError
+    attr_reader :status
+
+    def initialize(status)
+      super()
+
+      status.created_at = Time.now.utc
+      status.id = Mastodon::Snowflake.id_at(status.created_at)
+      status.in_reply_to_account_id = status.thread&.account_id
+
+      status.delete # Make sure this is not persisted
+
+      @status = DummyStatus.new(status)
+    end
+  end
+
+  def local_preflight_check!(status)
+    return unless spammy_texts.any? { |spammy_text| status.text.include?(spammy_text) }
+    return unless suspicious_reply_or_mention?(status)
+    return unless status.account.created_at >= ACCOUNT_AGE_EXEMPTION.ago
+
+    report_if_needed!(status.account)
+
+    raise SilentlyDrop, status
+  end
+
+  private
+
+  def spammy_texts
+    redis.smembers('antispam:spammy_texts')
+  end
+
+  def suspicious_reply_or_mention?(status)
+    parent = status.thread
+    return true if parent.present? && !Follow.exists?(account_id: parent.account_id, target_account: status.account_id)
+
+    account_ids = status.mentions.map(&:account_id).uniq
+    !Follow.exists?(account_id: account_ids, target_account_id: status.account.id)
+  end
+
+  def report_if_needed!(account)
+    return if Report.unresolved.exists?(account: Account.representative, target_account: account)
+
+    Report.create!(account: Account.representative, target_account: account, category: :spam, comment: 'Account automatically reported for posting a banned URL')
+  end
+end

app/lib/feed_manager.rb

@@ -58,6 +58,7 @@ class FeedManager
   # @param [Boolean] update
   # @return [Boolean]
   def push_to_home(account, status, update: false)
+    return false unless account.user&.signed_in_recently?
     return false unless add_to_feed(:home, account.id, status, aggregate_reblogs: account.user&.aggregates_reblogs?, update: update)
 
     trim(:home, account.id)
@@ -84,6 +85,8 @@ class FeedManager
   # @return [Boolean]
   def push_to_list(list, status, update: false)
     return false if filter_from_list?(status, list) || !add_to_feed(:list, list.id, status, aggregate_reblogs: list.account.user&.aggregates_reblogs?, update: update)
+    return false unless list.account.user&.signed_in_recently?
+    return false unless add_to_feed(:list, list.id, status, aggregate_reblogs: list.account.user&.aggregates_reblogs?)
 
     trim(:list, list.id)
     PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}", { 'update' => update }) if push_update_required?("timeline:list:#{list.id}")

app/lib/link_details_extractor.rb

@@ -157,7 +157,7 @@ class LinkDetailsExtractor
   end
 
   def title
-    html_entities.decode(structured_data&.headline || opengraph_tag('og:title') || document.xpath('//title').map(&:content).first)&.strip
+    html_entities.decode(structured_data&.headline || opengraph_tag('og:title') || head.at_xpath('title')&.content)&.strip
   end
 
   def description
@@ -205,11 +205,11 @@ class LinkDetailsExtractor
   end
 
   def language
-    valid_locale_or_nil(structured_data&.language || opengraph_tag('og:locale') || document.xpath('//html').pick('lang'))
+    valid_locale_or_nil(structured_data&.language || opengraph_tag('og:locale') || document.root.attr('lang'))
   end
 
   def icon
-    valid_url_or_nil(structured_data&.publisher_icon || link_tag('apple-touch-icon') || link_tag('shortcut icon'))
+    valid_url_or_nil(structured_data&.publisher_icon || link_tag('apple-touch-icon') || link_tag('icon'))
   end
 
   private
@@ -237,18 +237,20 @@ class LinkDetailsExtractor
   end
 
   def link_tag(name)
-    document.xpath("//link[@rel=\"#{name}\"]").pick('href')
+    head.at_xpath("//link[nokogiri:link_rel_include(@rel, '#{name}')]", NokogiriHandler)&.attr('href')
   end
 
   def opengraph_tag(name)
-    document.xpath("//meta[@property=\"#{name}\" or @name=\"#{name}\"]").pick('content')
+    head.at_xpath("//meta[nokogiri:casecmp(@property, '#{name}') or nokogiri:casecmp(@name, '#{name}')]", NokogiriHandler)&.attr('content')
   end
 
   def meta_tag(name)
-    document.xpath("//meta[@name=\"#{name}\"]").pick('content')
+    head.at_xpath("//meta[nokogiri:casecmp(@name, '#{name}')]", NokogiriHandler)&.attr('content')
   end
 
   def structured_data
+    return @structured_data if defined?(@structured_data)
+
     # Some publications have more than one JSON-LD definition on the page,
     # and some of those definitions aren't valid JSON either, so we have
     # to loop through here until we find something that is the right type
@@ -273,6 +275,10 @@ class LinkDetailsExtractor
     @document ||= detect_encoding_and_parse_document
   end
 
+  def head
+    @head ||= document.at_xpath('/html/head')
+  end
+
   def detect_encoding_and_parse_document
     html = nil
     encoding = nil

app/lib/nokogiri_handler.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class NokogiriHandler
+  class << self
+    # See "set of space-separated tokens" in the HTML5 spec.
+    WHITE_SPACE = /[ \x09\x0A\x0C\x0D]+/
+
+    def link_rel_include(token_list, token)
+      token_list.to_s.downcase.split(WHITE_SPACE).include?(token.downcase)
+    end
+
+    def casecmp(str1, str2)
+      str1.to_s.casecmp?(str2.to_s)
+    end
+  end
+end

app/lib/oauth_pre_authorization_extension.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module OauthPreAuthorizationExtension
-  extend ActiveSupport::Concern
-
-  included do
-    validate :code_challenge_method_s256, error: Doorkeeper::Errors::InvalidCodeChallengeMethod
-  end
-
-  def validate_code_challenge_method_s256
-    code_challenge.blank? || code_challenge_method == 'S256'
-  end
-end

app/lib/status_reach_finder.rb

@@ -43,19 +43,23 @@ class StatusReachFinder
 
   def reached_account_inboxes
     reject_domains = @status.limited_visibility? ? banned_domains : banned_domains + friend_domains
-    Account.where(id: reached_account_ids).where.not(domain: reject_domains).inboxes
+    scope = Account.where(id: reached_account_ids).where.not(domain: reject_domains)
+    inboxes_without_suspended_for(scope)
   end
 
   def reached_account_inboxes_for_misskey
-    Account.where(id: reached_account_ids, domain: banned_domains_for_misskey - friend_domains).inboxes
+    scope = Account.where(id: reached_account_ids, domain: banned_domains_for_misskey - friend_domains)
+    inboxes_without_suspended_for(scope)
   end
 
   def reached_account_inboxes_for_friend
-    Account.where(id: reached_account_ids, domain: friend_domains).inboxes
+    scope = Account.where(id: reached_account_ids, domain: friend_domains)
+    inboxes_without_suspended_for(scope)
   end
 
   def reached_account_inboxes_for_sending_domain_block
-    Account.where(id: reached_account_ids, domain: banned_domains_of_status(@status)).inboxes
+    scope = Account.where(id: reached_account_ids, domain: banned_domains_of_status(@status))
+    inboxes_without_suspended_for(scope)
   end
 
   def reached_account_ids
@@ -115,13 +119,8 @@ class StatusReachFinder
   end
 
   def followers_inboxes
-    if @status.in_reply_to_local_account? && distributable?
-      @status.account.followers.or(@status.thread.account.followers.not_domain_blocked_by_account(@status.account)).where.not(domain: banned_domains + friend_domains).inboxes
-    elsif @status.direct_visibility? || @status.limited_visibility?
-      []
-    else
-      @status.account.followers.where.not(domain: banned_domains + friend_domains).inboxes
-    end
+    scope = followers_scope
+    inboxes_without_suspended_for(scope)
   end
 
   def followers_inboxes_for_misskey
@@ -224,4 +223,19 @@ class StatusReachFinder
     from_domain_block = DomainBlock.where(detect_invalid_subscription: true).pluck(:domain)
     (from_info + from_domain_block).uniq
   end
+
+  def followers_scope
+    if @status.in_reply_to_local_account? && distributable?
+      @status.account.followers.or(@status.thread.account.followers.not_domain_blocked_by_account(@status.account)).where.not(domain: banned_domains + friend_domains)
+    elsif @status.direct_visibility? || @status.limited_visibility?
+      Account.none
+    else
+      @status.account.followers.where.not(domain: banned_domains + friend_domains)
+    end
+  end
+
+  def inboxes_without_suspended_for(scope)
+    scope.merge!(Account.without_suspended) unless unsafe?
+    scope.inboxes
+  end
 end

app/lib/video_metadata_extractor.rb

@@ -46,6 +46,9 @@ class VideoMetadataExtractor
         # For some video streams the frame_rate reported by `ffprobe` will be 0/0, but for these streams we
         # should use `r_frame_rate` instead. Video screencast generated by Gnome Screencast have this issue.
         @frame_rate ||= @r_frame_rate
+        # If the video has not been re-encoded by ffmpeg, it may contain rotation information,
+        # and we need to simulate applying it to the dimensions
+        @width, @height = @height, @width if video_stream[:side_data_list]&.any? { |x| x[:rotation].abs == 90 }
       end
 
       if (audio_stream = audio_streams.first)