Merge commit '9c2d5b534f' into upstream-20250314

app/services/activitypub/fetch_all_replies_service.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+class ActivityPub::FetchAllRepliesService < ActivityPub::FetchRepliesService
+  include JsonLdHelper
+
+  # Limit of replies to fetch per status
+  MAX_REPLIES = (ENV['FETCH_REPLIES_MAX_SINGLE'] || 500).to_i
+
+  def call(status_uri, collection_or_uri, max_pages: 1, request_id: nil)
+    @status_uri = status_uri
+
+    super
+  end
+
+  private
+
+  def filter_replies(items)
+    # Find all statuses that we *shouldn't* update the replies for, and use that as a filter.
+    # We don't assume that we have the statuses before they're created,
+    # hence the negative filter -
+    # "keep all these uris except the ones we already have"
+    # instead of
+    # "keep all these uris that match some conditions on existing Status objects"
+    #
+    # Typically we assume the number of replies we *shouldn't* fetch is smaller than the
+    # replies we *should* fetch, so we also minimize the number of uris we should load here.
+    uris = items.map { |item| value_or_id(item) }
+
+    # Expand collection to get replies in the DB that were
+    # - not included in the collection,
+    # - that we have locally
+    # - but we have no local followers and thus don't get updates/deletes for
+    parent_id = Status.where(uri: @status_uri).pick(:id)
+    unless parent_id.nil?
+      unsubscribed_replies = Status
+                             .where.not(uri: uris)
+                             .where(in_reply_to_id: parent_id)
+                             .unsubscribed
+                             .pluck(:uri)
+      uris.concat(unsubscribed_replies)
+    end
+
+    dont_update = Status.where(uri: uris).should_not_fetch_replies.pluck(:uri)
+
+    # touch all statuses that already exist and that we're about to update
+    Status.where(uri: uris).should_fetch_replies.touch_all(:fetched_replies_at)
+
+    # Reject all statuses that we already have in the db
+    uris = (uris - dont_update).take(MAX_REPLIES)
+
+    Rails.logger.debug { "FetchAllRepliesService - #{@collection_or_uri}: Fetching filtered statuses: #{uris}" }
+    uris
+  end
+end

app/services/activitypub/fetch_featured_collection_service.rb

@@ -33,7 +33,7 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService
     return collection_or_uri if collection_or_uri.is_a?(Hash)
     return if non_matching_uri_hosts?(@account.uri, collection_or_uri)
 
-    fetch_resource_without_id_validation(collection_or_uri, local_follower, true)
+    fetch_resource_without_id_validation(collection_or_uri, local_follower, raise_on_error: :temporary)
   end
 
   def process_items(items)

app/services/activitypub/fetch_featured_tags_collection_service.rb

@@ -45,7 +45,7 @@ class ActivityPub::FetchFeaturedTagsCollectionService < BaseService
     return collection_or_uri if collection_or_uri.is_a?(Hash)
     return if non_matching_uri_hosts?(@account.uri, collection_or_uri)
 
-    fetch_resource_without_id_validation(collection_or_uri, local_follower, true)
+    fetch_resource_without_id_validation(collection_or_uri, local_follower, raise_on_error: :temporary)
   end
 
   def process_items(items)

app/services/activitypub/fetch_remote_status_service.rb

@@ -13,7 +13,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService
 
     @request_id = request_id || "#{Time.now.utc.to_i}-status-#{uri}"
     @json = if prefetched_body.nil?
-              fetch_resource(uri, true, on_behalf_of)
+              fetch_status(uri, true, on_behalf_of)
             else
               body_to_json(prefetched_body, compare_id: uri)
             end
@@ -80,4 +80,20 @@ class ActivityPub::FetchRemoteStatusService < BaseService
   def expected_object_type?
     equals_or_includes_any?(@json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES)
   end
+
+  def fetch_status(uri, id_is_known, on_behalf_of = nil)
+    begin
+      fetch_resource(uri, id_is_known, on_behalf_of, raise_on_error: :all)
+    rescue Mastodon::UnexpectedResponseError => e
+      return unless e.response.code == 404
+
+      # If this is a 404 from a public status from a remote account, delete it
+      existing_status = Status.remote.find_by(uri: uri)
+      if existing_status&.distributable?
+        Rails.logger.debug { "FetchRemoteStatusService - Got 404 for orphaned status with URI #{uri}, deleting" }
+        Tombstone.find_or_create_by(uri: uri, account: existing_status.account)
+        RemoveStatusService.new.call(existing_status, redraft: false)
+      end
+    end
+  end
 end

app/services/activitypub/fetch_replies_service.rb

@@ -3,39 +3,59 @@
 class ActivityPub::FetchRepliesService < BaseService
   include JsonLdHelper
 
-  def call(parent_status, collection_or_uri, allow_synchronous_requests: true, request_id: nil)
-    @account = parent_status.account
+  # Limit of fetched replies
+  MAX_REPLIES = 5
+
+  def call(reference_uri, collection_or_uri, max_pages: 1, allow_synchronous_requests: true, request_id: nil)
+    @reference_uri = reference_uri
     @allow_synchronous_requests = allow_synchronous_requests
 
-    @items = collection_items(collection_or_uri)
+    @items, n_pages = collection_items(collection_or_uri, max_pages: max_pages)
     return if @items.nil?
 
-    FetchReplyWorker.push_bulk(filtered_replies) { |reply_uri| [reply_uri, { 'request_id' => request_id }] }
+    @items = filter_replies(@items)
+    FetchReplyWorker.push_bulk(@items) { |reply_uri| [reply_uri, { 'request_id' => request_id }] }
 
-    @items
+    [@items, n_pages]
   end
 
   private
 
-  def collection_items(collection_or_uri)
+  def collection_items(collection_or_uri, max_pages: 1)
     collection = fetch_collection(collection_or_uri)
     return unless collection.is_a?(Hash)
 
     collection = fetch_collection(collection['first']) if collection['first'].present?
     return unless collection.is_a?(Hash)
 
+    items = []
+    n_pages = 1
+    while collection.is_a?(Hash)
+      items.concat(as_array(collection_page_items(collection)))
+
+      break if items.size >= MAX_REPLIES
+      break if n_pages >= max_pages
+
+      collection = collection['next'].present? ? fetch_collection(collection['next']) : nil
+      n_pages += 1
+    end
+
+    [items, n_pages]
+  end
+
+  def collection_page_items(collection)
     case collection['type']
     when 'Collection', 'CollectionPage'
-      as_array(collection['items'])
+      collection['items']
     when 'OrderedCollection', 'OrderedCollectionPage'
-      as_array(collection['orderedItems'])
+      collection['orderedItems']
     end
   end
 
   def fetch_collection(collection_or_uri)
     return collection_or_uri if collection_or_uri.is_a?(Hash)
     return unless @allow_synchronous_requests
-    return if non_matching_uri_hosts?(@account.uri, collection_or_uri)
+    return if non_matching_uri_hosts?(@reference_uri, collection_or_uri)
 
     # NOTE: For backward compatibility reasons, Mastodon signs outgoing
     # queries incorrectly by default.
@@ -45,19 +65,19 @@ class ActivityPub::FetchRepliesService < BaseService
     #
     # Therefore, retry with correct signatures if this fails.
     begin
-      fetch_resource_without_id_validation(collection_or_uri, nil, true)
+      fetch_resource_without_id_validation(collection_or_uri, nil, raise_on_error: :temporary)
     rescue Mastodon::UnexpectedResponseError => e
       raise unless e.response && e.response.code == 401 && Addressable::URI.parse(collection_or_uri).query.present?
 
-      fetch_resource_without_id_validation(collection_or_uri, nil, true, request_options: { omit_query_string: false })
+      fetch_resource_without_id_validation(collection_or_uri, nil, raise_on_error: :temporary, request_options: { omit_query_string: false })
     end
   end
 
-  def filtered_replies
+  def filter_replies(items)
     # Only fetch replies to the same server as the original status to avoid
     # amplification attacks.
 
     # Also limit to 5 fetched replies to limit potential for DoS.
-    @items.map { |item| value_or_id(item) }.reject { |uri| non_matching_uri_hosts?(@account.uri, uri) }.take(5)
+    items.map { |item| value_or_id(item) }.reject { |uri| non_matching_uri_hosts?(@reference_uri, uri) }.take(MAX_REPLIES)
   end
 end

app/services/activitypub/synchronize_followers_service.rb

@@ -69,6 +69,6 @@ class ActivityPub::SynchronizeFollowersService < BaseService
     return collection_or_uri if collection_or_uri.is_a?(Hash)
     return if non_matching_uri_hosts?(@account.uri, collection_or_uri)
 
-    fetch_resource_without_id_validation(collection_or_uri, nil, true)
+    fetch_resource_without_id_validation(collection_or_uri, nil, raise_on_error: :temporary)
   end
 end