diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index a884e14154..1739a5ff26 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -493,6 +493,10 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     references = @object['references'].nil? ? [] : ActivityPub::FetchReferencesService.new.call(@status, @object['references'])
     quote = @object['quote'] || @object['quoteUrl'] || @object['quoteURL'] || @object['_misskey_quote']
     references << quote if quote
+
+    return unless ProcessReferencesService.need_process?(@status, [], references)
+
+    Rails.cache.write("status_reference:#{@status.id}", true, expires_in: 10.minutes)
     ProcessReferencesWorker.perform_async(@status.id, [], references)
   end
 
diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb
index d53dd71f4a..3603026f39 100644
--- a/app/serializers/rest/status_serializer.rb
+++ b/app/serializers/rest/status_serializer.rb
@@ -103,7 +103,9 @@ class REST::StatusSerializer < ActiveModel::Serializer
   end
 
   def status_references_count
-    status_reference_ids.size
+    return status_reference_ids.size if status_reference_ids.any?
+
+    Rails.cache.exist?("status_reference:#{object.id}") ? 1 : 0
   end
 
   def reblogs_count
diff --git a/app/services/activitypub/process_status_update_service.rb b/app/services/activitypub/process_status_update_service.rb
index 418b757248..a043d9e00c 100644
--- a/app/services/activitypub/process_status_update_service.rb
+++ b/app/services/activitypub/process_status_update_service.rb
@@ -254,6 +254,10 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
     references = @json['references'].nil? ? [] : ActivityPub::FetchReferencesService.new.call(@status, @json['references'])
     quote = @json['quote'] || @json['quoteUrl'] || @json['quoteURL'] || @json['_misskey_quote']
     references << quote if quote
+
+    return unless ProcessReferencesService.need_process?(@status, [], references)
+
+    Rails.cache.write("status_reference:#{@status.id}", true, expires_in: 10.minutes)
     ProcessReferencesWorker.perform_async(@status.id, [], references)
   end
 
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index 2e5b765d86..291138ee36 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -186,8 +186,12 @@ class PostStatusService < BaseService
   def postprocess_status!
     @account.user.update!(settings_attributes: { default_privacy: @options[:visibility] }) if @account.user&.setting_stay_privacy && !@status.reply? && %i(public public_unlisted login unlisted private).include?(@status.visibility.to_sym) && @status.visibility.to_s != @account.user&.setting_default_privacy && !@dtl
 
+    if ProcessReferencesService.need_process?(@status, @reference_ids, [])
+      Rails.cache.write("status_reference:#{@status.id}", true, expires_in: 10.minutes)
+      ProcessReferencesWorker.perform_async(@status.id, @reference_ids, [])
+    end
+
     process_hashtags_service.call(@status)
-    ProcessReferencesWorker.perform_async(@status.id, @reference_ids, [])
     Trends.tags.register(@status)
     LinkCrawlWorker.perform_async(@status.id)
     DistributionWorker.perform_async(@status.id)
diff --git a/app/services/process_references_service.rb b/app/services/process_references_service.rb
index 9c8b3813f5..54edd4ff6b 100644
--- a/app/services/process_references_service.rb
+++ b/app/services/process_references_service.rb
@@ -2,6 +2,7 @@
 
 class ProcessReferencesService < BaseService
   include Payloadable
+  include FormattingHelper
 
   DOMAIN = ENV['WEB_DOMAIN'] || ENV.fetch('LOCAL_DOMAIN', nil)
   REFURL_EXP = /(RT|QT|BT|RN|RE)((:|;)?\s+|:|;)(#{URI::DEFAULT_PARSER.make_regexp(%w(http https))})/
@@ -23,9 +24,15 @@ class ProcessReferencesService < BaseService
       @status.save!
     end
 
+    Rails.cache.delete("status_reference:#{@status.id}")
+
     create_notifications!
   end
 
+  def self.need_process?(status, reference_parameters, urls)
+    reference_parameters.any? || (urls || []).any? || FormattingHelper.extract_status_plain_text(status).scan(REFURL_EXP).pluck(3).uniq.any?
+  end
+
   private
 
   def references
diff --git a/app/services/update_status_service.rb b/app/services/update_status_service.rb
index ccf7b4827f..e6d4d4e1f4 100644
--- a/app/services/update_status_service.rb
+++ b/app/services/update_status_service.rb
@@ -161,6 +161,10 @@ class UpdateStatusService < BaseService
 
   def update_references!
     reference_ids = (@options[:status_reference_ids] || []).map(&:to_i).filter(&:positive?)
+
+    return unless ProcessReferencesService.need_process?(@status, reference_ids, [])
+
+    Rails.cache.write("status_reference:#{@status.id}", true, expires_in: 10.minutes)
     ProcessReferencesWorker.perform_async(@status.id, reference_ids, [])
   end