Add optional fetching with processing status references

app/services/process_references_service.rb

@@ -3,30 +3,39 @@
 class ProcessReferencesService < BaseService
   include Payloadable
   include FormattingHelper
+  include Redisable
+  include Lockable
 
   DOMAIN = ENV['WEB_DOMAIN'] || ENV.fetch('LOCAL_DOMAIN', nil)
   REFURL_EXP = /(RT|QT|BT|RN|RE)((:|;)?\s+|:|;)(#{URI::DEFAULT_PARSER.make_regexp(%w(http https))})/
   MAX_REFERENCES = 5
 
-  def call(status, reference_parameters, urls: nil)
+  def call(status, reference_parameters, urls: nil, fetch_remote: true, no_fetch_urls: nil)
     @status = status
     @reference_parameters = reference_parameters || []
     @urls = urls || []
+    @no_fetch_urls = no_fetch_urls || []
+    @fetch_remote = fetch_remote
+    @again = false
 
-    @references_count = old_references.size
+    with_redis_lock("process_status_refs:#{@status.id}") do
+      @references_count = old_references.size
 
-    return unless added_references.size.positive? || removed_references.size.positive?
+      if added_references.size.positive? || removed_references.size.positive?
+        StatusReference.transaction do
+          remove_old_references
+          add_references
 
-    StatusReference.transaction do
-      remove_old_references
-      add_references
+          @status.save!
+        end
 
-      @status.save!
+        create_notifications!
+      end
+
+      Rails.cache.delete("status_reference:#{@status.id}")
     end
 
-    Rails.cache.delete("status_reference:#{@status.id}")
-
-    create_notifications!
+    launch_worker if @again
   end
 
   def self.need_process?(status, reference_parameters, urls)
@@ -37,13 +46,13 @@ class ProcessReferencesService < BaseService
     return unless need_process?(status, reference_parameters, urls)
 
     Rails.cache.write("status_reference:#{status.id}", true, expires_in: 10.minutes)
-    ProcessReferencesWorker.perform_async(status.id, reference_parameters, urls)
+    ProcessReferencesWorker.perform_async(status.id, reference_parameters, urls, [])
   end
 
   def self.call_service(status, reference_parameters, urls)
     return unless need_process?(status, reference_parameters, urls)
 
-    ProcessReferencesService.new.call(status, reference_parameters || [], urls: urls || [])
+    ProcessReferencesService.new.call(status, reference_parameters || [], urls: urls || [], fetch_remote: false)
   end
 
   private
@@ -65,14 +74,22 @@ class ProcessReferencesService < BaseService
   end
 
   def scan_text!
-    text = @status.account.local? ? @status.text : @status.text.gsub(%r{</?[^>]*>}, '')
-    @scan_text = fetch_statuses!(text.scan(REFURL_EXP).pluck(3).uniq).map(&:id).uniq.filter { |status_id| !status_id.zero? }
+    text = extract_status_plain_text(@status)
+    statuses = fetch_statuses!(text.scan(REFURL_EXP).pluck(3).uniq)
+
+    @again = true if !@fetch_remote && statuses.any?(&:nil?)
+
+    @scan_text = statuses.compact.map(&:id).uniq.filter { |status_id| !status_id.zero? }
   end
 
   def fetch_statuses!(urls)
-    (urls + @urls)
-      .map { |url| ResolveURLService.new.call(url, on_behalf_of: @status.account) }
-      .filter { |status| status }
+    target_urls = urls + @urls
+
+    target_urls.map do |url|
+      status = ResolveURLService.new.call(url, on_behalf_of: @status.account, fetch_remote: @fetch_remote && @no_fetch_urls.exclude?(url))
+      @no_fetch_urls << url if !@fetch_remote && status.present?
+      status
+    end
   end
 
   def add_references
@@ -112,4 +129,8 @@ class ProcessReferencesService < BaseService
       @references_count -= 1
     end
   end
+
+  def launch_worker
+    ProcessReferencesWorker.perform_async(@status.id, @reference_parameters, @urls, @no_fetch_urls)
+  end
 end

app/services/resolve_url_service.rb

@@ -6,13 +6,13 @@ class ResolveURLService < BaseService
 
   USERNAME_STATUS_RE = %r{/@(?<username>#{Account::USERNAME_RE})/(?<status_id>[0-9]+)\Z}
 
-  def call(url, on_behalf_of: nil)
+  def call(url, on_behalf_of: nil, fetch_remote: true)
     @url          = url
     @on_behalf_of = on_behalf_of
 
     if local_url?
       process_local_url
-    elsif !fetched_resource.nil?
+    elsif fetch_remote && !fetched_resource.nil?
       process_url
     else
       process_url_from_db

app/views/settings/preferences/other/show.html.haml

@@ -15,7 +15,7 @@
       = ff.input :lock_follow_from_bot, wrapper: :with_label, kmyblue: true, label: I18n.t('simple_form.labels.defaults.setting_lock_follow_from_bot')
 
     .fields-group
-      = ff.input :single_ref_to_quote, wrapper: :with_label, kmyblue: true, label: I18n.t('simple_form.labels.defaults.setting_single_ref_to_quote')
+      = ff.input :single_ref_to_quote, wrapper: :with_label, kmyblue: true, label: I18n.t('simple_form.labels.defaults.setting_single_ref_to_quote'), hint: I18n.t('simple_form.labels.defaults.setting_single_ref_to_quote')
 
     %h4= t 'preferences.posting_defaults'
 

app/workers/process_references_worker.rb

@@ -3,8 +3,8 @@
 class ProcessReferencesWorker
   include Sidekiq::Worker
 
-  def perform(status_id, ids, urls)
-    ProcessReferencesService.new.call(Status.find(status_id), ids || [], urls: urls || [])
+  def perform(status_id, ids, urls, no_fetch_urls)
+    ProcessReferencesService.new.call(Status.find(status_id), ids || [], urls: urls || [], no_fetch_urls: no_fetch_urls)
   rescue ActiveRecord::RecordNotFound
     true
   end

config/locales/simple_form.en.yml

@@ -71,6 +71,7 @@ en:
         setting_dtl_menu: Show DTL menu on web
         setting_emoji_reaction_policy: Even with this setting, users on other servers are free to put their stamp on the post and share it within the same server. If you simply want to remove the stamp from your own screen, you can disable it from the appearance settings
         setting_enable_emoji_reaction: If turn off, other users still can react your posts
+        setting_single_ref_to_quote: If this server does not have target post, target server maybe cannot read your quote
         setting_use_blurhash: Gradients are based on the colors of the hidden visuals but obfuscate any details
         setting_use_pending_items: Hide timeline updates behind a click instead of automatically scrolling the feed
         username: You can use letters, numbers, and underscores

config/locales/simple_form.ja.yml

@@ -82,6 +82,7 @@ ja:
         setting_public_post_to_unlisted: 未対応のサードパーティアプリからもローカル公開で投稿できますが、公開投稿はWeb以外できなくなります
         setting_reject_unlisted_subscription: Misskeyやそのフォーク(Calckeyなど)は、フォローしていないアカウントの「未収載」投稿を **購読・検索** することができます。これはkmyblueの挙動と異なります。そのようなサーバーに、指定した公開範囲の投稿を「フォロワーのみ」として配送します。ただし構造上、完璧な対応は困難でたまに未収載として配信されること、ご理解ください
         setting_show_application: 投稿するのに使用したアプリが投稿の詳細ビューに表示されるようになります
+        setting_single_ref_to_quote: 当サーバーがまだ対象投稿を取り込んでいない場合、引用が相手に正常に認識されない場合があります
         setting_stop_emoji_reaction_streaming: 通信容量の節約に役立ちます
         setting_unsafe_limited_distribution: Mastodon 3.5、4.0、4.1のサーバーにも限定投稿（相互のみ）が届くようになりますが、安全でない方法で送信します
         setting_use_blurhash: ぼかしはメディアの色を元に生成されますが、細部は見えにくくなっています