From 0838f1ac8f584f89757c4271a5144883b3eefb0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?KMY=EF=BC=88=E9=9B=AA=E3=81=82=E3=81=99=E3=81=8B=EF=BC=89?=
 <tt@kmycode.net>
Date: Mon, 11 Dec 2023 14:02:44 +0900
Subject: [PATCH 1/8] =?UTF-8?q?Add:=20Sharkey=E3=82=92=E3=82=B9=E3=82=BF?=
 =?UTF-8?q?=E3=83=B3=E3=83=97=E5=88=A9=E7=94=A8=E5=8F=AF=E8=83=BD=E3=81=AA?=
 =?UTF-8?q?=E3=82=B5=E3=83=BC=E3=83=90=E3=83=BC=E3=81=AB=E8=BF=BD=E5=8A=A0?=
 =?UTF-8?q?=20(#345)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 app/lib/activitypub/activity/follow.rb | 2 +-
 app/lib/status_reach_finder.rb         | 2 +-
 app/models/instance_info.rb            | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/app/lib/activitypub/activity/follow.rb b/app/lib/activitypub/activity/follow.rb
index fdded98440..c8cf16f822 100644
--- a/app/lib/activitypub/activity/follow.rb
+++ b/app/lib/activitypub/activity/follow.rb
@@ -103,7 +103,7 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity
     info = instance_info
     return false if info.nil?
 
-    %w(misskey calckey firefish meisskey cherrypick).include?(info.software)
+    %w(misskey calckey firefish meisskey cherrypick sharkey).include?(info.software)
   end
 
   def instance_info
diff --git a/app/lib/status_reach_finder.rb b/app/lib/status_reach_finder.rb
index 8f20b94675..5ce2ff17a4 100644
--- a/app/lib/status_reach_finder.rb
+++ b/app/lib/status_reach_finder.rb
@@ -216,7 +216,7 @@ class StatusReachFinder
     return [] if status.public_searchability?
     return [] unless (status.public_unlisted_visibility? && status.account.user&.setting_reject_public_unlisted_subscription) || (status.unlisted_visibility? && status.account.user&.setting_reject_unlisted_subscription)
 
-    from_info = InstanceInfo.where(software: %w(misskey calckey cherrypick)).pluck(:domain)
+    from_info = InstanceInfo.where(software: %w(misskey calckey cherrypick sharkey)).pluck(:domain)
     from_domain_block = DomainBlock.where(detect_invalid_subscription: true).pluck(:domain)
     (from_info + from_domain_block).uniq
   end
diff --git a/app/models/instance_info.rb b/app/models/instance_info.rb
index dcdc0aacac..79c70fe06f 100644
--- a/app/models/instance_info.rb
+++ b/app/models/instance_info.rb
@@ -21,6 +21,7 @@ class InstanceInfo < ApplicationRecord
     calckey
     cherrypick
     meisskey
+    sharkey
     firefish
     renedon
     fedibird

From 9e22306574d0ebb59d3591e849ff25011f4f8518 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?KMY=EF=BC=88=E9=9B=AA=E3=81=82=E3=81=99=E3=81=8B=EF=BC=89?=
 <tt@kmycode.net>
Date: Mon, 11 Dec 2023 14:13:33 +0900
Subject: [PATCH 2/8] =?UTF-8?q?Fix:=20DistributionWorker=E3=81=A7String=20?=
 =?UTF-8?q?to=20Integer=E3=81=AE=E3=82=A8=E3=83=A9=E3=83=BC=E3=81=8C?=
 =?UTF-8?q?=E5=87=BA=E3=82=8B=E5=95=8F=E9=A1=8C=20(#344)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Fix: DeliveryWorkerでString to Integerのエラーが出る問題

* Test: Mastodon v3向け
---
 app/models/instance_info.rb | 1 +
 spec/models/status_spec.rb  | 7 ++++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/app/models/instance_info.rb b/app/models/instance_info.rb
index 79c70fe06f..62da02a743 100644
--- a/app/models/instance_info.rb
+++ b/app/models/instance_info.rb
@@ -40,6 +40,7 @@ class InstanceInfo < ApplicationRecord
     return false if info.nil?
 
     return true if EMOJI_REACTION_AVAILABLE_SOFTWARES.include?(info['software'])
+    return false if info.data['metadata'].nil? || !info.data['metadata'].is_a?(Hash)
 
     features = info.data.dig('metadata', 'features')
     return false if features.nil? || !features.is_a?(Array)
diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb
index 4e10f3d316..c1732376d9 100644
--- a/spec/models/status_spec.rb
+++ b/spec/models/status_spec.rb
@@ -463,7 +463,7 @@ RSpec.describe Status do
   describe '.emoji_reaction_availables_map' do
     subject { described_class.emoji_reaction_availables_map(domains) }
 
-    let(:domains) { %w(features_available.com features_unavailable.com features_invalid.com features_nil.com no_info.com mastodon.com misskey.com) }
+    let(:domains) { %w(features_available.com features_unavailable.com features_invalid.com features_nil.com no_info.com mastodon.com misskey.com old_mastodon.com) }
 
     before do
       Fabricate(:instance_info, domain: 'features_available.com', software: 'mastodon', data: { metadata: { features: ['emoji_reaction'] } })
@@ -472,6 +472,7 @@ RSpec.describe Status do
       Fabricate(:instance_info, domain: 'features_nil.com', software: 'mastodon', data: { metadata: { features: nil } })
       Fabricate(:instance_info, domain: 'mastodon.com', software: 'mastodon')
       Fabricate(:instance_info, domain: 'misskey.com', software: 'misskey')
+      Fabricate(:instance_info, domain: 'old_mastodon.com', software: 'mastodon', data: { metadata: [] })
     end
 
     it 'availables if features contains emoji_reaction' do
@@ -497,6 +498,10 @@ RSpec.describe Status do
     it 'availables if misskey server' do
       expect(subject['misskey.com']).to be true
     end
+
+    it 'unavailables if old mastodon server' do
+      expect(subject['old_mastodon.com']).to be false
+    end
   end
 
   describe '.tagged_with' do

From 94070aa41ef5dbf5cc942d7768f59bdefd3aea69 Mon Sep 17 00:00:00 2001
From: KMY <tt@kmycode.net>
Date: Tue, 12 Dec 2023 09:32:34 +0900
Subject: [PATCH 3/8] =?UTF-8?q?Fix:=20=E7=94=BB=E5=83=8F=E3=81=AE=E4=B8=A6?=
 =?UTF-8?q?=E3=81=B3=E9=A0=86=E3=81=8C=E5=A4=89=E3=82=8F=E3=82=8B=E5=A0=B4?=
 =?UTF-8?q?=E5=90=88=20(#339)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Fix: 画像の並び順が変わる場合

* Fix
---
 app/models/status.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/models/status.rb b/app/models/status.rb
index 2b24ff50e5..55f8755ee5 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -83,7 +83,7 @@ class Status < ApplicationRecord
   has_many :mentions, dependent: :destroy, inverse_of: :status
   has_many :mentioned_accounts, through: :mentions, source: :account, class_name: 'Account'
   has_many :active_mentions, -> { active }, class_name: 'Mention', inverse_of: :status
-  has_many :media_attachments, dependent: :nullify
+  has_many :media_attachments, -> { order('id asc') }, dependent: :nullify, inverse_of: false
   has_many :reference_objects, class_name: 'StatusReference', inverse_of: :status, dependent: :destroy
   has_many :references, through: :reference_objects, class_name: 'Status', source: :target_status
   has_many :referenced_by_status_objects, foreign_key: 'target_status_id', class_name: 'StatusReference', inverse_of: :target_status, dependent: :destroy

From bea82f2279ec57089922900e284e74af467e487f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?KMY=EF=BC=88=E9=9B=AA=E3=81=82=E3=81=99=E3=81=8B=EF=BC=89?=
 <tt@kmycode.net>
Date: Tue, 12 Dec 2023 09:52:08 +0900
Subject: [PATCH 4/8] =?UTF-8?q?Fix:=20#284=20`FetchInstanceInfoWorker`?=
 =?UTF-8?q?=E3=81=8C=E5=8E=9F=E5=9B=A0=E3=81=A7Sidekiq=E3=81=AEJob?=
 =?UTF-8?q?=E3=81=8C=E8=A9=B0=E3=81=BE=E3=82=8B=E5=95=8F=E9=A1=8C=20(#342)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Fix: #284 `FetchInstanceInfoWorker`が原因でSidekiqのJobが詰まる問題

* Fix: InstanceInfoを取得するタイミング

* Fix test

* Fix test

* Fix: HTTPコード

* 調整
---
 .../activitypub/process_account_service.rb    |  5 +--
 .../activitypub/fetch_instance_info_worker.rb | 34 +++++++++----------
 .../update_instance_info_scheduler.rb         | 15 --------
 config/sidekiq.yml                            |  4 ---
 spec/lib/activitypub/activity/update_spec.rb  |  1 +
 .../process_account_service_spec.rb           |  4 +++
 .../fetch_instance_info_worker_spec.rb        | 20 +++++++++++
 .../update_instance_info_scheduler_spec.rb    | 19 -----------
 8 files changed, 44 insertions(+), 58 deletions(-)
 delete mode 100644 app/workers/scheduler/update_instance_info_scheduler.rb
 delete mode 100644 spec/workers/scheduler/update_instance_info_scheduler_spec.rb

diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index f3b1c84615..a1a07eae04 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -46,7 +46,6 @@ class ActivityPub::ProcessAccountService < BaseService
         end
 
         create_account
-        fetch_instance_info
       end
 
       update_account
@@ -66,6 +65,8 @@ class ActivityPub::ProcessAccountService < BaseService
       check_links! if @account.fields.any?(&:requires_verification?)
     end
 
+    fetch_instance_info
+
     @account
   rescue Oj::ParseError
     nil
@@ -210,7 +211,7 @@ class ActivityPub::ProcessAccountService < BaseService
   end
 
   def fetch_instance_info
-    ActivityPub::FetchInstanceInfoWorker.perform_async(@account.domain) unless InstanceInfo.exists?(domain: @account.domain)
+    ActivityPub::FetchInstanceInfoWorker.perform_async(@account.domain) unless Rails.cache.exist?("fetch_instance_info:#{@account.domain}", expires_in: 1.day)
   end
 
   def actor_type
diff --git a/app/workers/activitypub/fetch_instance_info_worker.rb b/app/workers/activitypub/fetch_instance_info_worker.rb
index 57cbd97d10..bc9a1a4815 100644
--- a/app/workers/activitypub/fetch_instance_info_worker.rb
+++ b/app/workers/activitypub/fetch_instance_info_worker.rb
@@ -8,28 +8,32 @@ class ActivityPub::FetchInstanceInfoWorker
 
   sidekiq_options queue: 'push', retry: 2
 
-  class Error < StandardError; end
-  class RequestError < Error; end
-  class DeadError < Error; end
-
   SUPPORTED_NOTEINFO_RELS = ['http://nodeinfo.diaspora.software/ns/schema/2.0', 'http://nodeinfo.diaspora.software/ns/schema/2.1'].freeze
 
   def perform(domain)
     @instance = Instance.find_by(domain: domain)
     return if !@instance || @instance.unavailable_domain.present?
 
-    with_redis_lock("instance_info:#{domain}") do
-      link = nodeinfo_link
-      return if link.nil?
-
-      update_info!(link)
+    Rails.cache.fetch("fetch_instance_info:#{@instance.domain}", expires_in: 1.day, race_condition_ttl: 1.hour) do
+      fetch!
     end
-  rescue ActivityPub::FetchInstanceInfoWorker::DeadError
+
     true
   end
 
   private
 
+  def fetch!
+    link = nodeinfo_link
+    return if link.nil?
+
+    update_info!(link)
+
+    true
+  rescue Mastodon::UnexpectedResponseError
+    true
+  end
+
   def nodeinfo_link
     nodeinfo = fetch_json("https://#{@instance.domain}/.well-known/nodeinfo")
     return nil if nodeinfo.nil? || !nodeinfo.key?('links')
@@ -63,15 +67,9 @@ class ActivityPub::FetchInstanceInfoWorker
 
   def fetch_json(url)
     build_request(url).perform do |response|
-      if [200, 203].include?(response.code)
-        raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response)
+      raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response)
 
-        body_to_json(response.body_with_limit)
-      elsif [400, 401, 403, 404, 410].include?(response.code)
-        raise ActivityPub::FetchInstanceInfoWorker::DeadError, "Request for #{@instance.domain} returned HTTP #{response.code}"
-      else
-        raise ActivityPub::FetchInstanceInfoWorker::RequestError, "Request for #{@instance.domain} returned HTTP #{response.code}"
-      end
+      body_to_json(response.body_with_limit)
     end
   end
 
diff --git a/app/workers/scheduler/update_instance_info_scheduler.rb b/app/workers/scheduler/update_instance_info_scheduler.rb
deleted file mode 100644
index f5b2852859..0000000000
--- a/app/workers/scheduler/update_instance_info_scheduler.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-class Scheduler::UpdateInstanceInfoScheduler
-  include Sidekiq::Worker
-
-  sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
-
-  def perform
-    Instance.select(:domain).reorder(nil).find_in_batches do |instances|
-      ActivityPub::FetchInstanceInfoWorker.push_bulk(instances) do |instance|
-        [instance.domain]
-      end
-    end
-  end
-end
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index 6102dcaeaf..538987c9ac 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -63,10 +63,6 @@
       interval: 30 seconds
       class: Scheduler::SidekiqHealthScheduler
       queue: scheduler
-    update_instance_info_scheduler:
-      cron: '0 0 * * *'
-      class: Scheduler::UpdateInstanceInfoScheduler
-      queue: scheduler
     software_update_check_scheduler:
       interval: 30 minutes
       class: Scheduler::SoftwareUpdateCheckScheduler
diff --git a/spec/lib/activitypub/activity/update_spec.rb b/spec/lib/activitypub/activity/update_spec.rb
index 87e96d2d1b..6c84c5836a 100644
--- a/spec/lib/activitypub/activity/update_spec.rb
+++ b/spec/lib/activitypub/activity/update_spec.rb
@@ -55,6 +55,7 @@ RSpec.describe ActivityPub::Activity::Update do
         stub_request(:get, actor_json[:following]).to_return(status: 404)
         stub_request(:get, actor_json[:featured]).to_return(status: 404)
         stub_request(:get, actor_json[:featuredTags]).to_return(status: 404)
+        stub_request(:get, 'https://example.com/.well-known/nodeinfo').to_return(status: 404)
 
         subject.perform
       end
diff --git a/spec/services/activitypub/process_account_service_spec.rb b/spec/services/activitypub/process_account_service_spec.rb
index f0885ff672..6dc3ced27e 100644
--- a/spec/services/activitypub/process_account_service_spec.rb
+++ b/spec/services/activitypub/process_account_service_spec.rb
@@ -5,6 +5,10 @@ require 'rails_helper'
 RSpec.describe ActivityPub::ProcessAccountService, type: :service do
   subject { described_class.new }
 
+  before do
+    stub_request(:get, 'https://example.com/.well-known/nodeinfo').to_return(status: 404)
+  end
+
   context 'with searchability' do
     subject { described_class.new.call('alice', 'example.com', payload) }
 
diff --git a/spec/workers/activitypub/fetch_instance_info_worker_spec.rb b/spec/workers/activitypub/fetch_instance_info_worker_spec.rb
index f6dacff5fc..9dc9594041 100644
--- a/spec/workers/activitypub/fetch_instance_info_worker_spec.rb
+++ b/spec/workers/activitypub/fetch_instance_info_worker_spec.rb
@@ -67,9 +67,22 @@ describe ActivityPub::FetchInstanceInfoWorker do
       Instance.refresh
     end
 
+    it 'does not update immediately' do
+      stub_request(:get, 'https://example.com/nodeinfo/2.0').to_return(status: 200, body: nodeinfo_json)
+      subject.perform('example.com')
+      stub_request(:get, 'https://example.com/nodeinfo/2.0').to_return(status: 200, body: new_nodeinfo_json)
+      subject.perform('example.com')
+
+      info = InstanceInfo.find_by(domain: 'example.com')
+      expect(info).to_not be_nil
+      expect(info.software).to eq 'mastodon'
+      expect(info.version).to eq '4.2.0-beta1'
+    end
+
     it 'performs a mastodon instance' do
       stub_request(:get, 'https://example.com/nodeinfo/2.0').to_return(status: 200, body: nodeinfo_json)
       subject.perform('example.com')
+      Rails.cache.delete('fetch_instance_info:example.com')
       stub_request(:get, 'https://example.com/nodeinfo/2.0').to_return(status: 200, body: new_nodeinfo_json)
       subject.perform('example.com')
 
@@ -93,5 +106,12 @@ describe ActivityPub::FetchInstanceInfoWorker do
       info = InstanceInfo.find_by(domain: 'example.com')
       expect(info).to be_nil
     end
+
+    it 'does not fetch again immediately' do
+      expect(subject.perform('example.com')).to be true
+      expect(subject.perform('example.com')).to be true
+
+      expect(a_request(:get, 'https://example.com/.well-known/nodeinfo')).to have_been_made.once
+    end
   end
 end
diff --git a/spec/workers/scheduler/update_instance_info_scheduler_spec.rb b/spec/workers/scheduler/update_instance_info_scheduler_spec.rb
deleted file mode 100644
index f3a190417f..0000000000
--- a/spec/workers/scheduler/update_instance_info_scheduler_spec.rb
+++ /dev/null
@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe Scheduler::UpdateInstanceInfoScheduler do
-  let(:worker) { described_class.new }
-
-  before do
-    stub_request(:get, 'https://example.com/.well-known/nodeinfo').to_return(status: 200, body: '{}')
-    Fabricate(:account, domain: 'example.com')
-    Instance.refresh
-  end
-
-  describe 'perform' do
-    it 'runs without error' do
-      expect { worker.perform }.to_not raise_error
-    end
-  end
-end

From acdf1ef2c9ff8c70290f8a8369c8061e49dfa2f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?KMY=EF=BC=88=E9=9B=AA=E3=81=82=E3=81=99=E3=81=8B=EF=BC=89?=
 <tt@kmycode.net>
Date: Wed, 13 Dec 2023 09:13:32 +0900
Subject: [PATCH 5/8] =?UTF-8?q?Fix:=20#355=20LTL=E3=81=AE=E3=82=A4?=
 =?UTF-8?q?=E3=83=B3=E3=83=87=E3=83=83=E3=82=AF=E3=82=B9=E3=81=8C=E9=81=A9?=
 =?UTF-8?q?=E5=88=87=E3=81=AB=E3=81=AF=E3=82=89=E3=82=8C=E3=81=A6=E3=81=84?=
 =?UTF-8?q?=E3=81=AA=E3=81=84=E5=95=8F=E9=A1=8C=20(#357)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Fix: #355 LTLのインデックスが適切にはられていない問題

* Migrate
---
 ...improve_index_for_public_timeline_speed.rb | 19 +++++++++++++++++++
 db/schema.rb                                  |  6 +++---
 2 files changed, 22 insertions(+), 3 deletions(-)
 create mode 100644 db/migrate/20231212225737_improve_index_for_public_timeline_speed.rb

diff --git a/db/migrate/20231212225737_improve_index_for_public_timeline_speed.rb b/db/migrate/20231212225737_improve_index_for_public_timeline_speed.rb
new file mode 100644
index 0000000000..bcfe83de03
--- /dev/null
+++ b/db/migrate/20231212225737_improve_index_for_public_timeline_speed.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class ImproveIndexForPublicTimelineSpeed < ActiveRecord::Migration[7.0]
+  disable_ddl_transaction!
+
+  def up
+    add_index :statuses, [:id, :account_id], name: :index_statuses_local_20231213, algorithm: :concurrently, order: { id: :desc }, where: '(local OR (uri IS NULL)) AND deleted_at IS NULL AND visibility IN (0, 10, 11) AND reblog_of_id IS NULL AND ((NOT reply) OR (in_reply_to_account_id = account_id))'
+    add_index :statuses, [:id, :account_id], name: :index_statuses_public_20231213, algorithm: :concurrently, order: { id: :desc }, where: 'deleted_at IS NULL AND visibility IN (0, 10, 11) AND reblog_of_id IS NULL AND ((NOT reply) OR (in_reply_to_account_id = account_id))'
+    remove_index :statuses, name: :index_statuses_local_20190824
+    remove_index :statuses, name: :index_statuses_public_20200119
+  end
+
+  def down
+    add_index :statuses, [:id, :account_id], name: :index_statuses_local_20190824, algorithm: :concurrently, order: { id: :desc }, where: '(local OR (uri IS NULL)) AND deleted_at IS NULL AND visibility = 0 AND reblog_of_id IS NULL AND ((NOT reply) OR (in_reply_to_account_id = account_id))'
+    add_index :statuses, [:id, :account_id], name: :index_statuses_public_20200119, algorithm: :concurrently, order: { id: :desc }, where: 'deleted_at IS NULL AND visibility = 0 AND reblog_of_id IS NULL AND ((NOT reply) OR (in_reply_to_account_id = account_id))'
+    remove_index :statuses, name: :index_statuses_local_20231213
+    remove_index :statuses, name: :index_statuses_public_20231213
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 4850882e11..62e7ed98fb 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.1].define(version: 2023_11_05_225839) do
+ActiveRecord::Schema[7.1].define(version: 2023_12_12_225737) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -1235,8 +1235,8 @@ ActiveRecord::Schema[7.1].define(version: 2023_11_05_225839) do
     t.index ["account_id", "reblog_of_id", "deleted_at", "searchability"], name: "index_statuses_for_get_following_accounts_to_search", where: "((deleted_at IS NULL) AND (reblog_of_id IS NULL) AND (searchability = ANY (ARRAY[0, 10, 1])))"
     t.index ["account_id"], name: "index_statuses_on_account_id"
     t.index ["deleted_at"], name: "index_statuses_on_deleted_at", where: "(deleted_at IS NOT NULL)"
-    t.index ["id", "account_id"], name: "index_statuses_local_20190824", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
-    t.index ["id", "account_id"], name: "index_statuses_public_20200119", order: { id: :desc }, where: "((deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
+    t.index ["id", "account_id"], name: "index_statuses_local_20231213", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = ANY (ARRAY[0, 10, 11])) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
+    t.index ["id", "account_id"], name: "index_statuses_public_20231213", order: { id: :desc }, where: "((deleted_at IS NULL) AND (visibility = ANY (ARRAY[0, 10, 11])) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
     t.index ["in_reply_to_account_id"], name: "index_statuses_on_in_reply_to_account_id", where: "(in_reply_to_account_id IS NOT NULL)"
     t.index ["in_reply_to_id"], name: "index_statuses_on_in_reply_to_id", where: "(in_reply_to_id IS NOT NULL)"
     t.index ["quote_of_id", "account_id"], name: "index_statuses_on_quote_of_id_and_account_id"

From 7b847f4eb32f5279ee85538d3d52017d05fc76e2 Mon Sep 17 00:00:00 2001
From: KMY <tt@kmycode.net>
Date: Fri, 15 Dec 2023 08:21:36 +0900
Subject: [PATCH 6/8] =?UTF-8?q?Fix:=20=E4=B8=80=E9=83=A8=E3=82=B5=E3=83=BC?=
 =?UTF-8?q?=E3=83=90=E3=83=BC=E3=81=AE=E6=8A=95=E7=A8=BF=E3=81=8C=E5=8F=82?=
 =?UTF-8?q?=E7=85=A7=E5=BC=95=E7=94=A8=E3=81=A7=E3=81=8D=E3=81=AA=E3=81=84?=
 =?UTF-8?q?=E5=95=8F=E9=A1=8C=E3=83=BB=E5=8F=82=E7=85=A7=E5=BC=95=E7=94=A8?=
 =?UTF-8?q?=E6=99=82=E3=80=81=E3=83=95=E3=82=A7=E3=83=83=E3=83=81=E3=82=92?=
 =?UTF-8?q?=E5=9F=BA=E6=9C=AC=E3=81=97=E3=81=AA=E3=81=84=E3=82=88=E3=81=86?=
 =?UTF-8?q?=E5=A4=89=E6=9B=B4?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 app/lib/activitypub/activity/create.rb        |  2 +-
 app/lib/activitypub/tag_manager.rb            |  6 +-
 app/services/process_references_service.rb    | 13 +++-
 app/services/resolve_url_service.rb           |  9 ++-
 .../20231214225249_index_to_statuses_url.rb   | 13 ++++
 db/schema.rb                                  |  3 +-
 .../process_references_service_spec.rb        | 70 +++++++++++++++++++
 7 files changed, 105 insertions(+), 11 deletions(-)
 create mode 100644 db/migrate/20231214225249_index_to_statuses_url.rb

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 204899fa39..09b064c9b2 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -505,7 +505,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     url = quote
 
     if url.present?
-      ResolveURLService.new.call(url, on_behalf_of: @account, local_only: true).present?
+      ActivityPub::TagManager.instance.uri_to_resource(url, Status)&.local?
     else
       false
     end
diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb
index ef3bb8310d..2754299378 100644
--- a/app/lib/activitypub/tag_manager.rb
+++ b/app/lib/activitypub/tag_manager.rb
@@ -208,7 +208,7 @@ class ActivityPub::TagManager
     uri_to_resource(uri, Account)
   end
 
-  def uri_to_resource(uri, klass)
+  def uri_to_resource(uri, klass, url: false)
     return if uri.nil?
 
     if local_uri?(uri)
@@ -221,7 +221,9 @@ class ActivityPub::TagManager
     elsif OStatus::TagManager.instance.local_id?(uri)
       klass.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, klass.to_s))
     else
-      klass.find_by(uri: uri.split('#').first)
+      resource   = klass.find_by(uri: uri.split('#').first)
+      resource ||= klass.where('uri != url').find_by(url: uri.split('#').first) if url
+      resource
     end
   rescue ActiveRecord::RecordNotFound
     nil
diff --git a/app/services/process_references_service.rb b/app/services/process_references_service.rb
index 40405c6ee7..f18a0dc76e 100644
--- a/app/services/process_references_service.rb
+++ b/app/services/process_references_service.rb
@@ -108,7 +108,16 @@ class ProcessReferencesService < BaseService
   end
 
   def url_to_status(url)
-    ResolveURLService.new.call(url, on_behalf_of: @status.account, fetch_remote: @fetch_remote && @no_fetch_urls.exclude?(url))
+    status   = ActivityPub::TagManager.instance.uri_to_resource(url, Status, url: true)
+    status ||= ResolveURLService.new.call(url, on_behalf_of: @status.account) if @fetch_remote && @no_fetch_urls.exclude?(url)
+    referrable?(status) ? status : nil
+  end
+
+  def referrable?(target_status)
+    return false if target_status.nil?
+    return @referrable if defined?(@referrable)
+
+    @referrable = StatusPolicy.new(@status.account, target_status).show?
   end
 
   def quote_status_ids
@@ -116,7 +125,7 @@ class ProcessReferencesService < BaseService
   end
 
   def quotable?(target_status)
-    target_status.account.allow_quote? && (!@status.local? || StatusPolicy.new(@status.account, target_status).quote?)
+    target_status.account.allow_quote? && StatusPolicy.new(@status.account, target_status).quote?
   end
 
   def add_references
diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb
index 1194afc368..19a94e77ad 100644
--- a/app/services/resolve_url_service.rb
+++ b/app/services/resolve_url_service.rb
@@ -6,16 +6,15 @@ class ResolveURLService < BaseService
 
   USERNAME_STATUS_RE = %r{/@(?<username>#{Account::USERNAME_RE})/(?<status_id>[0-9]+)\Z}
 
-  def call(url, on_behalf_of: nil, fetch_remote: true, local_only: false)
+  def call(url, on_behalf_of: nil)
     @url          = url
     @on_behalf_of = on_behalf_of
-    @fetch_remote = fetch_remote
 
     if local_url?
       process_local_url
-    elsif !local_only && fetch_remote && !fetched_resource.nil?
+    elsif !fetched_resource.nil?
       process_url
-    elsif !local_only
+    else
       process_url_from_db
     end
   end
@@ -38,7 +37,7 @@ class ResolveURLService < BaseService
       return account unless account.nil?
     end
 
-    return unless @on_behalf_of.present? && (!@fetch_remote || [401, 403, 404].include?(fetch_resource_service.response_code))
+    return unless @on_behalf_of.present? && [401, 403, 404].include?(fetch_resource_service.response_code)
 
     # It may happen that the resource is a private toot, and thus not fetchable,
     # but we can return the toot if we already know about it.
diff --git a/db/migrate/20231214225249_index_to_statuses_url.rb b/db/migrate/20231214225249_index_to_statuses_url.rb
new file mode 100644
index 0000000000..ab3eff41ab
--- /dev/null
+++ b/db/migrate/20231214225249_index_to_statuses_url.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class IndexToStatusesURL < ActiveRecord::Migration[7.1]
+  disable_ddl_transaction!
+
+  def up
+    add_index :statuses, :url, name: :index_statuses_on_url, algorithm: :concurrently, opclass: :text_pattern_ops, where: 'url IS NOT NULL AND url <> uri'
+  end
+
+  def down
+    remove_index :statuses, name: :index_statuses_on_url
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 62e7ed98fb..3fae126a34 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.1].define(version: 2023_12_12_225737) do
+ActiveRecord::Schema[7.1].define(version: 2023_12_14_225249) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -1242,6 +1242,7 @@ ActiveRecord::Schema[7.1].define(version: 2023_12_12_225737) do
     t.index ["quote_of_id", "account_id"], name: "index_statuses_on_quote_of_id_and_account_id"
     t.index ["reblog_of_id", "account_id"], name: "index_statuses_on_reblog_of_id_and_account_id"
     t.index ["uri"], name: "index_statuses_on_uri", unique: true, opclass: :text_pattern_ops, where: "(uri IS NOT NULL)"
+    t.index ["url"], name: "index_statuses_on_url", opclass: :text_pattern_ops, where: "((url IS NOT NULL) AND ((url)::text <> (uri)::text))"
   end
 
   create_table "statuses_tags", primary_key: ["tag_id", "status_id"], force: :cascade do |t|
diff --git a/spec/services/process_references_service_spec.rb b/spec/services/process_references_service_spec.rb
index 37818b2d45..b4c063788a 100644
--- a/spec/services/process_references_service_spec.rb
+++ b/spec/services/process_references_service_spec.rb
@@ -241,6 +241,76 @@ RSpec.describe ProcessReferencesService, type: :service do
         end
       end
     end
+
+    context 'when already fetched remote post' do
+      let(:account) { Fabricate(:account, followers_url: 'http://example.com/followers', domain: 'example.com', uri: 'https://example.com/actor') }
+      let!(:remote_status) { Fabricate(:status, account: account, uri: 'https://example.com/test_post', url: 'https://example.com/test_post', text: 'Lorem ipsum') }
+      let(:object_json) do
+        {
+          id: 'https://example.com/test_post',
+          to: ActivityPub::TagManager::COLLECTIONS[:public],
+          '@context': ActivityPub::TagManager::CONTEXT,
+          type: 'Note',
+          actor: account.uri,
+          attributedTo: account.uri,
+          content: 'Lorem ipsum',
+        }
+      end
+      let(:text) { 'BT:https://example.com/test_post' }
+
+      shared_examples 'reference once' do |uri, url|
+        it 'reference it' do
+          expect(subject.size).to eq 1
+          expect(subject[0][1]).to eq 'BT'
+
+          status = Status.find_by(id: subject[0][0])
+          expect(status).to_not be_nil
+          expect(status.id).to eq remote_status.id
+          expect(status.uri).to eq uri
+          expect(status.url).to eq url
+        end
+      end
+
+      before do
+        stub_request(:get, 'https://example.com/test_post').to_return(status: 200, body: Oj.dump(object_json), headers: { 'Content-Type' => 'application/activity+json' })
+      end
+
+      it_behaves_like 'reference once', 'https://example.com/test_post', 'https://example.com/test_post'
+
+      context 'when uri and url is difference and url is not accessable' do
+        let(:remote_status) { Fabricate(:status, account: account, uri: 'https://example.com/test_post', url: 'https://example.com/test_post_ohagi', text: 'Lorem ipsum') }
+        let(:text) { 'BT:https://example.com/test_post_ohagi' }
+        let(:object_json) do
+          {
+            id: 'https://example.com/test_post',
+            url: 'https://example.com/test_post_ohagi',
+            to: ActivityPub::TagManager::COLLECTIONS[:public],
+            '@context': ActivityPub::TagManager::CONTEXT,
+            type: 'Note',
+            actor: account.uri,
+            attributedTo: account.uri,
+            content: 'Lorem ipsum',
+          }
+        end
+
+        before do
+          stub_request(:get, 'https://example.com/test_post_ohagi').to_return(status: 404)
+        end
+
+        it_behaves_like 'reference once', 'https://example.com/test_post', 'https://example.com/test_post_ohagi'
+
+        it 'do not request to uri' do
+          subject
+          expect(a_request(:get, 'https://example.com/test_post_ohagi')).to_not have_been_made
+        end
+
+        context 'when url and uri is specified at the same time' do
+          let(:text) { 'BT:https://example.com/test_post_ohagi BT:https://example.com/test_post' }
+
+          it_behaves_like 'reference once', 'https://example.com/test_post', 'https://example.com/test_post_ohagi'
+        end
+      end
+    end
   end
 
   describe 'editing new status' do

From 86d4b95be90ed84f8ddbf73739525d8183e8a19d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?KMY=EF=BC=88=E9=9B=AA=E3=81=82=E3=81=99=E3=81=8B=EF=BC=89?=
 <tt@kmycode.net>
Date: Fri, 15 Dec 2023 09:41:22 +0900
Subject: [PATCH 7/8] Merge pull request from GHSA-qg32-3vrh-w6mw

---
 app/controllers/concerns/cache_concern.rb | 10 ++++++++
 app/controllers/statuses_controller.rb    | 31 ++++++++++++++++++++---
 2 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/app/controllers/concerns/cache_concern.rb b/app/controllers/concerns/cache_concern.rb
index 2dfe5e263a..6beecc9791 100644
--- a/app/controllers/concerns/cache_concern.rb
+++ b/app/controllers/concerns/cache_concern.rb
@@ -172,6 +172,16 @@ module CacheConcern
   def render_with_cache(**options)
     raise ArgumentError, 'Only JSON render calls are supported' unless options.key?(:json) || block_given?
 
+    if options.delete(:cancel_cache)
+      if block_given?
+        options[:json] = yield
+      elsif options[:json].is_a?(Symbol)
+        options[:json] = send(options[:json])
+      end
+
+      return render(options)
+    end
+
     key        = options.delete(:key) || [[params[:controller], params[:action]].join('/'), options[:json].respond_to?(:cache_key) ? options[:json].cache_key : nil, options[:fields].nil? ? nil : options[:fields].join(',')].compact.join(':')
     expires_in = options.delete(:expires_in) || 3.minutes
     body       = Rails.cache.read(key, raw: true)
diff --git a/app/controllers/statuses_controller.rb b/app/controllers/statuses_controller.rb
index 7d18aef132..f5397c3b82 100644
--- a/app/controllers/statuses_controller.rb
+++ b/app/controllers/statuses_controller.rb
@@ -29,15 +29,15 @@ class StatusesController < ApplicationController
       end
 
       format.json do
-        expires_in 3.minutes, public: true if @status.distributable? && public_fetch_mode?
-        render_with_cache json: @status, content_type: 'application/activity+json', serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter
+        expires_in 3.minutes, public: true if @status.distributable? && public_fetch_mode? && !misskey_software?
+        render_with_cache json: @status, content_type: 'application/activity+json', serializer: status_activity_serializer, adapter: ActivityPub::Adapter, cancel_cache: misskey_software?
       end
     end
   end
 
   def activity
-    expires_in 3.minutes, public: @status.distributable? && public_fetch_mode?
-    render_with_cache json: ActivityPub::ActivityPresenter.from_status(@status), content_type: 'application/activity+json', serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter
+    expires_in 3.minutes, public: @status.distributable? && public_fetch_mode? && !misskey_software?
+    render_with_cache json: ActivityPub::ActivityPresenter.from_status(@status, for_misskey: misskey_software?), content_type: 'application/activity+json', serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter, cancel_cache: misskey_software?
   end
 
   def embed
@@ -71,6 +71,29 @@ class StatusesController < ApplicationController
     not_found
   end
 
+  def misskey_software?
+    return @misskey_software if defined?(@misskey_software)
+
+    @misskey_software = false
+
+    return false if !@status.local? || signed_request_account&.domain.blank?
+
+    info = InstanceInfo.find_by(domain: signed_request_account.domain)
+    return false if info.nil?
+
+    @misskey_software = %w(misskey calckey cherrypick sharkey).include?(info.software) &&
+                        ((@status.public_unlisted_visibility? && @status.account.user&.setting_reject_public_unlisted_subscription) ||
+                         (@status.unlisted_visibility? && @status.account.user&.setting_reject_unlisted_subscription))
+  end
+
+  def status_activity_serializer
+    if misskey_software?
+      ActivityPub::NoteForMisskeySerializer
+    else
+      ActivityPub::NoteSerializer
+    end
+  end
+
   def redirect_to_original
     redirect_to(ActivityPub::TagManager.instance.url_for(@status.reblog), allow_other_host: true) if @status.reblog?
   end

From 91857a6fa39a58621db04c52047b1be97747c185 Mon Sep 17 00:00:00 2001
From: KMY <tt@kmycode.net>
Date: Mon, 11 Dec 2023 14:04:07 +0900
Subject: [PATCH 8/8] Bump version to 9.4

---
 lib/mastodon/version.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb
index c0f90aad19..e6b982e532 100644
--- a/lib/mastodon/version.rb
+++ b/lib/mastodon/version.rb
@@ -9,7 +9,7 @@ module Mastodon
     end
 
     def kmyblue_minor
-      3
+      4
     end
 
     def kmyblue_flag