Merge pull request #346 from kmycode/kb-draft-9.4

Release: 9.4
This commit is contained in:
KMY(雪あすか) 2023-12-15 10:03:45 +09:00 committed by GitHub
commit efffdebdc9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 219 additions and 80 deletions

View file

@ -172,6 +172,16 @@ module CacheConcern
def render_with_cache(**options)
raise ArgumentError, 'Only JSON render calls are supported' unless options.key?(:json) || block_given?
if options.delete(:cancel_cache)
if block_given?
options[:json] = yield
elsif options[:json].is_a?(Symbol)
options[:json] = send(options[:json])
end
return render(options)
end
key = options.delete(:key) || [[params[:controller], params[:action]].join('/'), options[:json].respond_to?(:cache_key) ? options[:json].cache_key : nil, options[:fields].nil? ? nil : options[:fields].join(',')].compact.join(':')
expires_in = options.delete(:expires_in) || 3.minutes
body = Rails.cache.read(key, raw: true)

View file

@ -29,15 +29,15 @@ class StatusesController < ApplicationController
end
format.json do
expires_in 3.minutes, public: true if @status.distributable? && public_fetch_mode?
render_with_cache json: @status, content_type: 'application/activity+json', serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter
expires_in 3.minutes, public: true if @status.distributable? && public_fetch_mode? && !misskey_software?
render_with_cache json: @status, content_type: 'application/activity+json', serializer: status_activity_serializer, adapter: ActivityPub::Adapter, cancel_cache: misskey_software?
end
end
end
def activity
expires_in 3.minutes, public: @status.distributable? && public_fetch_mode?
render_with_cache json: ActivityPub::ActivityPresenter.from_status(@status), content_type: 'application/activity+json', serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter
expires_in 3.minutes, public: @status.distributable? && public_fetch_mode? && !misskey_software?
render_with_cache json: ActivityPub::ActivityPresenter.from_status(@status, for_misskey: misskey_software?), content_type: 'application/activity+json', serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter, cancel_cache: misskey_software?
end
def embed
@ -71,6 +71,29 @@ class StatusesController < ApplicationController
not_found
end
def misskey_software?
return @misskey_software if defined?(@misskey_software)
@misskey_software = false
return false if !@status.local? || signed_request_account&.domain.blank?
info = InstanceInfo.find_by(domain: signed_request_account.domain)
return false if info.nil?
@misskey_software = %w(misskey calckey cherrypick sharkey).include?(info.software) &&
((@status.public_unlisted_visibility? && @status.account.user&.setting_reject_public_unlisted_subscription) ||
(@status.unlisted_visibility? && @status.account.user&.setting_reject_unlisted_subscription))
end
def status_activity_serializer
if misskey_software?
ActivityPub::NoteForMisskeySerializer
else
ActivityPub::NoteSerializer
end
end
def redirect_to_original
redirect_to(ActivityPub::TagManager.instance.url_for(@status.reblog), allow_other_host: true) if @status.reblog?
end

View file

@ -505,7 +505,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
url = quote
if url.present?
ResolveURLService.new.call(url, on_behalf_of: @account, local_only: true).present?
ActivityPub::TagManager.instance.uri_to_resource(url, Status)&.local?
else
false
end

View file

@ -103,7 +103,7 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity
info = instance_info
return false if info.nil?
%w(misskey calckey firefish meisskey cherrypick).include?(info.software)
%w(misskey calckey firefish meisskey cherrypick sharkey).include?(info.software)
end
def instance_info

View file

@ -208,7 +208,7 @@ class ActivityPub::TagManager
uri_to_resource(uri, Account)
end
def uri_to_resource(uri, klass)
def uri_to_resource(uri, klass, url: false)
return if uri.nil?
if local_uri?(uri)
@ -221,7 +221,9 @@ class ActivityPub::TagManager
elsif OStatus::TagManager.instance.local_id?(uri)
klass.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, klass.to_s))
else
klass.find_by(uri: uri.split('#').first)
resource = klass.find_by(uri: uri.split('#').first)
resource ||= klass.where('uri != url').find_by(url: uri.split('#').first) if url
resource
end
rescue ActiveRecord::RecordNotFound
nil

View file

@ -216,7 +216,7 @@ class StatusReachFinder
return [] if status.public_searchability?
return [] unless (status.public_unlisted_visibility? && status.account.user&.setting_reject_public_unlisted_subscription) || (status.unlisted_visibility? && status.account.user&.setting_reject_unlisted_subscription)
from_info = InstanceInfo.where(software: %w(misskey calckey cherrypick)).pluck(:domain)
from_info = InstanceInfo.where(software: %w(misskey calckey cherrypick sharkey)).pluck(:domain)
from_domain_block = DomainBlock.where(detect_invalid_subscription: true).pluck(:domain)
(from_info + from_domain_block).uniq
end

View file

@ -21,6 +21,7 @@ class InstanceInfo < ApplicationRecord
calckey
cherrypick
meisskey
sharkey
firefish
renedon
fedibird
@ -39,6 +40,7 @@ class InstanceInfo < ApplicationRecord
return false if info.nil?
return true if EMOJI_REACTION_AVAILABLE_SOFTWARES.include?(info['software'])
return false if info.data['metadata'].nil? || !info.data['metadata'].is_a?(Hash)
features = info.data.dig('metadata', 'features')
return false if features.nil? || !features.is_a?(Array)

View file

@ -83,7 +83,7 @@ class Status < ApplicationRecord
has_many :mentions, dependent: :destroy, inverse_of: :status
has_many :mentioned_accounts, through: :mentions, source: :account, class_name: 'Account'
has_many :active_mentions, -> { active }, class_name: 'Mention', inverse_of: :status
has_many :media_attachments, dependent: :nullify
has_many :media_attachments, -> { order('id asc') }, dependent: :nullify, inverse_of: false
has_many :reference_objects, class_name: 'StatusReference', inverse_of: :status, dependent: :destroy
has_many :references, through: :reference_objects, class_name: 'Status', source: :target_status
has_many :referenced_by_status_objects, foreign_key: 'target_status_id', class_name: 'StatusReference', inverse_of: :target_status, dependent: :destroy

View file

@ -46,7 +46,6 @@ class ActivityPub::ProcessAccountService < BaseService
end
create_account
fetch_instance_info
end
update_account
@ -66,6 +65,8 @@ class ActivityPub::ProcessAccountService < BaseService
check_links! if @account.fields.any?(&:requires_verification?)
end
fetch_instance_info
@account
rescue Oj::ParseError
nil
@ -210,7 +211,7 @@ class ActivityPub::ProcessAccountService < BaseService
end
def fetch_instance_info
ActivityPub::FetchInstanceInfoWorker.perform_async(@account.domain) unless InstanceInfo.exists?(domain: @account.domain)
ActivityPub::FetchInstanceInfoWorker.perform_async(@account.domain) unless Rails.cache.exist?("fetch_instance_info:#{@account.domain}", expires_in: 1.day)
end
def actor_type

View file

@ -108,7 +108,16 @@ class ProcessReferencesService < BaseService
end
def url_to_status(url)
ResolveURLService.new.call(url, on_behalf_of: @status.account, fetch_remote: @fetch_remote && @no_fetch_urls.exclude?(url))
status = ActivityPub::TagManager.instance.uri_to_resource(url, Status, url: true)
status ||= ResolveURLService.new.call(url, on_behalf_of: @status.account) if @fetch_remote && @no_fetch_urls.exclude?(url)
referrable?(status) ? status : nil
end
def referrable?(target_status)
return false if target_status.nil?
return @referrable if defined?(@referrable)
@referrable = StatusPolicy.new(@status.account, target_status).show?
end
def quote_status_ids
@ -116,7 +125,7 @@ class ProcessReferencesService < BaseService
end
def quotable?(target_status)
target_status.account.allow_quote? && (!@status.local? || StatusPolicy.new(@status.account, target_status).quote?)
target_status.account.allow_quote? && StatusPolicy.new(@status.account, target_status).quote?
end
def add_references

View file

@ -6,16 +6,15 @@ class ResolveURLService < BaseService
USERNAME_STATUS_RE = %r{/@(?<username>#{Account::USERNAME_RE})/(?<status_id>[0-9]+)\Z}
def call(url, on_behalf_of: nil, fetch_remote: true, local_only: false)
def call(url, on_behalf_of: nil)
@url = url
@on_behalf_of = on_behalf_of
@fetch_remote = fetch_remote
if local_url?
process_local_url
elsif !local_only && fetch_remote && !fetched_resource.nil?
elsif !fetched_resource.nil?
process_url
elsif !local_only
else
process_url_from_db
end
end
@ -38,7 +37,7 @@ class ResolveURLService < BaseService
return account unless account.nil?
end
return unless @on_behalf_of.present? && (!@fetch_remote || [401, 403, 404].include?(fetch_resource_service.response_code))
return unless @on_behalf_of.present? && [401, 403, 404].include?(fetch_resource_service.response_code)
# It may happen that the resource is a private toot, and thus not fetchable,
# but we can return the toot if we already know about it.

View file

@ -8,28 +8,32 @@ class ActivityPub::FetchInstanceInfoWorker
sidekiq_options queue: 'push', retry: 2
class Error < StandardError; end
class RequestError < Error; end
class DeadError < Error; end
SUPPORTED_NOTEINFO_RELS = ['http://nodeinfo.diaspora.software/ns/schema/2.0', 'http://nodeinfo.diaspora.software/ns/schema/2.1'].freeze
def perform(domain)
@instance = Instance.find_by(domain: domain)
return if !@instance || @instance.unavailable_domain.present?
with_redis_lock("instance_info:#{domain}") do
link = nodeinfo_link
return if link.nil?
update_info!(link)
Rails.cache.fetch("fetch_instance_info:#{@instance.domain}", expires_in: 1.day, race_condition_ttl: 1.hour) do
fetch!
end
rescue ActivityPub::FetchInstanceInfoWorker::DeadError
true
end
private
def fetch!
link = nodeinfo_link
return if link.nil?
update_info!(link)
true
rescue Mastodon::UnexpectedResponseError
true
end
def nodeinfo_link
nodeinfo = fetch_json("https://#{@instance.domain}/.well-known/nodeinfo")
return nil if nodeinfo.nil? || !nodeinfo.key?('links')
@ -63,15 +67,9 @@ class ActivityPub::FetchInstanceInfoWorker
def fetch_json(url)
build_request(url).perform do |response|
if [200, 203].include?(response.code)
raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response)
body_to_json(response.body_with_limit)
elsif [400, 401, 403, 404, 410].include?(response.code)
raise ActivityPub::FetchInstanceInfoWorker::DeadError, "Request for #{@instance.domain} returned HTTP #{response.code}"
else
raise ActivityPub::FetchInstanceInfoWorker::RequestError, "Request for #{@instance.domain} returned HTTP #{response.code}"
end
end
end

View file

@ -1,15 +0,0 @@
# frozen_string_literal: true
class Scheduler::UpdateInstanceInfoScheduler
include Sidekiq::Worker
sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
def perform
Instance.select(:domain).reorder(nil).find_in_batches do |instances|
ActivityPub::FetchInstanceInfoWorker.push_bulk(instances) do |instance|
[instance.domain]
end
end
end
end

View file

@ -63,10 +63,6 @@
interval: 30 seconds
class: Scheduler::SidekiqHealthScheduler
queue: scheduler
update_instance_info_scheduler:
cron: '0 0 * * *'
class: Scheduler::UpdateInstanceInfoScheduler
queue: scheduler
software_update_check_scheduler:
interval: 30 minutes
class: Scheduler::SoftwareUpdateCheckScheduler

View file

@ -0,0 +1,19 @@
# frozen_string_literal: true
class ImproveIndexForPublicTimelineSpeed < ActiveRecord::Migration[7.0]
disable_ddl_transaction!
def up
add_index :statuses, [:id, :account_id], name: :index_statuses_local_20231213, algorithm: :concurrently, order: { id: :desc }, where: '(local OR (uri IS NULL)) AND deleted_at IS NULL AND visibility IN (0, 10, 11) AND reblog_of_id IS NULL AND ((NOT reply) OR (in_reply_to_account_id = account_id))'
add_index :statuses, [:id, :account_id], name: :index_statuses_public_20231213, algorithm: :concurrently, order: { id: :desc }, where: 'deleted_at IS NULL AND visibility IN (0, 10, 11) AND reblog_of_id IS NULL AND ((NOT reply) OR (in_reply_to_account_id = account_id))'
remove_index :statuses, name: :index_statuses_local_20190824
remove_index :statuses, name: :index_statuses_public_20200119
end
def down
add_index :statuses, [:id, :account_id], name: :index_statuses_local_20190824, algorithm: :concurrently, order: { id: :desc }, where: '(local OR (uri IS NULL)) AND deleted_at IS NULL AND visibility = 0 AND reblog_of_id IS NULL AND ((NOT reply) OR (in_reply_to_account_id = account_id))'
add_index :statuses, [:id, :account_id], name: :index_statuses_public_20200119, algorithm: :concurrently, order: { id: :desc }, where: 'deleted_at IS NULL AND visibility = 0 AND reblog_of_id IS NULL AND ((NOT reply) OR (in_reply_to_account_id = account_id))'
remove_index :statuses, name: :index_statuses_local_20231213
remove_index :statuses, name: :index_statuses_public_20231213
end
end

View file

@ -0,0 +1,13 @@
# frozen_string_literal: true
class IndexToStatusesURL < ActiveRecord::Migration[7.1]
disable_ddl_transaction!
def up
add_index :statuses, :url, name: :index_statuses_on_url, algorithm: :concurrently, opclass: :text_pattern_ops, where: 'url IS NOT NULL AND url <> uri'
end
def down
remove_index :statuses, name: :index_statuses_on_url
end
end

View file

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.1].define(version: 2023_11_05_225839) do
ActiveRecord::Schema[7.1].define(version: 2023_12_14_225249) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@ -1235,13 +1235,14 @@ ActiveRecord::Schema[7.1].define(version: 2023_11_05_225839) do
t.index ["account_id", "reblog_of_id", "deleted_at", "searchability"], name: "index_statuses_for_get_following_accounts_to_search", where: "((deleted_at IS NULL) AND (reblog_of_id IS NULL) AND (searchability = ANY (ARRAY[0, 10, 1])))"
t.index ["account_id"], name: "index_statuses_on_account_id"
t.index ["deleted_at"], name: "index_statuses_on_deleted_at", where: "(deleted_at IS NOT NULL)"
t.index ["id", "account_id"], name: "index_statuses_local_20190824", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
t.index ["id", "account_id"], name: "index_statuses_public_20200119", order: { id: :desc }, where: "((deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
t.index ["id", "account_id"], name: "index_statuses_local_20231213", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = ANY (ARRAY[0, 10, 11])) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
t.index ["id", "account_id"], name: "index_statuses_public_20231213", order: { id: :desc }, where: "((deleted_at IS NULL) AND (visibility = ANY (ARRAY[0, 10, 11])) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
t.index ["in_reply_to_account_id"], name: "index_statuses_on_in_reply_to_account_id", where: "(in_reply_to_account_id IS NOT NULL)"
t.index ["in_reply_to_id"], name: "index_statuses_on_in_reply_to_id", where: "(in_reply_to_id IS NOT NULL)"
t.index ["quote_of_id", "account_id"], name: "index_statuses_on_quote_of_id_and_account_id"
t.index ["reblog_of_id", "account_id"], name: "index_statuses_on_reblog_of_id_and_account_id"
t.index ["uri"], name: "index_statuses_on_uri", unique: true, opclass: :text_pattern_ops, where: "(uri IS NOT NULL)"
t.index ["url"], name: "index_statuses_on_url", opclass: :text_pattern_ops, where: "((url IS NOT NULL) AND ((url)::text <> (uri)::text))"
end
create_table "statuses_tags", primary_key: ["tag_id", "status_id"], force: :cascade do |t|

View file

@ -9,7 +9,7 @@ module Mastodon
end
def kmyblue_minor
3
4
end
def kmyblue_flag

View file

@ -55,6 +55,7 @@ RSpec.describe ActivityPub::Activity::Update do
stub_request(:get, actor_json[:following]).to_return(status: 404)
stub_request(:get, actor_json[:featured]).to_return(status: 404)
stub_request(:get, actor_json[:featuredTags]).to_return(status: 404)
stub_request(:get, 'https://example.com/.well-known/nodeinfo').to_return(status: 404)
subject.perform
end

View file

@ -463,7 +463,7 @@ RSpec.describe Status do
describe '.emoji_reaction_availables_map' do
subject { described_class.emoji_reaction_availables_map(domains) }
let(:domains) { %w(features_available.com features_unavailable.com features_invalid.com features_nil.com no_info.com mastodon.com misskey.com) }
let(:domains) { %w(features_available.com features_unavailable.com features_invalid.com features_nil.com no_info.com mastodon.com misskey.com old_mastodon.com) }
before do
Fabricate(:instance_info, domain: 'features_available.com', software: 'mastodon', data: { metadata: { features: ['emoji_reaction'] } })
@ -472,6 +472,7 @@ RSpec.describe Status do
Fabricate(:instance_info, domain: 'features_nil.com', software: 'mastodon', data: { metadata: { features: nil } })
Fabricate(:instance_info, domain: 'mastodon.com', software: 'mastodon')
Fabricate(:instance_info, domain: 'misskey.com', software: 'misskey')
Fabricate(:instance_info, domain: 'old_mastodon.com', software: 'mastodon', data: { metadata: [] })
end
it 'availables if features contains emoji_reaction' do
@ -497,6 +498,10 @@ RSpec.describe Status do
it 'availables if misskey server' do
expect(subject['misskey.com']).to be true
end
it 'unavailables if old mastodon server' do
expect(subject['old_mastodon.com']).to be false
end
end
describe '.tagged_with' do

View file

@ -5,6 +5,10 @@ require 'rails_helper'
RSpec.describe ActivityPub::ProcessAccountService, type: :service do
subject { described_class.new }
before do
stub_request(:get, 'https://example.com/.well-known/nodeinfo').to_return(status: 404)
end
context 'with searchability' do
subject { described_class.new.call('alice', 'example.com', payload) }

View file

@ -241,6 +241,76 @@ RSpec.describe ProcessReferencesService, type: :service do
end
end
end
context 'when already fetched remote post' do
let(:account) { Fabricate(:account, followers_url: 'http://example.com/followers', domain: 'example.com', uri: 'https://example.com/actor') }
let!(:remote_status) { Fabricate(:status, account: account, uri: 'https://example.com/test_post', url: 'https://example.com/test_post', text: 'Lorem ipsum') }
let(:object_json) do
{
id: 'https://example.com/test_post',
to: ActivityPub::TagManager::COLLECTIONS[:public],
'@context': ActivityPub::TagManager::CONTEXT,
type: 'Note',
actor: account.uri,
attributedTo: account.uri,
content: 'Lorem ipsum',
}
end
let(:text) { 'BT:https://example.com/test_post' }
shared_examples 'reference once' do |uri, url|
it 'reference it' do
expect(subject.size).to eq 1
expect(subject[0][1]).to eq 'BT'
status = Status.find_by(id: subject[0][0])
expect(status).to_not be_nil
expect(status.id).to eq remote_status.id
expect(status.uri).to eq uri
expect(status.url).to eq url
end
end
before do
stub_request(:get, 'https://example.com/test_post').to_return(status: 200, body: Oj.dump(object_json), headers: { 'Content-Type' => 'application/activity+json' })
end
it_behaves_like 'reference once', 'https://example.com/test_post', 'https://example.com/test_post'
context 'when uri and url is difference and url is not accessable' do
let(:remote_status) { Fabricate(:status, account: account, uri: 'https://example.com/test_post', url: 'https://example.com/test_post_ohagi', text: 'Lorem ipsum') }
let(:text) { 'BT:https://example.com/test_post_ohagi' }
let(:object_json) do
{
id: 'https://example.com/test_post',
url: 'https://example.com/test_post_ohagi',
to: ActivityPub::TagManager::COLLECTIONS[:public],
'@context': ActivityPub::TagManager::CONTEXT,
type: 'Note',
actor: account.uri,
attributedTo: account.uri,
content: 'Lorem ipsum',
}
end
before do
stub_request(:get, 'https://example.com/test_post_ohagi').to_return(status: 404)
end
it_behaves_like 'reference once', 'https://example.com/test_post', 'https://example.com/test_post_ohagi'
it 'do not request to uri' do
subject
expect(a_request(:get, 'https://example.com/test_post_ohagi')).to_not have_been_made
end
context 'when url and uri is specified at the same time' do
let(:text) { 'BT:https://example.com/test_post_ohagi BT:https://example.com/test_post' }
it_behaves_like 'reference once', 'https://example.com/test_post', 'https://example.com/test_post_ohagi'
end
end
end
end
describe 'editing new status' do

View file

@ -67,9 +67,22 @@ describe ActivityPub::FetchInstanceInfoWorker do
Instance.refresh
end
it 'does not update immediately' do
stub_request(:get, 'https://example.com/nodeinfo/2.0').to_return(status: 200, body: nodeinfo_json)
subject.perform('example.com')
stub_request(:get, 'https://example.com/nodeinfo/2.0').to_return(status: 200, body: new_nodeinfo_json)
subject.perform('example.com')
info = InstanceInfo.find_by(domain: 'example.com')
expect(info).to_not be_nil
expect(info.software).to eq 'mastodon'
expect(info.version).to eq '4.2.0-beta1'
end
it 'performs a mastodon instance' do
stub_request(:get, 'https://example.com/nodeinfo/2.0').to_return(status: 200, body: nodeinfo_json)
subject.perform('example.com')
Rails.cache.delete('fetch_instance_info:example.com')
stub_request(:get, 'https://example.com/nodeinfo/2.0').to_return(status: 200, body: new_nodeinfo_json)
subject.perform('example.com')
@ -93,5 +106,12 @@ describe ActivityPub::FetchInstanceInfoWorker do
info = InstanceInfo.find_by(domain: 'example.com')
expect(info).to be_nil
end
it 'does not fetch again immediately' do
expect(subject.perform('example.com')).to be true
expect(subject.perform('example.com')).to be true
expect(a_request(:get, 'https://example.com/.well-known/nodeinfo')).to have_been_made.once
end
end
end

View file

@ -1,19 +0,0 @@
# frozen_string_literal: true
require 'rails_helper'
describe Scheduler::UpdateInstanceInfoScheduler do
let(:worker) { described_class.new }
before do
stub_request(:get, 'https://example.com/.well-known/nodeinfo').to_return(status: 200, body: '{}')
Fabricate(:account, domain: 'example.com')
Instance.refresh
end
describe 'perform' do
it 'runs without error' do
expect { worker.perform }.to_not raise_error
end
end
end