From d29b71bfd94f8260469ecde3fa0638dc1a97aff2 Mon Sep 17 00:00:00 2001 From: KMY Date: Sat, 5 Aug 2023 09:10:55 +0900 Subject: [PATCH] Add instance info support --- .../admin/domain_blocks_controller.rb | 9 ++- .../api/v1/admin/domain_blocks_controller.rb | 6 +- app/lib/activitypub/activity/create.rb | 13 +++- app/lib/status_reach_finder.rb | 9 ++- app/models/instance.rb | 1 + app/models/instance_info.rb | 17 +++++ .../activitypub/process_account_service.rb | 18 ++++- app/views/admin/instances/show.html.haml | 4 + app/workers/fetch_instance_info_worker.rb | 75 +++++++++++++++++++ .../update_instance_info_scheduler.rb | 15 ++++ config/sidekiq.yml | 4 + .../20230804222017_create_instance_infoes.rb | 14 ++++ db/schema.rb | 12 ++- 13 files changed, 181 insertions(+), 16 deletions(-) create mode 100644 app/models/instance_info.rb create mode 100644 app/workers/fetch_instance_info_worker.rb create mode 100644 app/workers/scheduler/update_instance_info_scheduler.rb create mode 100644 db/migrate/20230804222017_create_instance_infoes.rb diff --git a/app/controllers/admin/domain_blocks_controller.rb b/app/controllers/admin/domain_blocks_controller.rb index 96bf2048c0..5920dcdee8 100644 --- a/app/controllers/admin/domain_blocks_controller.rb +++ b/app/controllers/admin/domain_blocks_controller.rb @@ -88,15 +88,18 @@ module Admin end def update_params - params.require(:domain_block).permit(:severity, :reject_media, :reject_favourite, :reject_reply, :reject_reply_exclude_followers, :reject_send_not_public_searchability, :reject_send_public_unlisted, :reject_send_dissubscribable, :reject_send_media, :reject_send_sensitive, :reject_hashtag, :reject_straight_follow, :reject_new_follow, :detect_invalid_subscription, :reject_reports, :private_comment, :public_comment, :obfuscate, :hidden, :hidden_anonymous) + params.require(:domain_block).permit(:severity, :reject_media, :reject_favourite, :reject_reply, :reject_reply_exclude_followers, :reject_send_not_public_searchability, :reject_send_public_unlisted, :reject_send_dissubscribable, :reject_send_media, :reject_send_sensitive, :reject_hashtag, + :reject_straight_follow, :reject_new_follow, :detect_invalid_subscription, :reject_reports, :private_comment, :public_comment, :obfuscate, :hidden, :hidden_anonymous) end def resource_params - params.require(:domain_block).permit(:domain, :severity, :reject_media, :reject_favourite, :reject_reply, :reject_reply_exclude_followers, :reject_send_not_public_searchability, :reject_send_public_unlisted, :reject_send_dissubscribable, :reject_send_media, :reject_send_sensitive, :reject_hashtag, :reject_straight_follow, :reject_new_follow, :detect_invalid_subscription, :reject_reports, :private_comment, :public_comment, :obfuscate, :hidden, :hidden_anonymous) + params.require(:domain_block).permit(:domain, :severity, :reject_media, :reject_favourite, :reject_reply, :reject_reply_exclude_followers, :reject_send_not_public_searchability, :reject_send_public_unlisted, :reject_send_dissubscribable, :reject_send_media, :reject_send_sensitive, :reject_hashtag, + :reject_straight_follow, :reject_new_follow, :detect_invalid_subscription, :reject_reports, :private_comment, :public_comment, :obfuscate, :hidden, :hidden_anonymous) end def form_domain_block_batch_params - params.require(:form_domain_block_batch).permit(domain_blocks_attributes: [:enabled, :domain, :severity, :reject_media, :reject_favourite, :reject_reply, :reject_reply_exclude_followers, :reject_send_not_public_searchability, :reject_send_public_unlisted, :reject_send_dissubscribable, :reject_send_media, :reject_send_sensitive, :reject_hashtag, :reject_straight_follow, :reject_new_follow, :detect_invalid_subscription, :reject_reports, :private_comment, :public_comment, :obfuscate, :hidden, :hidden_anonymous]) + params.require(:form_domain_block_batch).permit(domain_blocks_attributes: [:enabled, :domain, :severity, :reject_media, :reject_favourite, :reject_reply, :reject_reply_exclude_followers, :reject_send_not_public_searchability, :reject_send_public_unlisted, :reject_send_dissubscribable, :reject_send_media, + :reject_send_sensitive, :reject_hashtag, :reject_straight_follow, :reject_new_follow, :detect_invalid_subscription, :reject_reports, :private_comment, :public_comment, :obfuscate, :hidden, :hidden_anonymous]) end def action_from_button diff --git a/app/controllers/api/v1/admin/domain_blocks_controller.rb b/app/controllers/api/v1/admin/domain_blocks_controller.rb index ef0f4e464a..bd0660dbaa 100644 --- a/app/controllers/api/v1/admin/domain_blocks_controller.rb +++ b/app/controllers/api/v1/admin/domain_blocks_controller.rb @@ -69,7 +69,8 @@ class Api::V1::Admin::DomainBlocksController < Api::BaseController end def domain_block_params - params.permit(:severity, :reject_media, :reject_favourite, :reject_reply, :reject_reply_exclude_followers, :reject_reports, :reject_send_not_public_searchability, :reject_send_public_unlisted, :reject_send_dissubscribable, :reject_send_media, :reject_send_sensitive, :reject_hashtag, :reject_straight_follow, :reject_new_follow, :detect_invalid_subscription, :private_comment, :public_comment, :obfuscate, :hidden, :hidden_anonymous) + params.permit(:severity, :reject_media, :reject_favourite, :reject_reply, :reject_reply_exclude_followers, :reject_reports, :reject_send_not_public_searchability, :reject_send_public_unlisted, :reject_send_dissubscribable, :reject_send_media, :reject_send_sensitive, :reject_hashtag, :reject_straight_follow, + :reject_new_follow, :detect_invalid_subscription, :private_comment, :public_comment, :obfuscate, :hidden, :hidden_anonymous) end def insert_pagination_headers @@ -101,6 +102,7 @@ class Api::V1::Admin::DomainBlocksController < Api::BaseController end def resource_params - params.permit(:domain, :severity, :reject_media, :reject_favourite, :reject_reply, :reject_reply_exclude_followers, :reject_send_not_public_searchability, :reject_send_public_unlisted, :reject_send_dissubscribable, :reject_send_media, :reject_send_sensitive, :reject_hashtag, :reject_straight_follow, :reject_new_follow, :detect_invalid_subscription, :reject_reports, :private_comment, :public_comment, :obfuscate, :hidden, :hidden_anonymous) + params.permit(:domain, :severity, :reject_media, :reject_favourite, :reject_reply, :reject_reply_exclude_followers, :reject_send_not_public_searchability, :reject_send_public_unlisted, :reject_send_dissubscribable, :reject_send_media, :reject_send_sensitive, :reject_hashtag, :reject_straight_follow, + :reject_new_follow, :detect_invalid_subscription, :reject_reports, :private_comment, :public_comment, :obfuscate, :hidden, :hidden_anonymous) end end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index c5a01093ed..cc2536578f 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -507,7 +507,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity SCAN_SEARCHABILITY_FEDIBIRD_RE = /searchable_by_(all_users|followers_only|reacted_users_only|nobody)/ def searchability - searchability_from_audience || searchability_from_bio || (marked_as_misskey_searchability? ? misskey_searchability : nil) + searchability_from_audience || searchability_from_bio || (misskey_software? ? misskey_searchability : nil) end def searchability_from_bio @@ -528,8 +528,15 @@ class ActivityPub::Activity::Create < ActivityPub::Activity searchability end - def marked_as_misskey_searchability? - @marked_as_misskey_searchability ||= DomainBlock.detect_invalid_subscription?(@account.domain) + def instance_info + @instance_info ||= InstanceInfo.find_by(@account.domain) + end + + def misskey_software? + info = instance_info + return DomainBlock.detect_invalid_subscription?(@account.domain) if info.nil? + + %w(misskey calckey firefish).include?(info.software) || DomainBlock.detect_invalid_subscription?(@account.domain) end def misskey_searchability diff --git a/app/lib/status_reach_finder.rb b/app/lib/status_reach_finder.rb index 8d621b3fe8..6cce4e04fb 100644 --- a/app/lib/status_reach_finder.rb +++ b/app/lib/status_reach_finder.rb @@ -153,9 +153,10 @@ class StatusReachFinder end def banned_domains_for_misskey_of_status(status) - blocks = DomainBlock.where(domain: nil) - blocks = blocks.or(DomainBlock.where(detect_invalid_subscription: true)) if status.public_unlisted_visibility? && status.account.user&.setting_reject_public_unlisted_subscription - blocks = blocks.or(DomainBlock.where(detect_invalid_subscription: true)) if status.unlisted_visibility? && status.account.user&.setting_reject_unlisted_subscription - blocks.pluck(:domain).uniq + return [] unless (status.public_unlisted_visibility? && status.account.user&.setting_reject_public_unlisted_subscription) || (status.unlisted_visibility? && status.account.user&.setting_reject_unlisted_subscription) + + from_info = InstanceInfo.where(software: %w(misskey calckey firefish)).pluck(:domain) + from_domain_block = DomainBlock.where(detect_invalid_subscription: true).pluck(:domain) + (from_info + from_domain_block).uniq end end diff --git a/app/models/instance.rb b/app/models/instance.rb index 17ee0cbb1e..0fb1d3e96a 100644 --- a/app/models/instance.rb +++ b/app/models/instance.rb @@ -19,6 +19,7 @@ class Instance < ApplicationRecord belongs_to :domain_block belongs_to :domain_allow belongs_to :unavailable_domain # skipcq: RB-RL1031 + belongs_to :instance_info end scope :searchable, -> { where.not(domain: DomainBlock.select(:domain)) } diff --git a/app/models/instance_info.rb b/app/models/instance_info.rb new file mode 100644 index 0000000000..297134884f --- /dev/null +++ b/app/models/instance_info.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: instance_infos +# +# id :bigint(8) not null, primary key +# domain :string default(""), not null +# software :string default(""), not null +# version :string default(""), not null +# data :jsonb not null +# created_at :datetime not null +# updated_at :datetime not null +# + +class InstanceInfo < ApplicationRecord +end diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 11283b889b..18af495e35 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -46,6 +46,7 @@ class ActivityPub::ProcessAccountService < BaseService end create_account + fetch_instance_info end update_account @@ -207,6 +208,10 @@ class ActivityPub::ProcessAccountService < BaseService AccountMergingWorker.perform_async(@account.id) end + def fetch_instance_info + FetchInstanceInfoWorker.perform_async(@account.domain) unless InstanceInfo.exists?(domain: @account.domain) + end + def actor_type if @json['type'].is_a?(Array) @json['type'].find { |type| ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(type) } @@ -258,7 +263,7 @@ class ActivityPub::ProcessAccountService < BaseService bio = searchability_from_bio return bio unless bio.nil? - return marked_as_misskey_searchability? ? :public : :private + return misskey_software? ? :public : :private end if audience_searchable_by.any? { |uri| ActivityPub::TagManager.instance.public_collection?(uri) } @@ -288,8 +293,15 @@ class ActivityPub::ProcessAccountService < BaseService searchability end - def marked_as_misskey_searchability? - domain_block&.detect_invalid_subscription + def instance_info + @instance_info ||= InstanceInfo.find_by(@domain) + end + + def misskey_software? + info = instance_info + return domain_block&.detect_invalid_subscription if info.nil? + + %w(misskey calckey firefish).include?(info.software) || domain_block&.detect_invalid_subscription end def subscribable_by diff --git a/app/views/admin/instances/show.html.haml b/app/views/admin/instances/show.html.haml index 3e4c41f737..9612f1e58d 100644 --- a/app/views/admin/instances/show.html.haml +++ b/app/views/admin/instances/show.html.haml @@ -1,6 +1,10 @@ - content_for :page_title do = @instance.domain +-if @instance.instance_info.present? + %p + = "#{@instance.instance_info.software} #{@instance.instance_info.version}" + - content_for :header_tags do = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' diff --git a/app/workers/fetch_instance_info_worker.rb b/app/workers/fetch_instance_info_worker.rb new file mode 100644 index 0000000000..78de3490f4 --- /dev/null +++ b/app/workers/fetch_instance_info_worker.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +class FetchInstanceInfoWorker + include Sidekiq::Worker + include JsonLdHelper + include Redisable + include Lockable + + class Error < StandardError; end + class GoneError < Error; end + class RequestError < Error; end + + def perform(domain) + @instance = Instance.find_by(domain: domain) + return if !@instance || @instance.unavailable_domain.present? + + with_redis_lock("instance_info:#{domain}") do + link = nodeinfo_link + return if link.nil? + + update_info!(link) + end + end + + private + + def nodeinfo_link + nodeinfo = fetch_json("https://#{@instance.domain}/.well-known/nodeinfo") + return nil if nodeinfo.nil? || !nodeinfo.key?('links') + + nodeinfo_links = nodeinfo['links'] + return nil if !nodeinfo_links.is_a?(Array) || nodeinfo_links.blank? + + nodeinfo_link = nodeinfo_links.find { |item| item.key?('rel') && item.key?('href') && item['rel'] == 'http://nodeinfo.diaspora.software/ns/schema/2.0' } + return nil if nodeinfo_link.nil? || nodeinfo_link['href'].nil? || !nodeinfo_link['href'].start_with?('http') + + nodeinfo_link['href'] + end + + def update_info!(url) + content = fetch_json(url) + return nil if content.nil? || !content.key?('software') || !content['software'].key?('name') + + software = content['software']['name'] + version = content['software'].key?('version') ? content['software']['version'] : '' + + exists = @instance.instance_info + if exists.nil? + InstanceInfo.create!(domain: @instance.domain, software: software, version: version, data: content) + else + exists.software = software + exists.version = version + exists.data = content + exists.save! + end + end + + def fetch_json(url) + build_request(url).perform do |response| + if [200, 203].include?(response.code) + raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) + + body_to_json(response.body_with_limit) + elsif response.code == 410 + raise FetchInstanceInfoWorker::GoneError, "#{domain} is gone from the server" + else + raise FetchInstanceInfoWorker::RequestError, "Request for #{domain} returned HTTP #{response.code}" + end + end + end + + def build_request(url) + Request.new(:get, url).add_headers('Accept' => 'application/jrd+json, application/json') + end +end diff --git a/app/workers/scheduler/update_instance_info_scheduler.rb b/app/workers/scheduler/update_instance_info_scheduler.rb new file mode 100644 index 0000000000..e4d1230a78 --- /dev/null +++ b/app/workers/scheduler/update_instance_info_scheduler.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class Scheduler::UpdateInstanceInfoScheduler + include Sidekiq::Worker + + sidekiq_options retry: 1 + + def perform + Instance.select(:domain).reorder(nil).find_in_batches do |instances| + FetchInstanceInfoWorker.push_bulk(instances) do |instance| + [instance.domain] + end + end + end +end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index fdd1d73e1b..abd90944cf 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -62,3 +62,7 @@ interval: 30 seconds class: Scheduler::SidekiqHealthScheduler queue: scheduler + update_instance_info_scheduler: + cron: '0 0 * * *' + class: Scheduler::UpdateInstanceInfoScheduler + queue: scheduler diff --git a/db/migrate/20230804222017_create_instance_infoes.rb b/db/migrate/20230804222017_create_instance_infoes.rb new file mode 100644 index 0000000000..c1d1ccbdc9 --- /dev/null +++ b/db/migrate/20230804222017_create_instance_infoes.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class CreateInstanceInfoes < ActiveRecord::Migration[7.0] + def change + create_table :instance_infos do |t| + t.string :domain, null: false, default: '', index: { unique: true } + t.string :software, null: false, default: '' + t.string :version, null: false, default: '' + t.jsonb :data, null: false, default: {} + t.datetime :created_at, null: false + t.datetime :updated_at, null: false + end + end +end diff --git a/db/schema.rb b/db/schema.rb index a763d08f09..d3ac065448 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2023_07_24_160715) do +ActiveRecord::Schema[7.0].define(version: 2023_08_04_222017) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -631,6 +631,16 @@ ActiveRecord::Schema[7.0].define(version: 2023_07_24_160715) do t.boolean "overwrite", default: false, null: false end + create_table "instance_infos", force: :cascade do |t| + t.string "domain", default: "", null: false + t.string "software", default: "", null: false + t.string "version", default: "", null: false + t.jsonb "data", default: {}, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["domain"], name: "index_instance_infos_on_domain", unique: true + end + create_table "invites", force: :cascade do |t| t.bigint "user_id", null: false t.string "code", default: "", null: false