gitea/nas
1
0
Fork 2
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge commit '7742f440fa' into kbtopic-follow-9.0-to-patch

Browse source
This commit is contained in:
KMY 2023-11-28 12:06:25 +09:00
commit c4410ac88d
862 changed files with 28942 additions and 21611 deletions
Download patch file Download diff file Expand all files Collapse all files

67
lib/mastodon/cli/main.rb
View file

@ -69,7 +69,6 @@ module Mastodon::CLI
desc 'ohagi SUBCOMMAND ...ARGS', 'Ohagis'
subcommand 'ohagi', Ohagi
option :dry_run, type: :boolean
desc 'self-destruct', 'Erase the server from the federation'
long_desc <<~LONG_DESC
Erase the server from the federation by broadcasting account delete
@ -96,55 +95,37 @@ module Mastodon::CLI
prompt = TTY::Prompt.new
exit(1) unless prompt.ask('Type in the domain of the server to confirm:', required: true) == Rails.configuration.x.local_domain
if SelfDestructHelper.self_destruct?
prompt.ok('Self-destruct mode is already enabled for this Mastodon server')
unless dry_run?
prompt.warn('This operation WILL NOT be reversible. It can also take a long time.')
prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.')
prompt.warn('A running Sidekiq process is required. Do not shut it down until queues clear.')
pending_accounts = Account.local.without_suspended.count + Account.local.suspended.joins(:deletion_request).count
sidekiq_stats = Sidekiq::Stats.new
exit(1) if prompt.no?('Are you sure you want to proceed?')
end
inboxes = Account.inboxes
processed = 0
Setting.registrations_mode = 'none' unless dry_run?
if inboxes.empty?
Account.local.without_suspended.in_batches.update_all(suspended_at: Time.now.utc, suspension_origin: :local) unless dry_run?
prompt.ok('It seems like your server has not federated with anything')
prompt.ok('You can shut it down and delete it any time')
return
end
prompt.warn('Do NOT interrupt this process...')
delete_account = lambda do |account|
payload = ActiveModelSerializers::SerializableResource.new(
account,
serializer: ActivityPub::DeleteActorSerializer,
adapter: ActivityPub::Adapter
).as_json
json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(account))
unless dry_run?
ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url|
[json, account.id, inbox_url]
end
account.suspend!(block_email: false)
if pending_accounts.positive?
prompt.warn("#{pending_accounts} accounts are still pending deletion.")
elsif sidekiq_stats.enqueued.positive?
prompt.warn('Deletion notices are still being processed')
elsif sidekiq_stats.retry_size.positive?
prompt.warn('At least one delivery attempt for each deletion notice has been made, but some have failed and are scheduled for retry')
else
prompt.ok('Every deletion notice has been sent! You can safely delete all data and decomission your servers!')
end
processed += 1
exit(0)
end
Account.local.without_suspended.find_each { |account| delete_account.call(account) }
Account.local.suspended.joins(:deletion_request).find_each { |account| delete_account.call(account) }
exit(1) unless prompt.ask('Type in the domain of the server to confirm:', required: true) == Rails.configuration.x.local_domain
prompt.ok("Queued #{inboxes.size * processed} items into Sidekiq for #{processed} accounts#{dry_run_mode_suffix}")
prompt.ok('Wait until Sidekiq processes all items, then you can shut everything down and delete the data')
prompt.warn('This operation WILL NOT be reversible.')
prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.')
prompt.warn('The deletion process itself may take a long time, and will be handled by Sidekiq, so do not shut it down until it has finished (you will be able to re-run this command to see the state of the self-destruct process).')
exit(1) if prompt.no?('Are you sure you want to proceed?')
self_destruct_value = Rails.application.message_verifier('self-destruct').generate(Rails.configuration.x.local_domain)
prompt.ok('To switch Mastodon to self-destruct mode, add the following variable to your evironment (e.g. by adding a line to your `.env.production`) and restart all Mastodon processes:')
prompt.ok(" SELF_DESTRUCT=#{self_destruct_value}")
prompt.ok("\nYou can re-run this command to see the state of the self-destruct process.")
rescue TTY::Reader::InputInterrupt
exit(1)
end

80
lib/mastodon/cli/maintenance.rb
View file

@ -136,24 +136,24 @@ module Mastodon::CLI
Mastodon has to be stopped to run this task, which will take a long time and may be destructive.
LONG_DESC
def fix_duplicates
if ActiveRecord::Migrator.current_version < MIN_SUPPORTED_VERSION
say 'Your version of the database schema is too old and is not supported by this script.', :red
say 'Please update to at least Mastodon 3.0.0 before running this script.', :red
exit(1)
elsif ActiveRecord::Migrator.current_version > MAX_SUPPORTED_VERSION
say 'Your version of the database schema is more recent than this script, this may cause unexpected errors.', :yellow
exit(1) unless yes?('Continue anyway? (Yes/No)')
end
verify_system_ready!
if Sidekiq::ProcessSet.new.any?
say 'It seems Sidekiq is running. All Mastodon processes need to be stopped when using this script.', :red
exit(1)
end
process_deduplications
say 'This task will take a long time to run and is potentially destructive.', :yellow
say 'Please make sure to stop Mastodon and have a backup.', :yellow
exit(1) unless yes?('Continue? (Yes/No)')
deduplication_cleanup_tasks
say 'Finished!'
end
private
def verify_system_ready!
verify_schema_version!
verify_sidekiq_not_active!
verify_backup_warning!
end
def process_deduplications
deduplicate_users!
deduplicate_account_domain_blocks!
deduplicate_account_identity_proofs!
@ -173,14 +173,44 @@ module Mastodon::CLI
deduplicate_webauthn_credentials!
deduplicate_webhooks!
deduplicate_software_updates!
Scenic.database.refresh_materialized_view('instances', concurrently: true, cascade: false) if ActiveRecord::Migrator.current_version >= 2020_12_06_004238
Rails.cache.clear
say 'Finished!'
end
private
def deduplication_cleanup_tasks
refresh_instances_view if schema_has_instances_view?
Rails.cache.clear
end
def refresh_instances_view
Scenic.database.refresh_materialized_view('instances', concurrently: true, cascade: false)
end
def schema_has_instances_view?
ActiveRecord::Migrator.current_version >= 2020_12_06_004238
end
def verify_schema_version!
if ActiveRecord::Migrator.current_version < MIN_SUPPORTED_VERSION
say 'Your version of the database schema is too old and is not supported by this script.', :red
say 'Please update to at least Mastodon 3.0.0 before running this script.', :red
exit(1)
elsif ActiveRecord::Migrator.current_version > MAX_SUPPORTED_VERSION
say 'Your version of the database schema is more recent than this script, this may cause unexpected errors.', :yellow
exit(1) unless yes?('Continue anyway? (Yes/No)')
end
end
def verify_sidekiq_not_active!
if Sidekiq::ProcessSet.new.any?
say 'It seems Sidekiq is running. All Mastodon processes need to be stopped when using this script.', :red
exit(1)
end
end
def verify_backup_warning!
say 'This task will take a long time to run and is potentially destructive.', :yellow
say 'Please make sure to stop Mastodon and have a backup.', :yellow
exit(1) unless yes?('Continue? (Yes/No)')
end
def deduplicate_accounts!
remove_index_if_exists!(:accounts, 'index_accounts_on_username_and_domain_lower')
@ -224,7 +254,7 @@ module Mastodon::CLI
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse
ref_user = users.shift
say "Multiple users registered with e-mail address #{ref_user.email}.", :yellow
say "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}", :yellow
say "e-mail will be disabled for the following accounts: #{user.map { |user| user.account.acct }.join(', ')}", :yellow
say 'Please reach out to them and set another address with `tootctl account modify` or delete them.', :yellow
users.each_with_index do |user, index|
@ -253,7 +283,7 @@ module Mastodon::CLI
def deduplicate_users_process_confirmation_token
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:created_at).reverse.drop(1)
say "Unsetting confirmation token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}", :yellow
say "Unsetting confirmation token for those accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow
users.each do |user|
user.update!(confirmation_token: nil)
@ -265,7 +295,7 @@ module Mastodon::CLI
if ActiveRecord::Migrator.current_version < 2022_01_18_183010
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
say "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}", :yellow
say "Unsetting remember token for those accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow
users.each do |user|
user.update!(remember_token: nil)
@ -277,7 +307,7 @@ module Mastodon::CLI
def deduplicate_users_process_password_token
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE reset_password_token IS NOT NULL GROUP BY reset_password_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
say "Unsetting password reset token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}", :yellow
say "Unsetting password reset token for those accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow
users.each do |user|
user.update!(reset_password_token: nil)

63
lib/mastodon/migration_helpers.rb
View file

@ -158,10 +158,8 @@ module Mastodon
'in the body of your migration class'
end
if supports_drop_index_concurrently?
options = options.merge({ algorithm: :concurrently })
disable_statement_timeout
end
options = options.merge({ algorithm: :concurrently })
disable_statement_timeout
remove_index(table_name, **options.merge({ column: column_name }))
end
@ -182,28 +180,12 @@ module Mastodon
'in the body of your migration class'
end
if supports_drop_index_concurrently?
options = options.merge({ algorithm: :concurrently })
disable_statement_timeout
end
options = options.merge({ algorithm: :concurrently })
disable_statement_timeout
remove_index(table_name, **options.merge({ name: index_name }))
end
# Only available on Postgresql >= 9.2
def supports_drop_index_concurrently?
version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
version >= 90_200
end
# Only available on Postgresql >= 11
def supports_add_column_with_default?
version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
version >= 110_000
end
# Adds a foreign key with only minimal locking on the tables involved.
#
# This method only requires minimal locking when using PostgreSQL. When
@ -420,42 +402,7 @@ module Mastodon
# This method can also take a block which is passed directly to the
# `update_column_in_batches` method.
def add_column_with_default(table, column, type, default:, limit: nil, allow_null: false, &block)
if supports_add_column_with_default?
add_column(table, column, type, default: default, limit: limit, null: allow_null)
return
end
if transaction_open?
raise 'add_column_with_default can not be run inside a transaction, ' \
'you can disable transactions by calling disable_ddl_transaction! ' \
'in the body of your migration class'
end
disable_statement_timeout
transaction do
if limit
add_column(table, column, type, default: nil, limit: limit)
else
add_column(table, column, type, default: nil)
end
# Changing the default before the update ensures any newly inserted
# rows already use the proper default value.
change_column_default(table, column, default)
end
begin
update_column_in_batches(table, column, default, &block)
change_column_null(table, column, false) unless allow_null
# We want to rescue _all_ exceptions here, even those that don't inherit
# from StandardError.
rescue Exception => error # rubocop: disable all
remove_column(table, column)
raise error
end
add_column(table, column, type, default: default, limit: limit, null: allow_null)
end
# Renames a column without requiring downtime.

6
lib/mastodon/redis_config.rb
View file

@ -35,9 +35,11 @@ REDIS_CACHE_PARAMS = {
url: ENV['CACHE_REDIS_URL'],
expires_in: 10.minutes,
namespace: cache_namespace,
pool_size: Sidekiq.server? ? Sidekiq[:concurrency] : Integer(ENV['MAX_THREADS'] || 5),
pool_timeout: 5,
connect_timeout: 5,
pool: {
size: Sidekiq.server? ? Sidekiq[:concurrency] : Integer(ENV['MAX_THREADS'] || 5),
timeout: 5,
},
}.freeze
REDIS_SIDEKIQ_PARAMS = {

2
lib/mastodon/version.rb
View file

@ -5,7 +5,7 @@ module Mastodon
module_function
def kmyblue_major
8
9
end
def kmyblue_minor