Merge commit 'e58c36d308' into kb_migration

lib/mastodon/cli/accounts.rb

@@ -1,18 +1,10 @@
 # frozen_string_literal: true
 
 require 'set'
-require_relative '../../config/boot'
-require_relative '../../config/environment'
-require_relative 'cli_helper'
-
-module Mastodon
-  class AccountsCLI < Thor
-    include CLIHelper
-
-    def self.exit_on_failure?
-      true
-    end
+require_relative 'base'
 
+module Mastodon::CLI
+  class Accounts < Base
     option :all, type: :boolean
     desc 'rotate [USERNAME]', 'Generate and broadcast new keys'
     long_desc <<-LONG_DESC

lib/mastodon/cli/base.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative '../../../config/boot'
+require_relative '../../../config/environment'
+
+require 'thor'
+require_relative 'helper'
+
+module Mastodon
+  module CLI
+    class Base < Thor
+      include CLI::Helper
+
+      def self.exit_on_failure?
+        true
+      end
+    end
+  end
+end

lib/mastodon/cli/cache.rb

@@ -1,17 +1,9 @@
 # frozen_string_literal: true
 
-require_relative '../../config/boot'
-require_relative '../../config/environment'
-require_relative 'cli_helper'
-
-module Mastodon
-  class CacheCLI < Thor
-    include CLIHelper
-
-    def self.exit_on_failure?
-      true
-    end
+require_relative 'base'
 
+module Mastodon::CLI
+  class Cache < Base
     desc 'clear', 'Clear out the cache storage'
     def clear
       Rails.cache.clear

lib/mastodon/cli/canonical_email_blocks.rb

@@ -1,18 +1,10 @@
 # frozen_string_literal: true
 
 require 'concurrent'
-require_relative '../../config/boot'
-require_relative '../../config/environment'
-require_relative 'cli_helper'
-
-module Mastodon
-  class CanonicalEmailBlocksCLI < Thor
-    include CLIHelper
-
-    def self.exit_on_failure?
-      true
-    end
+require_relative 'base'
 
+module Mastodon::CLI
+  class CanonicalEmailBlocks < Base
     desc 'find EMAIL', 'Find a given e-mail address in the canonical e-mail blocks'
     long_desc <<-LONG_DESC
       When suspending a local user, a hash of a "canonical" version of their e-mail

lib/mastodon/cli/domains.rb

@@ -1,18 +1,10 @@
 # frozen_string_literal: true
 
 require 'concurrent'
-require_relative '../../config/boot'
-require_relative '../../config/environment'
-require_relative 'cli_helper'
-
-module Mastodon
-  class DomainsCLI < Thor
-    include CLIHelper
-
-    def self.exit_on_failure?
-      true
-    end
+require_relative 'base'
 
+module Mastodon::CLI
+  class Domains < Base
     option :concurrency, type: :numeric, default: 5, aliases: [:c]
     option :verbose, type: :boolean, aliases: [:v]
     option :dry_run, type: :boolean

lib/mastodon/cli/email_domain_blocks.rb

@@ -1,18 +1,10 @@
 # frozen_string_literal: true
 
 require 'concurrent'
-require_relative '../../config/boot'
-require_relative '../../config/environment'
-require_relative 'cli_helper'
-
-module Mastodon
-  class EmailDomainBlocksCLI < Thor
-    include CLIHelper
-
-    def self.exit_on_failure?
-      true
-    end
+require_relative 'base'
 
+module Mastodon::CLI
+  class EmailDomainBlocks < Base
     desc 'list', 'List blocked e-mail domains'
     def list
       EmailDomainBlock.where(parent_id: nil).order(id: 'DESC').find_each do |entry|

lib/mastodon/cli/emoji.rb

@@ -1,16 +1,10 @@
 # frozen_string_literal: true
 
 require 'rubygems/package'
-require_relative '../../config/boot'
-require_relative '../../config/environment'
-require_relative 'cli_helper'
-
-module Mastodon
-  class EmojiCLI < Thor
-    def self.exit_on_failure?
-      true
-    end
+require_relative 'base'
 
+module Mastodon::CLI
+  class Emoji < Base
     option :prefix
     option :suffix
     option :overwrite, type: :boolean

lib/mastodon/cli/feeds.rb

@@ -1,18 +1,11 @@
 # frozen_string_literal: true
 
-require_relative '../../config/boot'
-require_relative '../../config/environment'
-require_relative 'cli_helper'
+require_relative 'base'
 
-module Mastodon
-  class FeedsCLI < Thor
-    include CLIHelper
+module Mastodon::CLI
+  class Feeds < Base
     include Redisable
 
-    def self.exit_on_failure?
-      true
-    end
-
     option :all, type: :boolean, default: false
     option :concurrency, type: :numeric, default: 5, aliases: [:c]
     option :verbose, type: :boolean, aliases: [:v]

lib/mastodon/cli/helper.rb

@@ -9,8 +9,8 @@ HttpLog.configuration.logger = dev_null
 Paperclip.options[:log]      = false
 Chewy.logger                 = dev_null
 
-module Mastodon
-  module CLIHelper
+module Mastodon::CLI
+  module Helper
     def dry_run?
       options[:dry_run]
     end

lib/mastodon/cli/ip_blocks.rb

@@ -1,16 +1,10 @@
 # frozen_string_literal: true
 
 require 'rubygems/package'
-require_relative '../../config/boot'
-require_relative '../../config/environment'
-require_relative 'cli_helper'
-
-module Mastodon
-  class IpBlocksCLI < Thor
-    def self.exit_on_failure?
-      true
-    end
+require_relative 'base'
 
+module Mastodon::CLI
+  class IpBlocks < Base
     option :severity, required: true, enum: %w(no_access sign_up_requires_approval sign_up_block), desc: 'Severity of the block'
     option :comment, aliases: [:c], desc: 'Optional comment'
     option :duration, aliases: [:d], type: :numeric, desc: 'Duration of the block in seconds'

lib/mastodon/cli/main.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+require_relative 'accounts'
+require_relative 'cache'
+require_relative 'canonical_email_blocks'
+require_relative 'domains'
+require_relative 'email_domain_blocks'
+require_relative 'emoji'
+require_relative 'feeds'
+require_relative 'ip_blocks'
+require_relative 'maintenance'
+require_relative 'media'
+require_relative 'preview_cards'
+require_relative 'search'
+require_relative 'settings'
+require_relative 'statuses'
+require_relative 'upgrade'
+
+module Mastodon::CLI
+  class Main < Base
+    desc 'media SUBCOMMAND ...ARGS', 'Manage media files'
+    subcommand 'media', Media
+
+    desc 'emoji SUBCOMMAND ...ARGS', 'Manage custom emoji'
+    subcommand 'emoji', Emoji
+
+    desc 'accounts SUBCOMMAND ...ARGS', 'Manage accounts'
+    subcommand 'accounts', Accounts
+
+    desc 'feeds SUBCOMMAND ...ARGS', 'Manage feeds'
+    subcommand 'feeds', Feeds
+
+    desc 'search SUBCOMMAND ...ARGS', 'Manage the search engine'
+    subcommand 'search', Search
+
+    desc 'settings SUBCOMMAND ...ARGS', 'Manage dynamic settings'
+    subcommand 'settings', Settings
+
+    desc 'statuses SUBCOMMAND ...ARGS', 'Manage statuses'
+    subcommand 'statuses', Statuses
+
+    desc 'domains SUBCOMMAND ...ARGS', 'Manage account domains'
+    subcommand 'domains', Domains
+
+    desc 'preview_cards SUBCOMMAND ...ARGS', 'Manage preview cards'
+    subcommand 'preview_cards', PreviewCards
+
+    desc 'cache SUBCOMMAND ...ARGS', 'Manage cache'
+    subcommand 'cache', Cache
+
+    desc 'upgrade SUBCOMMAND ...ARGS', 'Various version upgrade utilities'
+    subcommand 'upgrade', Upgrade
+
+    desc 'email_domain_blocks SUBCOMMAND ...ARGS', 'Manage e-mail domain blocks'
+    subcommand 'email_domain_blocks', EmailDomainBlocks
+
+    desc 'ip_blocks SUBCOMMAND ...ARGS', 'Manage IP blocks'
+    subcommand 'ip_blocks', IpBlocks
+
+    desc 'canonical_email_blocks SUBCOMMAND ...ARGS', 'Manage canonical e-mail blocks'
+    subcommand 'canonical_email_blocks', CanonicalEmailBlocks
+
+    desc 'maintenance SUBCOMMAND ...ARGS', 'Various maintenance utilities'
+    subcommand 'maintenance', Maintenance
+
+    option :dry_run, type: :boolean
+    desc 'self-destruct', 'Erase the server from the federation'
+    long_desc <<~LONG_DESC
+      Erase the server from the federation by broadcasting account delete
+      activities to all known other servers. This allows a "clean exit" from
+      running a Mastodon server, as it leaves next to no cache behind on
+      other servers.
+
+      This command is always interactive and requires confirmation twice.
+
+      No local data is actually deleted, because emptying the
+      database or removing files is much faster through other, external
+      means, such as e.g. deleting the entire VPS. However, because other
+      servers will delete data about local users, but no local data will be
+      updated (such as e.g. followers), there will be a state mismatch
+      that will lead to glitches and issues if you then continue to run and use
+      the server.
+
+      So either you know exactly what you are doing, or you are starting
+      from a blank slate afterwards by manually clearing out all the local
+      data!
+    LONG_DESC
+    def self_destruct
+      require 'tty-prompt'
+
+      prompt = TTY::Prompt.new
+
+      exit(1) unless prompt.ask('Type in the domain of the server to confirm:', required: true) == Rails.configuration.x.local_domain
+
+      unless options[:dry_run]
+        prompt.warn('This operation WILL NOT be reversible. It can also take a long time.')
+        prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.')
+        prompt.warn('A running Sidekiq process is required. Do not shut it down until queues clear.')
+
+        exit(1) if prompt.no?('Are you sure you want to proceed?')
+      end
+
+      inboxes   = Account.inboxes
+      processed = 0
+      dry_run   = options[:dry_run] ? ' (DRY RUN)' : ''
+
+      Setting.registrations_mode = 'none' unless options[:dry_run]
+
+      if inboxes.empty?
+        Account.local.without_suspended.in_batches.update_all(suspended_at: Time.now.utc, suspension_origin: :local) unless options[:dry_run]
+        prompt.ok('It seems like your server has not federated with anything')
+        prompt.ok('You can shut it down and delete it any time')
+        return
+      end
+
+      prompt.warn('Do NOT interrupt this process...')
+
+      delete_account = lambda do |account|
+        payload = ActiveModelSerializers::SerializableResource.new(
+          account,
+          serializer: ActivityPub::DeleteActorSerializer,
+          adapter: ActivityPub::Adapter
+        ).as_json
+
+        json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(account))
+
+        unless options[:dry_run]
+          ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url|
+            [json, account.id, inbox_url]
+          end
+
+          account.suspend!(block_email: false)
+        end
+
+        processed += 1
+      end
+
+      Account.local.without_suspended.find_each { |account| delete_account.call(account) }
+      Account.local.suspended.joins(:deletion_request).find_each { |account| delete_account.call(account) }
+
+      prompt.ok("Queued #{inboxes.size * processed} items into Sidekiq for #{processed} accounts#{dry_run}")
+      prompt.ok('Wait until Sidekiq processes all items, then you can shut everything down and delete the data')
+    rescue TTY::Reader::InputInterrupt
+      exit(1)
+    end
+
+    map %w(--version -v) => :version
+
+    desc 'version', 'Show version'
+    def version
+      say(Mastodon::Version.to_s)
+    end
+  end
+end

lib/mastodon/cli/maintenance.rb

@@ -1,18 +1,10 @@
 # frozen_string_literal: true
 
 require 'tty-prompt'
-require_relative '../../config/boot'
-require_relative '../../config/environment'
-require_relative 'cli_helper'
-
-module Mastodon
-  class MaintenanceCLI < Thor
-    include CLIHelper
-
-    def self.exit_on_failure?
-      true
-    end
+require_relative 'base'
 
+module Mastodon::CLI
+  class Maintenance < Base
     MIN_SUPPORTED_VERSION = 2019_10_01_213028
     MAX_SUPPORTED_VERSION = 2022_11_04_133904
 

lib/mastodon/cli/media.rb

@@ -1,20 +1,13 @@
 # frozen_string_literal: true
 
-require_relative '../../config/boot'
-require_relative '../../config/environment'
-require_relative 'cli_helper'
+require_relative 'base'
 
-module Mastodon
-  class MediaCLI < Thor
+module Mastodon::CLI
+  class Media < Base
     include ActionView::Helpers::NumberHelper
-    include CLIHelper
 
     VALID_PATH_SEGMENTS_SIZE = [7, 10].freeze
 
-    def self.exit_on_failure?
-      true
-    end
-
     option :days, type: :numeric, default: 7, aliases: [:d]
     option :prune_profiles, type: :boolean, default: false
     option :remove_headers, type: :boolean, default: false

lib/mastodon/cli/preview_cards.rb

@@ -1,18 +1,11 @@
 # frozen_string_literal: true
 
 require 'tty-prompt'
-require_relative '../../config/boot'
-require_relative '../../config/environment'
-require_relative 'cli_helper'
+require_relative 'base'
 
-module Mastodon
-  class PreviewCardsCLI < Thor
+module Mastodon::CLI
+  class PreviewCards < Base
     include ActionView::Helpers::NumberHelper
-    include CLIHelper
-
-    def self.exit_on_failure?
-      true
-    end
 
     option :days, type: :numeric, default: 180
     option :concurrency, type: :numeric, default: 5, aliases: [:c]

lib/mastodon/cli/search.rb

@@ -1,13 +1,9 @@
 # frozen_string_literal: true
 
-require_relative '../../config/boot'
-require_relative '../../config/environment'
-require_relative 'cli_helper'
-
-module Mastodon
-  class SearchCLI < Thor
-    include CLIHelper
+require_relative 'base'
 
+module Mastodon::CLI
+  class Search < Base
     # Indices are sorted by amount of data to be expected in each, so that
     # smaller indices can go online sooner
     INDICES = [

lib/mastodon/cli/settings.rb

@@ -1,15 +1,9 @@
 # frozen_string_literal: true
 
-require_relative '../../config/boot'
-require_relative '../../config/environment'
-require_relative 'cli_helper'
-
-module Mastodon
-  class RegistrationsCLI < Thor
-    def self.exit_on_failure?
-      true
-    end
+require_relative 'base'
 
+module Mastodon::CLI
+  class Registrations < Base
     desc 'open', 'Open registrations'
     def open
       Setting.registrations_mode = 'open'
@@ -37,8 +31,8 @@ module Mastodon
     end
   end
 
-  class SettingsCLI < Thor
+  class Settings < Base
     desc 'registrations SUBCOMMAND ...ARGS', 'Manage state of registrations'
-    subcommand 'registrations', RegistrationsCLI
+    subcommand 'registrations', Registrations
   end
 end

lib/mastodon/cli/statuses.rb

@@ -1,18 +1,11 @@
 # frozen_string_literal: true
 
-require_relative '../../config/boot'
-require_relative '../../config/environment'
-require_relative 'cli_helper'
+require_relative 'base'
 
-module Mastodon
-  class StatusesCLI < Thor
-    include CLIHelper
+module Mastodon::CLI
+  class Statuses < Base
     include ActionView::Helpers::NumberHelper
 
-    def self.exit_on_failure?
-      true
-    end
-
     option :days, type: :numeric, default: 90
     option :batch_size, type: :numeric, default: 1_000, aliases: [:b], desc: 'Number of records in each batch'
     option :continue, type: :boolean, default: false, desc: 'If remove is not completed, execute from the previous continuation'

lib/mastodon/cli/upgrade.rb

@@ -1,17 +1,9 @@
 # frozen_string_literal: true
 
-require_relative '../../config/boot'
-require_relative '../../config/environment'
-require_relative 'cli_helper'
-
-module Mastodon
-  class UpgradeCLI < Thor
-    include CLIHelper
-
-    def self.exit_on_failure?
-      true
-    end
+require_relative 'base'
 
+module Mastodon::CLI
+  class Upgrade < Base
     CURRENT_STORAGE_SCHEMA_VERSION = 1
 
     option :dry_run, type: :boolean, default: false