diff --git a/app/lib/importer/base_importer.rb b/app/lib/importer/base_importer.rb index a21557d303..ef8c8b38d4 100644 --- a/app/lib/importer/base_importer.rb +++ b/app/lib/importer/base_importer.rb @@ -3,9 +3,10 @@ class Importer::BaseImporter # @param [Integer] batch_size # @param [Concurrent::ThreadPoolExecutor] executor - def initialize(batch_size:, executor:) + def initialize(batch_size:, executor:, full: true) @batch_size = batch_size @executor = executor + @full = full @wait_for = Concurrent::Set.new end diff --git a/app/lib/importer/statuses_index_importer.rb b/app/lib/importer/statuses_index_importer.rb index 1922f65f6d..bb74bb8d89 100644 --- a/app/lib/importer/statuses_index_importer.rb +++ b/app/lib/importer/statuses_index_importer.rb @@ -21,7 +21,7 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter to_index.map do |object| # This is unlikely to happen, but the post may have been # un-interacted with since it was queued for indexing - if object.searchable_by.empty? + if object.searchable_by.empty? && %w(public private).exclude?(object.searchability) deleted += 1 { delete: { _id: object.id } } else @@ -49,13 +49,15 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter end def scopes - [ + targets = [ local_statuses_scope, local_mentions_scope, local_favourites_scope, local_votes_scope, local_bookmarks_scope, ] + targets << remote_searchable_scope if @full + targets end def local_mentions_scope @@ -77,4 +79,8 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter def local_statuses_scope Status.local.select('"statuses"."id", COALESCE("statuses"."reblog_of_id", "statuses"."id") AS status_id') end + + def remote_searchable_scope + Status.remote_dynamic_searchability.select('"statuses"."id", COALESCE("statuses"."reblog_of_id", "statuses"."id") AS status_id') + end end diff --git a/app/models/concerns/status_search_concern.rb b/app/models/concerns/status_search_concern.rb index d1632b7b84..376f82e509 100644 --- a/app/models/concerns/status_search_concern.rb +++ b/app/models/concerns/status_search_concern.rb @@ -5,6 +5,7 @@ module StatusSearchConcern included do scope :indexable, -> { without_reblogs.where(visibility: [:public, :login], searchability: nil).joins(:account).where(account: { indexable: true }) } + scope :remote_dynamic_searchability, -> { remote.where(searchability: [:public, :private]) } end def searchable_by @@ -41,6 +42,10 @@ module StatusSearchConcern @bookmarked_by ||= local_bookmarked.pluck(:id) end + def bookmark_categoried_by + @bookmark_categoried_by ||= local_bookmark_categoried.pluck(:id).uniq + end + def emoji_reacted_by @emoji_reacted_by ||= local_emoji_reacted.pluck(:id) end diff --git a/lib/mastodon/cli/search.rb b/lib/mastodon/cli/search.rb index 25a595aadd..f8697ffded 100644 --- a/lib/mastodon/cli/search.rb +++ b/lib/mastodon/cli/search.rb @@ -20,6 +20,7 @@ module Mastodon::CLI option :import, type: :boolean, default: true, desc: 'Import data from the database to the index' option :clean, type: :boolean, default: true, desc: 'Remove outdated documents from the index' option :reset_chewy, type: :boolean, default: false, desc: "Reset Chewy's internal index" + option :full, type: :boolean, default: true, desc: 'Import full data over Mastodon default importer' desc 'deploy', 'Create or upgrade Elasticsearch indices and populate them' long_desc <<~LONG_DESC If Elasticsearch is empty, this command will create the necessary indices @@ -41,7 +42,7 @@ module Mastodon::CLI end pool = Concurrent::FixedThreadPool.new(options[:concurrency], max_queue: options[:concurrency] * 10) - importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool) } + importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool, full: options[:full]) } progress = ProgressBar.create(total: nil, format: '%t%c/%u |%b%i| %e (%r docs/s)', autofinish: false) Chewy::Stash::Specification.reset! if options[:reset_chewy]