Add importing remote post

app/lib/importer/base_importer.rb

@@ -3,9 +3,10 @@
 class Importer::BaseImporter
   # @param [Integer] batch_size
   # @param [Concurrent::ThreadPoolExecutor] executor
-  def initialize(batch_size:, executor:)
+  def initialize(batch_size:, executor:, full: true)
     @batch_size = batch_size
     @executor   = executor
+    @full       = full
     @wait_for   = Concurrent::Set.new
   end
 

app/lib/importer/statuses_index_importer.rb

@@ -21,7 +21,7 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter
             to_index.map do |object|
               # This is unlikely to happen, but the post may have been
               # un-interacted with since it was queued for indexing
-              if object.searchable_by.empty?
+              if object.searchable_by.empty? && %w(public private).exclude?(object.searchability)
                 deleted += 1
                 { delete: { _id: object.id } }
               else
@@ -49,13 +49,15 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter
   end
 
   def scopes
-    [
+    targets = [
       local_statuses_scope,
       local_mentions_scope,
       local_favourites_scope,
       local_votes_scope,
       local_bookmarks_scope,
     ]
+    targets << remote_searchable_scope if @full
+    targets
   end
 
   def local_mentions_scope
@@ -77,4 +79,8 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter
   def local_statuses_scope
     Status.local.select('"statuses"."id", COALESCE("statuses"."reblog_of_id", "statuses"."id") AS status_id')
   end
+
+  def remote_searchable_scope
+    Status.remote_dynamic_searchability.select('"statuses"."id", COALESCE("statuses"."reblog_of_id", "statuses"."id") AS status_id')
+  end
 end

app/models/concerns/status_search_concern.rb

@@ -5,6 +5,7 @@ module StatusSearchConcern
 
   included do
     scope :indexable, -> { without_reblogs.where(visibility: [:public, :login], searchability: nil).joins(:account).where(account: { indexable: true }) }
+    scope :remote_dynamic_searchability, -> { remote.where(searchability: [:public, :private]) }
   end
 
   def searchable_by
@@ -41,6 +42,10 @@ module StatusSearchConcern
     @bookmarked_by ||= local_bookmarked.pluck(:id)
   end
 
+  def bookmark_categoried_by
+    @bookmark_categoried_by ||= local_bookmark_categoried.pluck(:id).uniq
+  end
+
   def emoji_reacted_by
     @emoji_reacted_by ||= local_emoji_reacted.pluck(:id)
   end

lib/mastodon/cli/search.rb

@@ -20,6 +20,7 @@ module Mastodon::CLI
     option :import, type: :boolean, default: true, desc: 'Import data from the database to the index'
     option :clean, type: :boolean, default: true, desc: 'Remove outdated documents from the index'
     option :reset_chewy, type: :boolean, default: false, desc: "Reset Chewy's internal index"
+    option :full, type: :boolean, default: true, desc: 'Import full data over Mastodon default importer'
     desc 'deploy', 'Create or upgrade Elasticsearch indices and populate them'
     long_desc <<~LONG_DESC
       If Elasticsearch is empty, this command will create the necessary indices
@@ -41,7 +42,7 @@ module Mastodon::CLI
                 end
 
       pool      = Concurrent::FixedThreadPool.new(options[:concurrency], max_queue: options[:concurrency] * 10)
-      importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool) }
+      importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool, full: options[:full]) }
       progress  = ProgressBar.create(total: nil, format: '%t%c/%u |%b%i| %e (%r docs/s)', autofinish: false)
 
       Chewy::Stash::Specification.reset! if options[:reset_chewy]