Merge remote-tracking branch 'parent/main' into kb_migration

app/chewy/accounts_index.rb

@@ -62,6 +62,6 @@ class AccountsIndex < Chewy::Index
     field(:last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at })
     field(:display_name, type: 'text', analyzer: 'verbatim') { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
     field(:username, type: 'text', analyzer: 'verbatim', value: ->(account) { [account.username, account.domain].compact.join('@') }) { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
-    field(:text, type: 'text', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' }
+    field(:text, type: 'text', analyzer: 'whitespace', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' }
   end
 end

app/chewy/public_statuses_index.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+class PublicStatusesIndex < Chewy::Index
+  settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: {
+    filter: {
+      english_stop: {
+        type: 'stop',
+        stopwords: '_english_',
+      },
+
+      english_stemmer: {
+        type: 'stemmer',
+        language: 'english',
+      },
+
+      english_possessive_stemmer: {
+        type: 'stemmer',
+        language: 'possessive_english',
+      },
+    },
+
+    analyzer: {
+      content: {
+        tokenizer: 'uax_url_email',
+        filter: %w(
+          english_possessive_stemmer
+          lowercase
+          asciifolding
+          cjk_width
+          english_stop
+          english_stemmer
+        ),
+      },
+    },
+  }
+
+  index_scope ::Status.unscoped
+                      .kept
+                      .indexable
+                      .includes(:media_attachments, :preloadable_poll, :preview_cards)
+
+  root date_detection: false do
+    field(:id, type: 'keyword')
+    field(:account_id, type: 'long')
+    field(:text, type: 'text', analyzer: 'whitespace', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
+    field(:language, type: 'keyword')
+    field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties })
+    field(:created_at, type: 'date')
+  end
+end

app/chewy/statuses_index.rb

@@ -1,23 +1,24 @@
 # frozen_string_literal: true
 
 class StatusesIndex < Chewy::Index
-  include FormattingHelper
-
   settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: {
     filter: {
       english_stop: {
         type: 'stop',
         stopwords: '_english_',
       },
+
       english_stemmer: {
         type: 'stemmer',
         language: 'english',
       },
+
       english_possessive_stemmer: {
         type: 'stemmer',
         language: 'possessive_english',
       },
     },
+
     analyzer: {
       content: {
         tokenizer: 'uax_url_email',
@@ -35,7 +36,7 @@ class StatusesIndex < Chewy::Index
 
   # We do not use delete_if option here because it would call a method that we
   # expect to be called with crutches without crutches, causing n+1 queries
-  index_scope ::Status.unscoped.kept.without_reblogs.includes(:media_attachments, :preloadable_poll)
+  index_scope ::Status.unscoped.kept.without_reblogs.includes(:media_attachments, :preloadable_poll, :preview_cards)
 
   crutch :mentions do |collection|
     data = ::Mention.where(status_id: collection.map(&:id)).where(account: Account.local, silent: false).pluck(:status_id, :account_id)
@@ -68,14 +69,13 @@ class StatusesIndex < Chewy::Index
   end
 
   root date_detection: false do
-    field :id, type: 'long'
-    field :account_id, type: 'long'
-
-    field :text, type: 'text', value: ->(status) { status.searchable_text } do
-      field :stemmed, type: 'text', analyzer: 'content'
-    end
-
-    field :searchable_by, type: 'long', value: ->(status, crutches) { status.searchable_by(crutches) }
-    field :searchability, type: 'keyword', value: ->(status) { status.compute_searchability }
+    field(:id, type: 'keyword')
+    field(:account_id, type: 'long')
+    field(:text, type: 'text', analyzer: 'whitespace', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
+    field(:searchable_by, type: 'long', value: ->(status, crutches) { status.searchable_by(crutches) })
+    field(:searchability, type: 'keyword', value: ->(status) { status.compute_searchability })
+    field(:language, type: 'keyword')
+    field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties })
+    field(:created_at, type: 'date')
   end
 end

app/controllers/api/v1/accounts/credentials_controller.rb

@@ -32,6 +32,7 @@ class Api::V1::Accounts::CredentialsController < Api::BaseController
       :searchability,
       :dissubscribable,
       :hide_collections,
+      :indexable,
       fields_attributes: [:name, :value]
     )
   end

app/controllers/settings/privacy_controller.rb

@@ -18,7 +18,7 @@ class Settings::PrivacyController < Settings::BaseController
   private
 
   def account_params
-    params.require(:account).permit(:discoverable, :unlocked, :show_collections, :dissubscribable, settings: UserSettings.keys)
+    params.require(:account).permit(:discoverable, :unlocked, :indexable, :show_collections, :dissubscribable, settings: UserSettings.keys)
   end
 
   def set_account

app/javascript/mastodon/components/column.jsx

@@ -16,7 +16,19 @@ export default class Column extends PureComponent {
   };
 
   scrollTop () {
-    const scrollable = this.props.bindToDocument ? document.scrollingElement : this.node.querySelector('.scrollable');
+    let scrollable = null;
+
+    if (this.props.bindToDocument) {
+      scrollable = document.scrollingElement;
+    } else {
+      scrollable = this.node.querySelector('.scrollable');
+
+      // Some columns have nested `.scrollable` containers, with the outer one
+      // being a wrapper while the actual scrollable content is deeper.
+      if (scrollable.classList.contains('scrollable--flex')) {
+        scrollable = scrollable?.querySelector('.scrollable') || scrollable;
+      }
+   }
 
     if (!scrollable) {
       return;

app/javascript/mastodon/features/ui/components/report_modal.jsx

@@ -62,7 +62,7 @@ class ReportModal extends ImmutablePureComponent {
     dispatch(submitReport({
       account_id: accountId,
       status_ids: selectedStatusIds.toArray(),
-      selected_domains: selectedDomains.toArray(),
+      forward_to_domains: selectedDomains.toArray(),
       comment,
       forward: selectedDomains.size > 0,
       category,

app/lib/importer/public_statuses_index_importer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class Importer::PublicStatusesIndexImporter < Importer::BaseImporter
+  def import!
+    indexable_statuses_scope.find_in_batches(batch_size: @batch_size) do |batch|
+      in_work_unit(batch.map(&:status_id)) do |status_ids|
+        bulk = ActiveRecord::Base.connection_pool.with_connection do
+          Chewy::Index::Import::BulkBuilder.new(index, to_index: Status.includes(:media_attachments, :preloadable_poll).where(id: status_ids)).bulk_body
+        end
+
+        indexed = 0
+        deleted = 0
+
+        bulk.map! do |entry|
+          if entry[:index]
+            indexed += 1
+          else
+            deleted += 1
+          end
+          entry
+        end
+
+        Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
+
+        [indexed, deleted]
+      end
+    end
+
+    wait!
+  end
+
+  private
+
+  def index
+    PublicStatusesIndex
+  end
+
+  def indexable_statuses_scope
+    Status.indexable.select('"statuses"."id", COALESCE("statuses"."reblog_of_id", "statuses"."id") AS status_id')
+  end
+end

app/lib/search_query_transformer.rb

@@ -36,7 +36,7 @@ class SearchQueryTransformer < Parslet::Transform
     def clause_to_filter(clause)
       case clause
       when PrefixClause
-        { term: { clause.filter => clause.term } }
+        { clause.type => { clause.filter => clause.term } }
       else
         raise "Unexpected clause type: #{clause}"
       end
@@ -47,12 +47,10 @@ class SearchQueryTransformer < Parslet::Transform
     class << self
       def symbol(str)
         case str
-        when '+'
+        when '+', nil
           :must
         when '-'
           :must_not
-        when nil
-          :should
         else
           raise "Unknown operator: #{str}"
         end
@@ -81,23 +79,52 @@ class SearchQueryTransformer < Parslet::Transform
   end
 
   class PrefixClause
-    attr_reader :filter, :operator, :term
+    attr_reader :type, :filter, :operator, :term
 
     def initialize(prefix, term)
       @operator = :filter
+
       case prefix
+      when 'has', 'is'
+        @filter = :properties
+        @type = :term
+        @term = term
+      when 'language'
+        @filter = :language
+        @type = :term
+        @term = term
       when 'from'
         @filter = :account_id
-
-        username, domain = term.gsub(/\A@/, '').split('@')
-        domain           = nil if TagManager.instance.local_domain?(domain)
-        account          = Account.find_remote!(username, domain)
-
-        @term = account.id
+        @type = :term
+        @term = account_id_from_term(term)
+      when 'before'
+        @filter = :created_at
+        @type = :range
+        @term = { lt: term }
+      when 'after'
+        @filter = :created_at
+        @type = :range
+        @term = { gt: term }
+      when 'during'
+        @filter = :created_at
+        @type = :range
+        @term = { gte: term, lte: term }
       else
         raise Mastodon::SyntaxError
       end
     end
+
+    private
+
+    def account_id_from_term(term)
+      username, domain = term.gsub(/\A@/, '').split('@')
+      domain = nil if TagManager.instance.local_domain?(domain)
+      account = Account.find_remote(username, domain)
+
+      # If the account is not found, we want to return empty results, so return
+      # an ID that does not exist
+      account&.id || -1
+    end
   end
 
   rule(clause: subtree(:clause)) do

app/lib/vacuum/statuses_vacuum.rb

@@ -20,7 +20,10 @@ class Vacuum::StatusesVacuum
       statuses.direct_visibility
               .includes(mentions: :account)
               .find_each(&:unlink_from_conversations!)
-      remove_from_search_index(statuses.ids) if Chewy.enabled?
+      if Chewy.enabled?
+        remove_from_index(statuses.ids, 'chewy:queue:StatusesIndex')
+        remove_from_index(statuses.ids, 'chewy:queue:PublicStatusesIndex')
+      end
 
       # Foreign keys take care of most associated records for us.
       # Media attachments will be orphaned.
@@ -38,7 +41,7 @@ class Vacuum::StatusesVacuum
     Mastodon::Snowflake.id_at(@retention_period.ago, with_random: false)
   end
 
-  def remove_from_search_index(status_ids)
-    with_redis { |redis| redis.sadd('chewy:queue:StatusesIndex', status_ids) }
+  def remove_from_index(status_ids, index)
+    with_redis { |redis| redis.sadd(index, status_ids) }
   end
 end

app/models/account.rb

@@ -86,6 +86,7 @@ class Account < ApplicationRecord
   include DomainMaterializable
   include AccountMerging
   include AccountSearch
+  include AccountStatusesSearch
 
   enum protocol: { ostatus: 0, activitypub: 1 }
   enum suspension_origin: { local: 0, remote: 1 }, _prefix: true

app/models/concerns/account_statuses_search.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module AccountStatusesSearch
+  extend ActiveSupport::Concern
+
+  included do
+    after_update_commit :enqueue_update_public_statuses_index, if: :saved_change_to_indexable?
+    after_destroy_commit :enqueue_remove_from_public_statuses_index, if: :indexable?
+  end
+
+  def enqueue_update_public_statuses_index
+    if indexable?
+      enqueue_add_to_public_statuses_index
+    else
+      enqueue_remove_from_public_statuses_index
+    end
+  end
+
+  def enqueue_add_to_public_statuses_index
+    return unless Chewy.enabled?
+
+    AddToPublicStatusesIndexWorker.perform_async(id)
+  end
+
+  def enqueue_remove_from_public_statuses_index
+    return unless Chewy.enabled?
+
+    RemoveFromPublicStatusesIndexWorker.perform_async(id)
+  end
+
+  def add_to_public_statuses_index!
+    return unless Chewy.enabled?
+
+    statuses.indexable.find_in_batches do |batch|
+      PublicStatusesIndex.import(query: batch)
+    end
+  end
+
+  def remove_from_public_statuses_index!
+    return unless Chewy.enabled?
+
+    PublicStatusesIndex.filter(term: { account_id: id }).delete_all
+  end
+end

app/models/concerns/status_search_concern.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module StatusSearchConcern
+  extend ActiveSupport::Concern
+
+  included do
+    scope :indexable, -> { without_reblogs.where(visibility: [:public, :login], searchability: nil).joins(:account).where(account: { indexable: true }) }
+  end
+
+  def searchable_by(preloaded = nil)
+    ids = []
+
+    ids << account_id if local?
+
+    if preloaded.nil?
+      ids += mentions.joins(:account).merge(Account.local).active.pluck(:account_id)
+      ids += favourites.joins(:account).merge(Account.local).pluck(:account_id)
+      ids += emoji_reactions.joins(:account).merge(Account.local).pluck(:account_id)
+      ids += reblogs.joins(:account).merge(Account.local).pluck(:account_id)
+      ids += bookmarks.joins(:account).merge(Account.local).pluck(:account_id)
+      ids += poll.votes.joins(:account).merge(Account.local).pluck(:account_id) if poll.present?
+    else
+      ids += preloaded.mentions[id] || []
+      ids += preloaded.favourites[id] || []
+      ids += preloaded.emoji_reactions[id] || []
+      ids += preloaded.reblogs[id] || []
+      ids += preloaded.bookmarks[id] || []
+      ids += preloaded.votes[id] || []
+    end
+
+    ids.uniq
+  end
+
+  def searchable_text
+    [
+      spoiler_text,
+      FormattingHelper.extract_status_plain_text(self),
+      preloadable_poll&.options&.join("\n\n"),
+      ordered_media_attachments.map(&:description).join("\n\n"),
+    ].compact.join("\n\n")
+  end
+
+  def searchable_properties
+    [].tap do |properties|
+      properties << 'image' if ordered_media_attachments.any?(&:image?)
+      properties << 'video' if ordered_media_attachments.any?(&:video?)
+      properties << 'audio' if ordered_media_attachments.any?(&:audio?)
+      properties << 'media' if with_media?
+      properties << 'poll' if with_poll?
+      properties << 'link' if with_preview_card?
+      properties << 'embed' if preview_cards.any?(&:video?)
+      properties << 'sensitive' if sensitive?
+      properties << 'reply' if reply?
+    end
+  end
+end

app/models/status.rb

@@ -42,6 +42,7 @@ class Status < ApplicationRecord
   include StatusSnapshotConcern
   include RateLimitable
   include StatusSafeReblogInsert
+  include StatusSearchConcern
 
   rate_limit by: :account, family: :statuses
 
@@ -52,6 +53,7 @@ class Status < ApplicationRecord
   attr_accessor :override_timestamps
 
   update_index('statuses', :proper)
+  update_index('public_statuses', :proper)
 
   enum visibility: { public: 0, unlisted: 1, private: 2, direct: 3, limited: 4, public_unlisted: 10, login: 11 }, _suffix: :visibility
   enum searchability: { public: 0, private: 1, direct: 2, limited: 3, unsupported: 4, public_unlisted: 10 }, _suffix: :searchability
@@ -183,39 +185,6 @@ class Status < ApplicationRecord
     "v3:#{super}"
   end
 
-  def searchable_by(preloaded = nil)
-    ids = []
-
-    ids << account_id if local?
-
-    if preloaded.nil?
-      ids += mentions.joins(:account).merge(Account.local).active.pluck(:account_id)
-      ids += favourites.joins(:account).merge(Account.local).pluck(:account_id)
-      ids += emoji_reactions.joins(:account).merge(Account.local).pluck(:account_id)
-      ids += reblogs.joins(:account).merge(Account.local).pluck(:account_id)
-      ids += bookmarks.joins(:account).merge(Account.local).pluck(:account_id)
-      ids += poll.votes.joins(:account).merge(Account.local).pluck(:account_id) if poll.present?
-    else
-      ids += preloaded.mentions[id] || []
-      ids += preloaded.favourites[id] || []
-      ids += preloaded.emoji_reactions[id] || []
-      ids += preloaded.reblogs[id] || []
-      ids += preloaded.bookmarks[id] || []
-      ids += preloaded.votes[id] || []
-    end
-
-    ids.uniq
-  end
-
-  def searchable_text
-    [
-      spoiler_text,
-      FormattingHelper.extract_status_plain_text(self),
-      preloadable_poll ? preloadable_poll.options.join("\n\n") : nil,
-      ordered_media_attachments.map(&:description).join("\n\n"),
-    ].compact.join("\n\n")
-  end
-
   def to_log_human_identifier
     account.acct
   end
@@ -295,6 +264,10 @@ class Status < ApplicationRecord
     preview_cards.any?
   end
 
+  def with_poll?
+    preloadable_poll.present?
+  end
+
   def non_sensitive_with_media?
     !sensitive? && with_media?
   end

app/serializers/activitypub/actor_serializer.rb

@@ -8,13 +8,13 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
 
   context_extensions :manually_approves_followers, :featured, :also_known_as,
                      :moved_to, :property_value, :discoverable, :olm, :suspended, :searchable_by, :subscribable_by,
-                     :other_setting, :memorial
+                     :other_setting, :memorial, :indexable
 
   attributes :id, :type, :following, :followers,
              :inbox, :outbox, :featured, :featured_tags,
              :preferred_username, :name, :summary,
              :url, :manually_approves_followers,
-             :discoverable, :published, :searchable_by, :subscribable_by, :other_setting, :memorial
+             :discoverable, :indexable, :published, :memorial, :searchable_by, :subscribable_by, :other_setting
 
   has_one :public_key, serializer: ActivityPub::PublicKeySerializer
 
@@ -107,6 +107,10 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
     object.suspended? ? false : (object.discoverable || false)
   end
 
+  def indexable
+    object.suspended? ? false : (object.indexable || false)
+  end
+
   def name
     object.suspended? ? object.username : (object.display_name.presence || object.username)
   end

app/services/batched_remove_status_service.rb

@@ -35,7 +35,10 @@ class BatchedRemoveStatusService < BaseService
 
     # Since we skipped all callbacks, we also need to manually
     # deindex the statuses
-    Chewy.strategy.current.update(StatusesIndex, statuses_and_reblogs) if Chewy.enabled?
+    if Chewy.enabled?
+      Chewy.strategy.current.update(StatusesIndex, statuses_and_reblogs)
+      Chewy.strategy.current.update(PublicStatusesIndex, statuses_and_reblogs)
+    end
 
     return if options[:skip_side_effects]
 

app/services/search_service.rb

@@ -41,41 +41,16 @@ class SearchService < BaseService
   end
 
   def perform_statuses_search!
-    privacy_definition = parsed_query.apply(StatusesIndex.filter(terms: { searchability: %w(public private direct) }).filter(term: { searchable_by: @account.id }))
-
-    # 'direct' searchability posts are NOT in here because it's already added at previous line.
-    case @searchability
-    when 'public'
-      privacy_definition = privacy_definition.or(StatusesIndex.filter(term: { searchability: 'public' }))
-      privacy_definition = privacy_definition.or(StatusesIndex.filter(term: { searchability: 'private' }).filter(terms: { account_id: following_account_ids })) unless following_account_ids.empty?
-      privacy_definition = privacy_definition.or(StatusesIndex.filter(term: { searchability: 'limited' }).filter(term: { account_id: @account.id }))
-    when 'private', 'direct'
-      privacy_definition = privacy_definition.or(StatusesIndex.filter(terms: { searchability: %w(public private) }).filter(terms: { account_id: following_account_ids })) unless following_account_ids.empty?
-      privacy_definition = privacy_definition.or(StatusesIndex.filter(term: { searchability: 'limited' }).filter(term: { account_id: @account.id }))
-    when 'limited'
-      privacy_definition = privacy_definition.or(StatusesIndex.filter(term: { searchability: 'limited' }).filter(term: { account_id: @account.id }))
-    end
-
-    definition = parsed_query.apply(StatusesIndex).order(id: :desc)
-    definition = definition.filter(term: { account_id: @options[:account_id] }) if @options[:account_id].present?
-
-    definition = definition.and(privacy_definition)
-
-    if @options[:min_id].present? || @options[:max_id].present?
-      range      = {}
-      range[:gt] = @options[:min_id].to_i if @options[:min_id].present?
-      range[:lt] = @options[:max_id].to_i if @options[:max_id].present?
-      definition = definition.filter(range: { id: range })
-    end
-
-    results             = definition.limit(@limit).offset(@offset).objects.compact
-    account_ids         = results.map(&:account_id)
-    account_domains     = results.map(&:account_domain)
-    account_relations   = @account.relations_map(account_ids, account_domains) # variable old name: preloaded_relations
-
-    results.reject { |status| StatusFilter.new(status, @account, account_relations).filtered? }
-  rescue Faraday::ConnectionFailed, Parslet::ParseFailed
-    []
+    StatusesSearchService.new.call(
+      @query,
+      @account,
+      limit: @limit,
+      offset: @offset,
+      account_id: @options[:account_id],
+      min_id: @options[:min_id],
+      max_id: @options[:max_id],
+      searchability: @searchability
+    )
   end
 
   def perform_hashtags_search!
@@ -132,17 +107,4 @@ class SearchService < BaseService
   def statuses_search?
     @options[:type].blank? || @options[:type] == 'statuses'
   end
-
-  def parsed_query
-    SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query))
-  end
-
-  def following_account_ids
-    return @following_account_ids if defined?(@following_account_ids)
-
-    account_exists_sql     = Account.where('accounts.id = follows.target_account_id').where(searchability: %w(public private)).reorder(nil).select(1).to_sql
-    status_exists_sql      = Status.where('statuses.account_id = follows.target_account_id').where(reblog_of_id: nil).where(searchability: %w(public private)).reorder(nil).select(1).to_sql
-    following_accounts     = Follow.where(account_id: @account.id).merge(Account.where("EXISTS (#{account_exists_sql})").or(Account.where("EXISTS (#{status_exists_sql})")))
-    @following_account_ids = following_accounts.pluck(:target_account_id)
-  end
 end

app/services/statuses_search_service.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+class StatusesSearchService < BaseService
+  def call(query, account = nil, options = {})
+    @query   = query&.strip
+    @account = account
+    @options = options
+    @limit   = options[:limit].to_i
+    @offset  = options[:offset].to_i
+    @searchability = options[:searchability]&.to_sym
+
+    status_search_results
+  end
+
+  private
+
+  def status_search_results
+    definition_should = [
+      publicly_searchable,
+      non_publicly_searchable,
+      searchability_limited,
+    ]
+    definition_should << searchability_public if %i(public).include?(@searchability)
+    definition_should << searchability_private if %i(public private).include?(@searchability)
+
+    definition = parsed_query.apply(
+      StatusesIndex.filter(
+        bool: {
+          should: definition_should,
+          minimum_should_match: 1,
+        }
+      )
+    )
+
+    # This is the best way to submit identical queries to multi-indexes though chewy
+    definition.instance_variable_get(:@parameters)[:indices].value[:indices] << PublicStatusesIndex
+
+    results             = definition.collapse(field: :id).order(_id: { order: :desc }).limit(@limit).offset(@offset).objects.compact
+    account_ids         = results.map(&:account_id)
+    account_domains     = results.map(&:account_domain)
+    preloaded_relations = @account.relations_map(account_ids, account_domains)
+
+    results.reject { |status| StatusFilter.new(status, @account, preloaded_relations).filtered? }
+  rescue Faraday::ConnectionFailed, Parslet::ParseFailed
+    []
+  end
+
+  def publicly_searchable
+    {
+      bool: {
+        must_not: {
+          exists: {
+            field: 'searchable_by',
+          },
+        },
+      },
+    }
+  end
+
+  def non_publicly_searchable
+    {
+      bool: {
+        must: [
+          {
+            exists: {
+              field: 'searchable_by',
+            },
+          },
+          {
+            exists: {
+              field: 'searchability',
+            },
+          },
+          {
+            term: { searchable_by: @account.id },
+          },
+        ],
+        must_not: [
+          {
+            term: { searchability: 'limited' },
+          },
+        ],
+      },
+    }
+  end
+
+  def searchability_public
+    {
+      bool: {
+        must: [
+          {
+            exists: {
+              field: 'searchability',
+            },
+          },
+          {
+            term: { searchability: 'public' },
+          },
+        ],
+      },
+    }
+  end
+
+  def searchability_private
+    {
+      bool: {
+        must: [
+          {
+            exists: {
+              field: 'searchability',
+            },
+          },
+          {
+            term: { searchability: 'private' },
+          },
+          {
+            terms: { account_id: following_account_ids },
+          },
+        ],
+      },
+    }
+  end
+
+  def searchability_limited
+    {
+      bool: {
+        must: [
+          {
+            exists: {
+              field: 'searchability',
+            },
+          },
+          {
+            term: { searchability: 'limited' },
+          },
+          {
+            term: { account_id: @account.id },
+          },
+        ],
+      },
+    }
+  end
+
+  def following_account_ids
+    return @following_account_ids if defined?(@following_account_ids)
+
+    account_exists_sql     = Account.where('accounts.id = follows.target_account_id').where(searchability: %w(public private)).reorder(nil).select(1).to_sql
+    status_exists_sql      = Status.where('statuses.account_id = follows.target_account_id').where(reblog_of_id: nil).where(searchability: %w(public private)).reorder(nil).select(1).to_sql
+    following_accounts     = Follow.where(account_id: @account.id).merge(Account.where("EXISTS (#{account_exists_sql})").or(Account.where("EXISTS (#{status_exists_sql})")))
+    @following_account_ids = following_accounts.pluck(:target_account_id)
+  end
+
+  def parsed_query
+    SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query))
+  end
+end

app/views/admin/accounts/index.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('admin.accounts.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 = form_tag admin_accounts_url, method: 'GET', class: 'simple_form' do
   .filters
     .filter-subset.filter-subset--with-select

app/views/admin/action_logs/index.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('admin.action_logs.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 = form_tag admin_action_logs_url, method: 'GET', class: 'simple_form' do
   = hidden_field_tag :target_account_id, params[:target_account_id] if params[:target_account_id].present?
 

app/views/admin/announcements/edit.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 = simple_form_for @announcement, url: admin_announcement_path(@announcement), html: { novalidate: false } do |f|
   = render 'shared/error_messages', object: @announcement
 

app/views/admin/announcements/new.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 = simple_form_for @announcement, url: admin_announcements_path, html: { novalidate: false } do |f|
   = render 'shared/error_messages', object: @announcement
 

app/views/admin/custom_emojis/index.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('admin.custom_emojis.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - if can?(:create, :custom_emoji)
   - content_for :heading_actions do
     = link_to t('admin.custom_emojis.upload'), new_admin_custom_emoji_path, class: 'button'

app/views/admin/dashboard/index.html.haml

@@ -1,6 +1,3 @@
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - content_for :page_title do
   = t('admin.dashboard.title')
 

app/views/admin/disputes/appeals/index.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('admin.disputes.appeals.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 .filters
   .filter-subset
     %strong= t('admin.tags.review')

app/views/admin/domain_allows/new.html.haml

@@ -1,6 +1,3 @@
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - content_for :page_title do
   = t('admin.domain_allows.add_new')
 

app/views/admin/domain_blocks/confirm_suspension.html.haml

@@ -1,6 +1,3 @@
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - content_for :page_title do
   = t('.title', domain: Addressable::IDNA.to_unicode(@domain_block.domain))
 

app/views/admin/domain_blocks/edit.html.haml

@@ -1,6 +1,3 @@
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - content_for :page_title do
   = t('admin.domain_blocks.edit')
 

app/views/admin/domain_blocks/new.html.haml

@@ -1,6 +1,3 @@
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - content_for :page_title do
   = t('.title')
 

app/views/admin/email_domain_blocks/index.html.haml

@@ -4,9 +4,6 @@
 - content_for :heading_actions do
   = link_to t('admin.email_domain_blocks.add_new'), new_admin_email_domain_block_path, class: 'button'
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 = form_for(@form, url: batch_admin_email_domain_blocks_path) do |f|
   = hidden_field_tag :page, params[:page] || 1
 

app/views/admin/email_domain_blocks/new.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 = simple_form_for @email_domain_block, url: admin_email_domain_blocks_path do |f|
   = render 'shared/error_messages', object: @email_domain_block
 

app/views/admin/export_domain_blocks/import.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('admin.export_domain_blocks.import.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 %p= t('admin.export_domain_blocks.import.description_html')
 
 - if defined?(@global_private_comment) && @global_private_comment.present?

app/views/admin/follow_recommendations/show.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('admin.follow_recommendations.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 %p= t('admin.follow_recommendations.description_html')
 
 %hr.spacer/

app/views/admin/instances/index.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('admin.instances.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - content_for :heading_actions do
   - if limited_federation_mode?
     = link_to t('admin.domain_allows.add_new'), new_admin_domain_allow_path, class: 'button', id: 'add-instance-button'

app/views/admin/instances/show.html.haml

@@ -1,13 +1,10 @@
 - content_for :page_title do
   = @instance.domain
-
+  
 - if @instance.instance_info.present?
   %p
     = "#{@instance.instance_info.software} #{@instance.instance_info.version}"
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - if current_user.can?(:view_dashboard)
   - content_for :heading_actions do
     = l(@time_period.first)

app/views/admin/ip_blocks/index.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('admin.ip_blocks.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - if can?(:create, :ip_block)
   - content_for :heading_actions do
     = link_to t('admin.ip_blocks.add_new'), new_admin_ip_block_path, class: 'button'

app/views/admin/reports/show.html.haml

@@ -1,7 +1,3 @@
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-  = javascript_pack_tag 'public', async: true, crossorigin: 'anonymous'
-
 - content_for :page_title do
   = t('admin.reports.report', id: @report.id)
 

app/views/admin/settings/about/show.html.haml

@@ -1,6 +1,3 @@
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - content_for :page_title do
   = t('admin.settings.about.title')
 

app/views/admin/settings/appearance/show.html.haml

@@ -1,6 +1,3 @@
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - content_for :page_title do
   = t('admin.settings.appearance.title')
 

app/views/admin/settings/branding/show.html.haml

@@ -1,6 +1,3 @@
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - content_for :page_title do
   = t('admin.settings.branding.title')
 

app/views/admin/settings/content_retention/show.html.haml

@@ -1,6 +1,3 @@
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - content_for :page_title do
   = t('admin.settings.content_retention.title')
 

app/views/admin/settings/discovery/show.html.haml

@@ -1,6 +1,3 @@
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - content_for :page_title do
   = t('admin.settings.discovery.title')
 

app/views/admin/settings/registrations/show.html.haml

@@ -1,6 +1,3 @@
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - content_for :page_title do
   = t('admin.settings.registrations.title')
 

app/views/admin/statuses/index.html.haml

@@ -1,6 +1,3 @@
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - content_for :page_title do
   = t('admin.statuses.title')
   \-

app/views/admin/statuses/show.html.haml

@@ -1,6 +1,3 @@
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - content_for :page_title do
   = t('statuses.title', name: display_name(@account), quote: truncate(@status.spoiler_text.presence || @status.text, length: 50, omission: '…', escape: false))
 

app/views/admin/tags/show.html.haml

@@ -1,6 +1,3 @@
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - content_for :page_title do
   = "##{@tag.display_name}"
 

app/views/admin/trends/links/index.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('admin.trends.links.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 %p= t('admin.trends.links.description_html')
 
 %hr.spacer/

app/views/admin/trends/links/preview_card_providers/index.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('admin.trends.preview_card_providers.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 %p= t('admin.trends.preview_card_providers.description_html')
 
 %hr.spacer/

app/views/admin/trends/statuses/index.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('admin.trends.statuses.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 %p= t('admin.trends.statuses.description_html')
 
 %hr.spacer/

app/views/admin/trends/tags/index.html.haml

@@ -1,9 +1,6 @@
 - content_for :page_title do
   = t('admin.trends.tags.title')
 
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 %p= t('admin.trends.tags.description_html')
 
 %hr.spacer/

app/views/filters/statuses/index.html.haml

@@ -1,6 +1,3 @@
-- content_for :header_tags do
-  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
-
 - content_for :page_title do
   = t('filters.statuses.index.title')
   \-

app/views/layouts/admin.html.haml

@@ -1,6 +1,7 @@
 - content_for :header_tags do
   = render_initial_state
   = javascript_pack_tag 'public', crossorigin: 'anonymous'
+  = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
 
 - content_for :content do
   .admin-wrapper

app/views/settings/preferences/notifications/show.html.haml

@@ -18,14 +18,19 @@
       = ff.input :'notification_emails.reblog', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.reblog')
       = ff.input :'notification_emails.favourite', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.favourite')
       = ff.input :'notification_emails.mention', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.mention')
-      = ff.input :'notification_emails.report', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.report') if current_user.can?(:manage_reports)
-      = ff.input :'notification_emails.appeal', as: :boolean, wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.appeal') if current_user.can?(:manage_appeals)
-      = ff.input :'notification_emails.pending_account', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.pending_account') if current_user.can?(:manage_users)
-      = ff.input :'notification_emails.trends', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.trending_tag') if current_user.can?(:manage_taxonomies)
 
     .fields-group
       = ff.input :always_send_emails, wrapper: :with_label, label: I18n.t('simple_form.labels.defaults.setting_always_send_emails'), hint: I18n.t('simple_form.hints.defaults.setting_always_send_emails')
 
+    - if current_user.can?(:manage_reports, :manage_appeals, :manage_users, :manage_taxonomies)
+      %h4= t 'notifications.administration_emails'
+
+      .fields-group
+        = ff.input :'notification_emails.report', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.report') if current_user.can?(:manage_reports)
+        = ff.input :'notification_emails.appeal', as: :boolean, wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.appeal') if current_user.can?(:manage_appeals)
+        = ff.input :'notification_emails.pending_account', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.pending_account') if current_user.can?(:manage_users)
+        = ff.input :'notification_emails.trends', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.trending_tag') if current_user.can?(:manage_taxonomies)
+
   %h4= t 'notifications.other_settings'
 
   .fields-group

app/views/settings/privacy/show.html.haml

@@ -31,6 +31,9 @@
 
   %p.lead= t('privacy.search_hint_html')
 
+  .fields-group
+    = f.input :indexable, as: :boolean, wrapper: :with_label
+
   = f.simple_fields_for :settings, current_user.settings do |ff|
     .fields-group
       = ff.input :indexable, wrapper: :with_label

app/workers/add_to_public_statuses_index_worker.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class AddToPublicStatusesIndexWorker
+  include Sidekiq::Worker
+
+  def perform(account_id)
+    account = Account.find(account_id)
+
+    return unless account.indexable?
+
+    account.add_to_public_statuses_index!
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end

app/workers/remove_from_public_statuses_index_worker.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class RemoveFromPublicStatusesIndexWorker
+  include Sidekiq::Worker
+
+  def perform(account_id)
+    account = Account.find(account_id)
+
+    return if account.indexable?
+
+    account.remove_from_public_statuses_index!
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end

app/workers/scheduler/indexing_scheduler.rb

@@ -23,6 +23,6 @@ class Scheduler::IndexingScheduler
   end
 
   def indexes
-    [AccountsIndex, TagsIndex, StatusesIndex]
+    [AccountsIndex, TagsIndex, PublicStatusesIndex, StatusesIndex]
   end
 end