diff --git a/.github/workflows/build-nightly.yml b/.github/workflows/build-nightly.yml index b700790d8a..5b34c1c3c6 100644 --- a/.github/workflows/build-nightly.yml +++ b/.github/workflows/build-nightly.yml @@ -16,7 +16,7 @@ jobs: env: TZ: Etc/UTC run: | - echo mastodon_version_suffix=nightly-$(date +'%Y-%m-%d')>> $GITHUB_OUTPUT + echo mastodon_version_suffix=nightly.$(date +'%Y-%m-%d')>> $GITHUB_OUTPUT outputs: suffix: ${{ steps.version_vars.outputs.mastodon_version_suffix }} @@ -29,8 +29,8 @@ jobs: push_to_images: | tootsuite/mastodon ghcr.io/mastodon/mastodon - # The `+` is important here, result will be v4.1.2+nightly-2022-03-05 - version_suffix: +${{ needs.compute-suffix.outputs.suffix }} + # The `-` is important here, result will be v4.1.2-nightly.2022-03-05 + version_suffix: -${{ needs.compute-suffix.outputs.suffix }} labels: | org.opencontainers.image.description=Nightly build image used for testing purposes flavor: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 107dfaca3f..fe66adc08a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,7 +101,7 @@ The following changelog entries focus on changes visible to users, administrator - **Change translation feature to cover Content Warnings, poll options and media descriptions** ([c960657](https://github.com/mastodon/mastodon/pull/24175), [S-H-GAMELINKS](https://github.com/mastodon/mastodon/pull/25251), [c960657](https://github.com/mastodon/mastodon/pull/26168), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26452)) - **Change account search to match by text when opted-in** ([jsgoldstein](https://github.com/mastodon/mastodon/pull/25599), [Gargron](https://github.com/mastodon/mastodon/pull/26378)) - **Change import feature to be clearer, less error-prone and more reliable** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/21054), [mgmn](https://github.com/mastodon/mastodon/pull/24874)) -- **Change local and federated timelines to be in a single “Live feeds” column** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25641), [Gargron](https://github.com/mastodon/mastodon/pull/25683), [mgmn](https://github.com/mastodon/mastodon/pull/25694), [Plastikmensch](https://github.com/mastodon/mastodon/pull/26247)) +- **Change local and federated timelines to be tabs of a single “Live feeds” column** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25641), [Gargron](https://github.com/mastodon/mastodon/pull/25683), [mgmn](https://github.com/mastodon/mastodon/pull/25694), [Plastikmensch](https://github.com/mastodon/mastodon/pull/26247)) - **Change user archive export to be faster and more reliable, and export `.zip` archives instead of `.tar.gz` ones** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23360), [TheEssem](https://github.com/mastodon/mastodon/pull/25034)) - **Change `mastodon-streaming` systemd unit files to be templated** ([e-nomem](https://github.com/mastodon/mastodon/pull/24751)) - **Change `statsd` integration to disable sidekiq metrics by default** ([mjankowski](https://github.com/mastodon/mastodon/pull/25265), [mjankowski](https://github.com/mastodon/mastodon/pull/25336), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26310)) @@ -189,6 +189,7 @@ The following changelog entries focus on changes visible to users, administrator - **Fix log-in flow when involving both OAuth and external authentication** ([CSDUMMI](https://github.com/mastodon/mastodon/pull/24073)) - **Fix broken links in account gallery** ([c960657](https://github.com/mastodon/mastodon/pull/24218)) - **Fix blocking subdomains of an already-blocked domain** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26392)) +- **Fix migration handler not updating lists** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24808)) - Fix uploading of video files for which `ffprobe` reports `0/0` average framerate ([NicolaiSoeborg](https://github.com/mastodon/mastodon/pull/26500)) - Fix cached posts including stale stats ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26409)) - Fix adding column with default value taking longer on Postgres >= 11 ([Gargron](https://github.com/mastodon/mastodon/pull/26375)) diff --git a/Gemfile b/Gemfile index 42ca7322cd..18452b1ce9 100644 --- a/Gemfile +++ b/Gemfile @@ -110,7 +110,7 @@ group :test do gem 'fuubar', '~> 2.5' # Extra RSpec extenion methods and helpers for sidekiq - gem 'rspec-sidekiq', '~> 3.1' + gem 'rspec-sidekiq', '~> 4.0' # Browser integration testing gem 'capybara', '~> 3.39' diff --git a/Gemfile.lock b/Gemfile.lock index d710e77025..87d52bf7f5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -635,10 +635,12 @@ GEM rspec-support (~> 3.12) rspec-retry (0.6.2) rspec-core (> 3.3) - rspec-sidekiq (3.1.0) - rspec-core (~> 3.0, >= 3.0.0) - sidekiq (>= 2.4.0) - rspec-support (3.12.0) + rspec-sidekiq (4.0.1) + rspec-core (~> 3.0) + rspec-expectations (~> 3.0) + rspec-mocks (~> 3.0) + sidekiq (>= 5, < 8) + rspec-support (3.12.1) rspec_chunked (0.6) rubocop (1.56.1) base64 (~> 0.1.1) @@ -914,7 +916,7 @@ DEPENDENCIES rqrcode (~> 2.2) rspec-rails (~> 6.0) rspec-retry (>= 0.6.2) - rspec-sidekiq (~> 3.1) + rspec-sidekiq (~> 4.0) rspec_chunked (~> 0.6) rubocop rubocop-capybara diff --git a/app/chewy/accounts_index.rb b/app/chewy/accounts_index.rb index c2e08d2e11..c00a936bc5 100644 --- a/app/chewy/accounts_index.rb +++ b/app/chewy/accounts_index.rb @@ -62,6 +62,6 @@ class AccountsIndex < Chewy::Index field(:last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at }) field(:display_name, type: 'text', analyzer: 'verbatim') { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' } field(:username, type: 'text', analyzer: 'verbatim', value: ->(account) { [account.username, account.domain].compact.join('@') }) { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' } - field(:text, type: 'text', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' } + field(:text, type: 'text', analyzer: 'whitespace', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' } end end diff --git a/app/chewy/public_statuses_index.rb b/app/chewy/public_statuses_index.rb new file mode 100644 index 0000000000..1fad5de3a1 --- /dev/null +++ b/app/chewy/public_statuses_index.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +class PublicStatusesIndex < Chewy::Index + settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: { + filter: { + english_stop: { + type: 'stop', + stopwords: '_english_', + }, + + english_stemmer: { + type: 'stemmer', + language: 'english', + }, + + english_possessive_stemmer: { + type: 'stemmer', + language: 'possessive_english', + }, + }, + + analyzer: { + content: { + tokenizer: 'uax_url_email', + filter: %w( + english_possessive_stemmer + lowercase + asciifolding + cjk_width + english_stop + english_stemmer + ), + }, + }, + } + + index_scope ::Status.unscoped + .kept + .indexable + .includes(:media_attachments, :preloadable_poll, :preview_cards) + + root date_detection: false do + field(:id, type: 'keyword') + field(:account_id, type: 'long') + field(:text, type: 'text', analyzer: 'whitespace', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') } + field(:language, type: 'keyword') + field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties }) + field(:created_at, type: 'date') + end +end diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb index a70436e502..f84a0c69ec 100644 --- a/app/chewy/statuses_index.rb +++ b/app/chewy/statuses_index.rb @@ -1,23 +1,24 @@ # frozen_string_literal: true class StatusesIndex < Chewy::Index - include FormattingHelper - settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: { filter: { english_stop: { type: 'stop', stopwords: '_english_', }, + english_stemmer: { type: 'stemmer', language: 'english', }, + english_possessive_stemmer: { type: 'stemmer', language: 'possessive_english', }, }, + analyzer: { content: { tokenizer: 'uax_url_email', @@ -35,7 +36,7 @@ class StatusesIndex < Chewy::Index # We do not use delete_if option here because it would call a method that we # expect to be called with crutches without crutches, causing n+1 queries - index_scope ::Status.unscoped.kept.without_reblogs.includes(:media_attachments, :preloadable_poll) + index_scope ::Status.unscoped.kept.without_reblogs.includes(:media_attachments, :preloadable_poll, :preview_cards) crutch :mentions do |collection| data = ::Mention.where(status_id: collection.map(&:id)).where(account: Account.local, silent: false).pluck(:status_id, :account_id) @@ -68,14 +69,13 @@ class StatusesIndex < Chewy::Index end root date_detection: false do - field :id, type: 'long' - field :account_id, type: 'long' - - field :text, type: 'text', value: ->(status) { status.searchable_text } do - field :stemmed, type: 'text', analyzer: 'content' - end - - field :searchable_by, type: 'long', value: ->(status, crutches) { status.searchable_by(crutches) } - field :searchability, type: 'keyword', value: ->(status) { status.compute_searchability } + field(:id, type: 'keyword') + field(:account_id, type: 'long') + field(:text, type: 'text', analyzer: 'whitespace', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') } + field(:searchable_by, type: 'long', value: ->(status, crutches) { status.searchable_by(crutches) }) + field(:searchability, type: 'keyword', value: ->(status) { status.compute_searchability }) + field(:language, type: 'keyword') + field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties }) + field(:created_at, type: 'date') end end diff --git a/app/controllers/api/v1/accounts/credentials_controller.rb b/app/controllers/api/v1/accounts/credentials_controller.rb index f26389f41c..011693c7c7 100644 --- a/app/controllers/api/v1/accounts/credentials_controller.rb +++ b/app/controllers/api/v1/accounts/credentials_controller.rb @@ -32,6 +32,7 @@ class Api::V1::Accounts::CredentialsController < Api::BaseController :searchability, :dissubscribable, :hide_collections, + :indexable, fields_attributes: [:name, :value] ) end diff --git a/app/controllers/settings/privacy_controller.rb b/app/controllers/settings/privacy_controller.rb index d1f9c533e0..e5c4e5a905 100644 --- a/app/controllers/settings/privacy_controller.rb +++ b/app/controllers/settings/privacy_controller.rb @@ -18,7 +18,7 @@ class Settings::PrivacyController < Settings::BaseController private def account_params - params.require(:account).permit(:discoverable, :unlocked, :show_collections, :dissubscribable, settings: UserSettings.keys) + params.require(:account).permit(:discoverable, :unlocked, :indexable, :show_collections, :dissubscribable, settings: UserSettings.keys) end def set_account diff --git a/app/javascript/mastodon/components/column.jsx b/app/javascript/mastodon/components/column.jsx index 7e7eaa4115..d737bd347c 100644 --- a/app/javascript/mastodon/components/column.jsx +++ b/app/javascript/mastodon/components/column.jsx @@ -16,7 +16,19 @@ export default class Column extends PureComponent { }; scrollTop () { - const scrollable = this.props.bindToDocument ? document.scrollingElement : this.node.querySelector('.scrollable'); + let scrollable = null; + + if (this.props.bindToDocument) { + scrollable = document.scrollingElement; + } else { + scrollable = this.node.querySelector('.scrollable'); + + // Some columns have nested `.scrollable` containers, with the outer one + // being a wrapper while the actual scrollable content is deeper. + if (scrollable.classList.contains('scrollable--flex')) { + scrollable = scrollable?.querySelector('.scrollable') || scrollable; + } + } if (!scrollable) { return; diff --git a/app/javascript/mastodon/features/ui/components/report_modal.jsx b/app/javascript/mastodon/features/ui/components/report_modal.jsx index 7ed5742588..fef1ced824 100644 --- a/app/javascript/mastodon/features/ui/components/report_modal.jsx +++ b/app/javascript/mastodon/features/ui/components/report_modal.jsx @@ -62,7 +62,7 @@ class ReportModal extends ImmutablePureComponent { dispatch(submitReport({ account_id: accountId, status_ids: selectedStatusIds.toArray(), - selected_domains: selectedDomains.toArray(), + forward_to_domains: selectedDomains.toArray(), comment, forward: selectedDomains.size > 0, category, diff --git a/app/lib/importer/public_statuses_index_importer.rb b/app/lib/importer/public_statuses_index_importer.rb new file mode 100644 index 0000000000..8e36e36f90 --- /dev/null +++ b/app/lib/importer/public_statuses_index_importer.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Importer::PublicStatusesIndexImporter < Importer::BaseImporter + def import! + indexable_statuses_scope.find_in_batches(batch_size: @batch_size) do |batch| + in_work_unit(batch.map(&:status_id)) do |status_ids| + bulk = ActiveRecord::Base.connection_pool.with_connection do + Chewy::Index::Import::BulkBuilder.new(index, to_index: Status.includes(:media_attachments, :preloadable_poll).where(id: status_ids)).bulk_body + end + + indexed = 0 + deleted = 0 + + bulk.map! do |entry| + if entry[:index] + indexed += 1 + else + deleted += 1 + end + entry + end + + Chewy::Index::Import::BulkRequest.new(index).perform(bulk) + + [indexed, deleted] + end + end + + wait! + end + + private + + def index + PublicStatusesIndex + end + + def indexable_statuses_scope + Status.indexable.select('"statuses"."id", COALESCE("statuses"."reblog_of_id", "statuses"."id") AS status_id') + end +end diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb index 8ca14de08f..be11597d2b 100644 --- a/app/lib/search_query_transformer.rb +++ b/app/lib/search_query_transformer.rb @@ -36,7 +36,7 @@ class SearchQueryTransformer < Parslet::Transform def clause_to_filter(clause) case clause when PrefixClause - { term: { clause.filter => clause.term } } + { clause.type => { clause.filter => clause.term } } else raise "Unexpected clause type: #{clause}" end @@ -47,12 +47,10 @@ class SearchQueryTransformer < Parslet::Transform class << self def symbol(str) case str - when '+' + when '+', nil :must when '-' :must_not - when nil - :should else raise "Unknown operator: #{str}" end @@ -81,23 +79,52 @@ class SearchQueryTransformer < Parslet::Transform end class PrefixClause - attr_reader :filter, :operator, :term + attr_reader :type, :filter, :operator, :term def initialize(prefix, term) @operator = :filter + case prefix + when 'has', 'is' + @filter = :properties + @type = :term + @term = term + when 'language' + @filter = :language + @type = :term + @term = term when 'from' @filter = :account_id - - username, domain = term.gsub(/\A@/, '').split('@') - domain = nil if TagManager.instance.local_domain?(domain) - account = Account.find_remote!(username, domain) - - @term = account.id + @type = :term + @term = account_id_from_term(term) + when 'before' + @filter = :created_at + @type = :range + @term = { lt: term } + when 'after' + @filter = :created_at + @type = :range + @term = { gt: term } + when 'during' + @filter = :created_at + @type = :range + @term = { gte: term, lte: term } else raise Mastodon::SyntaxError end end + + private + + def account_id_from_term(term) + username, domain = term.gsub(/\A@/, '').split('@') + domain = nil if TagManager.instance.local_domain?(domain) + account = Account.find_remote(username, domain) + + # If the account is not found, we want to return empty results, so return + # an ID that does not exist + account&.id || -1 + end end rule(clause: subtree(:clause)) do diff --git a/app/lib/vacuum/statuses_vacuum.rb b/app/lib/vacuum/statuses_vacuum.rb index 28c087b1c2..ad1de07380 100644 --- a/app/lib/vacuum/statuses_vacuum.rb +++ b/app/lib/vacuum/statuses_vacuum.rb @@ -20,7 +20,10 @@ class Vacuum::StatusesVacuum statuses.direct_visibility .includes(mentions: :account) .find_each(&:unlink_from_conversations!) - remove_from_search_index(statuses.ids) if Chewy.enabled? + if Chewy.enabled? + remove_from_index(statuses.ids, 'chewy:queue:StatusesIndex') + remove_from_index(statuses.ids, 'chewy:queue:PublicStatusesIndex') + end # Foreign keys take care of most associated records for us. # Media attachments will be orphaned. @@ -38,7 +41,7 @@ class Vacuum::StatusesVacuum Mastodon::Snowflake.id_at(@retention_period.ago, with_random: false) end - def remove_from_search_index(status_ids) - with_redis { |redis| redis.sadd('chewy:queue:StatusesIndex', status_ids) } + def remove_from_index(status_ids, index) + with_redis { |redis| redis.sadd(index, status_ids) } end end diff --git a/app/models/account.rb b/app/models/account.rb index 62c1bebb4f..d2538bb065 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -86,6 +86,7 @@ class Account < ApplicationRecord include DomainMaterializable include AccountMerging include AccountSearch + include AccountStatusesSearch enum protocol: { ostatus: 0, activitypub: 1 } enum suspension_origin: { local: 0, remote: 1 }, _prefix: true diff --git a/app/models/concerns/account_statuses_search.rb b/app/models/concerns/account_statuses_search.rb new file mode 100644 index 0000000000..563a871051 --- /dev/null +++ b/app/models/concerns/account_statuses_search.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module AccountStatusesSearch + extend ActiveSupport::Concern + + included do + after_update_commit :enqueue_update_public_statuses_index, if: :saved_change_to_indexable? + after_destroy_commit :enqueue_remove_from_public_statuses_index, if: :indexable? + end + + def enqueue_update_public_statuses_index + if indexable? + enqueue_add_to_public_statuses_index + else + enqueue_remove_from_public_statuses_index + end + end + + def enqueue_add_to_public_statuses_index + return unless Chewy.enabled? + + AddToPublicStatusesIndexWorker.perform_async(id) + end + + def enqueue_remove_from_public_statuses_index + return unless Chewy.enabled? + + RemoveFromPublicStatusesIndexWorker.perform_async(id) + end + + def add_to_public_statuses_index! + return unless Chewy.enabled? + + statuses.indexable.find_in_batches do |batch| + PublicStatusesIndex.import(query: batch) + end + end + + def remove_from_public_statuses_index! + return unless Chewy.enabled? + + PublicStatusesIndex.filter(term: { account_id: id }).delete_all + end +end diff --git a/app/models/concerns/status_search_concern.rb b/app/models/concerns/status_search_concern.rb new file mode 100644 index 0000000000..daf5c0a80f --- /dev/null +++ b/app/models/concerns/status_search_concern.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module StatusSearchConcern + extend ActiveSupport::Concern + + included do + scope :indexable, -> { without_reblogs.where(visibility: [:public, :login], searchability: nil).joins(:account).where(account: { indexable: true }) } + end + + def searchable_by(preloaded = nil) + ids = [] + + ids << account_id if local? + + if preloaded.nil? + ids += mentions.joins(:account).merge(Account.local).active.pluck(:account_id) + ids += favourites.joins(:account).merge(Account.local).pluck(:account_id) + ids += emoji_reactions.joins(:account).merge(Account.local).pluck(:account_id) + ids += reblogs.joins(:account).merge(Account.local).pluck(:account_id) + ids += bookmarks.joins(:account).merge(Account.local).pluck(:account_id) + ids += poll.votes.joins(:account).merge(Account.local).pluck(:account_id) if poll.present? + else + ids += preloaded.mentions[id] || [] + ids += preloaded.favourites[id] || [] + ids += preloaded.emoji_reactions[id] || [] + ids += preloaded.reblogs[id] || [] + ids += preloaded.bookmarks[id] || [] + ids += preloaded.votes[id] || [] + end + + ids.uniq + end + + def searchable_text + [ + spoiler_text, + FormattingHelper.extract_status_plain_text(self), + preloadable_poll&.options&.join("\n\n"), + ordered_media_attachments.map(&:description).join("\n\n"), + ].compact.join("\n\n") + end + + def searchable_properties + [].tap do |properties| + properties << 'image' if ordered_media_attachments.any?(&:image?) + properties << 'video' if ordered_media_attachments.any?(&:video?) + properties << 'audio' if ordered_media_attachments.any?(&:audio?) + properties << 'media' if with_media? + properties << 'poll' if with_poll? + properties << 'link' if with_preview_card? + properties << 'embed' if preview_cards.any?(&:video?) + properties << 'sensitive' if sensitive? + properties << 'reply' if reply? + end + end +end diff --git a/app/models/status.rb b/app/models/status.rb index d2cc1fb7a5..49a1a3ca3c 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -42,6 +42,7 @@ class Status < ApplicationRecord include StatusSnapshotConcern include RateLimitable include StatusSafeReblogInsert + include StatusSearchConcern rate_limit by: :account, family: :statuses @@ -52,6 +53,7 @@ class Status < ApplicationRecord attr_accessor :override_timestamps update_index('statuses', :proper) + update_index('public_statuses', :proper) enum visibility: { public: 0, unlisted: 1, private: 2, direct: 3, limited: 4, public_unlisted: 10, login: 11 }, _suffix: :visibility enum searchability: { public: 0, private: 1, direct: 2, limited: 3, unsupported: 4, public_unlisted: 10 }, _suffix: :searchability @@ -183,39 +185,6 @@ class Status < ApplicationRecord "v3:#{super}" end - def searchable_by(preloaded = nil) - ids = [] - - ids << account_id if local? - - if preloaded.nil? - ids += mentions.joins(:account).merge(Account.local).active.pluck(:account_id) - ids += favourites.joins(:account).merge(Account.local).pluck(:account_id) - ids += emoji_reactions.joins(:account).merge(Account.local).pluck(:account_id) - ids += reblogs.joins(:account).merge(Account.local).pluck(:account_id) - ids += bookmarks.joins(:account).merge(Account.local).pluck(:account_id) - ids += poll.votes.joins(:account).merge(Account.local).pluck(:account_id) if poll.present? - else - ids += preloaded.mentions[id] || [] - ids += preloaded.favourites[id] || [] - ids += preloaded.emoji_reactions[id] || [] - ids += preloaded.reblogs[id] || [] - ids += preloaded.bookmarks[id] || [] - ids += preloaded.votes[id] || [] - end - - ids.uniq - end - - def searchable_text - [ - spoiler_text, - FormattingHelper.extract_status_plain_text(self), - preloadable_poll ? preloadable_poll.options.join("\n\n") : nil, - ordered_media_attachments.map(&:description).join("\n\n"), - ].compact.join("\n\n") - end - def to_log_human_identifier account.acct end @@ -295,6 +264,10 @@ class Status < ApplicationRecord preview_cards.any? end + def with_poll? + preloadable_poll.present? + end + def non_sensitive_with_media? !sensitive? && with_media? end diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb index bc03d86a24..ed95a29bf7 100644 --- a/app/serializers/activitypub/actor_serializer.rb +++ b/app/serializers/activitypub/actor_serializer.rb @@ -8,13 +8,13 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer context_extensions :manually_approves_followers, :featured, :also_known_as, :moved_to, :property_value, :discoverable, :olm, :suspended, :searchable_by, :subscribable_by, - :other_setting, :memorial + :other_setting, :memorial, :indexable attributes :id, :type, :following, :followers, :inbox, :outbox, :featured, :featured_tags, :preferred_username, :name, :summary, :url, :manually_approves_followers, - :discoverable, :published, :searchable_by, :subscribable_by, :other_setting, :memorial + :discoverable, :indexable, :published, :memorial, :searchable_by, :subscribable_by, :other_setting has_one :public_key, serializer: ActivityPub::PublicKeySerializer @@ -107,6 +107,10 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer object.suspended? ? false : (object.discoverable || false) end + def indexable + object.suspended? ? false : (object.indexable || false) + end + def name object.suspended? ? object.username : (object.display_name.presence || object.username) end diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb index 191854e0fd..672d40f425 100644 --- a/app/services/batched_remove_status_service.rb +++ b/app/services/batched_remove_status_service.rb @@ -35,7 +35,10 @@ class BatchedRemoveStatusService < BaseService # Since we skipped all callbacks, we also need to manually # deindex the statuses - Chewy.strategy.current.update(StatusesIndex, statuses_and_reblogs) if Chewy.enabled? + if Chewy.enabled? + Chewy.strategy.current.update(StatusesIndex, statuses_and_reblogs) + Chewy.strategy.current.update(PublicStatusesIndex, statuses_and_reblogs) + end return if options[:skip_side_effects] diff --git a/app/services/search_service.rb b/app/services/search_service.rb index 0458ddbea6..a8f6d0dca4 100644 --- a/app/services/search_service.rb +++ b/app/services/search_service.rb @@ -41,41 +41,16 @@ class SearchService < BaseService end def perform_statuses_search! - privacy_definition = parsed_query.apply(StatusesIndex.filter(terms: { searchability: %w(public private direct) }).filter(term: { searchable_by: @account.id })) - - # 'direct' searchability posts are NOT in here because it's already added at previous line. - case @searchability - when 'public' - privacy_definition = privacy_definition.or(StatusesIndex.filter(term: { searchability: 'public' })) - privacy_definition = privacy_definition.or(StatusesIndex.filter(term: { searchability: 'private' }).filter(terms: { account_id: following_account_ids })) unless following_account_ids.empty? - privacy_definition = privacy_definition.or(StatusesIndex.filter(term: { searchability: 'limited' }).filter(term: { account_id: @account.id })) - when 'private', 'direct' - privacy_definition = privacy_definition.or(StatusesIndex.filter(terms: { searchability: %w(public private) }).filter(terms: { account_id: following_account_ids })) unless following_account_ids.empty? - privacy_definition = privacy_definition.or(StatusesIndex.filter(term: { searchability: 'limited' }).filter(term: { account_id: @account.id })) - when 'limited' - privacy_definition = privacy_definition.or(StatusesIndex.filter(term: { searchability: 'limited' }).filter(term: { account_id: @account.id })) - end - - definition = parsed_query.apply(StatusesIndex).order(id: :desc) - definition = definition.filter(term: { account_id: @options[:account_id] }) if @options[:account_id].present? - - definition = definition.and(privacy_definition) - - if @options[:min_id].present? || @options[:max_id].present? - range = {} - range[:gt] = @options[:min_id].to_i if @options[:min_id].present? - range[:lt] = @options[:max_id].to_i if @options[:max_id].present? - definition = definition.filter(range: { id: range }) - end - - results = definition.limit(@limit).offset(@offset).objects.compact - account_ids = results.map(&:account_id) - account_domains = results.map(&:account_domain) - account_relations = @account.relations_map(account_ids, account_domains) # variable old name: preloaded_relations - - results.reject { |status| StatusFilter.new(status, @account, account_relations).filtered? } - rescue Faraday::ConnectionFailed, Parslet::ParseFailed - [] + StatusesSearchService.new.call( + @query, + @account, + limit: @limit, + offset: @offset, + account_id: @options[:account_id], + min_id: @options[:min_id], + max_id: @options[:max_id], + searchability: @searchability + ) end def perform_hashtags_search! @@ -132,17 +107,4 @@ class SearchService < BaseService def statuses_search? @options[:type].blank? || @options[:type] == 'statuses' end - - def parsed_query - SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query)) - end - - def following_account_ids - return @following_account_ids if defined?(@following_account_ids) - - account_exists_sql = Account.where('accounts.id = follows.target_account_id').where(searchability: %w(public private)).reorder(nil).select(1).to_sql - status_exists_sql = Status.where('statuses.account_id = follows.target_account_id').where(reblog_of_id: nil).where(searchability: %w(public private)).reorder(nil).select(1).to_sql - following_accounts = Follow.where(account_id: @account.id).merge(Account.where("EXISTS (#{account_exists_sql})").or(Account.where("EXISTS (#{status_exists_sql})"))) - @following_account_ids = following_accounts.pluck(:target_account_id) - end end diff --git a/app/services/statuses_search_service.rb b/app/services/statuses_search_service.rb new file mode 100644 index 0000000000..97099ecaa5 --- /dev/null +++ b/app/services/statuses_search_service.rb @@ -0,0 +1,156 @@ +# frozen_string_literal: true + +class StatusesSearchService < BaseService + def call(query, account = nil, options = {}) + @query = query&.strip + @account = account + @options = options + @limit = options[:limit].to_i + @offset = options[:offset].to_i + @searchability = options[:searchability]&.to_sym + + status_search_results + end + + private + + def status_search_results + definition_should = [ + publicly_searchable, + non_publicly_searchable, + searchability_limited, + ] + definition_should << searchability_public if %i(public).include?(@searchability) + definition_should << searchability_private if %i(public private).include?(@searchability) + + definition = parsed_query.apply( + StatusesIndex.filter( + bool: { + should: definition_should, + minimum_should_match: 1, + } + ) + ) + + # This is the best way to submit identical queries to multi-indexes though chewy + definition.instance_variable_get(:@parameters)[:indices].value[:indices] << PublicStatusesIndex + + results = definition.collapse(field: :id).order(_id: { order: :desc }).limit(@limit).offset(@offset).objects.compact + account_ids = results.map(&:account_id) + account_domains = results.map(&:account_domain) + preloaded_relations = @account.relations_map(account_ids, account_domains) + + results.reject { |status| StatusFilter.new(status, @account, preloaded_relations).filtered? } + rescue Faraday::ConnectionFailed, Parslet::ParseFailed + [] + end + + def publicly_searchable + { + bool: { + must_not: { + exists: { + field: 'searchable_by', + }, + }, + }, + } + end + + def non_publicly_searchable + { + bool: { + must: [ + { + exists: { + field: 'searchable_by', + }, + }, + { + exists: { + field: 'searchability', + }, + }, + { + term: { searchable_by: @account.id }, + }, + ], + must_not: [ + { + term: { searchability: 'limited' }, + }, + ], + }, + } + end + + def searchability_public + { + bool: { + must: [ + { + exists: { + field: 'searchability', + }, + }, + { + term: { searchability: 'public' }, + }, + ], + }, + } + end + + def searchability_private + { + bool: { + must: [ + { + exists: { + field: 'searchability', + }, + }, + { + term: { searchability: 'private' }, + }, + { + terms: { account_id: following_account_ids }, + }, + ], + }, + } + end + + def searchability_limited + { + bool: { + must: [ + { + exists: { + field: 'searchability', + }, + }, + { + term: { searchability: 'limited' }, + }, + { + term: { account_id: @account.id }, + }, + ], + }, + } + end + + def following_account_ids + return @following_account_ids if defined?(@following_account_ids) + + account_exists_sql = Account.where('accounts.id = follows.target_account_id').where(searchability: %w(public private)).reorder(nil).select(1).to_sql + status_exists_sql = Status.where('statuses.account_id = follows.target_account_id').where(reblog_of_id: nil).where(searchability: %w(public private)).reorder(nil).select(1).to_sql + following_accounts = Follow.where(account_id: @account.id).merge(Account.where("EXISTS (#{account_exists_sql})").or(Account.where("EXISTS (#{status_exists_sql})"))) + @following_account_ids = following_accounts.pluck(:target_account_id) + end + + def parsed_query + SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query)) + end +end diff --git a/app/views/admin/accounts/index.html.haml b/app/views/admin/accounts/index.html.haml index 57d944b57a..da11e4361c 100644 --- a/app/views/admin/accounts/index.html.haml +++ b/app/views/admin/accounts/index.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('admin.accounts.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - = form_tag admin_accounts_url, method: 'GET', class: 'simple_form' do .filters .filter-subset.filter-subset--with-select diff --git a/app/views/admin/action_logs/index.html.haml b/app/views/admin/action_logs/index.html.haml index 4c78797c13..c4929cc422 100644 --- a/app/views/admin/action_logs/index.html.haml +++ b/app/views/admin/action_logs/index.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('admin.action_logs.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - = form_tag admin_action_logs_url, method: 'GET', class: 'simple_form' do = hidden_field_tag :target_account_id, params[:target_account_id] if params[:target_account_id].present? diff --git a/app/views/admin/announcements/edit.html.haml b/app/views/admin/announcements/edit.html.haml index c6c47586a0..df1ac455fb 100644 --- a/app/views/admin/announcements/edit.html.haml +++ b/app/views/admin/announcements/edit.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - = simple_form_for @announcement, url: admin_announcement_path(@announcement), html: { novalidate: false } do |f| = render 'shared/error_messages', object: @announcement diff --git a/app/views/admin/announcements/new.html.haml b/app/views/admin/announcements/new.html.haml index 57b7d5e0c6..cb39672e16 100644 --- a/app/views/admin/announcements/new.html.haml +++ b/app/views/admin/announcements/new.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - = simple_form_for @announcement, url: admin_announcements_path, html: { novalidate: false } do |f| = render 'shared/error_messages', object: @announcement diff --git a/app/views/admin/custom_emojis/index.html.haml b/app/views/admin/custom_emojis/index.html.haml index 3392224855..d103ab4445 100644 --- a/app/views/admin/custom_emojis/index.html.haml +++ b/app/views/admin/custom_emojis/index.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('admin.custom_emojis.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - if can?(:create, :custom_emoji) - content_for :heading_actions do = link_to t('admin.custom_emojis.upload'), new_admin_custom_emoji_path, class: 'button' diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml index ab7cb9de65..3597152e09 100644 --- a/app/views/admin/dashboard/index.html.haml +++ b/app/views/admin/dashboard/index.html.haml @@ -1,6 +1,3 @@ -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - content_for :page_title do = t('admin.dashboard.title') diff --git a/app/views/admin/disputes/appeals/index.html.haml b/app/views/admin/disputes/appeals/index.html.haml index 602414550e..7f04dd40fb 100644 --- a/app/views/admin/disputes/appeals/index.html.haml +++ b/app/views/admin/disputes/appeals/index.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('admin.disputes.appeals.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - .filters .filter-subset %strong= t('admin.tags.review') diff --git a/app/views/admin/domain_allows/new.html.haml b/app/views/admin/domain_allows/new.html.haml index 249a961cee..85ab7e4644 100644 --- a/app/views/admin/domain_allows/new.html.haml +++ b/app/views/admin/domain_allows/new.html.haml @@ -1,6 +1,3 @@ -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - content_for :page_title do = t('admin.domain_allows.add_new') diff --git a/app/views/admin/domain_blocks/confirm_suspension.html.haml b/app/views/admin/domain_blocks/confirm_suspension.html.haml index 1d28ba1579..e0e55e70f3 100644 --- a/app/views/admin/domain_blocks/confirm_suspension.html.haml +++ b/app/views/admin/domain_blocks/confirm_suspension.html.haml @@ -1,6 +1,3 @@ -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - content_for :page_title do = t('.title', domain: Addressable::IDNA.to_unicode(@domain_block.domain)) diff --git a/app/views/admin/domain_blocks/edit.html.haml b/app/views/admin/domain_blocks/edit.html.haml index 2561a6787d..8a06441508 100644 --- a/app/views/admin/domain_blocks/edit.html.haml +++ b/app/views/admin/domain_blocks/edit.html.haml @@ -1,6 +1,3 @@ -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - content_for :page_title do = t('admin.domain_blocks.edit') diff --git a/app/views/admin/domain_blocks/new.html.haml b/app/views/admin/domain_blocks/new.html.haml index 4f5fcf1c5f..606a784e12 100644 --- a/app/views/admin/domain_blocks/new.html.haml +++ b/app/views/admin/domain_blocks/new.html.haml @@ -1,6 +1,3 @@ -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - content_for :page_title do = t('.title') diff --git a/app/views/admin/email_domain_blocks/index.html.haml b/app/views/admin/email_domain_blocks/index.html.haml index b073e87162..9f16e0d5c3 100644 --- a/app/views/admin/email_domain_blocks/index.html.haml +++ b/app/views/admin/email_domain_blocks/index.html.haml @@ -4,9 +4,6 @@ - content_for :heading_actions do = link_to t('admin.email_domain_blocks.add_new'), new_admin_email_domain_block_path, class: 'button' -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - = form_for(@form, url: batch_admin_email_domain_blocks_path) do |f| = hidden_field_tag :page, params[:page] || 1 diff --git a/app/views/admin/email_domain_blocks/new.html.haml b/app/views/admin/email_domain_blocks/new.html.haml index 524b699688..fa1d950ad2 100644 --- a/app/views/admin/email_domain_blocks/new.html.haml +++ b/app/views/admin/email_domain_blocks/new.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - = simple_form_for @email_domain_block, url: admin_email_domain_blocks_path do |f| = render 'shared/error_messages', object: @email_domain_block diff --git a/app/views/admin/export_domain_blocks/import.html.haml b/app/views/admin/export_domain_blocks/import.html.haml index 804e61199e..01add232d1 100644 --- a/app/views/admin/export_domain_blocks/import.html.haml +++ b/app/views/admin/export_domain_blocks/import.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('admin.export_domain_blocks.import.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - %p= t('admin.export_domain_blocks.import.description_html') - if defined?(@global_private_comment) && @global_private_comment.present? diff --git a/app/views/admin/follow_recommendations/show.html.haml b/app/views/admin/follow_recommendations/show.html.haml index ebc4a2c6b8..dc65a72135 100644 --- a/app/views/admin/follow_recommendations/show.html.haml +++ b/app/views/admin/follow_recommendations/show.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('admin.follow_recommendations.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - %p= t('admin.follow_recommendations.description_html') %hr.spacer/ diff --git a/app/views/admin/instances/index.html.haml b/app/views/admin/instances/index.html.haml index 189dddcd29..7e43b4c538 100644 --- a/app/views/admin/instances/index.html.haml +++ b/app/views/admin/instances/index.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('admin.instances.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - content_for :heading_actions do - if limited_federation_mode? = link_to t('admin.domain_allows.add_new'), new_admin_domain_allow_path, class: 'button', id: 'add-instance-button' diff --git a/app/views/admin/instances/show.html.haml b/app/views/admin/instances/show.html.haml index 1ca9c7d72f..bdff098f5f 100644 --- a/app/views/admin/instances/show.html.haml +++ b/app/views/admin/instances/show.html.haml @@ -1,13 +1,10 @@ - content_for :page_title do = @instance.domain - + - if @instance.instance_info.present? %p = "#{@instance.instance_info.software} #{@instance.instance_info.version}" -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - if current_user.can?(:view_dashboard) - content_for :heading_actions do = l(@time_period.first) diff --git a/app/views/admin/ip_blocks/index.html.haml b/app/views/admin/ip_blocks/index.html.haml index 675c0aaad0..a48e4791a3 100644 --- a/app/views/admin/ip_blocks/index.html.haml +++ b/app/views/admin/ip_blocks/index.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('admin.ip_blocks.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - if can?(:create, :ip_block) - content_for :heading_actions do = link_to t('admin.ip_blocks.add_new'), new_admin_ip_block_path, class: 'button' diff --git a/app/views/admin/reports/show.html.haml b/app/views/admin/reports/show.html.haml index 2dd93d028c..2508bc2b5b 100644 --- a/app/views/admin/reports/show.html.haml +++ b/app/views/admin/reports/show.html.haml @@ -1,7 +1,3 @@ -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - = javascript_pack_tag 'public', async: true, crossorigin: 'anonymous' - - content_for :page_title do = t('admin.reports.report', id: @report.id) diff --git a/app/views/admin/settings/about/show.html.haml b/app/views/admin/settings/about/show.html.haml index 7b1b907ee2..1237c20fa8 100644 --- a/app/views/admin/settings/about/show.html.haml +++ b/app/views/admin/settings/about/show.html.haml @@ -1,6 +1,3 @@ -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - content_for :page_title do = t('admin.settings.about.title') diff --git a/app/views/admin/settings/appearance/show.html.haml b/app/views/admin/settings/appearance/show.html.haml index 1e73ab0a24..ed61774c9b 100644 --- a/app/views/admin/settings/appearance/show.html.haml +++ b/app/views/admin/settings/appearance/show.html.haml @@ -1,6 +1,3 @@ -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - content_for :page_title do = t('admin.settings.appearance.title') diff --git a/app/views/admin/settings/branding/show.html.haml b/app/views/admin/settings/branding/show.html.haml index 74a6fadf98..aee7306892 100644 --- a/app/views/admin/settings/branding/show.html.haml +++ b/app/views/admin/settings/branding/show.html.haml @@ -1,6 +1,3 @@ -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - content_for :page_title do = t('admin.settings.branding.title') diff --git a/app/views/admin/settings/content_retention/show.html.haml b/app/views/admin/settings/content_retention/show.html.haml index de34b5ee3c..5a67016148 100644 --- a/app/views/admin/settings/content_retention/show.html.haml +++ b/app/views/admin/settings/content_retention/show.html.haml @@ -1,6 +1,3 @@ -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - content_for :page_title do = t('admin.settings.content_retention.title') diff --git a/app/views/admin/settings/discovery/show.html.haml b/app/views/admin/settings/discovery/show.html.haml index 759bbdcebe..c48e0fdc62 100644 --- a/app/views/admin/settings/discovery/show.html.haml +++ b/app/views/admin/settings/discovery/show.html.haml @@ -1,6 +1,3 @@ -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - content_for :page_title do = t('admin.settings.discovery.title') diff --git a/app/views/admin/settings/registrations/show.html.haml b/app/views/admin/settings/registrations/show.html.haml index e06385bc81..168f109757 100644 --- a/app/views/admin/settings/registrations/show.html.haml +++ b/app/views/admin/settings/registrations/show.html.haml @@ -1,6 +1,3 @@ -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - content_for :page_title do = t('admin.settings.registrations.title') diff --git a/app/views/admin/statuses/index.html.haml b/app/views/admin/statuses/index.html.haml index d3d7cc160b..9163dee795 100644 --- a/app/views/admin/statuses/index.html.haml +++ b/app/views/admin/statuses/index.html.haml @@ -1,6 +1,3 @@ -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - content_for :page_title do = t('admin.statuses.title') \- diff --git a/app/views/admin/statuses/show.html.haml b/app/views/admin/statuses/show.html.haml index 8992cc6d5a..ceab7dee3e 100644 --- a/app/views/admin/statuses/show.html.haml +++ b/app/views/admin/statuses/show.html.haml @@ -1,6 +1,3 @@ -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - content_for :page_title do = t('statuses.title', name: display_name(@account), quote: truncate(@status.spoiler_text.presence || @status.text, length: 50, omission: '…', escape: false)) diff --git a/app/views/admin/tags/show.html.haml b/app/views/admin/tags/show.html.haml index 104190b588..71bce0c0cb 100644 --- a/app/views/admin/tags/show.html.haml +++ b/app/views/admin/tags/show.html.haml @@ -1,6 +1,3 @@ -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - content_for :page_title do = "##{@tag.display_name}" diff --git a/app/views/admin/trends/links/index.html.haml b/app/views/admin/trends/links/index.html.haml index 7b5122cf1c..e6ed9d95f6 100644 --- a/app/views/admin/trends/links/index.html.haml +++ b/app/views/admin/trends/links/index.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('admin.trends.links.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - %p= t('admin.trends.links.description_html') %hr.spacer/ diff --git a/app/views/admin/trends/links/preview_card_providers/index.html.haml b/app/views/admin/trends/links/preview_card_providers/index.html.haml index 025270c128..d9ad12fc96 100644 --- a/app/views/admin/trends/links/preview_card_providers/index.html.haml +++ b/app/views/admin/trends/links/preview_card_providers/index.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('admin.trends.preview_card_providers.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - %p= t('admin.trends.preview_card_providers.description_html') %hr.spacer/ diff --git a/app/views/admin/trends/statuses/index.html.haml b/app/views/admin/trends/statuses/index.html.haml index a42d60b009..bf04772f22 100644 --- a/app/views/admin/trends/statuses/index.html.haml +++ b/app/views/admin/trends/statuses/index.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('admin.trends.statuses.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - %p= t('admin.trends.statuses.description_html') %hr.spacer/ diff --git a/app/views/admin/trends/tags/index.html.haml b/app/views/admin/trends/tags/index.html.haml index 1ea34c9e3f..4730d20c18 100644 --- a/app/views/admin/trends/tags/index.html.haml +++ b/app/views/admin/trends/tags/index.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('admin.trends.tags.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - %p= t('admin.trends.tags.description_html') %hr.spacer/ diff --git a/app/views/filters/statuses/index.html.haml b/app/views/filters/statuses/index.html.haml index 886de58fa0..eaa39e170f 100644 --- a/app/views/filters/statuses/index.html.haml +++ b/app/views/filters/statuses/index.html.haml @@ -1,6 +1,3 @@ -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - - content_for :page_title do = t('filters.statuses.index.title') \- diff --git a/app/views/layouts/admin.html.haml b/app/views/layouts/admin.html.haml index e7a163c92a..d725e65aa9 100644 --- a/app/views/layouts/admin.html.haml +++ b/app/views/layouts/admin.html.haml @@ -1,6 +1,7 @@ - content_for :header_tags do = render_initial_state = javascript_pack_tag 'public', crossorigin: 'anonymous' + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - content_for :content do .admin-wrapper diff --git a/app/views/settings/preferences/notifications/show.html.haml b/app/views/settings/preferences/notifications/show.html.haml index 2a7abb9488..71f6415427 100644 --- a/app/views/settings/preferences/notifications/show.html.haml +++ b/app/views/settings/preferences/notifications/show.html.haml @@ -18,14 +18,19 @@ = ff.input :'notification_emails.reblog', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.reblog') = ff.input :'notification_emails.favourite', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.favourite') = ff.input :'notification_emails.mention', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.mention') - = ff.input :'notification_emails.report', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.report') if current_user.can?(:manage_reports) - = ff.input :'notification_emails.appeal', as: :boolean, wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.appeal') if current_user.can?(:manage_appeals) - = ff.input :'notification_emails.pending_account', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.pending_account') if current_user.can?(:manage_users) - = ff.input :'notification_emails.trends', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.trending_tag') if current_user.can?(:manage_taxonomies) .fields-group = ff.input :always_send_emails, wrapper: :with_label, label: I18n.t('simple_form.labels.defaults.setting_always_send_emails'), hint: I18n.t('simple_form.hints.defaults.setting_always_send_emails') + - if current_user.can?(:manage_reports, :manage_appeals, :manage_users, :manage_taxonomies) + %h4= t 'notifications.administration_emails' + + .fields-group + = ff.input :'notification_emails.report', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.report') if current_user.can?(:manage_reports) + = ff.input :'notification_emails.appeal', as: :boolean, wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.appeal') if current_user.can?(:manage_appeals) + = ff.input :'notification_emails.pending_account', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.pending_account') if current_user.can?(:manage_users) + = ff.input :'notification_emails.trends', wrapper: :with_label, label: I18n.t('simple_form.labels.notification_emails.trending_tag') if current_user.can?(:manage_taxonomies) + %h4= t 'notifications.other_settings' .fields-group diff --git a/app/views/settings/privacy/show.html.haml b/app/views/settings/privacy/show.html.haml index 96f9be6452..0bed511a8c 100644 --- a/app/views/settings/privacy/show.html.haml +++ b/app/views/settings/privacy/show.html.haml @@ -31,6 +31,9 @@ %p.lead= t('privacy.search_hint_html') + .fields-group + = f.input :indexable, as: :boolean, wrapper: :with_label + = f.simple_fields_for :settings, current_user.settings do |ff| .fields-group = ff.input :indexable, wrapper: :with_label diff --git a/app/workers/add_to_public_statuses_index_worker.rb b/app/workers/add_to_public_statuses_index_worker.rb new file mode 100644 index 0000000000..409e5e7086 --- /dev/null +++ b/app/workers/add_to_public_statuses_index_worker.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddToPublicStatusesIndexWorker + include Sidekiq::Worker + + def perform(account_id) + account = Account.find(account_id) + + return unless account.indexable? + + account.add_to_public_statuses_index! + rescue ActiveRecord::RecordNotFound + true + end +end diff --git a/app/workers/remove_from_public_statuses_index_worker.rb b/app/workers/remove_from_public_statuses_index_worker.rb new file mode 100644 index 0000000000..e108a5c209 --- /dev/null +++ b/app/workers/remove_from_public_statuses_index_worker.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class RemoveFromPublicStatusesIndexWorker + include Sidekiq::Worker + + def perform(account_id) + account = Account.find(account_id) + + return if account.indexable? + + account.remove_from_public_statuses_index! + rescue ActiveRecord::RecordNotFound + true + end +end diff --git a/app/workers/scheduler/indexing_scheduler.rb b/app/workers/scheduler/indexing_scheduler.rb index 2868a3b715..6c770d5a8f 100644 --- a/app/workers/scheduler/indexing_scheduler.rb +++ b/app/workers/scheduler/indexing_scheduler.rb @@ -23,6 +23,6 @@ class Scheduler::IndexingScheduler end def indexes - [AccountsIndex, TagsIndex, StatusesIndex] + [AccountsIndex, TagsIndex, PublicStatusesIndex, StatusesIndex] end end diff --git a/config/locales/en.yml b/config/locales/en.yml index a1cee1416a..90b47fe818 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1564,6 +1564,7 @@ en: update: subject: "%{name} edited a post" notifications: + administration_emails: Admin e-mail notifications email_events: Events for e-mail notifications email_events_hint: 'Select events that you want to receive notifications for:' other_settings: Other notifications settings diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml index adbd99c35c..148e8a109f 100644 --- a/config/locales/simple_form.en.yml +++ b/config/locales/simple_form.en.yml @@ -6,6 +6,7 @@ en: discoverable: Your public posts and profile may be featured or recommended in various areas of Mastodon and your profile may be suggested to other users. display_name: Your full name or your fun name. fields: Your homepage, pronouns, age, anything you want. + indexable: Your public posts may appear in search results on Mastodon. People who have interacted with your posts may be able to search them regardless. note: 'You can @mention other people or #hashtags.' show_collections: People will be able to browse through your follows and followers. People that you follow will see that you follow them regardless. unlocked: People will be able to follow you without requesting approval. Uncheck if you want to review follow requests and chose whether to accept or reject new followers. @@ -150,6 +151,7 @@ en: fields: name: Label value: Content + indexable: Include public posts in search results show_collections: Show follows and followers on profile unlocked: Automatically accept new followers account_alias: diff --git a/lib/mastodon/cli/search.rb b/lib/mastodon/cli/search.rb index 41862b5b6b..481e01d8e7 100644 --- a/lib/mastodon/cli/search.rb +++ b/lib/mastodon/cli/search.rb @@ -10,6 +10,7 @@ module Mastodon::CLI InstancesIndex, AccountsIndex, TagsIndex, + PublicStatusesIndex, StatusesIndex, ].freeze diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index ffcbc35f22..cb8ba67292 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -17,7 +17,7 @@ module Mastodon end def flags - ENV.fetch('MASTODON_VERSION_FLAGS', '-beta2') + ENV['MASTODON_VERSION_FLAGS'].presence || '-beta2' end def suffix diff --git a/spec/chewy/public_statuses_index_spec.rb b/spec/chewy/public_statuses_index_spec.rb new file mode 100644 index 0000000000..2f93d0ff02 --- /dev/null +++ b/spec/chewy/public_statuses_index_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe PublicStatusesIndex do + describe 'Searching the index' do + before do + mock_elasticsearch_response(described_class, raw_response) + end + + it 'returns results from a query' do + results = described_class.query(match: { name: 'status' }) + + expect(results).to eq [] + end + end + + def raw_response + { + took: 3, + hits: { + hits: [ + { + _id: '0', + _score: 1.6375021, + }, + ], + }, + } + end +end diff --git a/spec/lib/importer/public_statuses_index_importer_spec.rb b/spec/lib/importer/public_statuses_index_importer_spec.rb new file mode 100644 index 0000000000..bc7c038a97 --- /dev/null +++ b/spec/lib/importer/public_statuses_index_importer_spec.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Importer::PublicStatusesIndexImporter do + describe 'import!' do + let(:pool) { Concurrent::FixedThreadPool.new(5) } + let(:importer) { described_class.new(batch_size: 123, executor: pool) } + + before { Fabricate(:status, account: Fabricate(:account, indexable: true)) } + + it 'indexes relevant statuses' do + expect { importer.import! }.to update_index(PublicStatusesIndex) + end + end +end diff --git a/spec/lib/search_query_transformer_spec.rb b/spec/lib/search_query_transformer_spec.rb index b5290677cc..bd7c64e55c 100644 --- a/spec/lib/search_query_transformer_spec.rb +++ b/spec/lib/search_query_transformer_spec.rb @@ -9,8 +9,8 @@ describe SearchQueryTransformer do it 'sets attributes' do transformer = described_class.new.apply(parser) - expect(transformer.should_clauses.first).to be_a(SearchQueryTransformer::PhraseClause) - expect(transformer.must_clauses.first).to be_nil + expect(transformer.should_clauses.first).to be_nil + expect(transformer.must_clauses.first).to be_a(SearchQueryTransformer::PhraseClause) expect(transformer.must_not_clauses.first).to be_nil expect(transformer.filter_clauses.first).to be_nil end diff --git a/spec/models/concerns/account_statuses_search_spec.rb b/spec/models/concerns/account_statuses_search_spec.rb new file mode 100644 index 0000000000..46362936f4 --- /dev/null +++ b/spec/models/concerns/account_statuses_search_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe AccountStatusesSearch do + let(:account) { Fabricate(:account, indexable: indexable) } + + before do + allow(Chewy).to receive(:enabled?).and_return(true) + end + + describe '#enqueue_update_public_statuses_index' do + before do + allow(account).to receive(:enqueue_add_to_public_statuses_index) + allow(account).to receive(:enqueue_remove_from_public_statuses_index) + end + + context 'when account is indexable' do + let(:indexable) { true } + + it 'enqueues add_to_public_statuses_index and not to remove_from_public_statuses_index' do + account.enqueue_update_public_statuses_index + expect(account).to have_received(:enqueue_add_to_public_statuses_index) + expect(account).to_not have_received(:enqueue_remove_from_public_statuses_index) + end + end + + context 'when account is not indexable' do + let(:indexable) { false } + + it 'enqueues remove_from_public_statuses_index and not to add_to_public_statuses_index' do + account.enqueue_update_public_statuses_index + expect(account).to have_received(:enqueue_remove_from_public_statuses_index) + expect(account).to_not have_received(:enqueue_add_to_public_statuses_index) + end + end + end + + describe '#enqueue_add_to_public_statuses_index' do + let(:indexable) { true } + let(:worker) { AddToPublicStatusesIndexWorker } + + before do + allow(worker).to receive(:perform_async) + end + + it 'enqueues AddToPublicStatusesIndexWorker' do + account.enqueue_add_to_public_statuses_index + expect(worker).to have_received(:perform_async).with(account.id) + end + end + + describe '#enqueue_remove_from_public_statuses_index' do + let(:indexable) { false } + let(:worker) { RemoveFromPublicStatusesIndexWorker } + + before do + allow(worker).to receive(:perform_async) + end + + it 'enqueues RemoveFromPublicStatusesIndexWorker' do + account.enqueue_remove_from_public_statuses_index + expect(worker).to have_received(:perform_async).with(account.id) + end + end +end diff --git a/spec/workers/add_to_public_statuses_index_worker_spec.rb b/spec/workers/add_to_public_statuses_index_worker_spec.rb new file mode 100644 index 0000000000..fa15072241 --- /dev/null +++ b/spec/workers/add_to_public_statuses_index_worker_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe AddToPublicStatusesIndexWorker do + describe '#perform' do + let(:account) { Fabricate(:account, indexable: indexable) } + let(:account_id) { account.id } + + before do + allow(Account).to receive(:find).with(account_id).and_return(account) unless account.nil? + allow(account).to receive(:add_to_public_statuses_index!) unless account.nil? + end + + context 'when account is indexable' do + let(:indexable) { true } + + it 'adds the account to the public statuses index' do + subject.perform(account_id) + expect(account).to have_received(:add_to_public_statuses_index!) + end + end + + context 'when account is not indexable' do + let(:indexable) { false } + + it 'does not add the account to public statuses index' do + subject.perform(account_id) + expect(account).to_not have_received(:add_to_public_statuses_index!) + end + end + + context 'when account does not exist' do + let(:account) { nil } + let(:account_id) { 999 } + + it 'does not raise an error' do + expect { subject.perform(account_id) }.to_not raise_error + end + end + end +end diff --git a/spec/workers/remove_from_public_statuses_index_worker_spec.rb b/spec/workers/remove_from_public_statuses_index_worker_spec.rb new file mode 100644 index 0000000000..43ff211eaa --- /dev/null +++ b/spec/workers/remove_from_public_statuses_index_worker_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe RemoveFromPublicStatusesIndexWorker do + describe '#perform' do + let(:account) { Fabricate(:account, indexable: indexable) } + let(:account_id) { account.id } + + before do + allow(Account).to receive(:find).with(account_id).and_return(account) unless account.nil? + allow(account).to receive(:remove_from_public_statuses_index!) unless account.nil? + end + + context 'when account is not indexable' do + let(:indexable) { false } + + it 'removes the account from public statuses index' do + subject.perform(account_id) + expect(account).to have_received(:remove_from_public_statuses_index!) + end + end + + context 'when account is indexable' do + let(:indexable) { true } + + it 'does not remove the account from public statuses index' do + subject.perform(account_id) + expect(account).to_not have_received(:remove_from_public_statuses_index!) + end + end + + context 'when account does not exist' do + let(:account) { nil } + let(:account_id) { 999 } + + it 'does not raise an error' do + expect { subject.perform(account_id) }.to_not raise_error + end + end + end +end diff --git a/yarn.lock b/yarn.lock index 622647dbec..ca65df1383 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1755,10 +1755,10 @@ resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71" integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg== -"@redis/client@1.5.8": - version "1.5.8" - resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.8.tgz#a375ba7861825bd0d2dc512282b8bff7b98dbcb1" - integrity sha512-xzElwHIO6rBAqzPeVnCzgvrnBEcFL1P0w8P65VNLRkdVW8rOE58f52hdj0BDgmsdOm4f1EoXPZtH4Fh7M/qUpw== +"@redis/client@1.5.9": + version "1.5.9" + resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.9.tgz#c4ee81bbfedb4f1d9c7c5e9859661b9388fb4021" + integrity sha512-SffgN+P1zdWJWSXBvJeynvEnmnZrYmtKSRW00xl8pOPFOMJjxRR9u0frSxJpPR6Y4V+k54blJjGW7FgxbTI7bQ== dependencies: cluster-key-slot "1.1.2" generic-pool "3.9.0" @@ -1779,10 +1779,10 @@ resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.3.tgz#b5a6837522ce9028267fe6f50762a8bcfd2e998b" integrity sha512-4Dg1JjvCevdiCBTZqjhKkGoC5/BcB7k9j99kdMnaXFXg8x4eyOIVg9487CMv7/BUVkFLZCaIh8ead9mU15DNng== -"@redis/time-series@1.0.4": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.4.tgz#af85eb080f6934580e4d3b58046026b6c2b18717" - integrity sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng== +"@redis/time-series@1.0.5": + version "1.0.5" + resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.5.tgz#a6d70ef7a0e71e083ea09b967df0a0ed742bc6ad" + integrity sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg== "@reduxjs/toolkit@^1.9.5": version "1.9.5" @@ -10254,16 +10254,16 @@ redent@^4.0.0: strip-indent "^4.0.0" redis@^4.6.5: - version "4.6.7" - resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.7.tgz#c73123ad0b572776223f172ec78185adb72a6b57" - integrity sha512-KrkuNJNpCwRm5vFJh0tteMxW8SaUzkm5fBH7eL5hd/D0fAkzvapxbfGPP/r+4JAXdQuX7nebsBkBqA2RHB7Usw== + version "4.6.8" + resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.8.tgz#54c5992e8a5ba512506fe9f53142cadc405547e7" + integrity sha512-S7qNkPUYrsofQ0ztWlTHSaK0Qqfl1y+WMIxrzeAGNG+9iUZB4HGeBgkHxE6uJJ6iXrkvLd1RVJ2nvu6H1sAzfQ== dependencies: "@redis/bloom" "1.2.0" - "@redis/client" "1.5.8" + "@redis/client" "1.5.9" "@redis/graph" "1.1.0" "@redis/json" "1.0.4" "@redis/search" "1.1.3" - "@redis/time-series" "1.0.4" + "@redis/time-series" "1.0.5" redux-immutable@^4.0.0: version "4.0.0"