From 71c451a569d3847af2cd1a6643ca4cefcca92805 Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 15 Sep 2023 12:21:41 +0900 Subject: [PATCH 01/21] Cache status reference mark when posting --- app/lib/activitypub/activity/create.rb | 4 ++++ app/serializers/rest/status_serializer.rb | 4 +++- app/services/activitypub/process_status_update_service.rb | 4 ++++ app/services/post_status_service.rb | 6 +++++- app/services/process_references_service.rb | 7 +++++++ app/services/update_status_service.rb | 4 ++++ 6 files changed, 27 insertions(+), 2 deletions(-) diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index a884e14154..1739a5ff26 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -493,6 +493,10 @@ class ActivityPub::Activity::Create < ActivityPub::Activity references = @object['references'].nil? ? [] : ActivityPub::FetchReferencesService.new.call(@status, @object['references']) quote = @object['quote'] || @object['quoteUrl'] || @object['quoteURL'] || @object['_misskey_quote'] references << quote if quote + + return unless ProcessReferencesService.need_process?(@status, [], references) + + Rails.cache.write("status_reference:#{@status.id}", true, expires_in: 10.minutes) ProcessReferencesWorker.perform_async(@status.id, [], references) end diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb index d53dd71f4a..3603026f39 100644 --- a/app/serializers/rest/status_serializer.rb +++ b/app/serializers/rest/status_serializer.rb @@ -103,7 +103,9 @@ class REST::StatusSerializer < ActiveModel::Serializer end def status_references_count - status_reference_ids.size + return status_reference_ids.size if status_reference_ids.any? + + Rails.cache.exist?("status_reference:#{object.id}") ? 1 : 0 end def reblogs_count diff --git a/app/services/activitypub/process_status_update_service.rb b/app/services/activitypub/process_status_update_service.rb index 418b757248..a043d9e00c 100644 --- a/app/services/activitypub/process_status_update_service.rb +++ b/app/services/activitypub/process_status_update_service.rb @@ -254,6 +254,10 @@ class ActivityPub::ProcessStatusUpdateService < BaseService references = @json['references'].nil? ? [] : ActivityPub::FetchReferencesService.new.call(@status, @json['references']) quote = @json['quote'] || @json['quoteUrl'] || @json['quoteURL'] || @json['_misskey_quote'] references << quote if quote + + return unless ProcessReferencesService.need_process?(@status, [], references) + + Rails.cache.write("status_reference:#{@status.id}", true, expires_in: 10.minutes) ProcessReferencesWorker.perform_async(@status.id, [], references) end diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 2e5b765d86..291138ee36 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -186,8 +186,12 @@ class PostStatusService < BaseService def postprocess_status! @account.user.update!(settings_attributes: { default_privacy: @options[:visibility] }) if @account.user&.setting_stay_privacy && !@status.reply? && %i(public public_unlisted login unlisted private).include?(@status.visibility.to_sym) && @status.visibility.to_s != @account.user&.setting_default_privacy && !@dtl + if ProcessReferencesService.need_process?(@status, @reference_ids, []) + Rails.cache.write("status_reference:#{@status.id}", true, expires_in: 10.minutes) + ProcessReferencesWorker.perform_async(@status.id, @reference_ids, []) + end + process_hashtags_service.call(@status) - ProcessReferencesWorker.perform_async(@status.id, @reference_ids, []) Trends.tags.register(@status) LinkCrawlWorker.perform_async(@status.id) DistributionWorker.perform_async(@status.id) diff --git a/app/services/process_references_service.rb b/app/services/process_references_service.rb index 9c8b3813f5..54edd4ff6b 100644 --- a/app/services/process_references_service.rb +++ b/app/services/process_references_service.rb @@ -2,6 +2,7 @@ class ProcessReferencesService < BaseService include Payloadable + include FormattingHelper DOMAIN = ENV['WEB_DOMAIN'] || ENV.fetch('LOCAL_DOMAIN', nil) REFURL_EXP = /(RT|QT|BT|RN|RE)((:|;)?\s+|:|;)(#{URI::DEFAULT_PARSER.make_regexp(%w(http https))})/ @@ -23,9 +24,15 @@ class ProcessReferencesService < BaseService @status.save! end + Rails.cache.delete("status_reference:#{@status.id}") + create_notifications! end + def self.need_process?(status, reference_parameters, urls) + reference_parameters.any? || (urls || []).any? || FormattingHelper.extract_status_plain_text(status).scan(REFURL_EXP).pluck(3).uniq.any? + end + private def references diff --git a/app/services/update_status_service.rb b/app/services/update_status_service.rb index ccf7b4827f..e6d4d4e1f4 100644 --- a/app/services/update_status_service.rb +++ b/app/services/update_status_service.rb @@ -161,6 +161,10 @@ class UpdateStatusService < BaseService def update_references! reference_ids = (@options[:status_reference_ids] || []).map(&:to_i).filter(&:positive?) + + return unless ProcessReferencesService.need_process?(@status, reference_ids, []) + + Rails.cache.write("status_reference:#{@status.id}", true, expires_in: 10.minutes) ProcessReferencesWorker.perform_async(@status.id, reference_ids, []) end From a39b8cb90e444ac88f7ab6e65c2df04a177207a3 Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 15 Sep 2023 12:27:06 +0900 Subject: [PATCH 02/21] Refactor status reference flag --- app/lib/activitypub/activity/create.rb | 5 +---- app/services/activitypub/process_status_update_service.rb | 5 +---- app/services/post_status_service.rb | 6 +----- app/services/process_references_service.rb | 7 +++++++ app/services/update_status_service.rb | 5 +---- 5 files changed, 11 insertions(+), 17 deletions(-) diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 1739a5ff26..f991faa6a6 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -494,10 +494,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity quote = @object['quote'] || @object['quoteUrl'] || @object['quoteURL'] || @object['_misskey_quote'] references << quote if quote - return unless ProcessReferencesService.need_process?(@status, [], references) - - Rails.cache.write("status_reference:#{@status.id}", true, expires_in: 10.minutes) - ProcessReferencesWorker.perform_async(@status.id, [], references) + ProcessReferencesService.perform_worker_async(@status, [], references) end def join_group! diff --git a/app/services/activitypub/process_status_update_service.rb b/app/services/activitypub/process_status_update_service.rb index a043d9e00c..7ee444dea4 100644 --- a/app/services/activitypub/process_status_update_service.rb +++ b/app/services/activitypub/process_status_update_service.rb @@ -255,10 +255,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService quote = @json['quote'] || @json['quoteUrl'] || @json['quoteURL'] || @json['_misskey_quote'] references << quote if quote - return unless ProcessReferencesService.need_process?(@status, [], references) - - Rails.cache.write("status_reference:#{@status.id}", true, expires_in: 10.minutes) - ProcessReferencesWorker.perform_async(@status.id, [], references) + ProcessReferencesService.perform_worker_async(@status, [], references) end def expected_type? diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 291138ee36..e8725528d9 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -186,13 +186,9 @@ class PostStatusService < BaseService def postprocess_status! @account.user.update!(settings_attributes: { default_privacy: @options[:visibility] }) if @account.user&.setting_stay_privacy && !@status.reply? && %i(public public_unlisted login unlisted private).include?(@status.visibility.to_sym) && @status.visibility.to_s != @account.user&.setting_default_privacy && !@dtl - if ProcessReferencesService.need_process?(@status, @reference_ids, []) - Rails.cache.write("status_reference:#{@status.id}", true, expires_in: 10.minutes) - ProcessReferencesWorker.perform_async(@status.id, @reference_ids, []) - end - process_hashtags_service.call(@status) Trends.tags.register(@status) + ProcessReferencesService.perform_worker_async(@status, @reference_ids, []) LinkCrawlWorker.perform_async(@status.id) DistributionWorker.perform_async(@status.id) ActivityPub::DistributionWorker.perform_async(@status.id) diff --git a/app/services/process_references_service.rb b/app/services/process_references_service.rb index 54edd4ff6b..bd2b3dd42c 100644 --- a/app/services/process_references_service.rb +++ b/app/services/process_references_service.rb @@ -33,6 +33,13 @@ class ProcessReferencesService < BaseService reference_parameters.any? || (urls || []).any? || FormattingHelper.extract_status_plain_text(status).scan(REFURL_EXP).pluck(3).uniq.any? end + def self.perform_worker_async(status, reference_parameters, urls) + return unless need_process?(status, reference_parameters, urls) + + Rails.cache.write("status_reference:#{status.id}", true, expires_in: 10.minutes) + ProcessReferencesWorker.perform_async(status.id, reference_parameters, urls) + end + private def references diff --git a/app/services/update_status_service.rb b/app/services/update_status_service.rb index e6d4d4e1f4..8d99ebdd3d 100644 --- a/app/services/update_status_service.rb +++ b/app/services/update_status_service.rb @@ -162,10 +162,7 @@ class UpdateStatusService < BaseService def update_references! reference_ids = (@options[:status_reference_ids] || []).map(&:to_i).filter(&:positive?) - return unless ProcessReferencesService.need_process?(@status, reference_ids, []) - - Rails.cache.write("status_reference:#{@status.id}", true, expires_in: 10.minutes) - ProcessReferencesWorker.perform_async(@status.id, reference_ids, []) + ProcessReferencesService.perform_worker_async(@status, reference_ids, []) end def update_metadata! From 59a433a291b78c8ec5ffd6b4d28d4b08a870ee6b Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 15 Sep 2023 12:31:16 +0900 Subject: [PATCH 03/21] Add autientication in status_reference --- app/services/process_references_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/process_references_service.rb b/app/services/process_references_service.rb index bd2b3dd42c..c98a0434c7 100644 --- a/app/services/process_references_service.rb +++ b/app/services/process_references_service.rb @@ -65,7 +65,7 @@ class ProcessReferencesService < BaseService def fetch_statuses!(urls) (urls + @urls) - .map { |url| ResolveURLService.new.call(url) } + .map { |url| ResolveURLService.new.call(url, on_behalf_of: @status.account) } .filter { |status| status } end From 7b88422644f0d6c88d6b20f124b0ef8bbc6a6108 Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 15 Sep 2023 13:18:28 +0900 Subject: [PATCH 04/21] Add lists/circles index test --- spec/controllers/api/v1/circles_controller_spec.rb | 13 +++++++++++++ spec/controllers/api/v1/lists_controller_spec.rb | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/spec/controllers/api/v1/circles_controller_spec.rb b/spec/controllers/api/v1/circles_controller_spec.rb index 1aff55e65b..1e54a8af96 100644 --- a/spec/controllers/api/v1/circles_controller_spec.rb +++ b/spec/controllers/api/v1/circles_controller_spec.rb @@ -21,6 +21,19 @@ describe Api::V1::CirclesController do expect(response).to have_http_status(200) end end + + describe 'GET #index' do + it 'returns http success' do + circle_id = circle.id.to_s + Fabricate(:circle) + get :index + expect(response).to have_http_status(200) + + circle_ids = body_as_json.pluck(:id) + expect(circle_ids.size).to eq 1 + expect(circle_ids).to include circle_id + end + end end context 'with the wrong user context' do diff --git a/spec/controllers/api/v1/lists_controller_spec.rb b/spec/controllers/api/v1/lists_controller_spec.rb index b54f3f70fa..ed4edbf760 100644 --- a/spec/controllers/api/v1/lists_controller_spec.rb +++ b/spec/controllers/api/v1/lists_controller_spec.rb @@ -21,6 +21,19 @@ describe Api::V1::ListsController do expect(response).to have_http_status(200) end end + + describe 'GET #index' do + it 'returns http success' do + list_id = list.id.to_s + Fabricate(:list) + get :index + expect(response).to have_http_status(200) + + list_ids = body_as_json.pluck(:id) + expect(list_ids.size).to eq 1 + expect(list_ids).to include list_id + end + end end context 'with the wrong user context' do From 5fb6bce744c9e14b948d075d35dcb8ffaf665a47 Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 15 Sep 2023 18:50:45 +0900 Subject: [PATCH 05/21] Add my reacted statuses search --- app/chewy/accounts_index.rb | 2 +- app/chewy/public_statuses_index.rb | 2 +- app/chewy/statuses_index.rb | 13 ++++- .../features/compose/components/search.jsx | 4 +- app/lib/search_query_transformer.rb | 56 ++++++++++++++++++- app/models/bookmark_category_status.rb | 4 ++ app/models/concerns/status_search_concern.rb | 44 ++++++++++++--- app/models/status.rb | 8 +++ app/services/statuses_search_service.rb | 7 ++- 9 files changed, 124 insertions(+), 16 deletions(-) diff --git a/app/chewy/accounts_index.rb b/app/chewy/accounts_index.rb index ad3816e4d6..9503fb85a1 100644 --- a/app/chewy/accounts_index.rb +++ b/app/chewy/accounts_index.rb @@ -157,6 +157,6 @@ class AccountsIndex < Chewy::Index field(:domain, type: 'keyword', value: ->(account) { account.domain || '' }) field(:display_name, type: 'text', analyzer: 'verbatim') { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' } field(:username, type: 'text', analyzer: 'verbatim', value: ->(account) { [account.username, account.domain].compact.join('@') }) { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' } - field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' } + field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(account) { account.searchable_text }) end end diff --git a/app/chewy/public_statuses_index.rb b/app/chewy/public_statuses_index.rb index d152b52039..4bfe3f08ad 100644 --- a/app/chewy/public_statuses_index.rb +++ b/app/chewy/public_statuses_index.rb @@ -149,7 +149,7 @@ class PublicStatusesIndex < Chewy::Index root date_detection: false do field(:id, type: 'long') field(:account_id, type: 'long') - field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'sudachi_analyzer') } + field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(status) { status.searchable_text }) field(:tags, type: 'text', analyzer: 'hashtag', value: ->(status) { status.tags.map(&:display_name) }) field(:language, type: 'keyword') field(:domain, type: 'keyword', value: ->(status) { status.account.domain || '' }) diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb index 3b83e2f31c..3b0a38a466 100644 --- a/app/chewy/statuses_index.rb +++ b/app/chewy/statuses_index.rb @@ -160,17 +160,26 @@ class StatusesIndex < Chewy::Index if status.searchability == 'direct' status.searchable_by.empty? else - status.searchability == 'limited' ? status.account.domain.present? : false + status.searchability == 'limited' ? !status.local? : false end } root date_detection: false do field(:id, type: 'long') field(:account_id, type: 'long') - field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'sudachi_analyzer') } + field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(status) { status.searchable_text }) field(:tags, type: 'text', analyzer: 'hashtag', value: ->(status) { status.tags.map(&:display_name) }) field(:searchable_by, type: 'long', value: ->(status) { status.searchable_by }) + field(:mentioned_by, type: 'long', value: ->(status) { status.mentioned_by }) + field(:favourited_by, type: 'long', value: ->(status) { status.favourited_by }) + field(:reblogged_by, type: 'long', value: ->(status) { status.reblogged_by }) + field(:bookmarked_by, type: 'long', value: ->(status) { status.bookmarked_by }) + field(:bookmark_categoried_by, type: 'long', value: ->(status) { status.bookmark_categoried_by }) + field(:emoji_reacted_by, type: 'long', value: ->(status) { status.emoji_reacted_by }) + field(:referenced_by, type: 'long', value: ->(status) { status.referenced_by }) + field(:voted_by, type: 'long', value: ->(status) { status.voted_by }) field(:searchability, type: 'keyword', value: ->(status) { status.compute_searchability }) + field(:visibility, type: 'keyword', value: ->(status) { status.searchable_visibility }) field(:language, type: 'keyword') field(:domain, type: 'keyword', value: ->(status) { status.account.domain || '' }) field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties }) diff --git a/app/javascript/mastodon/features/compose/components/search.jsx b/app/javascript/mastodon/features/compose/components/search.jsx index 84d3306410..7584dbc819 100644 --- a/app/javascript/mastodon/features/compose/components/search.jsx +++ b/app/javascript/mastodon/features/compose/components/search.jsx @@ -59,13 +59,15 @@ class Search extends PureComponent { defaultOptions = [ { label: <>has: , action: e => { e.preventDefault(); this._insertText('has:') } }, { label: <>is: , action: e => { e.preventDefault(); this._insertText('is:') } }, + { label: <>my: , action: e => { e.preventDefault(); this._insertText('my:') } }, { label: <>language: , action: e => { e.preventDefault(); this._insertText('language:') } }, { label: <>from: , action: e => { e.preventDefault(); this._insertText('from:') } }, { label: <>domain: , action: e => { e.preventDefault(); this._insertText('domain:') } }, { label: <>before: , action: e => { e.preventDefault(); this._insertText('before:') } }, { label: <>during: , action: e => { e.preventDefault(); this._insertText('during:') } }, { label: <>after: , action: e => { e.preventDefault(); this._insertText('after:') } }, - { label: <>in: , action: e => { e.preventDefault(); this._insertText('in:') } } + { label: <>in: , action: e => { e.preventDefault(); this._insertText('in:') } }, + { label: <>order: , action: e => { e.preventDefault(); this._insertText('order:') } }, ]; setRef = c => { diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb index 62dd378866..32d30a3b0e 100644 --- a/app/lib/search_query_transformer.rb +++ b/app/lib/search_query_transformer.rb @@ -4,6 +4,7 @@ class SearchQueryTransformer < Parslet::Transform SUPPORTED_PREFIXES = %w( has is + my language from before @@ -11,6 +12,7 @@ class SearchQueryTransformer < Parslet::Transform during in domain + order ).freeze class Query @@ -34,6 +36,18 @@ class SearchQueryTransformer < Parslet::Transform search end + def order_by + return @order_by if @order_by + + @order_by = 'desc' + order_clauses.each { |clause| @order_by = clause.term } + @order_by + end + + def valid + must_clauses.any? || must_not_clauses.any? || filter_clauses.any? + end + private def clauses_by_operator @@ -56,6 +70,10 @@ class SearchQueryTransformer < Parslet::Transform clauses_by_operator.fetch(:filter, []) end + def order_clauses + clauses_by_operator.fetch(:order, []) + end + def indexes case @flags['in'] when 'library' @@ -214,6 +232,7 @@ class SearchQueryTransformer < Parslet::Transform if @term.start_with?('#') { match: { tags: { query: @term, operator: 'and' } } } else + # Memo for checking when manually merge # { multi_match: { type: 'most_fields', query: @term, fields: ['text', 'text.stemmed'], operator: 'and' } } { match_phrase: { text: { query: @term } } } end @@ -236,7 +255,7 @@ class SearchQueryTransformer < Parslet::Transform class PrefixClause attr_reader :operator, :prefix, :term - def initialize(prefix, operator, term, options = {}) + def initialize(prefix, operator, term, options = {}) # rubocop:disable Metrics/CyclomaticComplexity @prefix = prefix @negated = operator == '-' @options = options @@ -274,6 +293,39 @@ class SearchQueryTransformer < Parslet::Transform when 'in' @operator = :flag @term = term + when 'my' + @type = :term + @term = @options[:current_account]&.id + case term + when 'favourited', 'favorited', 'fav' + @filter = :favourited_by + when 'boosted', 'bt' + @filter = :reblogged_by + when 'replied', 'mentioned', 're' + @filter = :mentioned_by + when 'referenced', 'ref' + @filter = :referenced_by + when 'emoji_reacted', 'stamped', 'stamp' + @filter = :emoji_reacted_by + when 'bookmarked', 'bm' + @filter = :bookmarked_by + when 'categoried', 'bmc' + @filter = :bookmark_categoried_by + when 'voted', 'vote' + @filter = :voted_by + when 'interacted', 'act' + @filter = :searchable_by + else + raise "Unknown prefix: my:#{term}" + end + when 'order' + @operator = :order + @term = case term + when 'asc' + term + else + 'desc' + end else raise "Unknown prefix: #{prefix}" end @@ -302,7 +354,7 @@ class SearchQueryTransformer < Parslet::Transform end def domain_from_term(term) - return '' if %w(local me).include?(term) + return '' if ['local', 'me', Rails.configuration.x.local_domain].include?(term) term end diff --git a/app/models/bookmark_category_status.rb b/app/models/bookmark_category_status.rb index d932455ff5..8933d0e91f 100644 --- a/app/models/bookmark_category_status.rb +++ b/app/models/bookmark_category_status.rb @@ -13,6 +13,10 @@ # class BookmarkCategoryStatus < ApplicationRecord + include Paginable + + update_index('statuses', :status) if Chewy.enabled? + belongs_to :bookmark_category belongs_to :status belongs_to :bookmark diff --git a/app/models/concerns/status_search_concern.rb b/app/models/concerns/status_search_concern.rb index 69f1263f81..d1632b7b84 100644 --- a/app/models/concerns/status_search_concern.rb +++ b/app/models/concerns/status_search_concern.rb @@ -13,18 +13,48 @@ module StatusSearchConcern ids << account_id if local? - ids += local_mentioned.pluck(:id) - ids += local_favorited.pluck(:id) - ids += local_reblogged.pluck(:id) - ids += local_bookmarked.pluck(:id) - ids += local_emoji_reacted.pluck(:id) - ids += local_referenced.pluck(:id) - ids += preloadable_poll.local_voters.pluck(:id) if preloadable_poll.present? + ids += mentioned_by + ids += favourited_by + ids += reblogged_by + ids += bookmarked_by + ids += emoji_reacted_by + ids += referenced_by + ids += voted_by if preloadable_poll.present? ids.uniq end end + def mentioned_by + @mentioned_by ||= local_mentioned.pluck(:id) + end + + def favourited_by + @favourited_by ||= local_favorited.pluck(:id) + end + + def reblogged_by + @reblogged_by ||= local_reblogged.pluck(:id) + end + + def bookmarked_by + @bookmarked_by ||= local_bookmarked.pluck(:id) + end + + def emoji_reacted_by + @emoji_reacted_by ||= local_emoji_reacted.pluck(:id) + end + + def referenced_by + @referenced_by ||= local_referenced.pluck(:id) + end + + def voted_by + return [] if preloadable_poll.blank? + + @voted_by ||= preloadable_poll.local_voters.pluck(:id) + end + def searchable_text [ spoiler_text, diff --git a/app/models/status.rb b/app/models/status.rb index d4a28220d1..f9587b3c71 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -86,6 +86,7 @@ class Status < ApplicationRecord has_many :referenced_by_statuses, through: :referenced_by_status_objects, class_name: 'Status', source: :status has_many :capability_tokens, class_name: 'StatusCapabilityToken', inverse_of: :status, dependent: :destroy has_many :bookmark_category_relationships, class_name: 'BookmarkCategoryStatus', inverse_of: :status, dependent: :destroy + has_many :bookmark_categories, class_name: 'BookmarkCategory', through: :bookmark_category_relationships, source: :bookmark_category has_many :joined_bookmark_categories, class_name: 'BookmarkCategory', through: :bookmark_category_relationships, source: :bookmark_category # Those associations are used for the private search index @@ -93,6 +94,7 @@ class Status < ApplicationRecord has_many :local_favorited, -> { merge(Account.local) }, through: :favourites, source: :account has_many :local_reblogged, -> { merge(Account.local) }, through: :reblogs, source: :account has_many :local_bookmarked, -> { merge(Account.local) }, through: :bookmarks, source: :account + has_many :local_bookmark_categoried, -> { merge(Account.local) }, through: :bookmark_categories, source: :account has_many :local_emoji_reacted, -> { merge(Account.local) }, through: :emoji_reactions, source: :account has_many :local_referenced, -> { merge(Account.local) }, through: :referenced_by_statuses, source: :account @@ -435,6 +437,12 @@ class Status < ApplicationRecord compute_searchability end + def searchable_visibility + return limited_scope if limited_visibility? && !none_limited? + + visibility + end + class << self def selectable_visibilities visibilities.keys - %w(direct limited) diff --git a/app/services/statuses_search_service.rb b/app/services/statuses_search_service.rb index de5eda15c6..99921d1957 100644 --- a/app/services/statuses_search_service.rb +++ b/app/services/statuses_search_service.rb @@ -16,8 +16,11 @@ class StatusesSearchService < BaseService private def status_search_results - request = parsed_query.request - results = request.collapse(field: :id).order(id: { order: :desc }).limit(@limit).offset(@offset).objects.compact + query = parsed_query + request = query.request + return [] unless query.valid + + results = request.collapse(field: :id).order(id: { order: query.order_by }).limit(@limit).offset(@offset).objects.compact account_ids = results.map(&:account_id) account_domains = results.map(&:account_domain) preloaded_relations = @account.relations_map(account_ids, account_domains) From ddce464d70d84b5fb5a6ce3285e3549a5f53bdbf Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 15 Sep 2023 18:51:46 +0900 Subject: [PATCH 06/21] Add importing remote post --- app/lib/importer/base_importer.rb | 3 ++- app/lib/importer/statuses_index_importer.rb | 10 ++++++++-- app/models/concerns/status_search_concern.rb | 5 +++++ lib/mastodon/cli/search.rb | 3 ++- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/app/lib/importer/base_importer.rb b/app/lib/importer/base_importer.rb index a21557d303..ef8c8b38d4 100644 --- a/app/lib/importer/base_importer.rb +++ b/app/lib/importer/base_importer.rb @@ -3,9 +3,10 @@ class Importer::BaseImporter # @param [Integer] batch_size # @param [Concurrent::ThreadPoolExecutor] executor - def initialize(batch_size:, executor:) + def initialize(batch_size:, executor:, full: true) @batch_size = batch_size @executor = executor + @full = full @wait_for = Concurrent::Set.new end diff --git a/app/lib/importer/statuses_index_importer.rb b/app/lib/importer/statuses_index_importer.rb index 1922f65f6d..bb74bb8d89 100644 --- a/app/lib/importer/statuses_index_importer.rb +++ b/app/lib/importer/statuses_index_importer.rb @@ -21,7 +21,7 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter to_index.map do |object| # This is unlikely to happen, but the post may have been # un-interacted with since it was queued for indexing - if object.searchable_by.empty? + if object.searchable_by.empty? && %w(public private).exclude?(object.searchability) deleted += 1 { delete: { _id: object.id } } else @@ -49,13 +49,15 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter end def scopes - [ + targets = [ local_statuses_scope, local_mentions_scope, local_favourites_scope, local_votes_scope, local_bookmarks_scope, ] + targets << remote_searchable_scope if @full + targets end def local_mentions_scope @@ -77,4 +79,8 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter def local_statuses_scope Status.local.select('"statuses"."id", COALESCE("statuses"."reblog_of_id", "statuses"."id") AS status_id') end + + def remote_searchable_scope + Status.remote_dynamic_searchability.select('"statuses"."id", COALESCE("statuses"."reblog_of_id", "statuses"."id") AS status_id') + end end diff --git a/app/models/concerns/status_search_concern.rb b/app/models/concerns/status_search_concern.rb index d1632b7b84..376f82e509 100644 --- a/app/models/concerns/status_search_concern.rb +++ b/app/models/concerns/status_search_concern.rb @@ -5,6 +5,7 @@ module StatusSearchConcern included do scope :indexable, -> { without_reblogs.where(visibility: [:public, :login], searchability: nil).joins(:account).where(account: { indexable: true }) } + scope :remote_dynamic_searchability, -> { remote.where(searchability: [:public, :private]) } end def searchable_by @@ -41,6 +42,10 @@ module StatusSearchConcern @bookmarked_by ||= local_bookmarked.pluck(:id) end + def bookmark_categoried_by + @bookmark_categoried_by ||= local_bookmark_categoried.pluck(:id).uniq + end + def emoji_reacted_by @emoji_reacted_by ||= local_emoji_reacted.pluck(:id) end diff --git a/lib/mastodon/cli/search.rb b/lib/mastodon/cli/search.rb index 25a595aadd..f8697ffded 100644 --- a/lib/mastodon/cli/search.rb +++ b/lib/mastodon/cli/search.rb @@ -20,6 +20,7 @@ module Mastodon::CLI option :import, type: :boolean, default: true, desc: 'Import data from the database to the index' option :clean, type: :boolean, default: true, desc: 'Remove outdated documents from the index' option :reset_chewy, type: :boolean, default: false, desc: "Reset Chewy's internal index" + option :full, type: :boolean, default: true, desc: 'Import full data over Mastodon default importer' desc 'deploy', 'Create or upgrade Elasticsearch indices and populate them' long_desc <<~LONG_DESC If Elasticsearch is empty, this command will create the necessary indices @@ -41,7 +42,7 @@ module Mastodon::CLI end pool = Concurrent::FixedThreadPool.new(options[:concurrency], max_queue: options[:concurrency] * 10) - importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool) } + importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool, full: options[:full]) } progress = ProgressBar.create(total: nil, format: '%t%c/%u |%b%i| %e (%r docs/s)', autofinish: false) Chewy::Stash::Specification.reset! if options[:reset_chewy] From ed3bb8a7510693d85dfdc3879a3074751ee1330e Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 15 Sep 2023 18:52:26 +0900 Subject: [PATCH 07/21] Fix error when remove from bookmark category --- app/javascript/mastodon/reducers/bookmark_categories.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/javascript/mastodon/reducers/bookmark_categories.js b/app/javascript/mastodon/reducers/bookmark_categories.js index 3877303b12..ea9e4f4895 100644 --- a/app/javascript/mastodon/reducers/bookmark_categories.js +++ b/app/javascript/mastodon/reducers/bookmark_categories.js @@ -65,7 +65,10 @@ const appendToBookmarkCategoryStatusesById = (state, bookmarkCategoryId, statuse }; const removeStatusFromBookmarkCategoryById = (state, bookmarkCategoryId, status) => { - return state.updateIn([bookmarkCategoryId, 'items'], items => items.delete(status)); + if (state.getIn([bookmarkCategoryId, 'items'])) { + return state.updateIn([bookmarkCategoryId, 'items'], items => items.delete(status)); + } + return state; }; const removeStatusFromAllBookmarkCategories = (state, status) => { From bd1bc993c781619b4bf989023c881e2ac57b7019 Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 15 Sep 2023 18:53:16 +0900 Subject: [PATCH 08/21] Fix ohagi name mistake --- lib/mastodon/cli/ohagi.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mastodon/cli/ohagi.rb b/lib/mastodon/cli/ohagi.rb index 28c6c41953..dd78dea22a 100644 --- a/lib/mastodon/cli/ohagi.rb +++ b/lib/mastodon/cli/ohagi.rb @@ -25,8 +25,8 @@ module Mastodon::CLI say('Let the WAR begin.', :red) end - desc 'kokuraan', 'Ohagi is kokuraan' - def kokuraan + desc 'oguraan', 'Ohagi is oguraan' + def oguraan say('I hate you.', :yellow) end end From d1cbffc396af1db3b4ff5eb49acd42ac9291a581 Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 15 Sep 2023 22:26:53 +0900 Subject: [PATCH 09/21] Change search deploy full parameter default value --- lib/mastodon/cli/search.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mastodon/cli/search.rb b/lib/mastodon/cli/search.rb index f8697ffded..57e2895130 100644 --- a/lib/mastodon/cli/search.rb +++ b/lib/mastodon/cli/search.rb @@ -20,7 +20,7 @@ module Mastodon::CLI option :import, type: :boolean, default: true, desc: 'Import data from the database to the index' option :clean, type: :boolean, default: true, desc: 'Remove outdated documents from the index' option :reset_chewy, type: :boolean, default: false, desc: "Reset Chewy's internal index" - option :full, type: :boolean, default: true, desc: 'Import full data over Mastodon default importer' + option :full, type: :boolean, default: false, desc: 'Import full data over Mastodon default importer' desc 'deploy', 'Create or upgrade Elasticsearch indices and populate them' long_desc <<~LONG_DESC If Elasticsearch is empty, this command will create the necessary indices From 9bec798a80fd85320930624af873ee7b37fd6818 Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 15 Sep 2023 22:27:16 +0900 Subject: [PATCH 10/21] Receive emoji reaction as favourite when emoji reaction is disabled --- app/lib/activitypub/activity/like.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/activitypub/activity/like.rb b/app/lib/activitypub/activity/like.rb index c533d57d45..ebfac9a868 100644 --- a/app/lib/activitypub/activity/like.rb +++ b/app/lib/activitypub/activity/like.rb @@ -9,7 +9,7 @@ class ActivityPub::Activity::Like < ActivityPub::Activity return if @original_status.nil? || delete_arrived_first?(@json['id']) || reject_favourite? - if shortcode.nil? + if shortcode.nil? || !Setting.enable_emoji_reaction process_favourite else process_emoji_reaction From a8798ea4bd483ad137f17a145045d4ad77910cc0 Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 15 Sep 2023 22:28:06 +0900 Subject: [PATCH 11/21] Merge enable_emoji_reaction account and server settings --- app/serializers/initial_state_serializer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb index 7f8792d918..97f2a58092 100644 --- a/app/serializers/initial_state_serializer.rb +++ b/app/serializers/initial_state_serializer.rb @@ -48,7 +48,7 @@ class InitialStateSerializer < ActiveModel::Serializer store[:display_media] = object.current_account.user.setting_display_media store[:display_media_expand] = object.current_account.user.setting_display_media_expand store[:expand_spoilers] = object.current_account.user.setting_expand_spoilers - store[:enable_emoji_reaction] = object.current_account.user.setting_enable_emoji_reaction + store[:enable_emoji_reaction] = object.current_account.user.setting_enable_emoji_reaction && Setting.enable_emoji_reaction store[:show_emoji_reaction_on_timeline] = object.current_account.user.setting_show_emoji_reaction_on_timeline store[:enable_login_privacy] = object.current_account.user.setting_enable_login_privacy store[:enable_dtl_menu] = object.current_account.user.setting_enable_dtl_menu From c9d07de476b961acc651c69c11e1e37f580ba3ea Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 15 Sep 2023 22:28:24 +0900 Subject: [PATCH 12/21] Add signature to limited visibility --- app/services/concerns/payloadable.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/concerns/payloadable.rb b/app/services/concerns/payloadable.rb index bd9d9d74b5..73c397852e 100644 --- a/app/services/concerns/payloadable.rb +++ b/app/services/concerns/payloadable.rb @@ -17,7 +17,7 @@ module Payloadable payload = ActiveModelSerializers::SerializableResource.new(record, options.merge(serializer: serializer, adapter: ActivityPub::Adapter)).as_json object = record.respond_to?(:virtual_object) ? record.virtual_object : record - if (object.respond_to?(:sign?) && object.sign?) && signer && (always_sign || signing_enabled?) + if ((object.respond_to?(:sign?) && object.sign?) && signer && (always_sign || signing_enabled?)) || object.is_a?(String) ActivityPub::LinkedDataSignature.new(payload).sign!(signer, sign_with: sign_with) else payload From ef4d2702d21b2467d1b54e5a488a5fc1a5aa7211 Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 15 Sep 2023 22:33:05 +0900 Subject: [PATCH 13/21] Fix stamp is not visible in timeline when logout --- app/serializers/initial_state_serializer.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb index 97f2a58092..f8a3f924f7 100644 --- a/app/serializers/initial_state_serializer.rb +++ b/app/serializers/initial_state_serializer.rb @@ -66,6 +66,7 @@ class InitialStateSerializer < ActiveModel::Serializer store[:reduce_motion] = Setting.reduce_motion store[:use_blurhash] = Setting.use_blurhash store[:enable_emoji_reaction] = Setting.enable_emoji_reaction + store[:show_emoji_reaction_on_timeline] = Setting.enable_emoji_reaction end store[:disabled_account_id] = object.disabled_account.id.to_s if object.disabled_account From fc3950fe1cd663fc1ecb1da945d3dd7f4aa2ae58 Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 15 Sep 2023 22:33:05 +0900 Subject: [PATCH 14/21] Fix stamp is not visible in timeline when logout --- app/serializers/initial_state_serializer.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb index 4db1be6f73..68e3e4be8e 100644 --- a/app/serializers/initial_state_serializer.rb +++ b/app/serializers/initial_state_serializer.rb @@ -66,6 +66,7 @@ class InitialStateSerializer < ActiveModel::Serializer store[:reduce_motion] = Setting.reduce_motion store[:use_blurhash] = Setting.use_blurhash store[:enable_emoji_reaction] = Setting.enable_emoji_reaction + store[:show_emoji_reaction_on_timeline] = Setting.enable_emoji_reaction end store[:disabled_account_id] = object.disabled_account.id.to_s if object.disabled_account From d1fa9d8f448283efe4356306e8d835dc07544080 Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 15 Sep 2023 22:36:06 +0900 Subject: [PATCH 15/21] Bump version to 3.2 --- lib/mastodon/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index fbe320776e..651c11848a 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -9,7 +9,7 @@ module Mastodon end def kmyblue_minor - 1 + 2 end def major From 0ad0dcc8fd8a4c06bc3c5cbd5348a5e0b75ced7d Mon Sep 17 00:00:00 2001 From: KMY Date: Sat, 16 Sep 2023 08:29:24 +0900 Subject: [PATCH 16/21] Fix ES error --- app/lib/search_query_transformer.rb | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb index 32d30a3b0e..2ec68fd5f7 100644 --- a/app/lib/search_query_transformer.rb +++ b/app/lib/search_query_transformer.rb @@ -260,6 +260,7 @@ class SearchQueryTransformer < Parslet::Transform @negated = operator == '-' @options = options @operator = :filter + @statuses_index_only = false case prefix when 'has', 'is' @@ -296,6 +297,7 @@ class SearchQueryTransformer < Parslet::Transform when 'my' @type = :term @term = @options[:current_account]&.id + @statuses_index_only = true case term when 'favourited', 'favorited', 'fav' @filter = :favourited_by @@ -326,13 +328,33 @@ class SearchQueryTransformer < Parslet::Transform else 'desc' end + when 'searchability' + @filter = :searchablity + @type = :terms + @statuses_index_only = true + @term = case term + when 'public' + %w(public private direct limited) + when 'private' + %w(private direct limited) + when 'direct' + %w(direct limited) + else + %w(limited) + end else raise "Unknown prefix: #{prefix}" end end def to_query - if @negated + if @statuses_index_only + if @negated + { bool: { must_not: [_index => StatusesIndex.index_name, @type => { @filter => @term }] } } + else + { bool: { must: [_index => StatusesIndex.index_name, @type => { @filter => @term }] } } + end + elsif @negated { bool: { must_not: { @type => { @filter => @term } } } } else { @type => { @filter => @term } } From a7a6c62440b4e4e425debbcb37b7480dc90916d8 Mon Sep 17 00:00:00 2001 From: KMY Date: Sat, 16 Sep 2023 08:33:18 +0900 Subject: [PATCH 17/21] Sorry, --- app/lib/search_query_transformer.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb index 2ec68fd5f7..00a7d22eb9 100644 --- a/app/lib/search_query_transformer.rb +++ b/app/lib/search_query_transformer.rb @@ -350,9 +350,9 @@ class SearchQueryTransformer < Parslet::Transform def to_query if @statuses_index_only if @negated - { bool: { must_not: [_index => StatusesIndex.index_name, @type => { @filter => @term }] } } + { bool: { must_not: [{ term: { _index => StatusesIndex.index_name } }, { @type => { @filter => @term } }] } } else - { bool: { must: [_index => StatusesIndex.index_name, @type => { @filter => @term }] } } + { bool: { must: [{ term: { _index => StatusesIndex.index_name } }, { @type => { @filter => @term } }] } } end elsif @negated { bool: { must_not: { @type => { @filter => @term } } } } From 5b045f71988288e043c99af8f728cfc7d9d7722e Mon Sep 17 00:00:00 2001 From: KMY Date: Sat, 16 Sep 2023 08:51:17 +0900 Subject: [PATCH 18/21] Fix query miss --- app/lib/search_query_transformer.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb index 00a7d22eb9..81e07d77ce 100644 --- a/app/lib/search_query_transformer.rb +++ b/app/lib/search_query_transformer.rb @@ -350,9 +350,9 @@ class SearchQueryTransformer < Parslet::Transform def to_query if @statuses_index_only if @negated - { bool: { must_not: [{ term: { _index => StatusesIndex.index_name } }, { @type => { @filter => @term } }] } } + { bool: { must_not: [{ term: { _index: StatusesIndex.index_name } }, { @type => { @filter => @term } }] } } else - { bool: { must: [{ term: { _index => StatusesIndex.index_name } }, { @type => { @filter => @term } }] } } + { bool: { must: [{ term: { _index: StatusesIndex.index_name } }, { @type => { @filter => @term } }] } } end elsif @negated { bool: { must_not: { @type => { @filter => @term } } } } From 173b9f4fd7877b9af89a6eac408fc85dce654be1 Mon Sep 17 00:00:00 2001 From: KMY Date: Sat, 16 Sep 2023 10:31:43 +0900 Subject: [PATCH 19/21] Add searchability option --- app/lib/search_query_transformer.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb index 81e07d77ce..6521246803 100644 --- a/app/lib/search_query_transformer.rb +++ b/app/lib/search_query_transformer.rb @@ -13,6 +13,7 @@ class SearchQueryTransformer < Parslet::Transform in domain order + searchability ).freeze class Query From 0ffd8acacd749599006744da0cc9b6d9672fc458 Mon Sep 17 00:00:00 2001 From: KMY Date: Sat, 16 Sep 2023 10:53:09 +0900 Subject: [PATCH 20/21] Add search deploy date options --- app/lib/importer/base_importer.rb | 4 +++- app/lib/importer/public_statuses_index_importer.rb | 5 ++++- app/lib/importer/statuses_index_importer.rb | 2 ++ lib/mastodon/cli/search.rb | 4 +++- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/app/lib/importer/base_importer.rb b/app/lib/importer/base_importer.rb index ef8c8b38d4..96cacfc07b 100644 --- a/app/lib/importer/base_importer.rb +++ b/app/lib/importer/base_importer.rb @@ -3,10 +3,12 @@ class Importer::BaseImporter # @param [Integer] batch_size # @param [Concurrent::ThreadPoolExecutor] executor - def initialize(batch_size:, executor:, full: true) + def initialize(batch_size:, executor:, full: true, from: nil, to: nil) @batch_size = batch_size @executor = executor @full = full + @from = from.to_date if from.present? + @to = to.to_date if to.present? @wait_for = Concurrent::Set.new end diff --git a/app/lib/importer/public_statuses_index_importer.rb b/app/lib/importer/public_statuses_index_importer.rb index 7dfe998868..02134dc251 100644 --- a/app/lib/importer/public_statuses_index_importer.rb +++ b/app/lib/importer/public_statuses_index_importer.rb @@ -27,6 +27,9 @@ class Importer::PublicStatusesIndexImporter < Importer::BaseImporter end def scope - Status.indexable.reorder(nil) + to_index = Status.indexable.reorder(nil) + to_index = to_index.where('statuses.created_at >= ?', @from) if @from.present? + to_index = to_index.where('statuses.created_at < ?', @to) if @to.present? + to_index end end diff --git a/app/lib/importer/statuses_index_importer.rb b/app/lib/importer/statuses_index_importer.rb index bb74bb8d89..ca4b01d0e8 100644 --- a/app/lib/importer/statuses_index_importer.rb +++ b/app/lib/importer/statuses_index_importer.rb @@ -17,6 +17,8 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter bulk = ActiveRecord::Base.connection_pool.with_connection do to_index = index.adapter.default_scope.where(id: status_ids) + to_index = to_index.where('created_at >= ?', @from) if @from.present? + to_index = to_index.where('created_at < ?', @to) if @to.present? crutches = Chewy::Index::Crutch::Crutches.new index, to_index to_index.map do |object| # This is unlikely to happen, but the post may have been diff --git a/lib/mastodon/cli/search.rb b/lib/mastodon/cli/search.rb index 57e2895130..af70b6c2c7 100644 --- a/lib/mastodon/cli/search.rb +++ b/lib/mastodon/cli/search.rb @@ -21,6 +21,8 @@ module Mastodon::CLI option :clean, type: :boolean, default: true, desc: 'Remove outdated documents from the index' option :reset_chewy, type: :boolean, default: false, desc: "Reset Chewy's internal index" option :full, type: :boolean, default: false, desc: 'Import full data over Mastodon default importer' + option :from, type: :string, default: nil, desc: 'Statuses start date' + option :to, type: :string, default: nil, desc: 'Statuses end date' desc 'deploy', 'Create or upgrade Elasticsearch indices and populate them' long_desc <<~LONG_DESC If Elasticsearch is empty, this command will create the necessary indices @@ -42,7 +44,7 @@ module Mastodon::CLI end pool = Concurrent::FixedThreadPool.new(options[:concurrency], max_queue: options[:concurrency] * 10) - importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool, full: options[:full]) } + importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool, full: options[:full], from: options[:from], to: options[:to]) } progress = ProgressBar.create(total: nil, format: '%t%c/%u |%b%i| %e (%r docs/s)', autofinish: false) Chewy::Stash::Specification.reset! if options[:reset_chewy] From 6aaef5c34e4165daef1a69fa33d7ee5b1b8b5d95 Mon Sep 17 00:00:00 2001 From: KMY Date: Sat, 16 Sep 2023 13:42:55 +0900 Subject: [PATCH 21/21] Fix N+1 problem on statuses/public_statuses indices --- app/chewy/public_statuses_index.rb | 2 +- app/chewy/statuses_index.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/chewy/public_statuses_index.rb b/app/chewy/public_statuses_index.rb index 4bfe3f08ad..fc58f8f097 100644 --- a/app/chewy/public_statuses_index.rb +++ b/app/chewy/public_statuses_index.rb @@ -144,7 +144,7 @@ class PublicStatusesIndex < Chewy::Index index_scope ::Status.unscoped .kept .indexable - .includes(:media_attachments, :preloadable_poll, :preview_cards, :tags) + .includes(:media_attachments, :preloadable_poll, :preview_cards, :tags, :account) root date_detection: false do field(:id, type: 'long') diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb index 3b0a38a466..6a6621fb8c 100644 --- a/app/chewy/statuses_index.rb +++ b/app/chewy/statuses_index.rb @@ -145,6 +145,7 @@ class StatusesIndex < Chewy::Index settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: Rails.env.test? ? DEVELOPMENT_SETTINGS : PRODUCTION_SETTINGS index_scope ::Status.unscoped.kept.without_reblogs.includes( + :account, :media_attachments, :preview_cards, :local_mentioned,