Merge remote-tracking branch 'parent/main' into kb_migration

app/chewy/accounts_index.rb

@@ -31,12 +31,13 @@ class AccountsIndex < Chewy::Index
 
     analyzer: {
       natural: {
-        tokenizer: 'uax_url_email',
+        tokenizer: 'standard',
         filter: %w(
-          english_possessive_stemmer
           lowercase
           asciifolding
           cjk_width
+          elision
+          english_possessive_stemmer
           english_stop
           english_stemmer
         ),
@@ -52,7 +53,7 @@ class AccountsIndex < Chewy::Index
       },
 
       verbatim: {
-        tokenizer: 'standard',
+        tokenizer: 'uax_url_email',
         filter: %w(lowercase asciifolding cjk_width),
       },
 

app/chewy/public_statuses_index.rb

@@ -20,13 +20,19 @@ class PublicStatusesIndex < Chewy::Index
     },
 
     analyzer: {
-      content: {
+      verbatim: {
         tokenizer: 'uax_url_email',
+        filter: %w(lowercase),
+      },
+
+      content: {
+        tokenizer: 'standard',
         filter: %w(
-          english_possessive_stemmer
           lowercase
           asciifolding
           cjk_width
+          elision
+          english_possessive_stemmer
           english_stop
           english_stemmer
         ),
@@ -101,7 +107,7 @@ class PublicStatusesIndex < Chewy::Index
                       .includes(:media_attachments, :preloadable_poll, :preview_cards)
 
   root date_detection: false do
-    field(:id, type: 'keyword')
+    field(:id, type: 'long')
     field(:account_id, type: 'long')
     field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
     field(:language, type: 'keyword')

app/chewy/statuses_index.rb

@@ -19,13 +19,19 @@ class StatusesIndex < Chewy::Index
       },
     },
     analyzer: {
-      content: {
+      verbatim: {
         tokenizer: 'uax_url_email',
+        filter: %w(lowercase),
+      },
+
+      content: {
+        tokenizer: 'standard',
         filter: %w(
-          english_possessive_stemmer
           lowercase
           asciifolding
           cjk_width
+          elision
+          english_possessive_stemmer
           english_stop
           english_stemmer
         ),
@@ -133,7 +139,7 @@ class StatusesIndex < Chewy::Index
   end
 
   root date_detection: false do
-    field(:id, type: 'keyword')
+    field(:id, type: 'long')
     field(:account_id, type: 'long')
     field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
     field(:searchable_by, type: 'long', value: ->(status, crutches) { status.searchable_by(crutches) })

app/javascript/mastodon/components/hashtag_bar.tsx

@@ -10,8 +10,8 @@ import { groupBy, minBy } from 'lodash';
 
 import { getStatusContent } from './status_content';
 
-// About two lines on desktop
-const VISIBLE_HASHTAGS = 7;
+// Fit on a single line on desktop
+const VISIBLE_HASHTAGS = 3;
 
 // Those types are not correct, they need to be replaced once this part of the state is typed
 export type TagLike = Record<{ name: string }>;
@@ -210,7 +210,7 @@ const HashtagBar: React.FC<{
 
   const revealedHashtags = expanded
     ? hashtags
-    : hashtags.slice(0, VISIBLE_HASHTAGS - 1);
+    : hashtags.slice(0, VISIBLE_HASHTAGS);
 
   return (
     <div className='hashtag-bar'>

app/javascript/mastodon/features/compose/components/search.jsx

@@ -1,7 +1,7 @@
 import PropTypes from 'prop-types';
 import { PureComponent } from 'react';
 
-import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
+import { defineMessages, injectIntl, FormattedMessage, FormattedList } from 'react-intl';
 
 import classNames from 'classnames';
 
@@ -45,6 +45,16 @@ class Search extends PureComponent {
     options: [],
   };
 
+  defaultOptions = [
+    { label: <><mark>has:</mark> <FormattedList type='disjunction' value={['media', 'poll', 'embed']} /></>, action: e => { e.preventDefault(); this._insertText('has:') } },
+    { label: <><mark>is:</mark> <FormattedList type='disjunction' value={['reply', 'sensitive']} /></>, action: e => { e.preventDefault(); this._insertText('is:') } },
+    { label: <><mark>language:</mark> <FormattedMessage id='search_popout.language_code' defaultMessage='ISO language code' /></>, action: e => { e.preventDefault(); this._insertText('language:') } },
+    { label: <><mark>from:</mark> <FormattedMessage id='search_popout.user' defaultMessage='user' /></>, action: e => { e.preventDefault(); this._insertText('from:') } },
+    { label: <><mark>before:</mark> <FormattedMessage id='search_popout.specific_date' defaultMessage='specific date' /></>, action: e => { e.preventDefault(); this._insertText('before:') } },
+    { label: <><mark>during:</mark> <FormattedMessage id='search_popout.specific_date' defaultMessage='specific date' /></>, action: e => { e.preventDefault(); this._insertText('during:') } },
+    { label: <><mark>after:</mark> <FormattedMessage id='search_popout.specific_date' defaultMessage='specific date' /></>, action: e => { e.preventDefault(); this._insertText('after:') } },
+  ];
+
   setRef = c => {
     this.searchForm = c;
   };
@@ -70,7 +80,7 @@ class Search extends PureComponent {
 
   handleKeyDown = (e) => {
     const { selectedOption } = this.state;
-    const options = this._getOptions();
+    const options = this._getOptions().concat(this.defaultOptions);
 
     switch(e.key) {
     case 'Escape':
@@ -100,11 +110,9 @@ class Search extends PureComponent {
       if (selectedOption === -1) {
         this._submit();
       } else if (options.length > 0) {
-        options[selectedOption].action();
+        options[selectedOption].action(e);
       }
 
-      this._unfocus();
-
       break;
     case 'Delete':
       if (selectedOption > -1 && options.length > 0) {
@@ -147,6 +155,7 @@ class Search extends PureComponent {
 
     router.history.push(`/tags/${query}`);
     onClickSearchResult(query, 'hashtag');
+    this._unfocus();
   };
 
   handleAccountClick = () => {
@@ -157,6 +166,7 @@ class Search extends PureComponent {
 
     router.history.push(`/@${query}`);
     onClickSearchResult(query, 'account');
+    this._unfocus();
   };
 
   handleURLClick = () => {
@@ -164,6 +174,7 @@ class Search extends PureComponent {
     const { value, onOpenURL } = this.props;
 
     onOpenURL(value, router.history);
+    this._unfocus();
   };
 
   handleStatusSearch = () => {
@@ -182,6 +193,8 @@ class Search extends PureComponent {
     } else if (search.get('type') === 'hashtag') {
       router.history.push(`/tags/${search.get('q')}`);
     }
+
+    this._unfocus();
   };
 
   handleForgetRecentSearchClick = search => {
@@ -194,6 +207,18 @@ class Search extends PureComponent {
     document.querySelector('.ui').parentElement.focus();
   }
 
+  _insertText (text) {
+    const { value, onChange } = this.props;
+
+    if (value === '') {
+      onChange(text);
+    } else if (value[value.length - 1] === ' ') {
+      onChange(`${value}${text}`);
+    } else {
+      onChange(`${value} ${text}`);
+    }
+  }
+
   _submit (type) {
     const { onSubmit, openInRoute } = this.props;
     const { router } = this.context;
@@ -203,6 +228,8 @@ class Search extends PureComponent {
     if (openInRoute) {
       router.history.push('/search');
     }
+
+    this._unfocus();
   }
 
   _getOptions () {
@@ -325,6 +352,16 @@ class Search extends PureComponent {
               </div>
             </>
           )}
+
+          <h4><FormattedMessage id='search_popout.options' defaultMessage='Search options' /></h4>
+
+          <div className='search__popout__menu'>
+            {this.defaultOptions.map(({ key, label, action }, i) => (
+              <button key={key} onMouseDown={action} className={classNames('search__popout__menu__item', { selected: selectedOption === (options.length + i) })}>
+                {label}
+              </button>
+            ))}
+          </div>
         </div>
       </div>
     );

app/javascript/mastodon/locales/en.json

@@ -617,8 +617,12 @@
   "searchability.public.short": "Public",
   "searchability.unlisted.long": "Your followers and reactionners can find",
   "searchability.unlisted.short": "Followers and reactionners",
+  "search_popout.language_code": "ISO language code",
+  "search_popout.options": "Search options",
   "search_popout.quick_actions": "Quick actions",
   "search_popout.recent": "Recent searches",
+  "search_popout.specific_date": "specific date",
+  "search_popout.user": "user",
   "search_results.accounts": "Profiles",
   "search_results.all": "All",
   "search_results.hashtags": "Hashtags",

app/javascript/mastodon/test_setup.js

@@ -1 +1 @@
-import '@testing-library/jest-dom/extend-expect';
+import '@testing-library/jest-dom';

app/javascript/styles/mastodon/components.scss

@@ -5204,6 +5204,12 @@ a.status-card {
     }
 
     &__menu {
+      margin-bottom: 20px;
+
+      &:last-child {
+        margin-bottom: 0;
+      }
+
       &__message {
         color: $dark-text-color;
         padding: 0 10px;
@@ -9655,19 +9661,24 @@ noscript {
   display: flex;
   flex-wrap: wrap;
   font-size: 14px;
+  line-height: 18px;
   gap: 4px;
+  color: $darker-text-color;
 
   a {
     display: inline-flex;
-    color: $dark-text-color;
+    color: inherit;
     text-decoration: none;
 
-    &:hover {
-      text-decoration: none;
-
-      span {
-        text-decoration: underline;
-      }
+    &:hover span {
+      text-decoration: underline;
     }
   }
+
+  .link-button {
+    color: inherit;
+    font-size: inherit;
+    line-height: inherit;
+    padding: 0;
+  }
 }

app/lib/search_query_parser.rb

@@ -9,7 +9,7 @@ class SearchQueryParser < Parslet::Parser
   rule(:prefix)    { (term >> colon).as(:prefix) }
   rule(:shortcode) { (colon >> term >> colon.maybe).as(:shortcode) }
   rule(:phrase)    { (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) }
-  rule(:clause)    { (prefix.maybe >> operator.maybe >> (phrase | term | shortcode)).as(:clause) }
+  rule(:clause)    { (operator.maybe >> prefix.maybe >> (phrase | term | shortcode)).as(:clause) }
   rule(:query)     { (clause >> space.maybe).repeat.as(:query) }
   root(:query)
 end

app/lib/search_query_transformer.rb

@@ -36,7 +36,11 @@ class SearchQueryTransformer < Parslet::Transform
     def clause_to_filter(clause)
       case clause
       when PrefixClause
-        { clause.type => { clause.filter => clause.term } }
+        if clause.negated?
+          { bool: { must_not: { clause.type => { clause.filter => clause.term } } } }
+        else
+          { clause.type => { clause.filter => clause.term } }
+        end
       else
         raise "Unexpected clause type: #{clause}"
       end
@@ -81,7 +85,9 @@ class SearchQueryTransformer < Parslet::Transform
   class PrefixClause
     attr_reader :type, :filter, :operator, :term
 
-    def initialize(prefix, term)
+    def initialize(prefix, operator, term, options = {})
+      @negated  = operator == '-'
+      @options  = options
       @operator = :filter
 
       case prefix
@@ -100,23 +106,29 @@ class SearchQueryTransformer < Parslet::Transform
       when 'before'
         @filter = :created_at
         @type = :range
-        @term = { lt: term }
+        @term = { lt: term, time_zone: @options[:current_account]&.user_time_zone || 'UTC' }
       when 'after'
         @filter = :created_at
         @type = :range
-        @term = { gt: term }
+        @term = { gt: term, time_zone: @options[:current_account]&.user_time_zone || 'UTC' }
       when 'during'
         @filter = :created_at
         @type = :range
-        @term = { gte: term, lte: term }
+        @term = { gte: term, lte: term, time_zone: @options[:current_account]&.user_time_zone || 'UTC' }
       else
         raise Mastodon::SyntaxError
       end
     end
 
+    def negated?
+      @negated
+    end
+
     private
 
     def account_id_from_term(term)
+      return @options[:current_account]&.id || -1 if term == 'me'
+
       username, domain = term.gsub(/\A@/, '').split('@')
       domain = nil if TagManager.instance.local_domain?(domain)
       account = Account.find_remote(username, domain)
@@ -132,7 +144,7 @@ class SearchQueryTransformer < Parslet::Transform
     operator = clause[:operator]&.to_s
 
     if clause[:prefix]
-      PrefixClause.new(prefix, clause[:term].to_s)
+      PrefixClause.new(prefix, operator, clause[:term].to_s, current_account: current_account)
     elsif clause[:term]
       TermClause.new(prefix, operator, clause[:term].to_s)
     elsif clause[:shortcode]

app/models/concerns/account_statuses_search.rb

@@ -31,8 +31,8 @@ module AccountStatusesSearch
   def add_to_public_statuses_index!
     return unless Chewy.enabled?
 
-    statuses.indexable.find_in_batches do |batch|
-      PublicStatusesIndex.import(query: batch)
+    statuses.without_reblogs.where(visibility: :public).find_in_batches do |batch|
+      PublicStatusesIndex.import(batch)
     end
   end
 

app/models/media_attachment.rb

@@ -49,6 +49,7 @@ class MediaAttachment < ApplicationRecord
 
   MAX_VIDEO_MATRIX_LIMIT = 8_294_400 # 3840x2160px
   MAX_VIDEO_FRAME_RATE   = 120
+  MAX_VIDEO_FRAMES       = 36_000 # Approx. 5 minutes at 120 fps
 
   IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif .webp .heic .heif .avif).freeze
   VIDEO_FILE_EXTENSIONS = %w(.webm .mp4 .m4v .mov).freeze
@@ -103,17 +104,12 @@ class MediaAttachment < ApplicationRecord
     convert_options: {
       output: {
         'loglevel' => 'fatal',
-        'movflags' => 'faststart',
-        'pix_fmt' => 'yuv420p',
-        'vf' => 'scale=\'trunc(iw/2)*2:trunc(ih/2)*2\'',
-        'vsync' => 'cfr',
+        'preset' => 'veryfast',
         'c:v' => 'h264',
-        'maxrate' => '1300K',
-        'bufsize' => '1300K',
-        'b:v' => '1300K',
-        'frames:v' => 60 * 60 * 3,
-        'crf' => 18,
+        'c:a' => 'aac',
+        'b:a' => '192k',
         'map_metadata' => '-1',
+        'frames:v' => MAX_VIDEO_FRAMES,
       }.freeze,
     }.freeze,
   }.freeze
@@ -140,7 +136,7 @@ class MediaAttachment < ApplicationRecord
       convert_options: {
         output: {
           'loglevel' => 'fatal',
-          :vf => 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease',
+          :vf => 'scale=\'min(640\, iw):min(640\, ih)\':force_original_aspect_ratio=decrease',
         }.freeze,
       }.freeze,
       format: 'png',

app/services/search_service.rb

@@ -19,7 +19,7 @@ class SearchService < BaseService
         results.merge!(url_resource_results) unless url_resource.nil? || @offset.positive? || (@options[:type].present? && url_resource_symbol != @options[:type].to_sym)
       elsif @query.present?
         results[:accounts] = perform_accounts_search! if account_searchable?
-        results[:statuses] = perform_statuses_search! if full_text_searchable?
+        results[:statuses] = perform_statuses_search! if status_searchable?
         results[:hashtags] = perform_hashtags_search! if hashtag_searchable?
       end
     end
@@ -82,18 +82,16 @@ class SearchService < BaseService
     url_resource.class.name.downcase.pluralize.to_sym
   end
 
-  def full_text_searchable?
-    return false unless Chewy.enabled?
-
-    statuses_search? && !@account.nil? && !(@query.include?('@') && !@query.include?(' '))
+  def status_searchable?
+    Chewy.enabled? && status_search? && @account.present?
   end
 
   def account_searchable?
-    account_search? && !(@query.include?('@') && @query.include?(' '))
+    account_search?
   end
 
   def hashtag_searchable?
-    hashtag_search? && !@query.include?('@')
+    hashtag_search?
   end
 
   def account_search?
@@ -104,7 +102,7 @@ class SearchService < BaseService
     @options[:type].blank? || @options[:type] == 'hashtags'
   end
 
-  def statuses_search?
+  def status_search?
     @options[:type].blank? || @options[:type] == 'statuses'
   end
 end

app/services/statuses_search_service.rb

@@ -140,6 +140,6 @@ class StatusesSearchService < BaseService
   end
 
   def parsed_query
-    SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query))
+    SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query), current_account: @account)
   end
 end

app/workers/add_to_public_statuses_index_worker.rb

@@ -2,13 +2,20 @@
 
 class AddToPublicStatusesIndexWorker
   include Sidekiq::Worker
+  include DatabaseHelper
+
+  sidekiq_options queue: 'pull'
 
   def perform(account_id)
-    account = Account.find(account_id)
+    with_primary do
+      @account = Account.find(account_id)
+    end
 
-    return unless account.indexable?
+    return unless @account.indexable?
 
-    account.add_to_public_statuses_index!
+    with_read_replica do
+      @account.add_to_public_statuses_index!
+    end
   rescue ActiveRecord::RecordNotFound
     true
   end

app/workers/scheduler/indexing_scheduler.rb

@@ -3,6 +3,7 @@
 class Scheduler::IndexingScheduler
   include Sidekiq::Worker
   include Redisable
+  include DatabaseHelper
 
   sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
 
@@ -15,7 +16,10 @@ class Scheduler::IndexingScheduler
     indexes.each do |type|
       with_redis do |redis|
         redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids|
-          type.import!(ids)
+          with_read_replica do
+            type.import!(ids)
+          end
+
           redis.srem("chewy:queue:#{type.name}", ids)
         end
       end