Merge branch 'kb_migration' into kb_development

.devcontainer/Dockerfile

@@ -4,10 +4,6 @@ FROM mcr.microsoft.com/devcontainers/ruby:1-3.2-bullseye
 # Install Rails
 # RUN gem install rails webdrivers
 
-# Default value to allow debug server to serve content over GitHub Codespace's port forwarding service
-# The value is a comma-separated list of allowed domains
-ENV RAILS_DEVELOPMENT_HOSTS=".githubpreview.dev,.preview.app.github.dev,.app.github.dev"
-
 ARG NODE_VERSION="16"
 RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"
 

.devcontainer/codespaces/devcontainer.json

@@ -0,0 +1,49 @@
+{
+  "name": "Mastodon on GitHub Codespaces",
+  "dockerComposeFile": "../docker-compose.yml",
+  "service": "app",
+  "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+
+  "features": {
+    "ghcr.io/devcontainers/features/sshd:1": {}
+  },
+
+  "runServices": ["app", "db", "redis"],
+
+  "forwardPorts": [3000, 4000],
+
+  "portsAttributes": {
+    "3000": {
+      "label": "web",
+      "onAutoForward": "notify"
+    },
+    "4000": {
+      "label": "stream",
+      "onAutoForward": "silent"
+    }
+  },
+
+  "otherPortsAttributes": {
+    "onAutoForward": "silent"
+  },
+
+  "remoteEnv": {
+    "LOCAL_DOMAIN": "${localEnv:CODESPACE_NAME}-3000.app.github.dev",
+    "LOCAL_HTTPS": "true",
+    "STREAMING_API_BASE_URL": "https://${localEnv:CODESPACE_NAME}-4000.app.github.dev",
+    "DISABLE_FORGERY_REQUEST_PROTECTION": "true",
+    "ES_ENABLED": "",
+    "LIBRE_TRANSLATE_ENDPOINT": ""
+  },
+
+  "onCreateCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}",
+  "postCreateCommand": ".devcontainer/post-create.sh",
+  "waitFor": "postCreateCommand",
+
+  "customizations": {
+    "vscode": {
+      "settings": {},
+      "extensions": ["EditorConfig.EditorConfig", "webben.browserslist"]
+    }
+  }
+}

.devcontainer/devcontainer.json

@@ -1,5 +1,5 @@
 {
-  "name": "Mastodon",
+  "name": "Mastodon on local machine",
   "dockerComposeFile": "docker-compose.yml",
   "service": "app",
   "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
@@ -8,13 +8,23 @@
     "ghcr.io/devcontainers/features/sshd:1": {}
   },
 
-  "runServices": ["app", "db", "redis"],
-
   "forwardPorts": [3000, 4000],
 
-  "containerEnv": {
+  "portsAttributes": {
-    "ES_ENABLED": "",
+    "3000": {
-    "LIBRE_TRANSLATE_ENDPOINT": ""
+      "label": "web",
+      "onAutoForward": "notify",
+      "requireLocalPort": true
+    },
+    "4000": {
+      "label": "stream",
+      "onAutoForward": "silent",
+      "requireLocalPort": true
+    }
+  },
+
+  "otherPortsAttributes": {
+    "onAutoForward": "silent"
   },
 
   "onCreateCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}",

.devcontainer/docker-compose.yml

@@ -25,6 +25,7 @@ services:
     command: sleep infinity
     ports:
       - '127.0.0.1:3000:3000'
+      - '127.0.0.1:3035:3035'
       - '127.0.0.1:4000:4000'
     networks:
       - external_network

.github/workflows/build-container-image.yml

@@ -76,8 +76,6 @@ jobs:
         if: ${{ inputs.push_to_images != '' }}
         with:
           images: ${{ inputs.push_to_images }}
-          # Only tag with latest when ran against the latest stable branch
-          # This needs to be updated after each minor version release
           flavor: ${{ inputs.flavor }}
           tags: ${{ inputs.tags }}
           labels: ${{ inputs.labels }}
@@ -85,7 +83,9 @@ jobs:
       - uses: docker/build-push-action@v4
         with:
           context: .
-          build-args: MASTODON_VERSION_PRERELEASE=${{ inputs.version_prerelease }} MASTODON_VERSION_METADATA=${{ inputs.version_metadata }}
+          build-args: |
+            MASTODON_VERSION_PRERELEASE=${{ inputs.version_prerelease }}
+            MASTODON_VERSION_METADATA=${{ inputs.version_metadata }}
           platforms: ${{ inputs.platforms }}
           provenance: false
           builder: ${{ steps.buildx.outputs.name || steps.buildx-native.outputs.name }}

.github/workflows/build-releases.yml

@@ -17,6 +17,8 @@ jobs:
       push_to_images: |
         tootsuite/mastodon
         ghcr.io/mastodon/mastodon
+      # Only tag with latest when ran against the latest stable branch
+      # This needs to be updated after each minor version release
       flavor: |
         latest=${{ startsWith(github.ref, 'refs/tags/v4.1.') }}
       tags: |

Gemfile.lock

@@ -109,7 +109,7 @@ GEM
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-    addressable (2.8.4)
+    addressable (2.8.5)
       public_suffix (>= 2.0.2, < 6.0)
     aes_key_wrap (1.1.0)
     airbrussh (1.4.1)
@@ -124,8 +124,8 @@ GEM
     attr_required (1.0.1)
     awrence (1.2.1)
     aws-eventstream (1.2.0)
-    aws-partitions (1.793.0)
+    aws-partitions (1.809.0)
-    aws-sdk-core (3.180.3)
+    aws-sdk-core (3.181.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.5)
@@ -133,8 +133,8 @@ GEM
     aws-sdk-kms (1.71.0)
       aws-sdk-core (~> 3, >= 3.177.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.132.1)
+    aws-sdk-s3 (1.133.0)
-      aws-sdk-core (~> 3, >= 3.179.0)
+      aws-sdk-core (~> 3, >= 3.181.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.6)
     aws-sigv4 (1.6.0)
@@ -203,7 +203,7 @@ GEM
       activesupport
     cbor (0.5.9.6)
     charlock_holmes (0.7.7)
-    chewy (7.3.3)
+    chewy (7.3.4)
       activesupport (>= 5.2)
       elasticsearch (>= 7.12.0, < 7.14.0)
       elasticsearch-dsl
@@ -324,7 +324,7 @@ GEM
       ruby-progressbar (~> 1.4)
     globalid (1.1.0)
       activesupport (>= 5.0)
-    haml (6.1.1)
+    haml (6.1.2)
       temple (>= 0.8.2)
       thor
       tilt
@@ -333,7 +333,7 @@ GEM
       activesupport (>= 5.1)
       haml (>= 4.0.6)
       railties (>= 5.1)
-    haml_lint (0.49.3)
+    haml_lint (0.50.0)
       haml (>= 4.0, < 6.2)
       parallel (~> 1.10)
       rainbow
@@ -482,7 +482,7 @@ GEM
     nokogiri (1.15.4)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
-    oj (3.16.0)
+    oj (3.16.1)
     omniauth (2.1.1)
       hashie (>= 3.4.6)
       rack (>= 2.2.3)
@@ -519,7 +519,7 @@ GEM
     parslet (2.0.0)
     pastel (0.8.0)
       tty-color (~> 0.5)
-    pg (1.5.3)
+    pg (1.5.4)
     pghero (3.3.3)
       activerecord (>= 6)
     posix-spawn (0.3.15)
@@ -733,7 +733,7 @@ GEM
       net-ssh (>= 2.8.0)
     stackprof (0.2.25)
     statsd-ruby (1.5.0)
-    stoplight (3.0.1)
+    stoplight (3.0.2)
       redlock (~> 1.0)
     strong_migrations (0.8.0)
       activerecord (>= 5.2)
@@ -797,7 +797,7 @@ GEM
     webfinger (1.2.0)
       activesupport
       httpclient (>= 2.4)
-    webmock (3.18.1)
+    webmock (3.19.1)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)

Procfile.dev

@@ -1,4 +1,4 @@
 web: env PORT=3000 RAILS_ENV=development bundle exec puma -C config/puma.rb
 sidekiq: env PORT=3000 RAILS_ENV=development bundle exec sidekiq
 stream: env PORT=4000 yarn run start
-webpack: ./bin/webpack-dev-server --listen-host 0.0.0.0
+webpack: bin/webpack-dev-server

app/chewy/accounts_index.rb

@@ -31,12 +31,13 @@ class AccountsIndex < Chewy::Index
 
     analyzer: {
       natural: {
-        tokenizer: 'uax_url_email',
+        tokenizer: 'standard',
         filter: %w(
-          english_possessive_stemmer
           lowercase
           asciifolding
           cjk_width
+          elision
+          english_possessive_stemmer
           english_stop
           english_stemmer
         ),

app/chewy/public_statuses_index.rb

@@ -20,13 +20,19 @@ class PublicStatusesIndex < Chewy::Index
     },
 
     analyzer: {
-      content: {
+      verbatim: {
         tokenizer: 'uax_url_email',
+        filter: %w(lowercase),
+      },
+
+      content: {
+        tokenizer: 'standard',
         filter: %w(
-          english_possessive_stemmer
           lowercase
           asciifolding
           cjk_width
+          elision
+          english_possessive_stemmer
           english_stop
           english_stemmer
         ),
@@ -74,13 +80,20 @@ class PublicStatusesIndex < Chewy::Index
           english_stemmer
         ),
       },
+
       sudachi_analyzer: {
+        tokenizer: 'sudachi_tokenizer',
+        type: 'custom',
         filter: %w(
+          english_possessive_stemmer
+          lowercase
+          asciifolding
+          cjk_width
+          english_stop
+          english_stemmer
           my_posfilter
           sudachi_normalizedform
         ),
-        type: 'custom',
-        tokenizer: 'sudachi_tokenizer',
       },
     },
     tokenizer: {
@@ -101,7 +114,7 @@ class PublicStatusesIndex < Chewy::Index
                       .includes(:media_attachments, :preloadable_poll, :preview_cards)
 
   root date_detection: false do
-    field(:id, type: 'keyword')
+    field(:id, type: 'long')
     field(:account_id, type: 'long')
     field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
     field(:language, type: 'keyword')

app/chewy/statuses_index.rb

@@ -19,13 +19,19 @@ class StatusesIndex < Chewy::Index
       },
     },
     analyzer: {
-      content: {
+      verbatim: {
         tokenizer: 'uax_url_email',
+        filter: %w(lowercase),
+      },
+
+      content: {
+        tokenizer: 'standard',
         filter: %w(
-          english_possessive_stemmer
           lowercase
           asciifolding
           cjk_width
+          elision
+          english_possessive_stemmer
           english_stop
           english_stemmer
         ),
@@ -61,6 +67,11 @@ class StatusesIndex < Chewy::Index
       },
     },
     analyzer: {
+      verbatim: {
+        tokenizer: 'uax_url_email',
+        filter: %w(lowercase),
+      },
+
       content: {
         tokenizer: 'uax_url_email',
         filter: %w(
@@ -72,13 +83,20 @@ class StatusesIndex < Chewy::Index
           english_stemmer
         ),
       },
+
       sudachi_analyzer: {
+        tokenizer: 'sudachi_tokenizer',
+        type: 'custom',
         filter: %w(
+          english_possessive_stemmer
+          lowercase
+          asciifolding
+          cjk_width
+          english_stop
+          english_stemmer
           my_posfilter
           sudachi_normalizedform
         ),
-        type: 'custom',
-        tokenizer: 'sudachi_tokenizer',
       },
     },
     tokenizer: {
@@ -93,50 +111,30 @@ class StatusesIndex < Chewy::Index
 
   settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: PRODUCTION_SETTINGS
 
-  # We do not use delete_if option here because it would call a method that we
+  index_scope ::Status.unscoped.kept.without_reblogs.includes(
-  # expect to be called with crutches without crutches, causing n+1 queries
+    :media_attachments,
-  index_scope ::Status.unscoped.kept.without_reblogs.includes(:media_attachments, :preloadable_poll, :preview_cards)
+    :preview_cards,
-
+    :local_mentioned,
-  crutch :mentions do |collection|
+    :local_favorited,
-    data = ::Mention.where(status_id: collection.map(&:id)).where(account: Account.local, silent: false).pluck(:status_id, :account_id)
+    :local_reblogged,
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
+    :local_bookmarked,
-  end
+    :local_emoji_reacted,
-
+    :local_referenced,
-  crutch :favourites do |collection|
+    preloadable_poll: :local_voters
-    data = ::Favourite.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
+  ),
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
+              delete_if: lambda { |status|
-  end
+                           if status.searchability == 'direct'
-
+                             status.searchable_by.empty?
-  crutch :emoji_reactions do |collection|
+                           else
-    data = ::EmojiReaction.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
+                             status.searchability == 'limited' ? status.domain.present? : false
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
+                           end
-  end
+                         }
-
-  crutch :status_references do |collection|
-    data = ::StatusReference.joins(:status).where(target_status_id: collection.map(&:id), status: { account: Account.local }).pluck(:target_status_id, :account_id)
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-  end
-
-  crutch :reblogs do |collection|
-    data = ::Status.where(reblog_of_id: collection.map(&:id)).where(account: Account.local).pluck(:reblog_of_id, :account_id)
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-  end
-
-  crutch :bookmarks do |collection|
-    data = ::Bookmark.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-  end
-
-  crutch :votes do |collection|
-    data = ::PollVote.joins(:poll).where(poll: { status_id: collection.map(&:id) }).where(account: Account.local).pluck(:status_id, :account_id)
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-  end
 
   root date_detection: false do
-    field(:id, type: 'keyword')
+    field(:id, type: 'long')
     field(:account_id, type: 'long')
-    field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
+    field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'sudachi_analyzer') }
-    field(:searchable_by, type: 'long', value: ->(status, crutches) { status.searchable_by(crutches) })
+    field(:searchable_by, type: 'long', value: ->(status) { status.searchable_by })
     field(:searchability, type: 'keyword', value: ->(status) { status.compute_searchability })
     field(:language, type: 'keyword')
     field(:domain, type: 'keyword', value: ->(status) { status.account.domain || '' })

app/controllers/api/v1/statuses/translations_controller.rb

@@ -8,7 +8,15 @@ class Api::V1::Statuses::TranslationsController < Api::BaseController
   before_action :set_translation
 
   rescue_from TranslationService::NotConfiguredError, with: :not_found
-  rescue_from TranslationService::UnexpectedResponseError, TranslationService::QuotaExceededError, TranslationService::TooManyRequestsError, with: :service_unavailable
+  rescue_from TranslationService::UnexpectedResponseError, with: :service_unavailable
+
+  rescue_from TranslationService::QuotaExceededError do
+    render json: { error: I18n.t('translation.errors.quota_exceeded') }, status: 503
+  end
+
+  rescue_from TranslationService::TooManyRequestsError do
+    render json: { error: I18n.t('translation.errors.too_many_requests') }, status: 503
+  end
 
   def create
     render json: @translation, serializer: REST::TranslationSerializer

app/controllers/api/v1/timelines/tag_controller.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 class Api::V1::Timelines::TagController < Api::BaseController
+  before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: :show, if: :require_auth?
   before_action :load_tag
   after_action :insert_pagination_headers, unless: -> { @statuses.empty? }
 
@@ -12,6 +13,10 @@ class Api::V1::Timelines::TagController < Api::BaseController
 
   private
 
+  def require_auth?
+    !Setting.timeline_preview
+  end
+
   def load_tag
     @tag = Tag.find_normalized(params[:id])
   end

app/controllers/concerns/signature_verification.rb

@@ -119,6 +119,8 @@ module SignatureVerification
   private
 
   def fail_with!(message, **options)
+    Rails.logger.warn { "Signature verification failed: #{message}" }
+
     @signature_verification_failure_reason = { error: message }.merge(options)
     @signed_request_actor = nil
   end

app/controllers/settings/privacy_extra_controller.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class Settings::PrivacyExtraController < Settings::BaseController
+  before_action :set_account
+
+  def show; end
+
+  def update
+    if UpdateAccountService.new.call(@account, account_params.except(:settings))
+      current_user.update!(settings_attributes: account_params[:settings])
+      ActivityPub::UpdateDistributionWorker.perform_async(@account.id)
+      redirect_to settings_privacy_extra_path, notice: I18n.t('generic.changes_saved_msg')
+    else
+      render :show
+    end
+  end
+
+  private
+
+  def account_params
+    params.require(:account).permit(settings: UserSettings.keys)
+  end
+
+  def set_account
+    @account = current_account
+  end
+end

app/javascript/mastodon/actions/interactions.js

@@ -1,11 +1,16 @@
-import api from '../api';
+import api, { getLinks } from '../api';
 
+import { fetchRelationships } from './accounts';
 import { importFetchedAccounts, importFetchedStatus, importFetchedStatuses } from './importer';
 
 export const REBLOG_REQUEST = 'REBLOG_REQUEST';
 export const REBLOG_SUCCESS = 'REBLOG_SUCCESS';
 export const REBLOG_FAIL    = 'REBLOG_FAIL';
 
+export const REBLOGS_EXPAND_REQUEST = 'REBLOGS_EXPAND_REQUEST';
+export const REBLOGS_EXPAND_SUCCESS = 'REBLOGS_EXPAND_SUCCESS';
+export const REBLOGS_EXPAND_FAIL = 'REBLOGS_EXPAND_FAIL';
+
 export const FAVOURITE_REQUEST = 'FAVOURITE_REQUEST';
 export const FAVOURITE_SUCCESS = 'FAVOURITE_SUCCESS';
 export const FAVOURITE_FAIL    = 'FAVOURITE_FAIL';
@@ -34,6 +39,10 @@ export const FAVOURITES_FETCH_REQUEST = 'FAVOURITES_FETCH_REQUEST';
 export const FAVOURITES_FETCH_SUCCESS = 'FAVOURITES_FETCH_SUCCESS';
 export const FAVOURITES_FETCH_FAIL    = 'FAVOURITES_FETCH_FAIL';
 
+export const FAVOURITES_EXPAND_REQUEST = 'FAVOURITES_EXPAND_REQUEST';
+export const FAVOURITES_EXPAND_SUCCESS = 'FAVOURITES_EXPAND_SUCCESS';
+export const FAVOURITES_EXPAND_FAIL    = 'FAVOURITES_EXPAND_FAIL';
+
 export const STATUS_REFERENCES_FETCH_REQUEST = 'STATUS_REFERENCES_FETCH_REQUEST';
 export const STATUS_REFERENCES_FETCH_SUCCESS = 'STATUS_REFERENCES_FETCH_SUCCESS';
 export const STATUS_REFERENCES_FETCH_FAIL    = 'STATUS_REFERENCES_FETCH_FAIL';
@@ -374,8 +383,10 @@ export function fetchReblogs(id) {
     dispatch(fetchReblogsRequest(id));
 
     api(getState).get(`/api/v1/statuses/${id}/reblogged_by`).then(response => {
+      const next = getLinks(response).refs.find(link => link.rel === 'next');
       dispatch(importFetchedAccounts(response.data));
-      dispatch(fetchReblogsSuccess(id, response.data));
+      dispatch(fetchReblogsSuccess(id, response.data, next ? next.uri : null));
+      dispatch(fetchRelationships(response.data.map(item => item.id)));
     }).catch(error => {
       dispatch(fetchReblogsFail(id, error));
     });
@@ -389,17 +400,62 @@ export function fetchReblogsRequest(id) {
   };
 }
 
-export function fetchReblogsSuccess(id, accounts) {
+export function fetchReblogsSuccess(id, accounts, next) {
   return {
     type: REBLOGS_FETCH_SUCCESS,
     id,
     accounts,
+    next,
   };
 }
 
 export function fetchReblogsFail(id, error) {
   return {
     type: REBLOGS_FETCH_FAIL,
+    id,
+    error,
+  };
+}
+
+export function expandReblogs(id) {
+  return (dispatch, getState) => {
+    const url = getState().getIn(['user_lists', 'reblogged_by', id, 'next']);
+    if (url === null) {
+      return;
+    }
+
+    dispatch(expandReblogsRequest(id));
+
+    api(getState).get(url).then(response => {
+      const next = getLinks(response).refs.find(link => link.rel === 'next');
+
+      dispatch(importFetchedAccounts(response.data));
+      dispatch(expandReblogsSuccess(id, response.data, next ? next.uri : null));
+      dispatch(fetchRelationships(response.data.map(item => item.id)));
+    }).catch(error => dispatch(expandReblogsFail(id, error)));
+  };
+}
+
+export function expandReblogsRequest(id) {
+  return {
+    type: REBLOGS_EXPAND_REQUEST,
+    id,
+  };
+}
+
+export function expandReblogsSuccess(id, accounts, next) {
+  return {
+    type: REBLOGS_EXPAND_SUCCESS,
+    id,
+    accounts,
+    next,
+  };
+}
+
+export function expandReblogsFail(id, error) {
+  return {
+    type: REBLOGS_EXPAND_FAIL,
+    id,
     error,
   };
 }
@@ -409,8 +465,10 @@ export function fetchFavourites(id) {
     dispatch(fetchFavouritesRequest(id));
 
     api(getState).get(`/api/v1/statuses/${id}/favourited_by`).then(response => {
+      const next = getLinks(response).refs.find(link => link.rel === 'next');
       dispatch(importFetchedAccounts(response.data));
-      dispatch(fetchFavouritesSuccess(id, response.data));
+      dispatch(fetchFavouritesSuccess(id, response.data, next ? next.uri : null));
+      dispatch(fetchRelationships(response.data.map(item => item.id)));
     }).catch(error => {
       dispatch(fetchFavouritesFail(id, error));
     });
@@ -424,17 +482,62 @@ export function fetchFavouritesRequest(id) {
   };
 }
 
-export function fetchFavouritesSuccess(id, accounts) {
+export function fetchFavouritesSuccess(id, accounts, next) {
   return {
     type: FAVOURITES_FETCH_SUCCESS,
     id,
     accounts,
+    next,
   };
 }
 
 export function fetchFavouritesFail(id, error) {
   return {
     type: FAVOURITES_FETCH_FAIL,
+    id,
+    error,
+  };
+}
+
+export function expandFavourites(id) {
+  return (dispatch, getState) => {
+    const url = getState().getIn(['user_lists', 'favourited_by', id, 'next']);
+    if (url === null) {
+      return;
+    }
+
+    dispatch(expandFavouritesRequest(id));
+
+    api(getState).get(url).then(response => {
+      const next = getLinks(response).refs.find(link => link.rel === 'next');
+
+      dispatch(importFetchedAccounts(response.data));
+      dispatch(expandFavouritesSuccess(id, response.data, next ? next.uri : null));
+      dispatch(fetchRelationships(response.data.map(item => item.id)));
+    }).catch(error => dispatch(expandFavouritesFail(id, error)));
+  };
+}
+
+export function expandFavouritesRequest(id) {
+  return {
+    type: FAVOURITES_EXPAND_REQUEST,
+    id,
+  };
+}
+
+export function expandFavouritesSuccess(id, accounts, next) {
+  return {
+    type: FAVOURITES_EXPAND_SUCCESS,
+    id,
+    accounts,
+    next,
+  };
+}
+
+export function expandFavouritesFail(id, error) {
+  return {
+    type: FAVOURITES_EXPAND_FAIL,
+    id,
     error,
   };
 }

app/javascript/mastodon/components/hashtag_bar.tsx

@@ -10,8 +10,8 @@ import { groupBy, minBy } from 'lodash';
 
 import { getStatusContent } from './status_content';
 
-// About two lines on desktop
+// Fit on a single line on desktop
-const VISIBLE_HASHTAGS = 7;
+const VISIBLE_HASHTAGS = 3;
 
 // Those types are not correct, they need to be replaced once this part of the state is typed
 export type TagLike = Record<{ name: string }>;
@@ -210,7 +210,7 @@ const HashtagBar: React.FC<{
 
   const revealedHashtags = expanded
     ? hashtags
-    : hashtags.slice(0, VISIBLE_HASHTAGS - 1);
+    : hashtags.slice(0, VISIBLE_HASHTAGS);
 
   return (
     <div className='hashtag-bar'>

app/javascript/mastodon/components/status_content.jsx

@@ -241,7 +241,12 @@ class StatusContent extends PureComponent {
     const renderReadMore = this.props.onClick && status.get('collapsed');
     const contentLocale = intl.locale.replace(/[_-].*/, '');
     const targetLanguages = this.props.languages?.get(status.get('language') || 'und');
-    const renderTranslate = this.props.onTranslate && this.context.identity.signedIn && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('search_index').trim().length > 0 && status.get('language') && targetLanguages?.includes(contentLocale);
+    const renderTranslate = this.props.onTranslate &&
+        this.context.identity.signedIn &&
+        (['public', 'unlisted'].includes(status.get('visibility')) || status.getIn(['account', 'other_settings', 'translatable_private'])) &&
+        status.get('search_index').trim().length > 0 &&
+        status.get('language') &&
+        targetLanguages?.includes(contentLocale);
 
     const content = { __html: statusContent ?? getStatusContent(status) };
     const spoilerContent = { __html: status.getIn(['translation', 'spoilerHtml']) || status.get('spoilerHtml') };

app/javascript/mastodon/features/antennas/index.jsx

@@ -12,7 +12,6 @@ import { createSelector } from 'reselect';
 import { fetchAntennas } from 'mastodon/actions/antennas';
 import Column from 'mastodon/components/column';
 import ColumnHeader from 'mastodon/components/column_header';
-import { Icon }  from 'mastodon/components/icon';
 import { LoadingIndicator } from 'mastodon/components/loading_indicator';
 import ScrollableList from 'mastodon/components/scrollable_list';
 import ColumnLink from 'mastodon/features/ui/components/column_link';
@@ -23,6 +22,8 @@ import NewAntennaForm from './components/new_antenna_form';
 const messages = defineMessages({
   heading: { id: 'column.antennas', defaultMessage: 'Antennas' },
   subheading: { id: 'antennas.subheading', defaultMessage: 'Your antennas' },
+  insert_list: { id: 'antennas.insert_list', defaultMessage: 'List' },
+  insert_home: { id: 'antennas.insert_home', defaultMessage: 'Home' },
 });
 
 const getOrderedAntennas = createSelector([state => state.get('antennas')], antennas => {
@@ -77,14 +78,8 @@ class Antennas extends ImmutablePureComponent {
           bindToDocument={!multiColumn}
         >
           {antennas.map(antenna => (
-            <ColumnLink key={antenna.get('id')} to={`/antennast/${antenna.get('id')}`} icon='wifi' text={antenna.get('title')}>
+            <ColumnLink key={antenna.get('id')} to={`/antennast/${antenna.get('id')}`} icon='wifi' text={antenna.get('title')}
-              <span className='antenna-list-detail'>
+                        badge={antenna.get('insert_feeds') ? intl.formatMessage(antenna.get('list') ? messages.insert_list : messages.insert_home) : undefined} />
-                <span className='group'><Icon id='users' />{antenna.get('accounts_count')}</span>
-                <span className='group'><Icon id='sitemap' />{antenna.get('domains_count')}</span>
-                <span className='group'><Icon id='hashtag' />{antenna.get('tags_count')}</span>
-                <span className='group'><Icon id='paragraph' />{antenna.get('keywords_count')}</span>
-              </span>
-            </ColumnLink>
           ))}
         </ScrollableList>
 

app/javascript/mastodon/features/bookmark_category_adder/components/bookmark_category.jsx

@@ -55,7 +55,7 @@ class BookmarkCategory extends ImmutablePureComponent {
       <div className='list'>
         <div className='list__wrapper'>
           <div className='list__display-name'>
-            <Icon id='user-bookmarkCategory' className='column-link__icon' fixedWidth />
+            <Icon id='bookmark' className='column-link__icon' fixedWidth />
             {bookmarkCategory.get('title')}
           </div>
 

app/javascript/mastodon/features/compose/components/search.jsx

@@ -1,7 +1,7 @@
 import PropTypes from 'prop-types';
 import { PureComponent } from 'react';
 
-import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
+import { defineMessages, injectIntl, FormattedMessage, FormattedList } from 'react-intl';
 
 import classNames from 'classnames';
 
@@ -45,6 +45,17 @@ class Search extends PureComponent {
     options: [],
   };
 
+  defaultOptions = [
+    { label: <><mark>has:</mark> <FormattedList type='disjunction' value={['media', 'poll', 'embed']} /></>, action: e => { e.preventDefault(); this._insertText('has:') } },
+    { label: <><mark>is:</mark> <FormattedList type='disjunction' value={['reply', 'sensitive']} /></>, action: e => { e.preventDefault(); this._insertText('is:') } },
+    { label: <><mark>language:</mark> <FormattedMessage id='search_popout.language_code' defaultMessage='ISO language code' /></>, action: e => { e.preventDefault(); this._insertText('language:') } },
+    { label: <><mark>from:</mark> <FormattedMessage id='search_popout.user' defaultMessage='user' /></>, action: e => { e.preventDefault(); this._insertText('from:') } },
+    { label: <><mark>domain:</mark> <FormattedMessage id='search_popout.domain' defaultMessage='domain' /></>, action: e => { e.preventDefault(); this._insertText('domain:') } },
+    { label: <><mark>before:</mark> <FormattedMessage id='search_popout.specific_date' defaultMessage='specific date' /></>, action: e => { e.preventDefault(); this._insertText('before:') } },
+    { label: <><mark>during:</mark> <FormattedMessage id='search_popout.specific_date' defaultMessage='specific date' /></>, action: e => { e.preventDefault(); this._insertText('during:') } },
+    { label: <><mark>after:</mark> <FormattedMessage id='search_popout.specific_date' defaultMessage='specific date' /></>, action: e => { e.preventDefault(); this._insertText('after:') } },
+  ];
+
   setRef = c => {
     this.searchForm = c;
   };
@@ -70,7 +81,7 @@ class Search extends PureComponent {
 
   handleKeyDown = (e) => {
     const { selectedOption } = this.state;
-    const options = this._getOptions();
+    const options = this._getOptions().concat(this.defaultOptions);
 
     switch(e.key) {
     case 'Escape':
@@ -100,11 +111,9 @@ class Search extends PureComponent {
       if (selectedOption === -1) {
         this._submit();
       } else if (options.length > 0) {
-        options[selectedOption].action();
+        options[selectedOption].action(e);
       }
 
-      this._unfocus();
-
       break;
     case 'Delete':
       if (selectedOption > -1 && options.length > 0) {
@@ -147,6 +156,7 @@ class Search extends PureComponent {
 
     router.history.push(`/tags/${query}`);
     onClickSearchResult(query, 'hashtag');
+    this._unfocus();
   };
 
   handleAccountClick = () => {
@@ -157,6 +167,7 @@ class Search extends PureComponent {
 
     router.history.push(`/@${query}`);
     onClickSearchResult(query, 'account');
+    this._unfocus();
   };
 
   handleURLClick = () => {
@@ -164,6 +175,7 @@ class Search extends PureComponent {
     const { value, onOpenURL } = this.props;
 
     onOpenURL(value, router.history);
+    this._unfocus();
   };
 
   handleStatusSearch = () => {
@@ -182,6 +194,8 @@ class Search extends PureComponent {
     } else if (search.get('type') === 'hashtag') {
       router.history.push(`/tags/${search.get('q')}`);
     }
+
+    this._unfocus();
   };
 
   handleForgetRecentSearchClick = search => {
@@ -194,6 +208,18 @@ class Search extends PureComponent {
     document.querySelector('.ui').parentElement.focus();
   }
 
+  _insertText (text) {
+    const { value, onChange } = this.props;
+
+    if (value === '') {
+      onChange(text);
+    } else if (value[value.length - 1] === ' ') {
+      onChange(`${value}${text}`);
+    } else {
+      onChange(`${value} ${text}`);
+    }
+  }
+
   _submit (type) {
     const { onSubmit, openInRoute } = this.props;
     const { router } = this.context;
@@ -203,6 +229,8 @@ class Search extends PureComponent {
     if (openInRoute) {
       router.history.push('/search');
     }
+
+    this._unfocus();
   }
 
   _getOptions () {
@@ -325,6 +353,16 @@ class Search extends PureComponent {
               </div>
             </>
           )}
+
+          <h4><FormattedMessage id='search_popout.options' defaultMessage='Search options' /></h4>
+
+          <div className='search__popout__menu'>
+            {this.defaultOptions.map(({ key, label, action }, i) => (
+              <button key={key} onMouseDown={action} className={classNames('search__popout__menu__item', { selected: selectedOption === (options.length + i) })}>
+                {label}
+              </button>
+            ))}
+          </div>
         </div>
       </div>
     );

app/javascript/mastodon/features/favourites/index.jsx

@@ -8,7 +8,9 @@ import ImmutablePropTypes from 'react-immutable-proptypes';
 import ImmutablePureComponent from 'react-immutable-pure-component';
 import { connect } from 'react-redux';
 
-import { fetchFavourites } from 'mastodon/actions/interactions';
+import { debounce } from 'lodash';
+
+import { fetchFavourites, expandFavourites } from 'mastodon/actions/interactions';
 import ColumnHeader from 'mastodon/components/column_header';
 import { Icon }  from 'mastodon/components/icon';
 import { LoadingIndicator } from 'mastodon/components/loading_indicator';
@@ -21,7 +23,9 @@ const messages = defineMessages({
 });
 
 const mapStateToProps = (state, props) => ({
-  accountIds: state.getIn(['user_lists', 'favourited_by', props.params.statusId]),
+  accountIds: state.getIn(['user_lists', 'favourited_by', props.params.statusId, 'items']),
+  hasMore: !!state.getIn(['user_lists', 'favourited_by', props.params.statusId, 'next']),
+  isLoading: state.getIn(['user_lists', 'favourited_by', props.params.statusId, 'isLoading'], true),
 });
 
 class Favourites extends ImmutablePureComponent {
@@ -30,6 +34,8 @@ class Favourites extends ImmutablePureComponent {
     params: PropTypes.object.isRequired,
     dispatch: PropTypes.func.isRequired,
     accountIds: ImmutablePropTypes.list,
+    hasMore: PropTypes.bool,
+    isLoading: PropTypes.bool,
     multiColumn: PropTypes.bool,
     intl: PropTypes.object.isRequired,
   };
@@ -40,18 +46,16 @@ class Favourites extends ImmutablePureComponent {
     }
   }
 
-  UNSAFE_componentWillReceiveProps (nextProps) {
-    if (nextProps.params.statusId !== this.props.params.statusId && nextProps.params.statusId) {
-      this.props.dispatch(fetchFavourites(nextProps.params.statusId));
-    }
-  }
-
   handleRefresh = () => {
     this.props.dispatch(fetchFavourites(this.props.params.statusId));
   };
 
+  handleLoadMore = debounce(() => {
+    this.props.dispatch(expandFavourites(this.props.params.statusId));
+  }, 300, { leading: true });
+
   render () {
-    const { intl, accountIds, multiColumn } = this.props;
+    const { intl, accountIds, hasMore, isLoading, multiColumn } = this.props;
 
     if (!accountIds) {
       return (
@@ -75,6 +79,9 @@ class Favourites extends ImmutablePureComponent {
 
         <ScrollableList
           scrollKey='favourites'
+          onLoadMore={this.handleLoadMore}
+          hasMore={hasMore}
+          isLoading={isLoading}
           emptyMessage={emptyMessage}
           bindToDocument={!multiColumn}
         >

app/javascript/mastodon/features/reblogs/index.jsx

@@ -8,9 +8,11 @@ import ImmutablePropTypes from 'react-immutable-proptypes';
 import ImmutablePureComponent from 'react-immutable-pure-component';
 import { connect } from 'react-redux';
 
+import { debounce } from 'lodash';
+
 import { Icon }  from 'mastodon/components/icon';
 
-import { fetchReblogs } from '../../actions/interactions';
+import { fetchReblogs, expandReblogs } from '../../actions/interactions';
 import ColumnHeader from '../../components/column_header';
 import { LoadingIndicator } from '../../components/loading_indicator';
 import ScrollableList from '../../components/scrollable_list';
@@ -22,7 +24,9 @@ const messages = defineMessages({
 });
 
 const mapStateToProps = (state, props) => ({
-  accountIds: state.getIn(['user_lists', 'reblogged_by', props.params.statusId]),
+  accountIds: state.getIn(['user_lists', 'reblogged_by', props.params.statusId, 'items']),
+  hasMore: !!state.getIn(['user_lists', 'reblogged_by', props.params.statusId, 'next']),
+  isLoading: state.getIn(['user_lists', 'reblogged_by', props.params.statusId, 'isLoading'], true),
 });
 
 class Reblogs extends ImmutablePureComponent {
@@ -31,6 +35,8 @@ class Reblogs extends ImmutablePureComponent {
     params: PropTypes.object.isRequired,
     dispatch: PropTypes.func.isRequired,
     accountIds: ImmutablePropTypes.list,
+    hasMore: PropTypes.bool,
+    isLoading: PropTypes.bool,
     multiColumn: PropTypes.bool,
     intl: PropTypes.object.isRequired,
   };
@@ -39,20 +45,18 @@ class Reblogs extends ImmutablePureComponent {
     if (!this.props.accountIds) {
       this.props.dispatch(fetchReblogs(this.props.params.statusId));
     }
-  }
+  };
-
-  UNSAFE_componentWillReceiveProps(nextProps) {
-    if (nextProps.params.statusId !== this.props.params.statusId && nextProps.params.statusId) {
-      this.props.dispatch(fetchReblogs(nextProps.params.statusId));
-    }
-  }
 
   handleRefresh = () => {
     this.props.dispatch(fetchReblogs(this.props.params.statusId));
   };
 
+  handleLoadMore = debounce(() => {
+    this.props.dispatch(expandReblogs(this.props.params.statusId));
+  }, 300, { leading: true });
+
   render () {
-    const { intl, accountIds, multiColumn } = this.props;
+    const { intl, accountIds, hasMore, isLoading, multiColumn } = this.props;
 
     if (!accountIds) {
       return (
@@ -76,6 +80,9 @@ class Reblogs extends ImmutablePureComponent {
 
         <ScrollableList
           scrollKey='reblogs'
+          onLoadMore={this.handleLoadMore}
+          hasMore={hasMore}
+          isLoading={isLoading}
           emptyMessage={emptyMessage}
           bindToDocument={!multiColumn}
         >

app/javascript/mastodon/features/status/components/action_bar.jsx

@@ -23,11 +23,13 @@ const messages = defineMessages({
   mention: { id: 'status.mention', defaultMessage: 'Mention @{name}' },
   reply: { id: 'status.reply', defaultMessage: 'Reply' },
   reblog: { id: 'status.reblog', defaultMessage: 'Boost' },
+  cancel_reblog: { id: 'status.cancel_reblog_private', defaultMessage: 'Unboost' },
   reblog_private: { id: 'status.reblog_private', defaultMessage: 'Boost with original visibility' },
   cancel_reblog_private: { id: 'status.cancel_reblog_private', defaultMessage: 'Unboost' },
   cannot_reblog: { id: 'status.cannot_reblog', defaultMessage: 'This post cannot be boosted' },
   favourite: { id: 'status.favourite', defaultMessage: 'Favorite' },
   bookmark: { id: 'status.bookmark', defaultMessage: 'Bookmark' },
+  bookmark_category: { id: 'status.bookmark_category', defaultMessage: 'Bookmark category' },
   more: { id: 'status.more', defaultMessage: 'More' },
   mute: { id: 'status.mute', defaultMessage: 'Mute @{name}' },
   muteConversation: { id: 'status.mute_conversation', defaultMessage: 'Mute conversation' },
@@ -114,6 +116,10 @@ class ActionBar extends PureComponent {
     }
   };
 
+  handleBookmarkCategoryAdderClick = () => {
+    this.props.onBookmarkCategoryAdder(this.props.status);
+  };
+
   handleDeleteClick = () => {
     this.props.onDelete(this.props.status, this.context.router.history);
   };
@@ -237,11 +243,12 @@ class ActionBar extends PureComponent {
 
     if (signedIn) {
       menu.push(null);
-      menu.push({ text: intl.formatMessage(messages.reblog), action: this.handleReblogForceModalClick });
+      menu.push({ text: intl.formatMessage(status.get('reblogged') ? messages.cancel_reblog : messages.reblog), action: this.handleReblogForceModalClick });
 
       if (publicStatus) {
         menu.push({ text: intl.formatMessage(messages.reference), action: this.handleReference });
       }
+      menu.push({ text: intl.formatMessage(messages.bookmark_category), action: this.handleBookmarkCategoryAdderClick });
 
       if (writtenByMe) {
         if (pinnableStatus) {

app/javascript/mastodon/features/ui/components/navigation_panel.jsx

@@ -34,6 +34,7 @@ const messages = defineMessages({
   about: { id: 'navigation_bar.about', defaultMessage: 'About' },
   search: { id: 'navigation_bar.search', defaultMessage: 'Search' },
   advancedInterface: { id: 'navigation_bar.advanced_interface', defaultMessage: 'Open in advanced web interface' },
+  openedInClassicInterface: { id: 'navigation_bar.opened_in_classic_interface', defaultMessage: 'Posts, accounts, and other specific pages are opened by default in the classic web interface.' },
 });
 
 class NavigationPanel extends Component {
@@ -70,12 +71,17 @@ class NavigationPanel extends Component {
         <div className='navigation-panel__logo'>
           <Link to='/' className='column-link column-link--logo'><WordmarkLogo /></Link>
 
-          {transientSingleColumn && (
+          {transientSingleColumn ? (
-            <a href={`/deck${location.pathname}`} className='button button--block'>
+            <div class='switch-to-advanced'>
-              {intl.formatMessage(messages.advancedInterface)}
+              {intl.formatMessage(messages.openedInClassicInterface)}
-            </a>
+              {" "}
+              <a href={`/deck${location.pathname}`} class='switch-to-advanced__toggle'>
+                {intl.formatMessage(messages.advancedInterface)}
+              </a>
+            </div>
+          ) : (
+            <hr />
           )}
-          <hr />
         </div>
 
         {signedIn && (

app/javascript/mastodon/locales/en.json

@@ -421,6 +421,7 @@
   "navigation_bar.lists": "Lists",
   "navigation_bar.logout": "Logout",
   "navigation_bar.mutes": "Muted users",
+  "navigation_bar.opened_in_classic_interface": "Posts, accounts, and other specific pages are opened by default in the classic web interface.",
   "navigation_bar.personal": "Personal",
   "navigation_bar.pins": "Pinned posts",
   "navigation_bar.preferences": "Preferences",
@@ -617,8 +618,12 @@
   "searchability.public.short": "Public",
   "searchability.unlisted.long": "Your followers and reactionners can find",
   "searchability.unlisted.short": "Followers and reactionners",
+  "search_popout.language_code": "ISO language code",
+  "search_popout.options": "Search options",
   "search_popout.quick_actions": "Quick actions",
   "search_popout.recent": "Recent searches",
+  "search_popout.specific_date": "specific date",
+  "search_popout.user": "user",
   "search_results.accounts": "Profiles",
   "search_results.all": "All",
   "search_results.hashtags": "Hashtags",

app/javascript/mastodon/locales/fr.json

@@ -409,6 +409,7 @@
   "navigation_bar.lists": "Listes",
   "navigation_bar.logout": "Déconnexion",
   "navigation_bar.mutes": "Comptes masqués",
+  "navigation_bar.opened_in_classic_interface": "Les messages, les comptes et les pages spécifiques sont ouvertes dans l’interface classique.",
   "navigation_bar.personal": "Personnel",
   "navigation_bar.pins": "Messages épinglés",
   "navigation_bar.preferences": "Préférences",

app/javascript/mastodon/reducers/user_lists.js

@@ -45,8 +45,18 @@ import {
   BLOCKS_EXPAND_FAIL,
 } from '../actions/blocks';
 import {
+  REBLOGS_FETCH_REQUEST,
   REBLOGS_FETCH_SUCCESS,
+  REBLOGS_FETCH_FAIL,
+  REBLOGS_EXPAND_REQUEST,
+  REBLOGS_EXPAND_SUCCESS,
+  REBLOGS_EXPAND_FAIL,
+  FAVOURITES_FETCH_REQUEST,
   FAVOURITES_FETCH_SUCCESS,
+  FAVOURITES_FETCH_FAIL,
+  FAVOURITES_EXPAND_REQUEST,
+  FAVOURITES_EXPAND_SUCCESS,
+  FAVOURITES_EXPAND_FAIL,
   EMOJI_REACTIONS_FETCH_SUCCESS,
   STATUS_REFERENCES_FETCH_SUCCESS,
 } from '../actions/interactions';
@@ -138,9 +148,25 @@ export default function userLists(state = initialState, action) {
   case FOLLOWING_EXPAND_FAIL:
     return state.setIn(['following', action.id, 'isLoading'], false);
   case REBLOGS_FETCH_SUCCESS:
-    return state.setIn(['reblogged_by', action.id], ImmutableList(action.accounts.map(item => item.id)));
+    return normalizeList(state, ['reblogged_by', action.id], action.accounts, action.next);
+  case REBLOGS_EXPAND_SUCCESS:
+    return appendToList(state, ['reblogged_by', action.id], action.accounts, action.next);
+  case REBLOGS_FETCH_REQUEST:
+  case REBLOGS_EXPAND_REQUEST:
+    return state.setIn(['reblogged_by', action.id, 'isLoading'], true);
+  case REBLOGS_FETCH_FAIL:
+  case REBLOGS_EXPAND_FAIL:
+    return state.setIn(['reblogged_by', action.id, 'isLoading'], false);
   case FAVOURITES_FETCH_SUCCESS:
-    return state.setIn(['favourited_by', action.id], ImmutableList(action.accounts.map(item => item.id)));
+    return normalizeList(state, ['favourited_by', action.id], action.accounts, action.next);
+  case FAVOURITES_EXPAND_SUCCESS:
+    return appendToList(state, ['favourited_by', action.id], action.accounts, action.next);
+  case FAVOURITES_FETCH_REQUEST:
+  case FAVOURITES_EXPAND_REQUEST:
+    return state.setIn(['favourited_by', action.id, 'isLoading'], true);
+  case FAVOURITES_FETCH_FAIL:
+  case FAVOURITES_EXPAND_FAIL:
+    return state.setIn(['favourited_by', action.id, 'isLoading'], false);
   case EMOJI_REACTIONS_FETCH_SUCCESS:
     return state.setIn(['emoji_reactioned_by', action.id], ImmutableList(action.accounts));
   case STATUS_REFERENCES_FETCH_SUCCESS:

app/javascript/mastodon/test_setup.js

@@ -1 +1 @@
-import '@testing-library/jest-dom/extend-expect';
+import '@testing-library/jest-dom';

app/javascript/packs/public.jsx

@@ -7,8 +7,6 @@ import { defineMessages } from 'react-intl';
 
 import { delegate }  from '@rails/ujs';
 import axios from 'axios';
-import escapeTextContentForBrowser from 'escape-html';
-import { createBrowserHistory }  from 'history';
 import { throttle } from 'lodash';
 
 import { start } from '../mastodon/common';
@@ -48,23 +46,6 @@ window.addEventListener('message', e => {
 function loaded() {
   const { messages: localeData } = getLocale();
 
-  const scrollToDetailedStatus = () => {
-    const history = createBrowserHistory();
-    const detailedStatuses = document.querySelectorAll('.public-layout .detailed-status');
-    const location = history.location;
-
-    if (detailedStatuses.length === 1 && (!location.state || !location.state.scrolledToDetailedStatus)) {
-      detailedStatuses[0].scrollIntoView();
-      history.replace(location.pathname, { ...location.state, scrolledToDetailedStatus: true });
-    }
-  };
-
-  const getEmojiAnimationHandler = (swapTo) => {
-    return ({ target }) => {
-      target.src = target.getAttribute(swapTo);
-    };
-  };
-
   const locale = document.documentElement.lang;
 
   const dateTimeFormat = new Intl.DateTimeFormat(locale, {
@@ -158,27 +139,21 @@ function loaded() {
         const root = createRoot(content);
         root.render(<MediaContainer locale={locale} components={reactComponents} />);
         document.body.appendChild(content);
-        scrollToDetailedStatus();
       })
       .catch(error => {
         console.error(error);
-        scrollToDetailedStatus();
       });
-  } else {
-    scrollToDetailedStatus();
   }
 
-  delegate(document, '#user_account_attributes_username', 'input', throttle(() => {
+  delegate(document, '#user_account_attributes_username', 'input', throttle(({ target }) => {
-    const username = document.getElementById('user_account_attributes_username');
+    if (target.value && target.value.length > 0) {
-
+      axios.get('/api/v1/accounts/lookup', { params: { acct: target.value } }).then(() => {
-    if (username.value && username.value.length > 0) {
+        target.setCustomValidity(formatMessage(messages.usernameTaken));
-      axios.get('/api/v1/accounts/lookup', { params: { acct: username.value } }).then(() => {
-        username.setCustomValidity(formatMessage(messages.usernameTaken));
       }).catch(() => {
-        username.setCustomValidity('');
+        target.setCustomValidity('');
       });
     } else {
-      username.setCustomValidity('');
+      target.setCustomValidity('');
     }
   }, 500, { leading: false, trailing: true }));
 
@@ -196,9 +171,6 @@ function loaded() {
     }
   });
 
-  delegate(document, '.custom-emoji', 'mouseover', getEmojiAnimationHandler('data-original'));
-  delegate(document, '.custom-emoji', 'mouseout', getEmojiAnimationHandler('data-static'));
-
   delegate(document, '.status__content__spoiler-link', 'click', function() {
     const statusEl = this.parentNode.parentNode;
 
@@ -220,17 +192,6 @@ function loaded() {
   });
 }
 
-delegate(document, '#account_display_name', 'input', ({ target }) => {
-  const name = document.querySelector('.card .display-name strong');
-  if (name) {
-    if (target.value) {
-      name.innerHTML = emojify(escapeTextContentForBrowser(target.value));
-    } else {
-      name.textContent = target.dataset.default;
-    }
-  }
-});
-
 delegate(document, '#edit_profile input[type=file]', 'change', ({ target }) => {
   const avatar = document.getElementById(target.id + '-preview');
   const [file] = target.files || [];
@@ -239,33 +200,6 @@ delegate(document, '#edit_profile input[type=file]', 'change', ({ target }) => {
   avatar.src = url;
 });
 
-const getProfileAvatarAnimationHandler = (swapTo) => {
-  //animate avatar gifs on the profile page when moused over
-  return ({ target }) => {
-    const swapSrc = target.getAttribute(swapTo);
-    //only change the img source if autoplay is off and the image src is actually different
-    if(target.getAttribute('data-autoplay') !== 'true' && target.src !== swapSrc) {
-      target.src = swapSrc;
-    }
-  };
-};
-
-delegate(document, 'img#profile_page_avatar', 'mouseover', getProfileAvatarAnimationHandler('data-original'));
-
-delegate(document, 'img#profile_page_avatar', 'mouseout', getProfileAvatarAnimationHandler('data-static'));
-
-delegate(document, '#account_locked', 'change', ({ target }) => {
-  const lock = document.querySelector('.card .display-name i');
-
-  if (lock) {
-    if (target.checked) {
-      delete lock.dataset.hidden;
-    } else {
-      lock.dataset.hidden = 'true';
-    }
-  }
-});
-
 delegate(document, '.input-copy input', 'click', ({ target }) => {
   target.focus();
   target.select();
@@ -325,6 +259,9 @@ delegate(document, '.sidebar__toggle__icon', 'keydown', e => {
   }
 });
 
+delegate(document, '.custom-emoji', 'mouseover', ({ target }) => target.src = target.getAttribute('data-original'));
+delegate(document, '.custom-emoji', 'mouseout', ({ target }) => target.src = target.getAttribute('data-static'));
+
 // Empty the honeypot fields in JS in case something like an extension
 // automatically filled them.
 delegate(document, '#registration_new_user,#new_user', 'submit', () => {

app/javascript/styles/mastodon/components.scss

@@ -2441,6 +2441,7 @@ $ui-header-height: 55px;
 
   .filter-form {
     display: flex;
+    flex-wrap: wrap;
   }
 
   .autosuggest-textarea__textarea {
@@ -3330,6 +3331,22 @@ $ui-header-height: 55px;
   border-color: $ui-highlight-color;
 }
 
+.switch-to-advanced {
+  color: $classic-primary-color;
+  background-color: $classic-base-color;
+  padding: 15px;
+  border-radius: 4px;
+  margin-top: 4px;
+  margin-bottom: 12px;
+  font-size: 13px;
+  line-height: 18px;
+
+  .switch-to-advanced__toggle {
+    color: $ui-button-tertiary-color;
+    font-weight: bold;
+  }
+}
+
 .column-link {
   background: lighten($ui-base-color, 8%);
   color: $primary-text-color;
@@ -5204,6 +5221,12 @@ a.status-card {
     }
 
     &__menu {
+      margin-bottom: 20px;
+
+      &:last-child {
+        margin-bottom: 0;
+      }
+
       &__message {
         color: $dark-text-color;
         padding: 0 10px;
@@ -9655,19 +9678,24 @@ noscript {
   display: flex;
   flex-wrap: wrap;
   font-size: 14px;
+  line-height: 18px;
   gap: 4px;
+  color: $darker-text-color;
 
   a {
     display: inline-flex;
-    color: $dark-text-color;
+    color: inherit;
     text-decoration: none;
 
-    &:hover {
+    &:hover span {
-      text-decoration: none;
+      text-decoration: underline;
-
-      span {
-        text-decoration: underline;
-      }
     }
   }
+
+  .link-button {
+    color: inherit;
+    font-size: inherit;
+    line-height: inherit;
+    padding: 0;
+  }
 }

app/javascript/styles/mastodon/forms.scss

@@ -1186,14 +1186,14 @@ code {
   }
 
   li:first-child .label {
-    left: auto;
     inset-inline-start: 0;
+    inset-inline-end: auto;
     text-align: start;
     transform: none;
   }
 
   li:last-child .label {
-    left: auto;
+    inset-inline-start: auto;
     inset-inline-end: 0;
     text-align: end;
     transform: none;

app/lib/importer/accounts_index_importer.rb

@@ -4,10 +4,10 @@ class Importer::AccountsIndexImporter < Importer::BaseImporter
   def import!
     scope.includes(:account_stat).find_in_batches(batch_size: @batch_size) do |tmp|
       in_work_unit(tmp) do |accounts|
-        bulk = Chewy::Index::Import::BulkBuilder.new(index, to_index: accounts).bulk_body
+        bulk = build_bulk_body(accounts)
 
-        indexed = bulk.count { |entry| entry[:index] }
+        indexed = bulk.size
-        deleted = bulk.count { |entry| entry[:delete] }
+        deleted = 0
 
         Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
 

app/lib/importer/base_importer.rb

@@ -68,6 +68,14 @@ class Importer::BaseImporter
 
   protected
 
+  def build_bulk_body(to_import)
+    # Specialize `Chewy::Index::Import::BulkBuilder#bulk_body` to avoid a few
+    # inefficiencies, as none of our fields or join fields and we do not need
+    # `BulkBuilder`'s versatility.
+    crutches = Chewy::Index::Crutch::Crutches.new index, to_import
+    to_import.map { |object| { index: { _id: object.id, data: index.compose(object, crutches, fields: []) } } }
+  end
+
   def in_work_unit(...)
     work_unit = Concurrent::Promises.future_on(@executor, ...)
 

app/lib/importer/instances_index_importer.rb

@@ -4,10 +4,10 @@ class Importer::InstancesIndexImporter < Importer::BaseImporter
   def import!
     index.adapter.default_scope.find_in_batches(batch_size: @batch_size) do |tmp|
       in_work_unit(tmp) do |instances|
-        bulk = Chewy::Index::Import::BulkBuilder.new(index, to_index: instances).bulk_body
+        bulk = build_bulk_body(instances)
 
-        indexed = bulk.count { |entry| entry[:index] }
+        indexed = bulk.size
-        deleted = bulk.count { |entry| entry[:delete] }
+        deleted = 0
 
         Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
 

app/lib/importer/public_statuses_index_importer.rb

@@ -2,24 +2,15 @@
 
 class Importer::PublicStatusesIndexImporter < Importer::BaseImporter
   def import!
-    indexable_statuses_scope.find_in_batches(batch_size: @batch_size) do |batch|
+    scope.select(:id).find_in_batches(batch_size: @batch_size) do |batch|
-      in_work_unit(batch.map(&:status_id)) do |status_ids|
+      in_work_unit(batch.pluck(:id)) do |status_ids|
         bulk = ActiveRecord::Base.connection_pool.with_connection do
-          Chewy::Index::Import::BulkBuilder.new(index, to_index: Status.includes(:media_attachments, :preloadable_poll).where(id: status_ids)).bulk_body
+          build_bulk_body(index.adapter.default_scope.where(id: status_ids))
         end
 
-        indexed = 0
+        indexed = bulk.size
         deleted = 0
 
-        bulk.map! do |entry|
-          if entry[:index]
-            indexed += 1
-          else
-            deleted += 1
-          end
-          entry
-        end
-
         Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
 
         [indexed, deleted]
@@ -35,7 +26,7 @@ class Importer::PublicStatusesIndexImporter < Importer::BaseImporter
     PublicStatusesIndex
   end
 
-  def indexable_statuses_scope
+  def scope
-    Status.indexable.select('"statuses"."id", COALESCE("statuses"."reblog_of_id", "statuses"."id") AS status_id')
+    Status.indexable
   end
 end

app/lib/importer/statuses_index_importer.rb

@@ -13,32 +13,25 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter
 
       scope.find_in_batches(batch_size: @batch_size) do |tmp|
         in_work_unit(tmp.map(&:status_id)) do |status_ids|
-          bulk = ActiveRecord::Base.connection_pool.with_connection do
-            Chewy::Index::Import::BulkBuilder.new(index, to_index: Status.includes(:media_attachments, :preloadable_poll).where(id: status_ids)).bulk_body
-          end
-
-          indexed = 0
           deleted = 0
 
-          # We can't use the delete_if proc to do the filtering because delete_if
+          bulk = ActiveRecord::Base.connection_pool.with_connection do
-          # is called before rendering the data and we need to filter based
+            to_index = index.adapter.default_scope.where(id: status_ids)
-          # on the results of the filter, so this filtering happens here instead
+            crutches = Chewy::Index::Crutch::Crutches.new index, to_index
-          bulk.map! do |entry|
+            to_index.map do |object|
-            new_entry = if entry[:index] && entry.dig(:index, :data, 'searchable_by').blank?
+              # This is unlikely to happen, but the post may have been
-                          { delete: entry[:index].except(:data) }
+              # un-interacted with since it was queued for indexing
-                        else
+              if object.searchable_by.empty?
-                          entry
+                deleted += 1
-                        end
+                { delete: { _id: object.id } }
-
+              else
-            if new_entry[:index]
+                { index: { _id: object.id, data: index.compose(object, crutches, fields: []) } }
-              indexed += 1
+              end
-            else
-              deleted += 1
             end
-
-            new_entry
           end
 
+          indexed = bulk.size - deleted
+
           Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
 
           [indexed, deleted]

app/lib/importer/tags_index_importer.rb

@@ -4,10 +4,10 @@ class Importer::TagsIndexImporter < Importer::BaseImporter
   def import!
     index.adapter.default_scope.find_in_batches(batch_size: @batch_size) do |tmp|
       in_work_unit(tmp) do |tags|
-        bulk = Chewy::Index::Import::BulkBuilder.new(index, to_index: tags).bulk_body
+        bulk = build_bulk_body(tags)
 
-        indexed = bulk.count { |entry| entry[:index] }
+        indexed = bulk.size
-        deleted = bulk.count { |entry| entry[:delete] }
+        deleted = 0
 
         Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
 

app/lib/inline_renderer.rb

@@ -12,6 +12,9 @@ class InlineRenderer
     when :status
       serializer = REST::StatusSerializer
       preload_associations_for_status
+    when :status_internal
+      serializer = REST::StatusInternalSerializer
+      preload_associations_for_status
     when :notification
       serializer = REST::NotificationSerializer
     when :emoji_reaction

app/lib/plain_text_formatter.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 class PlainTextFormatter
-  include ActionView::Helpers::TextHelper
-
   NEWLINE_TAGS_RE = %r{(<br />|<br>|</p>)+}
 
   attr_reader :text, :local
@@ -18,7 +16,10 @@ class PlainTextFormatter
     if local?
       text
     else
-      html_entities.decode(strip_tags(insert_newlines)).chomp
+      node = Nokogiri::HTML.fragment(insert_newlines)
+      # Elements that are entirely removed with our Sanitize config
+      node.xpath('.//iframe|.//math|.//noembed|.//noframes|.//noscript|.//plaintext|.//script|.//style|.//svg|.//xmp').remove
+      node.text.chomp
     end
   end
 
@@ -27,8 +28,4 @@ class PlainTextFormatter
   def insert_newlines
     text.gsub(NEWLINE_TAGS_RE) { |match| "#{match}\n" }
   end
-
-  def html_entities
-    HTMLEntities.new
-  end
 end

app/lib/search_query_parser.rb

@@ -6,10 +6,10 @@ class SearchQueryParser < Parslet::Parser
   rule(:colon)     { str(':') }
   rule(:space)     { match('\s').repeat(1) }
   rule(:operator)  { (str('+') | str('-')).as(:operator) }
-  rule(:prefix)    { (term >> colon).as(:prefix) }
+  rule(:prefix)    { term >> colon }
   rule(:shortcode) { (colon >> term >> colon.maybe).as(:shortcode) }
   rule(:phrase)    { (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) }
-  rule(:clause)    { (prefix.maybe >> operator.maybe >> (phrase | term | shortcode)).as(:clause) }
+  rule(:clause)    { (operator.maybe >> prefix.maybe.as(:prefix) >> (phrase | term | shortcode)).as(:clause) | prefix.as(:clause) | quote.as(:junk) }
   rule(:query)     { (clause >> space.maybe).repeat.as(:query) }
   root(:query)
 end

app/lib/search_query_transformer.rb

@@ -1,46 +1,32 @@
 # frozen_string_literal: true
 
 class SearchQueryTransformer < Parslet::Transform
+  SUPPORTED_PREFIXES = %w(
+    has
+    is
+    language
+    from
+    before
+    after
+    during
+  ).freeze
+
   class Query
-    attr_reader :should_clauses, :must_not_clauses, :must_clauses, :filter_clauses
+    attr_reader :must_not_clauses, :must_clauses, :filter_clauses
 
     def initialize(clauses)
-      grouped = clauses.chunk(&:operator).to_h
+      grouped = clauses.compact.chunk(&:operator).to_h
-      @should_clauses = grouped.fetch(:should, [])
       @must_not_clauses = grouped.fetch(:must_not, [])
       @must_clauses = grouped.fetch(:must, [])
       @filter_clauses = grouped.fetch(:filter, [])
     end
 
     def apply(search)
-      should_clauses.each { |clause| search = search.query.should(clause_to_query(clause)) }
+      must_clauses.each { |clause| search = search.query.must(clause.to_query) }
-      must_clauses.each { |clause| search = search.query.must(clause_to_query(clause)) }
+      must_not_clauses.each { |clause| search = search.query.must_not(clause.to_query) }
-      must_not_clauses.each { |clause| search = search.query.must_not(clause_to_query(clause)) }
+      filter_clauses.each { |clause| search = search.filter(**clause.to_query) }
-      filter_clauses.each { |clause| search = search.filter(**clause_to_filter(clause)) }
       search.query.minimum_should_match(1)
     end
-
-    private
-
-    def clause_to_query(clause)
-      case clause
-      when TermClause
-        { match_phrase: { text: { query: clause.term } } }
-      when PhraseClause
-        { match_phrase: { text: { query: clause.phrase } } }
-      else
-        raise "Unexpected clause type: #{clause}"
-      end
-    end
-
-    def clause_to_filter(clause)
-      case clause
-      when PrefixClause
-        { clause.type => { clause.filter => clause.term } }
-      else
-        raise "Unexpected clause type: #{clause}"
-      end
-    end
   end
 
   class Operator
@@ -59,29 +45,39 @@ class SearchQueryTransformer < Parslet::Transform
   end
 
   class TermClause
-    attr_reader :prefix, :operator, :term
+    attr_reader :operator, :term
 
-    def initialize(prefix, operator, term)
+    def initialize(operator, term)
-      @prefix = prefix
       @operator = Operator.symbol(operator)
       @term = term
     end
+
+    def to_query
+      # { multi_match: { type: 'most_fields', query: @term, fields: ['text', 'text.stemmed'], operator: 'and' } }
+      { match_phrase: { text: { query: @phrase } } }
+    end
   end
 
   class PhraseClause
-    attr_reader :prefix, :operator, :phrase
+    attr_reader :operator, :phrase
 
-    def initialize(prefix, operator, phrase)
+    def initialize(operator, phrase)
-      @prefix = prefix
       @operator = Operator.symbol(operator)
       @phrase = phrase
     end
+
+    def to_query
+      { match_phrase: { text: { query: @phrase } } }
+    end
   end
 
   class PrefixClause
-    attr_reader :type, :filter, :operator, :term
+    attr_reader :operator, :prefix, :term
 
-    def initialize(prefix, term)
+    def initialize(prefix, operator, term, options = {})
+      @prefix = prefix
+      @negated = operator == '-'
+      @options = options
       @operator = :filter
 
       case prefix
@@ -92,31 +88,45 @@ class SearchQueryTransformer < Parslet::Transform
       when 'language'
         @filter = :language
         @type = :term
-        @term = term
+        @term = language_code_from_term(term)
       when 'from'
         @filter = :account_id
         @type = :term
         @term = account_id_from_term(term)
+      when 'domain'
+        @filter = :domain
+        @type = :term
+        @term = domain_from_term(term)
       when 'before'
         @filter = :created_at
         @type = :range
-        @term = { lt: term }
+        @term = { lt: term, time_zone: @options[:current_account]&.user_time_zone || 'UTC' }
       when 'after'
         @filter = :created_at
         @type = :range
-        @term = { gt: term }
+        @term = { gt: term, time_zone: @options[:current_account]&.user_time_zone || 'UTC' }
       when 'during'
         @filter = :created_at
         @type = :range
-        @term = { gte: term, lte: term }
+        @term = { gte: term, lte: term, time_zone: @options[:current_account]&.user_time_zone || 'UTC' }
       else
-        raise Mastodon::SyntaxError
+        raise "Unknown prefix: #{prefix}"
+      end
+    end
+
+    def to_query
+      if @negated
+        { bool: { must_not: { @type => { @filter => @term } } } }
+      else
+        { @type => { @filter => @term } }
       end
     end
 
     private
 
     def account_id_from_term(term)
+      return @options[:current_account]&.id || -1 if term == 'me'
+
       username, domain = term.gsub(/\A@/, '').split('@')
       domain = nil if TagManager.instance.local_domain?(domain)
       account = Account.find_remote(username, domain)
@@ -125,24 +135,54 @@ class SearchQueryTransformer < Parslet::Transform
       # an ID that does not exist
       account&.id || -1
     end
+
+    def domain_from_term(term)
+      return '' if %w(local me).include?(term)
+
+      term
+    end
+
+    def language_code_from_term(term)
+      language_code = term
+
+      return language_code if LanguagesHelper::SUPPORTED_LOCALES.key?(language_code.to_sym)
+
+      language_code = term.downcase
+
+      return language_code if LanguagesHelper::SUPPORTED_LOCALES.key?(language_code.to_sym)
+
+      language_code = term.split(/[_-]/).first.downcase
+
+      return language_code if LanguagesHelper::SUPPORTED_LOCALES.key?(language_code.to_sym)
+
+      term
+    end
   end
 
   rule(clause: subtree(:clause)) do
     prefix   = clause[:prefix][:term].to_s if clause[:prefix]
     operator = clause[:operator]&.to_s
 
-    if clause[:prefix]
+    if clause[:prefix] && SUPPORTED_PREFIXES.include?(prefix)
-      PrefixClause.new(prefix, clause[:term].to_s)
+      PrefixClause.new(prefix, operator, clause[:term].to_s, current_account: current_account)
+    elsif clause[:prefix]
+      TermClause.new(operator, "#{prefix} #{clause[:term]}")
     elsif clause[:term]
-      TermClause.new(prefix, operator, clause[:term].to_s)
+      TermClause.new(operator, clause[:term].to_s)
     elsif clause[:shortcode]
-      TermClause.new(prefix, operator, ":#{clause[:term]}:")
+      TermClause.new(operator, ":#{clause[:term]}:")
     elsif clause[:phrase]
-      PhraseClause.new(prefix, operator, clause[:phrase].is_a?(Array) ? clause[:phrase].map { |p| p[:term].to_s }.join(' ') : clause[:phrase].to_s)
+      PhraseClause.new(operator, clause[:phrase].is_a?(Array) ? clause[:phrase].map { |p| p[:term].to_s }.join(' ') : clause[:phrase].to_s)
     else
       raise "Unexpected clause type: #{clause}"
     end
   end
 
-  rule(query: sequence(:clauses)) { Query.new(clauses) }
+  rule(junk: subtree(:junk)) do
+    nil
+  end
+
+  rule(query: sequence(:clauses)) do
+    Query.new(clauses)
+  end
 end

app/models/account.rb

@@ -319,6 +319,17 @@ class Account < ApplicationRecord
     user&.setting_noai || (settings.present? && settings['noai']) || false
   end
 
+  def translatable_private?
+    user&.setting_translatable_private || (settings.present? && settings['translatable_private']) || false
+  end
+
+  def link_preview?
+    return user.setting_link_preview if local? && user.present?
+    return settings['link_preview'] if settings.present? && settings.key?('link_preview')
+
+    true
+  end
+
   def public_statuses_count
     hide_statuses_count? ? 0 : statuses_count
   end
@@ -384,10 +395,16 @@ class Account < ApplicationRecord
       'hide_statuses_count' => hide_statuses_count?,
       'hide_following_count' => hide_following_count?,
       'hide_followers_count' => hide_followers_count?,
-      'emoji_reaction_must_following' => emoji_reactions_must_following?,
+      'translatable_private' => translatable_private?,
-      'emoji_reaction_must_follower' => emoji_reactions_must_follower?,
+      'link_preview' => link_preview?,
-      'emoji_reaction_deny_from_all' => emoji_reactions_deny_from_all?,
     }
+    if Setting.enable_block_emoji_reaction_settings
+      config = config.merge({
+        'emoji_reaction_must_following' => emoji_reactions_must_following?,
+        'emoji_reaction_must_follower' => emoji_reactions_must_follower?,
+        'emoji_reaction_deny_from_all' => emoji_reactions_deny_from_all?,
+      })
+    end
     config = config.merge(settings) if settings.present?
     config
   end

app/models/concerns/account_statuses_search.rb

@@ -31,8 +31,8 @@ module AccountStatusesSearch
   def add_to_public_statuses_index!
     return unless Chewy.enabled?
 
-    statuses.indexable.find_in_batches do |batch|
+    statuses.without_reblogs.where(visibility: :public).find_in_batches do |batch|
-      PublicStatusesIndex.import(query: batch)
+      PublicStatusesIndex.import(batch)
     end
   end
 

app/models/concerns/has_user_settings.rb

@@ -68,7 +68,7 @@ module HasUserSettings
   end
 
   def setting_emoji_reaction_streaming_notify_impl2
-    settings['emoji_reaction_streaming_notify_impl2']
+    false
   end
 
   def setting_unfollow_modal
@@ -99,6 +99,14 @@ module HasUserSettings
     settings['noai']
   end
 
+  def setting_translatable_private
+    settings['translatable_private']
+  end
+
+  def setting_link_preview
+    settings['link_preview']
+  end
+
   def setting_hide_statuses_count
     settings['hide_statuses_count']
   end

app/models/concerns/status_search_concern.rb

@@ -7,30 +7,22 @@ module StatusSearchConcern
     scope :indexable, -> { without_reblogs.where(visibility: [:public, :login], searchability: nil).joins(:account).where(account: { indexable: true }) }
   end
 
-  def searchable_by(preloaded = nil)
+  def searchable_by
-    ids = []
+    @searchable_by ||= begin
+      ids = []
 
-    ids << account_id if local?
+      ids << account_id if local?
 
-    if preloaded.nil?
+      ids += local_mentioned.pluck(:id)
-      ids += mentions.joins(:account).merge(Account.local).active.pluck(:account_id)
+      ids += local_favorited.pluck(:id)
-      ids += favourites.joins(:account).merge(Account.local).pluck(:account_id)
+      ids += local_reblogged.pluck(:id)
-      ids += emoji_reactions.joins(:account).merge(Account.local).pluck(:account_id)
+      ids += local_bookmarked.pluck(:id)
-      ids += references.joins(:account).merge(Account.local).pluck(:account_id)
+      ids += local_emoji_reacted.pluck(:id)
-      ids += reblogs.joins(:account).merge(Account.local).pluck(:account_id)
+      ids += local_referenced.pluck(:id)
-      ids += bookmarks.joins(:account).merge(Account.local).pluck(:account_id)
+      ids += preloadable_poll.local_voters.pluck(:id) if preloadable_poll.present?
-      ids += poll.votes.joins(:account).merge(Account.local).pluck(:account_id) if poll.present?
+
-    else
+      ids.uniq
-      ids += preloaded.mentions[id] || []
-      ids += preloaded.favourites[id] || []
-      ids += preloaded.emoji_reactions[id] || []
-      ids += preloaded.status_references[id] || []
-      ids += preloaded.reblogs[id] || []
-      ids += preloaded.bookmarks[id] || []
-      ids += preloaded.votes[id] || []
     end
-
-    ids.uniq
   end
 
   def searchable_text

app/models/custom_filter.rb

@@ -101,7 +101,10 @@ class CustomFilter < ApplicationRecord
       next if filter.exclude_follows && following
       next if filter.exclude_localusers && status.account.local?
 
-      match = rules[:keywords].match(status.proper.searchable_text) if rules[:keywords].present?
+      if rules[:keywords].present?
+        match = rules[:keywords].match(status.proper.searchable_text)
+        match = rules[:keywords].match(status.proper.references.pluck(:text).join("\n\n")) if match.nil? && status.proper.references.exists?
+      end
       keyword_matches = [match.to_s] unless match.nil?
 
       status_matches = [status.id, status.reblog_of_id].compact & rules[:status_ids] if rules[:status_ids].present?

app/models/media_attachment.rb

@@ -49,6 +49,7 @@ class MediaAttachment < ApplicationRecord
 
   MAX_VIDEO_MATRIX_LIMIT = 8_294_400 # 3840x2160px
   MAX_VIDEO_FRAME_RATE   = 120
+  MAX_VIDEO_FRAMES       = 36_000 # Approx. 5 minutes at 120 fps
 
   IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif .webp .heic .heif .avif).freeze
   VIDEO_FILE_EXTENSIONS = %w(.webm .mp4 .m4v .mov).freeze
@@ -103,17 +104,14 @@ class MediaAttachment < ApplicationRecord
     convert_options: {
       output: {
         'loglevel' => 'fatal',
-        'movflags' => 'faststart',
+        'preset' => 'veryfast',
-        'pix_fmt' => 'yuv420p',
+        'movflags' => 'faststart', # Move metadata to start of file so playback can begin before download finishes
-        'vf' => 'scale=\'trunc(iw/2)*2:trunc(ih/2)*2\'',
+        'pix_fmt' => 'yuv420p', # Ensure color space for cross-browser compatibility
-        'vsync' => 'cfr',
         'c:v' => 'h264',
-        'maxrate' => '1300K',
+        'c:a' => 'aac',
-        'bufsize' => '1300K',
+        'b:a' => '192k',
-        'b:v' => '1300K',
-        'frames:v' => 60 * 60 * 3,
-        'crf' => 18,
         'map_metadata' => '-1',
+        'frames:v' => MAX_VIDEO_FRAMES,
       }.freeze,
     }.freeze,
   }.freeze
@@ -140,7 +138,7 @@ class MediaAttachment < ApplicationRecord
       convert_options: {
         output: {
           'loglevel' => 'fatal',
-          :vf => 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease',
+          :vf => 'scale=\'min(640\, iw):min(640\, ih)\':force_original_aspect_ratio=decrease',
         }.freeze,
       }.freeze,
       format: 'png',

app/models/poll.rb

@@ -28,6 +28,7 @@ class Poll < ApplicationRecord
 
   has_many :votes, class_name: 'PollVote', inverse_of: :poll, dependent: :delete_all
   has_many :voters, -> { group('accounts.id') }, through: :votes, class_name: 'Account', source: :account
+  has_many :local_voters, -> { group('accounts.id').merge(Account.local) }, through: :votes, class_name: 'Account', source: :account
 
   has_many :notifications, as: :activity, dependent: :destroy
 

app/models/status.rb

@@ -87,6 +87,14 @@ class Status < ApplicationRecord
   has_many :bookmark_category_relationships, class_name: 'BookmarkCategoryStatus', inverse_of: :status, dependent: :destroy
   has_many :joined_bookmark_categories, class_name: 'BookmarkCategory', through: :bookmark_category_relationships, source: :bookmark_category
 
+  # Those associations are used for the private search index
+  has_many :local_mentioned, -> { merge(Account.local) }, through: :active_mentions, source: :account
+  has_many :local_favorited, -> { merge(Account.local) }, through: :favourites, source: :account
+  has_many :local_reblogged, -> { merge(Account.local) }, through: :reblogs, source: :account
+  has_many :local_bookmarked, -> { merge(Account.local) }, through: :bookmarks, source: :account
+  has_many :local_emoji_reacted, -> { merge(Account.local) }, through: :emoji_reactions, source: :account
+  has_many :local_referenced, -> { merge(Account.local) }, through: :referenced_by_statuses, source: :account
+
   has_and_belongs_to_many :tags
   has_and_belongs_to_many :preview_cards
 

app/models/user_settings.rb

@@ -12,6 +12,8 @@ class UserSettings
   setting :theme, default: -> { ::Setting.theme }
   setting :noindex, default: -> { ::Setting.noindex }
   setting :noai, default: true
+  setting :translatable_private, default: false
+  setting :link_preview, default: true
   setting :bio_markdown, default: false
   setting :discoverable_local, default: false
   setting :hide_statuses_count, default: false

app/serializers/rest/status_internal_serializer.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class REST::StatusInternalSerializer < REST::StatusSerializer
+  attributes :reference_texts
+
+  def reference_texts
+    object.references.pluck(:text)
+  end
+end

app/serializers/webfinger_serializer.rb

@@ -18,18 +18,31 @@ class WebfingerSerializer < ActiveModel::Serializer
   end
 
   def links
-    if object.instance_actor?
+    [
-      [
+      { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: profile_page_href },
-        { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: about_more_url(instance_actor: true) },
+      { rel: 'self', type: 'application/activity+json', href: self_href },
-        { rel: 'self', type: 'application/activity+json', href: instance_actor_url },
+      { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" },
-        { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" },
+    ].tap do |x|
-      ]
+      x << { rel: 'http://webfinger.net/rel/avatar', type: object.avatar.content_type, href: full_asset_url(object.avatar_original_url) } if show_avatar?
-    else
-      [
-        { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: short_account_url(object) },
-        { rel: 'self', type: 'application/activity+json', href: account_url(object) },
-        { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" },
-      ]
     end
   end
+
+  private
+
+  def show_avatar?
+    media_present = object.avatar.present? && object.avatar.content_type.present?
+
+    # Show avatar only if an instance shows profiles to logged out users
+    allowed_by_config = ENV['DISALLOW_UNAUTHENTICATED_API_ACCESS'] != 'true' && !Rails.configuration.x.limited_federation_mode
+
+    media_present && allowed_by_config
+  end
+
+  def profile_page_href
+    object.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(object)
+  end
+
+  def self_href
+    object.instance_actor? ? instance_actor_url : account_url(object)
+  end
 end

app/services/fan_out_on_write_service.rb

@@ -195,7 +195,7 @@ class FanOutOnWriteService < BaseService
   end
 
   def rendered_status
-    @rendered_status ||= InlineRenderer.render(@status, nil, :status)
+    @rendered_status ||= InlineRenderer.render(@status, nil, :status_internal)
   end
 
   def update?

app/services/fetch_link_card_service.rb

@@ -20,7 +20,7 @@ class FetchLinkCardService < BaseService
     @status       = status
     @original_url = parse_urls
 
-    return if @original_url.nil? || @status.preview_cards.any?
+    return if @original_url.nil? || @status.preview_cards.any? || !@status.account.link_preview?
 
     @url = @original_url.to_s
 

app/services/post_status_service.rb

@@ -92,9 +92,10 @@ class PostStatusService < BaseService
   end
 
   def load_circle
-    return unless @options[:visibility] == 'circle'
+    return unless @options[:visibility] == 'circle' || (@options[:visibility] == 'limited' && @options[:circle_id].present?)
 
     @circle = @options[:circle_id].present? && Circle.find(@options[:circle_id])
+    @limited_scope = :circle
     raise ArgumentError if @circle.nil? || @circle.account_id != @account.id
   end
 

app/services/search_service.rb

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 
 class SearchService < BaseService
-  def call(query, account, limit, options = {})
+  QUOTE_EQUIVALENT_CHARACTERS = /[“”„«»「」『』《》]/
-    @query = query&.strip
 
+  def call(query, account, limit, options = {})
+    @query     = query&.strip&.gsub(QUOTE_EQUIVALENT_CHARACTERS, '"')
     @account   = account
     @options   = options
     @limit     = limit.to_i
@@ -19,7 +20,7 @@ class SearchService < BaseService
         results.merge!(url_resource_results) unless url_resource.nil? || @offset.positive? || (@options[:type].present? && url_resource_symbol != @options[:type].to_sym)
       elsif @query.present?
         results[:accounts] = perform_accounts_search! if account_searchable?
-        results[:statuses] = perform_statuses_search! if full_text_searchable?
+        results[:statuses] = perform_statuses_search! if status_searchable?
         results[:hashtags] = perform_hashtags_search! if hashtag_searchable?
       end
     end
@@ -82,18 +83,16 @@ class SearchService < BaseService
     url_resource.class.name.downcase.pluralize.to_sym
   end
 
-  def full_text_searchable?
+  def status_searchable?
-    return false unless Chewy.enabled?
+    Chewy.enabled? && status_search? && @account.present?
-
-    statuses_search? && !@account.nil? && !(@query.include?('@') && !@query.include?(' '))
   end
 
   def account_searchable?
-    account_search? && !(@query.include?('@') && @query.include?(' '))
+    account_search?
   end
 
   def hashtag_searchable?
-    hashtag_search? && !@query.include?('@')
+    hashtag_search?
   end
 
   def account_search?
@@ -104,7 +103,7 @@ class SearchService < BaseService
     @options[:type].blank? || @options[:type] == 'hashtags'
   end
 
-  def statuses_search?
+  def status_search?
     @options[:type].blank? || @options[:type] == 'statuses'
   end
 end

app/services/statuses_search_service.rb

@@ -140,6 +140,6 @@ class StatusesSearchService < BaseService
   end
 
   def parsed_query
-    SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query))
+    SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query), current_account: @account)
   end
 end

app/services/translate_status_service.rb

@@ -30,7 +30,7 @@ class TranslateStatusService < BaseService
   end
 
   def permitted?
-    return false unless @status.distributable? && TranslationService.configured?
+    return false unless (@status.distributable? || @status.account.translatable_private?) && TranslationService.configured?
 
     languages[@status.language]&.include?(@target_language)
   end

app/views/settings/preferences/other/show.html.haml

@@ -31,19 +31,16 @@
       = ff.input :stay_privacy, wrapper: :with_label, kmyblue: true, label: I18n.t('simple_form.labels.defaults.setting_stay_privacy')
 
     .fields-group
-      = ff.input :'web.enable_login_privacy', wrapper: :with_label, kmyblue: true, label: I18n.t('simple_form.labels.defaults.setting_enable_login_privacy'), hint: false
+      = ff.input :public_post_to_unlisted, wrapper: :with_label, kmyblue: true, label: I18n.t('simple_form.labels.defaults.setting_public_post_to_unlisted'), hint: I18n.t('simple_form.hints.defaults.setting_public_post_to_unlisted')
 
     .fields-group
       = ff.input :disallow_unlisted_public_searchability, wrapper: :with_label, kmyblue: true, label: I18n.t('simple_form.labels.defaults.setting_disallow_unlisted_public_searchability'), hint: I18n.t('simple_form.hints.defaults.setting_disallow_unlisted_public_searchability')
 
-    .fields-group
-      = ff.input :public_post_to_unlisted, wrapper: :with_label, kmyblue: true, label: I18n.t('simple_form.labels.defaults.setting_public_post_to_unlisted'), hint: I18n.t('simple_form.hints.defaults.setting_public_post_to_unlisted')
-
     .fields-group
       = ff.input :default_sensitive, wrapper: :with_label, label: I18n.t('simple_form.labels.defaults.setting_default_sensitive'), hint: I18n.t('simple_form.hints.defaults.setting_default_sensitive')
 
     .fields-group
-      = ff.input :emoji_reaction_streaming_notify_impl2, as: :boolean, wrapper: :with_label, kmyblue: true, label: I18n.t('simple_form.labels.defaults.setting_emoji_reaction_streaming_notify_impl2'), hint: I18n.t('simple_form.hints.defaults.setting_emoji_reaction_streaming_notify_impl2')
+      = ff.input :'web.enable_login_privacy', wrapper: :with_label, kmyblue: true, label: I18n.t('simple_form.labels.defaults.setting_enable_login_privacy'), hint: false
 
   %h4= t 'preferences.public_timelines'
 

app/views/settings/privacy/show.html.haml

@@ -21,6 +21,9 @@
     .fields-group
       = ff.input :discoverable_local, wrapper: :with_label, kmyblue: true, label: I18n.t('simple_form.labels.defaults.discoverable_local'), hint: I18n.t('simple_form.hints.defaults.discoverable_local')
 
+    .fields-group
+      = ff.input :noai, wrapper: :with_label, kmyblue: true, label: I18n.t('simple_form.labels.defaults.setting_noai'), hint: I18n.t('simple_form.hints.defaults.setting_noai')
+
   .fields-group
     = f.input :dissubscribable, as: :boolean, wrapper: :with_label, kmyblue: true, hint: t('simple_form.hints.defaults.dissubscribable')
 
@@ -54,15 +57,5 @@
     .fields-group
       = ff.input :show_application, wrapper: :with_label
 
-    %h4= t 'privacy.stop_deliver'
-
-    %p.lead= t('privacy.stop_deliver_hint_html')
-
-    .fields-group
-      = ff.input :reject_public_unlisted_subscription, kmyblue: true, as: :boolean, wrapper: :with_label, label: I18n.t('simple_form.labels.defaults.setting_reject_public_unlisted_subscription')
-
-    .fields-group
-      = ff.input :reject_unlisted_subscription, kmyblue: true, as: :boolean, wrapper: :with_label, label: I18n.t('simple_form.labels.defaults.setting_reject_unlisted_subscription'), hint: I18n.t('simple_form.hints.defaults.setting_reject_unlisted_subscription')
-
   .actions
     = f.button :button, t('generic.save_changes'), type: :submit

app/views/settings/privacy_extra/show.html.haml

@@ -0,0 +1,36 @@
+- content_for :page_title do
+  = t('privacy_extra.title')
+
+- content_for :heading do
+  %h2= t('settings.profile')
+  = render partial: 'settings/shared/profile_navigation'
+
+= simple_form_for @account, url: settings_privacy_extra_path, html: { method: :put } do |f|
+  = render 'shared/error_messages', object: @account
+
+  %p.lead= t('privacy_extra.hint_html')
+
+  %h4= t('privacy_extra.post_processing')
+
+  %p.lead= t('privacy_extra.post_processing_hint_html')
+
+  = f.simple_fields_for :settings, current_user.settings do |ff|
+    .fields-group
+      = ff.input :translatable_private, wrapper: :with_label, kmyblue: true, label: I18n.t('simple_form.labels.defaults.setting_translatable_private')
+
+    .fields-group
+      = ff.input :link_preview, wrapper: :with_label, kmyblue: true, label: I18n.t('simple_form.labels.defaults.setting_link_preview')
+
+  %h4= t 'privacy_extra.stop_deliver'
+
+  %p.lead= t('privacy_extra.stop_deliver_hint_html')
+
+  = f.simple_fields_for :settings, current_user.settings do |ff|
+    .fields-group
+      = ff.input :reject_public_unlisted_subscription, kmyblue: true, as: :boolean, wrapper: :with_label, label: I18n.t('simple_form.labels.defaults.setting_reject_public_unlisted_subscription')
+
+    .fields-group
+      = ff.input :reject_unlisted_subscription, kmyblue: true, as: :boolean, wrapper: :with_label, label: I18n.t('simple_form.labels.defaults.setting_reject_unlisted_subscription'), hint: I18n.t('simple_form.hints.defaults.setting_reject_unlisted_subscription')
+
+  .actions
+    = f.button :button, t('generic.save_changes'), type: :submit

app/views/settings/shared/_profile_navigation.html.haml

@@ -3,5 +3,6 @@
     :ruby
       primary.item :profile, safe_join([fa_icon('user fw'), t('settings.edit_profile')]), settings_profile_path
       primary.item :privacy, safe_join([fa_icon('lock fw'), t('privacy.title')]), settings_privacy_path
+      primary.item :privacy_extra, safe_join([fa_icon('lock fw'), t('privacy_extra.title')]), settings_privacy_extra_path
       primary.item :verification, safe_join([fa_icon('check fw'), t('verification.verification')]), settings_verification_path
       primary.item :featured_tags, safe_join([fa_icon('hashtag fw'), t('settings.featured_tags')]), settings_featured_tags_path

app/workers/add_to_public_statuses_index_worker.rb

@@ -2,13 +2,20 @@
 
 class AddToPublicStatusesIndexWorker
   include Sidekiq::Worker
+  include DatabaseHelper
+
+  sidekiq_options queue: 'pull'
 
   def perform(account_id)
-    account = Account.find(account_id)
+    with_primary do
+      @account = Account.find(account_id)
+    end
 
-    return unless account.indexable?
+    return unless @account.indexable?
 
-    account.add_to_public_statuses_index!
+    with_read_replica do
+      @account.add_to_public_statuses_index!
+    end
   rescue ActiveRecord::RecordNotFound
     true
   end

app/workers/scheduler/indexing_scheduler.rb

@@ -3,6 +3,7 @@
 class Scheduler::IndexingScheduler
   include Sidekiq::Worker
   include Redisable
+  include DatabaseHelper
 
   sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
 
@@ -15,7 +16,10 @@ class Scheduler::IndexingScheduler
     indexes.each do |type|
       with_redis do |redis|
         redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids|
-          type.import!(ids)
+          with_read_replica do
+            type.import!(ids)
+          end
+
           redis.srem("chewy:queue:#{type.name}", ids)
         end
       end

config/environments/development.rb

@@ -35,6 +35,8 @@ Rails.application.configure do
     config.cache_store = :null_store
   end
 
+  config.action_controller.forgery_protection_origin_check = ENV['DISABLE_FORGERY_REQUEST_PROTECTION'].nil?
+
   ActiveSupport::Logger.new(STDOUT).tap do |logger|
     logger.formatter = config.log_formatter
     config.logger = ActiveSupport::TaggedLogging.new(logger)

config/initializers/content_security_policy.rb

@@ -41,7 +41,8 @@ Rails.application.config.content_security_policy do |p|
   p.worker_src      :self, :blob, assets_host
 
   if Rails.env.development?
-    webpacker_urls = %w(ws http).map { |protocol| "#{protocol}#{Webpacker.dev_server.https? ? 's' : ''}://#{Webpacker.dev_server.host_with_port}" }
+    webpacker_public_host = ENV.fetch('WEBPACKER_DEV_SERVER_PUBLIC', Webpacker.config.dev_server[:public])
+    webpacker_urls = %w(ws http).map { |protocol| "#{protocol}#{Webpacker.dev_server.https? ? 's' : ''}://#{webpacker_public_host}" }
 
     p.connect_src :self, :data, :blob, assets_host, media_host, Rails.configuration.x.streaming_api_base_url, *webpacker_urls
     p.script_src  :self, :unsafe_inline, :unsafe_eval, assets_host, google_host, google_host2, google_host3

config/locales/en.yml

@@ -1619,6 +1619,13 @@ en:
     search_hint_html: Control how you want to be found. Do you want people to find you by what you've publicly posted about? Do you want people outside Mastodon to find your profile when searching the web? Please mind that total exclusion from all search engines cannot be guaranteed for public information.
     stop_deliver: Stop delivering
     title: Privacy and reach
+  privacy_extra:
+    hint_html: These settings are kmyblue original. You will receive additional privacy benefits by doing this setting.
+    post_processing_hint_html: 投稿された情報に対して、システムが追加で行うことができる操作を制御します。これらには、第三者のサイトへあなたの投稿に関する情報の送信を伴う設定も含まれます。
+    post_processing: 投稿の処理
+    stop_deliver: 配送停止
+    stop_deliver_hint_html: Mastodonの投稿を、他のソフトウェアでは自由に検索することができます。Mastodon内で行ったプライバシーの設定は無視され、あなたの投稿が意図しない人に見つかるおそれがあります。ここでは、他のサーバーやソフトウェアであなたの投稿が見つからないようにする設定が可能です。ただしリスクは伴います。
+    title: Privacy extra settings
   privacy_policy:
     title: Privacy Policy
   reactions:
@@ -1844,6 +1851,10 @@ en:
       default: "%b %d, %Y, %H:%M"
       month: "%b %Y"
       time: "%H:%M"
+  translation:
+    errors:
+      quota_exceeded: The server-wide usage quota for the translation service has been exceeded.
+      too_many_requests: There have been too many requests to the translation service recently.
   two_factor_authentication:
     add: Add
     disable: Disable 2FA

config/locales/ja.yml

@@ -1592,9 +1592,14 @@ ja:
     reach_hint_html: Control whether you want to be discovered and followed by new people. Do you want your posts to appear on the Explore screen? Do you want other people to see you in their follow recommendations? Do you want to accept all new followers automatically, or have granular control over each one?
     search: Search
     search_hint_html: Control how you want to be found. Do you want people to find you by what you've publicly posted about? Do you want people outside Mastodon to find your profile when searching the web? Please mind that total exclusion from all search engines cannot be guaranteed for public information.
+    title: Privacy and reach
+  privacy_extra:
+    hint_html: これらはkmyblue独自のプライバシー設定項目です。この機能を利用することで、あなたは追加の恩恵を受けることができます。なおこれらの設定の一部は他のサーバーにも送信されますが、kmyblue以外で対応が確認されているソフトウェアは現在確認できていません。他のサーバーではこれらの設定は無視されること、ご了承ください。
+    post_processing_hint_html: 投稿された情報に対して、システムが追加で行うことができる操作を制御します。これらには、第三者のサイトへあなたの投稿に関する情報の送信を伴う設定も含まれます。
+    post_processing: 投稿の処理
     stop_deliver: 配送停止
     stop_deliver_hint_html: Mastodonの投稿を、他のソフトウェアでは自由に検索することができます。Mastodon内で行ったプライバシーの設定は無視され、あなたの投稿が意図しない人に見つかるおそれがあります。ここでは、他のサーバーやソフトウェアであなたの投稿が見つからないようにする設定が可能です。ただしリスクは伴います。
-    title: Privacy and reach
+    title: プライバシー追加設定
   privacy_policy:
     title: プライバシーポリシー
   reactions:

config/locales/simple_form.en.yml

@@ -242,6 +242,7 @@ en:
         setting_hide_network: Hide your social graph
         setting_hide_recent_emojis: Hide recent emojis
         setting_hide_statuses_count: Hide statuses count
+        setting_link_preview: Generate post link preview card
         setting_noai: Set noai meta tags
         setting_public_post_to_unlisted: Convert public post to public unlisted if not using Web app
         setting_reduce_motion: Reduce motion in animations
@@ -253,6 +254,7 @@ en:
         setting_stop_emoji_reaction_streaming: Disable stamp streamings
         setting_system_font_ui: Use system's default font
         setting_theme: Site theme
+        setting_translatable_private: Allow other users translation of your private posts
         setting_trends: Show today's trends
         setting_unfollow_modal: Show confirmation dialog before unfollowing someone
         setting_unsafe_limited_distribution: Send limit posts with unsafe way to other servers

config/locales/simple_form.ja.yml

@@ -251,6 +251,7 @@ ja:
         setting_hide_network: 繋がりを隠す
         setting_hide_recent_emojis: 絵文字ピッカーで最近使用した絵文字を隠す（リアクションデッキのみを表示する）
         setting_hide_statuses_count: 投稿数を隠す
+        setting_link_preview: リンクのプレビューを生成する
         setting_stay_privacy: 投稿時に公開範囲を保存する
         setting_noai: 自分のコンテンツのAI学習利用に対して不快感を表明する
         setting_public_post_to_unlisted: サードパーティから公開範囲「公開」で投稿した場合、「ローカル公開」に変更する
@@ -263,6 +264,7 @@ ja:
         setting_stop_emoji_reaction_streaming: スタンプのストリーミングを停止する
         setting_system_font_ui: システムのデフォルトフォントを使う
         setting_theme: サイトテーマ
+        setting_translatable_private: 非公開投稿の翻訳を許可する
         setting_trends: 本日のトレンドタグを表示する
         setting_unfollow_modal: フォローを解除する前に確認ダイアログを表示する
         setting_unsafe_limited_distribution: 安全でない方法で限定投稿を他サーバーに配信する (非推奨)

config/routes/settings.rb

@@ -62,6 +62,7 @@ namespace :settings do
   resource :migration, only: [:show, :create]
   resource :verification, only: :show
   resource :privacy, only: [:show, :update], controller: 'privacy'
+  resource :privacy_extra, only: [:show, :update], controller: 'privacy_extra'
 
   namespace :migration do
     resource :redirect, only: [:new, :create, :destroy]

config/webpacker.yml

@@ -58,7 +58,7 @@ development:
   # Reference: https://webpack.js.org/configuration/dev-server/
   dev_server:
     https: false
-    host: localhost
+    host: 0.0.0.0
     port: 3035
     public: localhost:3035
     hmr: false

db/post_migrate/20230803082451_add_unique_index_on_preview_cards_statuses.rb

@@ -15,10 +15,22 @@ class AddUniqueIndexOnPreviewCardsStatuses < ActiveRecord::Migration[6.1]
 
   private
 
+  def supports_concurrent_reindex?
+    @supports_concurrent_reindex ||= begin
+      version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
+      version >= 12_000
+    end
+  end
+
   def deduplicate_and_reindex!
     deduplicate_preview_cards!
 
-    safety_assured { execute 'REINDEX INDEX CONCURRENTLY preview_cards_statuses_pkey' }
+    if supports_concurrent_reindex?
+      safety_assured { execute 'REINDEX INDEX CONCURRENTLY preview_cards_statuses_pkey' }
+    else
+      remove_index :preview_cards_statuses, name: :preview_cards_statuses_pkey
+      add_index :preview_cards_statuses, [:status_id, :preview_card_id], name: :preview_cards_statuses_pkey, algorithm: :concurrently, unique: true
+    end
   rescue ActiveRecord::RecordNotUnique
     retry
   end

dist/nginx.conf

@@ -36,7 +36,11 @@ server {
   server_name example.com;
 
   ssl_protocols TLSv1.2 TLSv1.3;
-  ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA;
+
+  # You can use https://ssl-config.mozilla.org/ to generate your cipher set.
+  # We recommend their "Intermediate" level.
+  ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305;
+
   ssl_prefer_server_ciphers on;
   ssl_session_cache shared:SSL:10m;
   ssl_session_tickets off;

lib/paperclip/transcoder.rb

@@ -4,6 +4,9 @@ module Paperclip
   # This transcoder is only to be used for the MediaAttachment model
   # to check when uploaded videos are actually gifv's
   class Transcoder < Paperclip::Processor
+    # This is the H.264 "High" value taken from https://www.dr-lex.be/info-stuff/videocalc.html
+    BITS_PER_PIXEL = 0.11
+
     def initialize(file, options = {}, attachment = nil)
       super
 
@@ -38,8 +41,11 @@ module Paperclip
         @output_options['vframes'] = 1
       when 'mp4'
         unless eligible_to_passthrough?(metadata)
-          @output_options['acodec'] = 'aac'
+          bitrate = (metadata.width * metadata.height * 30 * BITS_PER_PIXEL) / 1_000
-          @output_options['strict'] = 'experimental'
+
+          @output_options['b:v']     = "#{bitrate}k"
+          @output_options['maxrate'] = "#{bitrate + 192}k"
+          @output_options['bufsize'] = "#{bitrate * 5}k"
 
           if high_vfr?(metadata)
             @output_options['vsync'] = 'vfr'

package.json

@@ -153,7 +153,7 @@
   },
   "devDependencies": {
     "@formatjs/cli": "^6.1.1",
-    "@testing-library/jest-dom": "^5.16.5",
+    "@testing-library/jest-dom": "^6.0.0",
     "@testing-library/react": "^14.0.0",
     "@types/babel__core": "^7.20.1",
     "@types/emoji-mart": "^3.0.9",

spec/controllers/api/v1/timelines/tag_controller_spec.rb

@@ -5,36 +5,66 @@ require 'rails_helper'
 describe Api::V1::Timelines::TagController do
   render_views
 
-  let(:user) { Fabricate(:user) }
+  let(:user)   { Fabricate(:user) }
+  let(:token)  { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses') }
 
   before do
     allow(controller).to receive(:doorkeeper_token) { token }
   end
 
-  context 'with a user context' do
+  describe 'GET #show' do
-    let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id) }
+    subject do
+      get :show, params: { id: 'test' }
+    end
 
-    describe 'GET #show' do
+    before do
-      before do
+      PostStatusService.new.call(user.account, text: 'It is a #test')
-        PostStatusService.new.call(user.account, text: 'It is a #test')
+    end
+
+    context 'when the instance allows public preview' do
+      context 'when the user is not authenticated' do
+        let(:token) { nil }
+
+        it 'returns http success', :aggregate_failures do
+          subject
+
+          expect(response).to have_http_status(200)
+          expect(response.headers['Link'].links.size).to eq(2)
+        end
       end
 
-      it 'returns http success' do
+      context 'when the user is authenticated' do
-        get :show, params: { id: 'test' }
+        it 'returns http success', :aggregate_failures do
-        expect(response).to have_http_status(200)
+          subject
-        expect(response.headers['Link'].links.size).to eq(2)
+
+          expect(response).to have_http_status(200)
+          expect(response.headers['Link'].links.size).to eq(2)
+        end
       end
     end
-  end
 
-  context 'without a user context' do
+    context 'when the instance does not allow public preview' do
-    let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil) }
+      before do
+        Form::AdminSettings.new(timeline_preview: false).save
+      end
 
-    describe 'GET #show' do
+      context 'when the user is not authenticated' do
-      it 'returns http success' do
+        let(:token) { nil }
-        get :show, params: { id: 'test' }
+
-        expect(response).to have_http_status(200)
+        it 'returns http unauthorized' do
-        expect(response.headers['Link']).to be_nil
+          subject
+
+          expect(response).to have_http_status(401)
+        end
+      end
+
+      context 'when the user is authenticated' do
+        it 'returns http success', :aggregate_failures do
+          subject
+
+          expect(response).to have_http_status(200)
+          expect(response.headers['Link'].links.size).to eq(2)
+        end
       end
     end
   end

spec/controllers/well_known/webfinger_controller_spec.rb

@@ -3,6 +3,8 @@
 require 'rails_helper'
 
 describe WellKnown::WebfingerController do
+  include RoutingHelper
+
   render_views
 
   describe 'GET #show' do
@@ -167,5 +169,67 @@ describe WellKnown::WebfingerController do
         expect(response).to have_http_status(400)
       end
     end
+
+    context 'when an account has an avatar' do
+      let(:alice) { Fabricate(:account, username: 'alice', avatar: attachment_fixture('attachment.jpg')) }
+      let(:resource) { alice.to_webfinger_s }
+
+      it 'returns avatar in response' do
+        perform_show!
+
+        avatar_link = get_avatar_link(body_as_json)
+        expect(avatar_link).to_not be_nil
+        expect(avatar_link[:type]).to eq alice.avatar.content_type
+        expect(avatar_link[:href]).to eq full_asset_url(alice.avatar)
+      end
+
+      context 'with limited federation mode' do
+        before do
+          allow(Rails.configuration.x).to receive(:limited_federation_mode).and_return(true)
+        end
+
+        it 'does not return avatar in response' do
+          perform_show!
+
+          avatar_link = get_avatar_link(body_as_json)
+          expect(avatar_link).to be_nil
+        end
+      end
+
+      context 'when enabling DISALLOW_UNAUTHENTICATED_API_ACCESS' do
+        around do |example|
+          ClimateControl.modify DISALLOW_UNAUTHENTICATED_API_ACCESS: 'true' do
+            example.run
+          end
+        end
+
+        it 'does not return avatar in response' do
+          perform_show!
+
+          avatar_link = get_avatar_link(body_as_json)
+          expect(avatar_link).to be_nil
+        end
+      end
+    end
+
+    context 'when an account does not have an avatar' do
+      let(:alice) { Fabricate(:account, username: 'alice', avatar: nil) }
+      let(:resource) { alice.to_webfinger_s }
+
+      before do
+        perform_show!
+      end
+
+      it 'does not return avatar in response' do
+        avatar_link = get_avatar_link(body_as_json)
+        expect(avatar_link).to be_nil
+      end
+    end
+  end
+
+  private
+
+  def get_avatar_link(json)
+    json[:links].find { |link| link[:rel] == 'http://webfinger.net/rel/avatar' }
   end
 end

spec/lib/search_query_parser_spec.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+require 'parslet/rig/rspec'
+
+describe SearchQueryParser do
+  let(:parser) { described_class.new }
+
+  context 'with term' do
+    it 'consumes "hello"' do
+      expect(parser.term).to parse('hello')
+    end
+  end
+
+  context 'with prefix' do
+    it 'consumes "foo:"' do
+      expect(parser.prefix).to parse('foo:')
+    end
+  end
+
+  context 'with operator' do
+    it 'consumes "+"' do
+      expect(parser.operator).to parse('+')
+    end
+
+    it 'consumes "-"' do
+      expect(parser.operator).to parse('-')
+    end
+  end
+
+  context 'with shortcode' do
+    it 'consumes ":foo:"' do
+      expect(parser.shortcode).to parse(':foo:')
+    end
+  end
+
+  context 'with phrase' do
+    it 'consumes "hello world"' do
+      expect(parser.phrase).to parse('"hello world"')
+    end
+  end
+
+  context 'with clause' do
+    it 'consumes "foo"' do
+      expect(parser.clause).to parse('foo')
+    end
+
+    it 'consumes "-foo"' do
+      expect(parser.clause).to parse('-foo')
+    end
+
+    it 'consumes "foo:bar"' do
+      expect(parser.clause).to parse('foo:bar')
+    end
+
+    it 'consumes "-foo:bar"' do
+      expect(parser.clause).to parse('-foo:bar')
+    end
+
+    it 'consumes \'foo:"hello world"\'' do
+      expect(parser.clause).to parse('foo:"hello world"')
+    end
+
+    it 'consumes \'-foo:"hello world"\'' do
+      expect(parser.clause).to parse('-foo:"hello world"')
+    end
+
+    it 'consumes "foo:"' do
+      expect(parser.clause).to parse('foo:')
+    end
+
+    it 'consumes \'"\'' do
+      expect(parser.clause).to parse('"')
+    end
+  end
+
+  context 'with query' do
+    it 'consumes "hello -world"' do
+      expect(parser.query).to parse('hello -world')
+    end
+
+    it 'consumes \'foo "hello world"\'' do
+      expect(parser.query).to parse('foo "hello world"')
+    end
+
+    it 'consumes "foo:bar hello"' do
+      expect(parser.query).to parse('foo:bar hello')
+    end
+
+    it 'consumes \'"hello" world "\'' do
+      expect(parser.query).to parse('"hello" world "')
+    end
+
+    it 'consumes "foo:bar bar: hello"' do
+      expect(parser.query).to parse('foo:bar bar: hello')
+    end
+  end
+end

spec/lib/search_query_transformer_spec.rb

@@ -3,16 +3,57 @@
 require 'rails_helper'
 
 describe SearchQueryTransformer do
-  describe 'initialization' do
+  subject { described_class.new.apply(parser, current_account: nil) }
-    let(:parser) { SearchQueryParser.new.parse('query') }
 
-    it 'sets attributes' do
+  let(:parser) { SearchQueryParser.new.parse(query) }
-      transformer = described_class.new.apply(parser)
 
-      expect(transformer.should_clauses.first).to be_nil
+  context 'with "hello world"' do
-      expect(transformer.must_clauses.first).to be_a(SearchQueryTransformer::TermClause)
+    let(:query) { 'hello world' }
-      expect(transformer.must_not_clauses.first).to be_nil
+
-      expect(transformer.filter_clauses.first).to be_nil
+    it 'transforms clauses' do
+      expect(subject.must_clauses.map(&:term)).to match_array %w(hello world)
+      expect(subject.must_not_clauses).to be_empty
+      expect(subject.filter_clauses).to be_empty
+    end
+  end
+
+  context 'with "hello -world"' do
+    let(:query) { 'hello -world' }
+
+    it 'transforms clauses' do
+      expect(subject.must_clauses.map(&:term)).to match_array %w(hello)
+      expect(subject.must_not_clauses.map(&:term)).to match_array %w(world)
+      expect(subject.filter_clauses).to be_empty
+    end
+  end
+
+  context 'with "hello is:reply"' do
+    let(:query) { 'hello is:reply' }
+
+    it 'transforms clauses' do
+      expect(subject.must_clauses.map(&:term)).to match_array %w(hello)
+      expect(subject.must_not_clauses).to be_empty
+      expect(subject.filter_clauses.map(&:term)).to match_array %w(reply)
+    end
+  end
+
+  context 'with "foo: bar"' do
+    let(:query) { 'foo: bar' }
+
+    it 'transforms clauses' do
+      expect(subject.must_clauses.map(&:term)).to match_array %w(foo bar)
+      expect(subject.must_not_clauses).to be_empty
+      expect(subject.filter_clauses).to be_empty
+    end
+  end
+
+  context 'with "foo:bar"' do
+    let(:query) { 'foo:bar' }
+
+    it 'transforms clauses' do
+      expect(subject.must_clauses.map(&:term)).to contain_exactly('foo bar')
+      expect(subject.must_not_clauses).to be_empty
+      expect(subject.filter_clauses).to be_empty
     end
   end
 end

spec/services/search_service_spec.rb

@@ -83,15 +83,6 @@ describe SearchService, type: :service do
           expect(Tag).to have_received(:search_for).with('tag', 10, 0, exclude_unreviewed: nil)
           expect(results).to eq empty_results.merge(hashtags: [tag])
         end
-
-        it 'does not include tag when starts with @ character' do
-          query = '@username'
-          allow(Tag).to receive(:search_for)
-
-          results = subject.call(query, nil, 10)
-          expect(Tag).to_not have_received(:search_for)
-          expect(results).to eq empty_results
-        end
       end
     end
   end

streaming/index.js

@@ -763,6 +763,11 @@ const startServer = async () => {
     const listener = message => {
       const { event, payload } = message;
 
+      // reference_texts property is not working if ProcessReferencesWorker is
+      // used on PostStatusService and so on. (Asynchronous processing)
+      const reference_texts = payload.reference_texts || [];
+      delete payload.reference_texts;
+
       // Streaming only needs to apply filtering to some channels and only to
       // some events. This is because majority of the filtering happens on the
       // Ruby on Rails side when producing the event for streaming.
@@ -908,7 +913,7 @@ const startServer = async () => {
           if (req.cachedFilters) {
             const status = payload;
             // TODO: Calculate searchableContent in Ruby on Rails:
-            const searchableContent = ([status.spoiler_text || '', status.content].concat((status.poll && status.poll.options) ? status.poll.options.map(option => option.title) : [])).concat(status.media_attachments.map(att => att.description)).join('\n\n').replace(/<br\s*\/?>/g, '\n').replace(/<\/p><p>/g, '\n\n');
+            const searchableContent = ([status.spoiler_text || '', status.content, ...(reference_texts || [])].concat((status.poll && status.poll.options) ? status.poll.options.map(option => option.title) : [])).concat(status.media_attachments.map(att => att.description)).join('\n\n').replace(/<br\s*\/?>/g, '\n').replace(/<\/p><p>/g, '\n\n');
             const searchableTextContent = JSDOM.fragment(searchableContent).textContent;
 
             const now = new Date();