Merge commit 'aa98c8fbeb' into kb_migration

app/controllers/relationships_controller.rb

@@ -19,6 +19,8 @@ class RelationshipsController < ApplicationController
     @form.save
   rescue ActionController::ParameterMissing
     # Do nothing
+  rescue Mastodon::NotPermittedError, ActiveRecord::RecordNotFound
+    flash[:alert] = I18n.t('relationships.follow_failure') if action_from_button == 'follow'
   ensure
     redirect_to relationships_path(filter_params)
   end
@@ -60,8 +62,8 @@ class RelationshipsController < ApplicationController
       'unfollow'
     elsif params[:remove_from_followers]
       'remove_from_followers'
-    elsif params[:block_domains]
-      'block_domains'
+    elsif params[:block_domains] || params[:remove_domains_from_followers]
+      'remove_domains_from_followers'
     end
   end
 

app/javascript/mastodon/components/status_content.jsx

@@ -6,7 +6,7 @@ import { Link } from 'react-router-dom';
 import classnames from 'classnames';
 import PollContainer from 'mastodon/containers/poll_container';
 import Icon from 'mastodon/components/icon';
-import { autoPlayGif, languages as preloadedLanguages, translationEnabled } from 'mastodon/initial_state';
+import { autoPlayGif, languages as preloadedLanguages } from 'mastodon/initial_state';
 
 const MAX_HEIGHT = 706; // 22px * 32 (+ 2px padding at the top)
 
@@ -220,7 +220,7 @@ class StatusContent extends React.PureComponent {
 
     const hidden = this.props.onExpandedToggle ? !this.props.expanded : this.state.hidden;
     const renderReadMore = this.props.onClick && status.get('collapsed');
-    const renderTranslate = translationEnabled && this.context.identity.signedIn && this.props.onTranslate && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('contentHtml').length > 0 && status.get('language') !== null && intl.locale !== status.get('language');
+    const renderTranslate = this.props.onTranslate && status.get('translatable');
 
     const content = { __html: status.get('translation') ? status.getIn(['translation', 'content']) : status.get('contentHtml') };
     const spoilerContent = { __html: status.get('spoilerHtml') };

app/javascript/mastodon/features/ui/components/header.jsx

@@ -22,8 +22,8 @@ const mapDispatchToProps = (dispatch) => ({
   },
 });
 
-export default @connect(null, mapDispatchToProps)
-@withRouter
+export default @withRouter
+@connect(null, mapDispatchToProps)
 class Header extends React.PureComponent {
 
   static contextTypes = {

app/javascript/mastodon/features/ui/components/navigation_panel.jsx

@@ -82,8 +82,8 @@ class NavigationPanel extends React.Component {
         {signedIn && (
           <React.Fragment>
             <ColumnLink transparent to='/conversations' icon='at' text={intl.formatMessage(messages.direct)} />
-            <ColumnLink transparent to='/favourites' icon='star' text={intl.formatMessage(messages.favourites)} />
             <ColumnLink transparent to='/bookmarks' icon='bookmark' text={intl.formatMessage(messages.bookmarks)} />
+            <ColumnLink transparent to='/favourites' icon='star' text={intl.formatMessage(messages.favourites)} />
             <ColumnLink transparent to='/lists' icon='list-ul' text={intl.formatMessage(messages.lists)} />
 
             <ListPanel />

app/javascript/mastodon/initial_state.js

@@ -80,7 +80,6 @@
  * @property {boolean} use_blurhash
  * @property {boolean=} use_pending_items
  * @property {string} version
- * @property {boolean} translation_enabled
  */
 
 /**
@@ -132,7 +131,6 @@ export const unfollowModal = getMeta('unfollow_modal');
 export const useBlurhash = getMeta('use_blurhash');
 export const usePendingItems = getMeta('use_pending_items');
 export const version = getMeta('version');
-export const translationEnabled = getMeta('translation_enabled');
 export const languages = initialState?.languages;
 export const statusPageUrl = getMeta('status_page_url');
 

app/javascript/styles/application.scss

@@ -23,3 +23,4 @@
 @import 'mastodon/dashboard';
 @import 'mastodon/rtl';
 @import 'mastodon/accessibility';
+@import 'mastodon/rich_text';

app/javascript/styles/mastodon/rich_text.scss

@@ -0,0 +1,64 @@
+.status__content__text,
+.e-content,
+.reply-indicator__content {
+  pre,
+  blockquote {
+    margin-bottom: 20px;
+    white-space: pre-wrap;
+    unicode-bidi: plaintext;
+
+    &:last-child {
+      margin-bottom: 0;
+    }
+  }
+
+  blockquote {
+    padding-left: 10px;
+    border-left: 3px solid $darker-text-color;
+    color: $darker-text-color;
+    white-space: normal;
+
+    p:last-child {
+      margin-bottom: 0;
+    }
+  }
+
+  & > ul,
+  & > ol {
+    margin-bottom: 20px;
+  }
+
+  b,
+  strong {
+    font-weight: 700;
+  }
+
+  em,
+  i {
+    font-style: italic;
+  }
+
+  ul,
+  ol {
+    margin-left: 2em;
+
+    p {
+      margin: 0;
+    }
+  }
+
+  ul {
+    list-style-type: disc;
+  }
+
+  ol {
+    list-style-type: decimal;
+  }
+}
+
+.reply-indicator__content {
+  blockquote {
+    border-left-color: $inverted-text-color;
+    color: $inverted-text-color;
+  }
+}

app/lib/translation_service.rb

@@ -21,6 +21,10 @@ class TranslationService
     ENV['DEEPL_API_KEY'].present? || ENV['LIBRE_TRANSLATE_ENDPOINT'].present?
   end
 
+  def supported?(_source_language, _target_language)
+    false
+  end
+
   def translate(_text, _source_language, _target_language)
     raise NotImplementedError
   end

app/lib/translation_service/deepl.rb

@@ -11,33 +11,53 @@ class TranslationService::DeepL < TranslationService
   end
 
   def translate(text, source_language, target_language)
-    request(text, source_language, target_language).perform do |res|
+    form = { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' }
+    request(:post, '/v2/translate', form: form) do |res|
+      transform_response(res.body_with_limit)
+    end
+  end
+
+  def supported?(source_language, target_language)
+    source_language.in?(languages('source')) && target_language.in?(languages('target'))
+  end
+
+  private
+
+  def languages(type)
+    Rails.cache.fetch("translation_service/deepl/languages/#{type}", expires_in: 7.days, race_condition_ttl: 1.minute) do
+      request(:get, "/v2/languages?type=#{type}") do |res|
+        # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so
+        # they are supported but not returned by the API.
+        extra = type == 'source' ? [nil] : %w(en pt)
+        languages = Oj.load(res.body_with_limit).map { |language| language['language'].downcase }
+
+        languages + extra
+      end
+    end
+  end
+
+  def request(verb, path, **options)
+    req = Request.new(verb, "#{base_url}#{path}", **options)
+    req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}")
+    req.perform do |res|
       case res.code
       when 429
         raise TooManyRequestsError
       when 456
         raise QuotaExceededError
       when 200...300
-        transform_response(res.body_with_limit)
+        yield res
       else
         raise UnexpectedResponseError
       end
     end
   end
 
-  private
-
-  def request(text, source_language, target_language)
-    req = Request.new(:post, endpoint_url, form: { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' })
-    req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}")
-    req
-  end
-
-  def endpoint_url
+  def base_url
     if @plan == 'free'
-      'https://api-free.deepl.com/v2/translate'
+      'https://api-free.deepl.com'
     else
-      'https://api.deepl.com/v2/translate'
+      'https://api.deepl.com'
     end
   end
 

app/lib/translation_service/libre_translate.rb

@@ -9,29 +9,45 @@ class TranslationService::LibreTranslate < TranslationService
   end
 
   def translate(text, source_language, target_language)
-    request(text, source_language, target_language).perform do |res|
+    body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key)
+    request(:post, '/translate', body: body) do |res|
+      transform_response(res.body_with_limit, source_language)
+    end
+  end
+
+  def supported?(source_language, target_language)
+    languages.key?(source_language) && languages[source_language].include?(target_language)
+  end
+
+  private
+
+  def languages
+    Rails.cache.fetch('translation_service/libre_translate/languages', expires_in: 7.days, race_condition_ttl: 1.minute) do
+      request(:get, '/languages') do |res|
+        languages = Oj.load(res.body_with_limit).to_h { |language| [language['code'], language['targets']] }
+        languages[nil] = languages.values.flatten.uniq
+        languages
+      end
+    end
+  end
+
+  def request(verb, path, **options)
+    req = Request.new(verb, "#{@base_url}#{path}", allow_local: true, **options)
+    req.add_headers('Content-Type': 'application/json')
+    req.perform do |res|
       case res.code
       when 429
         raise TooManyRequestsError
       when 403
         raise QuotaExceededError
       when 200...300
-        transform_response(res.body_with_limit, source_language)
+        yield res
       else
         raise UnexpectedResponseError
       end
     end
   end
 
-  private
-
-  def request(text, source_language, target_language)
-    body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key)
-    req = Request.new(:post, "#{@base_url}/translate", body: body, allow_local: true)
-    req.add_headers('Content-Type': 'application/json')
-    req
-  end
-
   def transform_response(str, source_language)
     json = Oj.load(str, mode: :strict)
 

app/mailers/application_mailer.rb

@@ -7,9 +7,17 @@ class ApplicationMailer < ActionMailer::Base
   helper :instance
   helper :formatting
 
+  after_action :set_autoreply_headers!
+
   protected
 
   def locale_for_account(account, &block)
     I18n.with_locale(account.user_locale || I18n.default_locale, &block)
   end
+
+  def set_autoreply_headers!
+    headers['Precedence'] = 'list'
+    headers['X-Auto-Response-Suppress'] = 'All'
+    headers['Auto-Submitted'] = 'auto-generated'
+  end
 end

app/models/account_filter.rb

@@ -17,13 +17,13 @@ class AccountFilter
   attr_reader :params
 
   def initialize(params)
-    @params = params
+    @params = params.to_h.symbolize_keys
   end
 
   def results
     scope = Account.includes(:account_stat, user: [:ips, :invite_request]).without_instance_actor.reorder(nil)
 
-    params.each do |key, value|
+    relevant_params.each do |key, value|
       next if key.to_s == 'page'
 
       scope.merge!(scope_for(key, value)) if value.present?
@@ -34,6 +34,16 @@ class AccountFilter
 
   private
 
+  def relevant_params
+    params.tap do |args|
+      args.delete(:origin) if origin_is_remote_and_domain_present?
+    end
+  end
+
+  def origin_is_remote_and_domain_present?
+    params[:origin] == 'remote' && params[:by_domain].present?
+  end
+
   def scope_for(key, value)
     case key.to_s
     when 'origin'
@@ -94,7 +104,15 @@ class AccountFilter
   def order_scope(value)
     case value.to_s
     when 'active'
-      accounts_with_users.left_joins(:account_stat).order(Arel.sql('coalesce(users.current_sign_in_at, account_stats.last_status_at, to_timestamp(0)) desc, accounts.id desc'))
+      accounts_with_users
+        .left_joins(:account_stat)
+        .order(
+          Arel.sql(
+            <<~SQL.squish
+              COALESCE(users.current_sign_in_at, account_stats.last_status_at, to_timestamp(0)) DESC, accounts.id DESC
+            SQL
+          )
+        )
     when 'recent'
       Account.recent
     else

app/models/concerns/omniauthable.rb

@@ -61,7 +61,7 @@ module Omniauthable
         user.account.avatar_remote_url = nil
       end
 
-      user.skip_confirmation! if email_is_verified
+      user.confirm! if email_is_verified
       user.save!
       user
     end

app/models/form/account_batch.rb

@@ -17,8 +17,8 @@ class Form::AccountBatch
       unfollow!
     when 'remove_from_followers'
       remove_from_followers!
-    when 'block_domains'
-      block_domains!
+    when 'remove_domains_from_followers'
+      remove_domains_from_followers!
     when 'approve'
       approve!
     when 'reject'
@@ -35,9 +35,15 @@ class Form::AccountBatch
   private
 
   def follow!
+    error = nil
+
     accounts.each do |target_account|
       FollowService.new.call(current_account, target_account)
+    rescue Mastodon::NotPermittedError, ActiveRecord::RecordNotFound => e
+      error ||= e
     end
+
+    raise error if error.present?
   end
 
   def unfollow!
@@ -50,10 +56,8 @@ class Form::AccountBatch
     RemoveFromFollowersService.new.call(current_account, account_ids)
   end
 
-  def block_domains!
-    AfterAccountDomainBlockWorker.push_bulk(account_domains) do |domain|
-      [current_account.id, domain]
-    end
+  def remove_domains_from_followers!
+    RemoveDomainsFromFollowersService.new.call(current_account, account_domains)
   end
 
   def account_domains

app/models/status.rb

@@ -235,6 +235,16 @@ class Status < ApplicationRecord
     public_visibility? || unlisted_visibility?
   end
 
+  def translatable?
+    translate_target_locale = I18n.locale.to_s.split(/[_-]/).first
+
+    distributable? &&
+      content.present? &&
+      language != translate_target_locale &&
+      TranslationService.configured? &&
+      TranslationService.configured.supported?(language, translate_target_locale)
+  end
+
   alias sign? distributable?
 
   def with_media?

app/serializers/initial_state_serializer.rb

@@ -30,7 +30,6 @@ class InitialStateSerializer < ActiveModel::Serializer
       timeline_preview: Setting.timeline_preview,
       activity_api_enabled: Setting.activity_api_enabled,
       single_user_mode: Rails.configuration.x.single_user_mode,
-      translation_enabled: TranslationService.configured?,
       trends_as_landing_page: Setting.trends_as_landing_page,
       status_page_url: Setting.status_page_url,
     }

app/serializers/rest/status_serializer.rb

@@ -4,7 +4,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
   include FormattingHelper
 
   attributes :id, :created_at, :in_reply_to_id, :in_reply_to_account_id,
-             :sensitive, :spoiler_text, :visibility, :language,
+             :sensitive, :spoiler_text, :visibility, :language, :translatable,
              :uri, :url, :replies_count, :reblogs_count,
              :favourites_count, :emoji_reactions, :edited_at
 
@@ -50,6 +50,10 @@ class REST::StatusSerializer < ActiveModel::Serializer
     object.account.user_shows_application? || (current_user? && current_user.account_id == object.account_id)
   end
 
+  def translatable
+    current_user? && object.translatable?
+  end
+
   def visibility
     # This visibility is masked behind "private"
     # to avoid API changes because there are no

app/services/follow_migration_service.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class FollowMigrationService < FollowService
+  # Follow an account with the same settings as another account, and unfollow the old account once the request is sent
+  # @param [Account] source_account From which to follow
+  # @param [Account] target_account Account to follow
+  # @param [Account] old_target_account Account to unfollow once the follow request has been sent to the new one
+  # @option [Boolean] bypass_locked Whether to immediately follow the new account even if it is locked
+  def call(source_account, target_account, old_target_account, bypass_locked: false)
+    @old_target_account = old_target_account
+
+    follow    = source_account.active_relationships.find_by(target_account: old_target_account)
+    reblogs   = follow&.show_reblogs?
+    notify    = follow&.notify?
+    languages = follow&.languages
+
+    super(source_account, target_account, reblogs: reblogs, notify: notify, languages: languages, bypass_locked: bypass_locked, bypass_limit: true)
+  end
+
+  private
+
+  def request_follow!
+    follow_request = @source_account.request_follow!(@target_account, **follow_options.merge(rate_limit: @options[:with_rate_limit], bypass_limit: @options[:bypass_limit]))
+
+    if @target_account.local?
+      LocalNotificationWorker.perform_async(@target_account.id, follow_request.id, follow_request.class.name, 'follow_request')
+      UnfollowService.new.call(@source_account, @old_target_account, skip_unmerge: true)
+    elsif @target_account.activitypub?
+      ActivityPub::MigratedFollowDeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url, @old_target_account.id)
+    end
+
+    follow_request
+  end
+
+  def direct_follow!
+    follow = super
+    UnfollowService.new.call(@source_account, @old_target_account, skip_unmerge: true)
+    follow
+  end
+end

app/services/remove_domains_from_followers_service.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class RemoveDomainsFromFollowersService < BaseService
+  include Payloadable
+
+  def call(source_account, target_domains)
+    source_account.passive_relationships.where(account_id: Account.where(domain: target_domains)).find_each do |follow|
+      follow.destroy
+
+      create_notification(follow) if source_account.local? && !follow.account.local? && follow.account.activitypub?
+    end
+  end
+
+  private
+
+  def create_notification(follow)
+    ActivityPub::DeliveryWorker.perform_async(build_json(follow), follow.target_account_id, follow.account.inbox_url)
+  end
+
+  def build_json(follow)
+    Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer))
+  end
+end

app/services/translate_status_service.rb

@@ -6,7 +6,7 @@ class TranslateStatusService < BaseService
   include FormattingHelper
 
   def call(status, target_language)
-    raise Mastodon::NotPermittedError unless status.public_visibility? || status.unlisted_visibility?
+    raise Mastodon::NotPermittedError unless status.translatable?
 
     @status = status
     @content = status_content_format(@status)

app/views/admin/statuses/show.html.haml

@@ -34,7 +34,7 @@
           %td
             - if @status.trend.allowed?
               %abbr{ title: t('admin.trends.tags.current_score', score: @status.trend.score) }= t('admin.trends.tags.trending_rank', rank: @status.trend.rank)
-            - elsif @status.trend.requires_review?
+            - elsif @status.requires_review?
               = t('admin.trends.pending_review')
             - else
               = t('admin.trends.not_allowed_to_trend')

app/views/relationships/show.html.haml

@@ -48,7 +48,7 @@
 
         = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_followers')]), name: :remove_from_followers, class: 'table-action-link', type: :submit, data: { confirm: t('relationships.confirm_remove_selected_followers') } unless following_relationship?
 
-        = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_domains')]), name: :block_domains, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } if followed_by_relationship?
+        = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_domains')]), name: :remove_domains_from_followers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } if followed_by_relationship?
     .batch-table__body
       - if @accounts.empty?
         = nothing_here 'nothing-here--under-tabs'

app/workers/activitypub/delivery_worker.rb

@@ -10,6 +10,16 @@ class ActivityPub::DeliveryWorker
 
   sidekiq_options queue: 'push', retry: 16, dead: false
 
+  # Unfortunately, we cannot control Sidekiq's jitter, so add our own
+  sidekiq_retry_in do |count|
+    # This is Sidekiq's default delay
+    delay  = (count**4) + 15
+    # Our custom jitter, that will be added to Sidekiq's built-in one.
+    # Sidekiq's built-in jitter is `rand(10) * (count + 1)`
+    jitter = rand(0.5 * (count**4))
+    delay + jitter
+  end
+
   HEADERS = { 'Content-Type' => 'application/activity+json' }.freeze
 
   def perform(json, source_account_id, inbox_url, options = {})

app/workers/activitypub/migrated_follow_delivery_worker.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class ActivityPub::MigratedFollowDeliveryWorker < ActivityPub::DeliveryWorker
+  def perform(json, source_account_id, inbox_url, old_target_account_id, options = {})
+    super(json, source_account_id, inbox_url, options)
+    unfollow_old_account!(old_target_account_id)
+  end
+
+  private
+
+  def unfollow_old_account!(old_target_account_id)
+    old_target_account = Account.find(old_target_account_id)
+    UnfollowService.new.call(@source_account, old_target_account, skip_unmerge: true)
+  rescue
+    true
+  end
+end

app/workers/unfollow_follow_worker.rb

@@ -10,13 +10,7 @@ class UnfollowFollowWorker
     old_target_account = Account.find(old_target_account_id)
     new_target_account = Account.find(new_target_account_id)
 
-    follow    = follower_account.active_relationships.find_by(target_account: old_target_account)
-    reblogs   = follow&.show_reblogs?
-    notify    = follow&.notify?
-    languages = follow&.languages
-
-    FollowService.new.call(follower_account, new_target_account, reblogs: reblogs, notify: notify, languages: languages, bypass_locked: bypass_locked, bypass_limit: true)
-    UnfollowService.new.call(follower_account, old_target_account, skip_unmerge: true)
+    FollowMigrationService.new.call(follower_account, new_target_account, old_target_account, bypass_locked: bypass_locked)
   rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError
     true
   end