Merge commit '45ba9ada34' into kb_migration_development

Gemfile

@@ -160,3 +160,5 @@ gem 'cocoon', '~> 1.2'
 
 gem 'net-http', '~> 0.3.2'
 gem 'rubyzip', '~> 2.3'
+
+gem 'hcaptcha', '~> 7.1'

Gemfile.lock

@@ -312,6 +312,8 @@ GEM
       sysexits (~> 1.1)
     hashdiff (1.0.1)
     hashie (5.0.0)
+    hcaptcha (7.1.0)
+      json
     highline (2.1.0)
     hiredis (0.6.3)
     hkdf (0.3.0)
@@ -696,7 +698,7 @@ GEM
       unicode-display_width (>= 1.1.1, < 3)
     terrapin (0.6.0)
       climate_control (>= 0.0.3, < 1.0)
-    thor (1.2.1)
+    thor (1.2.2)
     tilt (2.1.0)
     timeout (0.3.2)
     tpm-key_attestation (0.12.0)
@@ -806,6 +808,7 @@ DEPENDENCIES
   fuubar (~> 2.5)
   haml-rails (~> 2.0)
   haml_lint
+  hcaptcha (~> 7.1)
   hiredis (~> 0.6)
   htmlentities (~> 4.3)
   http (~> 5.1)

app/controllers/api/v1/emails/confirmations_controller.rb

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 
 class Api::V1::Emails::ConfirmationsController < Api::BaseController
-  before_action -> { doorkeeper_authorize! :write, :'write:accounts' }
-  before_action :require_user_owned_by_application!
-  before_action :require_user_not_confirmed!
+  before_action -> { authorize_if_got_token! :read, :'read:accounts' }, only: :check
+  before_action -> { doorkeeper_authorize! :write, :'write:accounts' }, except: :check
+  before_action :require_user_owned_by_application!, except: :check
+  before_action :require_user_not_confirmed!, except: :check
 
   def create
     current_user.update!(email: params[:email]) if params.key?(:email)
@@ -12,6 +13,10 @@ class Api::V1::Emails::ConfirmationsController < Api::BaseController
     render_empty
   end
 
+  def check
+    render json: current_user.confirmed?
+  end
+
   private
 
   def require_user_owned_by_application!

app/controllers/api/v1/statuses/reblogs_controller.rb

@@ -2,6 +2,8 @@
 
 class Api::V1::Statuses::ReblogsController < Api::BaseController
   include Authorization
+  include Redisable
+  include Lockable
 
   before_action -> { doorkeeper_authorize! :write, :'write:statuses' }
   before_action :require_user!
@@ -10,7 +12,9 @@ class Api::V1::Statuses::ReblogsController < Api::BaseController
   override_rate_limit_headers :create, family: :statuses
 
   def create
-    @status = ReblogService.new.call(current_account, @reblog, reblog_params)
+    with_redis_lock("reblog:#{current_account.id}:#{@reblog.id}") do
+      @status = ReblogService.new.call(current_account, @reblog, reblog_params)
+    end
 
     render json: @status, serializer: REST::StatusSerializer
   end

app/controllers/auth/confirmations_controller.rb

@@ -1,21 +1,63 @@
 # frozen_string_literal: true
 
 class Auth::ConfirmationsController < Devise::ConfirmationsController
+  include CaptchaConcern
+
   layout 'auth'
 
   before_action :set_body_classes
+  before_action :set_confirmation_user!, only: [:show, :confirm_captcha]
   before_action :require_unconfirmed!
 
+  before_action :extend_csp_for_captcha!, only: [:show, :confirm_captcha]
+  before_action :require_captcha_if_needed!, only: [:show]
+
   skip_before_action :require_functional!
 
+  def show
+    old_session_values = session.to_hash
+    reset_session
+    session.update old_session_values.except('session_id')
+
+    super
+  end
+
   def new
     super
 
     resource.email = current_user.unconfirmed_email || current_user.email if user_signed_in?
   end
 
+  def confirm_captcha
+    check_captcha! do |message|
+      flash.now[:alert] = message
+      render :captcha
+      return
+    end
+
+    show
+  end
+
   private
 
+  def require_captcha_if_needed!
+    render :captcha if captcha_required?
+  end
+
+  def set_confirmation_user!
+    # We need to reimplement looking up the user because
+    # Devise::ConfirmationsController#show looks up and confirms in one
+    # step.
+    confirmation_token = params[:confirmation_token]
+    return if confirmation_token.nil?
+
+    @confirmation_user = User.find_first_by_auth_conditions(confirmation_token: confirmation_token)
+  end
+
+  def captcha_user_bypass?
+    return true if @confirmation_user.nil? || @confirmation_user.confirmed?
+  end
+
   def require_unconfirmed!
     if user_signed_in? && current_user.confirmed? && current_user.unconfirmed_email.blank?
       redirect_to(current_user.approved? ? root_path : edit_user_registration_path)

app/controllers/concerns/captcha_concern.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module CaptchaConcern
+  extend ActiveSupport::Concern
+  include Hcaptcha::Adapters::ViewMethods
+
+  included do
+    helper_method :render_captcha
+  end
+
+  def captcha_available?
+    ENV['HCAPTCHA_SECRET_KEY'].present? && ENV['HCAPTCHA_SITE_KEY'].present?
+  end
+
+  def captcha_enabled?
+    captcha_available? && Setting.captcha_enabled
+  end
+
+  def captcha_user_bypass?
+    false
+  end
+
+  def captcha_required?
+    captcha_enabled? && !captcha_user_bypass?
+  end
+
+  def check_captcha!
+    return true unless captcha_required?
+
+    if verify_hcaptcha
+      true
+    else
+      if block_given?
+        message = flash[:hcaptcha_error]
+        flash.delete(:hcaptcha_error)
+        yield message
+      end
+      false
+    end
+  end
+
+  def extend_csp_for_captcha!
+    policy = request.content_security_policy
+    return unless captcha_required? && policy.present?
+
+    %w(script_src frame_src style_src connect_src).each do |directive|
+      values = policy.send(directive)
+      values << 'https://hcaptcha.com' unless values.include?('https://hcaptcha.com') || values.include?('https:')
+      values << 'https://*.hcaptcha.com' unless values.include?('https://*.hcaptcha.com') || values.include?('https:')
+      policy.send(directive, *values)
+    end
+  end
+
+  def render_captcha
+    return unless captcha_required?
+
+    hcaptcha_tags
+  end
+end

app/helpers/admin/settings_helper.rb

@@ -1,4 +1,7 @@
 # frozen_string_literal: true
 
 module Admin::SettingsHelper
+  def captcha_available?
+    ENV['HCAPTCHA_SECRET_KEY'].present? && ENV['HCAPTCHA_SITE_KEY'].present?
+  end
 end

app/javascript/mastodon/components/status_list.jsx

@@ -26,6 +26,7 @@ export default class StatusList extends ImmutablePureComponent {
     alwaysPrepend: PropTypes.bool,
     withCounters: PropTypes.bool,
     timelineId: PropTypes.string,
+    lastId: PropTypes.string,
   };
 
   static defaultProps = {
@@ -55,7 +56,8 @@ export default class StatusList extends ImmutablePureComponent {
   };
 
   handleLoadOlder = debounce(() => {
-    this.props.onLoadMore(this.props.statusIds.size > 0 ? this.props.statusIds.last() : undefined);
+    const { statusIds, lastId, onLoadMore } = this.props;
+    onLoadMore(lastId || (statusIds.size > 0 ? statusIds.last() : undefined));
   }, 300, { leading: true });
 
   _selectChild (index, align_top) {

app/javascript/mastodon/features/ui/containers/status_list_container.js

@@ -37,6 +37,7 @@ const makeMapStateToProps = () => {
 
   const mapStateToProps = (state, { timelineId }) => ({
     statusIds: getStatusIds(state, { type: timelineId }),
+    lastId:    state.getIn(['timelines', timelineId, 'items'])?.last(),
     isLoading: state.getIn(['timelines', timelineId, 'isLoading'], true),
     isPartial: state.getIn(['timelines', timelineId, 'isPartial'], false),
     hasMore:   state.getIn(['timelines', timelineId, 'hasMore']),

app/javascript/packs/sign_up.js

@@ -0,0 +1,15 @@
+import './public-path';
+import ready from '../mastodon/ready';
+import axios from 'axios';
+
+ready(() => {
+  setInterval(() => {
+    axios.get('/api/v1/emails/check_confirmation').then((response) => {
+      if (response.data) {
+        window.location = '/start';
+      }
+    }).catch(error => {
+      console.error(error);
+    });
+  }, 5000);
+});

app/javascript/styles/mastodon/components.scss

@@ -3172,7 +3172,7 @@ $ui-header-height: 55px;
 
   &.active {
     transition: none;
-    box-shadow: 0 0 0 2px rgba(lighten($highlight-text-color, 8%), 0.7);
+    box-shadow: 0 0 0 6px rgba(lighten($highlight-text-color, 8%), 0.7);
   }
 }
 

app/javascript/styles/mastodon/forms.scss

@@ -136,6 +136,10 @@ code {
     line-height: 22px;
     color: $secondary-text-color;
     margin-bottom: 30px;
+
+    a {
+      color: $highlight-text-color;
+    }
   }
 
   .rules-list {
@@ -1039,6 +1043,10 @@ code {
   }
 }
 
+.simple_form .h-captcha {
+  text-align: center;
+}
+
 .permissions-list {
   &__item {
     padding: 15px;

app/models/form/account_batch.rb

@@ -123,7 +123,18 @@ class Form::AccountBatch
       account: current_account,
       action: :suspend
     )
+
     Admin::SuspensionWorker.perform_async(account.id)
+
+    # Suspending a single account closes their associated reports, so
+    # mass-suspending would be consistent.
+    Report.where(target_account: account).unresolved.find_each do |report|
+      authorize(report, :update?)
+      log_action(:resolve, report)
+      report.resolve!(current_account)
+    rescue Mastodon::NotPermittedError
+      # This should not happen, but just in case, do not fail early
+    end
   end
 
   def approve_account(account)

app/models/form/admin_settings.rb

@@ -33,6 +33,7 @@ class Form::AdminSettings
     content_cache_retention_period
     backups_retention_period
     status_page_url
+    captcha_enabled
   ).freeze
 
   INTEGER_KEYS = %i(
@@ -52,6 +53,7 @@ class Form::AdminSettings
     trendable_by_default
     noindex
     require_invite_text
+    captcha_enabled
   ).freeze
 
   UPLOAD_KEYS = %i(

app/validators/vote_validator.rb

@@ -3,8 +3,8 @@
 class VoteValidator < ActiveModel::Validator
   def validate(vote)
     vote.errors.add(:base, I18n.t('polls.errors.expired')) if vote.poll_expired?
-
     vote.errors.add(:base, I18n.t('polls.errors.invalid_choice')) if invalid_choice?(vote)
+    vote.errors.add(:base, I18n.t('polls.errors.self_vote')) if self_vote?(vote)
 
     vote.errors.add(:base, I18n.t('polls.errors.already_voted')) if additional_voting_not_allowed?(vote)
   end
@@ -27,6 +27,10 @@ class VoteValidator < ActiveModel::Validator
     vote.choice.negative? || vote.choice >= vote.poll.options.size
   end
 
+  def self_vote?(vote)
+    vote.account_id == vote.poll.account_id
+  end
+
   def already_voted_for_same_choice_on_multiple_poll?(vote)
     if vote.persisted?
       account_votes_on_same_poll(vote).where(choice: vote.choice).where.not(poll_votes: { id: vote }).exists?

app/views/admin/settings/registrations/show.html.haml

@@ -20,6 +20,10 @@
     .fields-row__column.fields-row__column-6.fields-group
       = f.input :require_invite_text, as: :boolean, wrapper: :with_label, disabled: !approved_registrations?
 
+  - if captcha_available?
+    .fields-group
+      = f.input :captcha_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.captcha_enabled.title'), hint: t('admin.settings.captcha_enabled.desc_html')
+
   .fields-group
     = f.input :closed_registrations_message, as: :text, wrapper: :with_block_label, input_html: { rows: 2 }
 

app/views/auth/confirmations/captcha.html.haml

@@ -0,0 +1,15 @@
+- content_for :page_title do
+  = t('auth.captcha_confirmation.title')
+
+= form_tag auth_captcha_confirmation_url, method: 'POST', class: 'simple_form' do
+  = render 'auth/shared/progress', stage: 'confirm'
+
+  = hidden_field_tag :confirmation_token, params[:confirmation_token]
+
+  %p.lead= t('auth.captcha_confirmation.hint_html')
+
+  .field-group
+    = render_captcha
+
+  .actions
+    %button.button= t('challenge.confirm')

app/views/auth/setup/show.html.haml

@@ -1,6 +1,8 @@
 - content_for :page_title do
   = t('auth.setup.title')
 
+= javascript_pack_tag 'sign_up', crossorigin: 'anonymous'
+
 = simple_form_for(@user, url: auth_setup_path) do |f|
   = render 'auth/shared/progress', stage: 'confirm'
 

config/locales/en.yml

@@ -770,6 +770,9 @@ en:
       branding:
         preamble: Your server's branding differentiates it from other servers in the network. This information may be displayed across a variety of environments, such as Mastodon's web interface, native applications, in link previews on other websites and within messaging apps, and so on. For this reason, it is best to keep this information clear, short and concise.
         title: Branding
+      captcha_enabled:
+        desc_html: This relies on external scripts from hCaptcha, which may be a security and privacy concern. In addition, <strong>this can make the registration process significantly less accessible to some (especially disabled) people</strong>. For these reasons, please consider alternative measures such as approval-based or invite-based registration.
+        title: Require new users to solve a CAPTCHA to confirm their account
       content_retention:
         preamble: Control how user-generated content is stored in Mastodon.
         title: Content retention
@@ -1071,6 +1074,9 @@ en:
     your_token: Your access token
   auth:
     apply_for_account: Request an account
+    captcha_confirmation:
+      hint_html: Just one more step! To confirm your account, this server requires you to solve a CAPTCHA. You can <a href="/about/more">contact the server administrator</a> if you have questions or need assistance with confirming your account.
+      title: User verification
     change_password: Password
     cloudflare_with_registering: With cloudflare on auth
     confirmations:
@@ -1537,6 +1543,7 @@ en:
       expired: The poll has already ended
       invalid_choice: The chosen vote option does not exist
       over_character_limit: cannot be longer than %{max} characters each
+      self_vote: You cannot vote in your own polls
       too_few_options: must have more than one item
       too_many_options: can't contain more than %{max} items
   preferences:

config/routes.rb

@@ -72,6 +72,7 @@ Rails.application.routes.draw do
       resource :setup, only: [:show, :update], controller: :setup
       resource :challenge, only: [:create], controller: :challenges
       get 'sessions/security_key_options', to: 'sessions#webauthn_options'
+      post 'captcha_confirmation', to: 'confirmations#confirm_captcha', as: :captcha_confirmation
     end
   end
 

config/routes/api.rb

@@ -117,6 +117,7 @@ namespace :api, format: false do
 
     namespace :emails do
       resources :confirmations, only: [:create]
+      get :check_confirmation, to: 'confirmations#check'
     end
 
     resource :instance, only: [:show] do

config/settings.yml

@@ -37,6 +37,7 @@ defaults: &defaults
   show_domain_blocks_rationale: 'disabled'
   require_invite_text: false
   backups_retention_period: 7
+  captcha_enabled: false
 
 development:
   <<: *defaults

package.json

@@ -82,7 +82,7 @@
     "npmlog": "^7.0.1",
     "path-complete-extname": "^1.0.0",
     "pg": "^8.5.0",
-    "pg-connection-string": "^2.5.0",
+    "pg-connection-string": "^2.6.0",
     "postcss": "^8.4.23",
     "postcss-loader": "^4.3.0",
     "prop-types": "^15.8.1",

spec/controllers/api/v1/emails/confirmations_controller_spec.rb

@@ -63,4 +63,72 @@ RSpec.describe Api::V1::Emails::ConfirmationsController do
       end
     end
   end
+
+  describe '#check' do
+    let(:scopes) { 'read' }
+
+    context 'with an oauth token' do
+      before do
+        allow(controller).to receive(:doorkeeper_token) { token }
+      end
+
+      context 'when the account is not confirmed' do
+        it 'returns http success' do
+          get :check
+          expect(response).to have_http_status(200)
+        end
+
+        it 'returns false' do
+          get :check
+          expect(body_as_json).to be false
+        end
+      end
+
+      context 'when the account is confirmed' do
+        let(:confirmed_at) { Time.now.utc }
+
+        it 'returns http success' do
+          get :check
+          expect(response).to have_http_status(200)
+        end
+
+        it 'returns true' do
+          get :check
+          expect(body_as_json).to be true
+        end
+      end
+    end
+
+    context 'with an authentication cookie' do
+      before do
+        sign_in user, scope: :user
+      end
+
+      context 'when the account is not confirmed' do
+        it 'returns http success' do
+          get :check
+          expect(response).to have_http_status(200)
+        end
+
+        it 'returns false' do
+          get :check
+          expect(body_as_json).to be false
+        end
+      end
+
+      context 'when the account is confirmed' do
+        let(:confirmed_at) { Time.now.utc }
+
+        it 'returns http success' do
+          get :check
+          expect(response).to have_http_status(200)
+        end
+
+        it 'returns true' do
+          get :check
+          expect(body_as_json).to be true
+        end
+      end
+    end
+  end
 end

spec/models/form/account_batch_spec.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe Form::AccountBatch do
+  let(:account_batch) { described_class.new }
+
+  describe '#save' do
+    subject           { account_batch.save }
+
+    let(:account)     { Fabricate(:user, role: UserRole.find_by(name: 'Admin')).account }
+    let(:account_ids) { [] }
+    let(:query)       { Account.none }
+
+    before do
+      account_batch.assign_attributes(
+        action: action,
+        current_account: account,
+        account_ids: account_ids,
+        query: query,
+        select_all_matching: select_all_matching
+      )
+    end
+
+    context 'when action is "suspend"' do
+      let(:action) { 'suspend' }
+
+      let(:target_account)  { Fabricate(:account) }
+      let(:target_account2) { Fabricate(:account) }
+
+      before do
+        Fabricate(:report, target_account: target_account)
+        Fabricate(:report, target_account: target_account2)
+      end
+
+      context 'when accounts are passed as account_ids' do
+        let(:select_all_matching) { '0' }
+        let(:account_ids)         { [target_account.id, target_account2.id] }
+
+        it 'suspends the expected users' do
+          expect { subject }.to change { [target_account.reload.suspended?, target_account2.reload.suspended?] }.from([false, false]).to([true, true])
+        end
+
+        it 'closes open reports targeting the suspended users' do
+          expect { subject }.to change { Report.unresolved.where(target_account: [target_account, target_account2]).count }.from(2).to(0)
+        end
+      end
+
+      context 'when accounts are passed as a query' do
+        let(:select_all_matching) { '1' }
+        let(:query)               { Account.where(id: [target_account.id, target_account2.id]) }
+
+        it 'suspends the expected users' do
+          expect { subject }.to change { [target_account.reload.suspended?, target_account2.reload.suspended?] }.from([false, false]).to([true, true])
+        end
+
+        it 'closes open reports targeting the suspended users' do
+          expect { subject }.to change { Report.unresolved.where(target_account: [target_account, target_account2]).count }.from(2).to(0)
+        end
+      end
+    end
+  end
+end

yarn.lock

@@ -8902,15 +8902,10 @@ performance-now@^2.1.0:
   resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
   integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
 
-pg-connection-string@^2.4.0:
-  version "2.4.0"
-  resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.4.0.tgz#c979922eb47832999a204da5dbe1ebf2341b6a10"
-  integrity sha512-3iBXuv7XKvxeMrIgym7njT+HlZkwZqqGX4Bu9cci8xHZNT+Um1gWKqCsAzcC0d95rcKMU5WBg6YRUcHyV0HZKQ==
-
-pg-connection-string@^2.5.0:
-  version "2.5.0"
-  resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34"
-  integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==
+pg-connection-string@^2.4.0, pg-connection-string@^2.6.0:
+  version "2.6.0"
+  resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.0.tgz#12a36cc4627df19c25cc1b9b736cc39ee1f73ae8"
+  integrity sha512-x14ibktcwlHKoHxx9X3uTVW9zIGR41ZB6QNhHb21OPNdCCO3NaRnpJuwKIQSR4u+Yqjx4HCvy7Hh7VSy1U4dGg==
 
 pg-int8@1.0.1:
   version "1.0.1"