Merge pull request #996 from kmycode/kb-draft-17.4

Release: 17.4

CHANGELOG.md

@@ -2,6 +2,88 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.3.7] - 2025-04-02
+
+### Add
+
+- Add delay to profile updates to debounce them (#34137 by @ClearlyClaire)
+- Add support for paginating partial collections in `SynchronizeFollowersService` (#34272 and #34277 by @ClearlyClaire)
+
+### Changed
+
+- Change account suspensions to be federated to recently-followed accounts as well (#34294 by @ClearlyClaire)
+- Change `AccountReachFinder` to consider statuses based on suspension date (#32805 and #34291 by @ClearlyClaire and @mjankowski)
+- Change user archive signed URL TTL from 10 seconds to 1 hour (#34254 by @ClearlyClaire)
+
+### Fixed
+
+- Fix static version of animated PNG emojis not being properly extracted (#34337 by @ClearlyClaire)
+- Fix filters not applying in detailed view, favourites and bookmarks (#34259 and #34260 by @ClearlyClaire)
+- Fix handling of malformed/unusual HTML (#34201 by @ClearlyClaire)
+- Fix `CacheBuster` being queued for missing media attachments (#34253 by @ClearlyClaire)
+- Fix incorrect URL being used when cache busting (#34189 by @ClearlyClaire)
+- Fix streaming server refusing unix socket path in `DATABASE_URL` (#34091 by @ClearlyClaire)
+- Fix “x” hotkey not working on boosted filtered posts (#33758 by @ClearlyClaire)
+
+## [4.3.6] - 2025-03-13
+
+### Security
+
+- Update dependency `omniauth-saml`
+- Update dependency `rack`
+
+### Fixed
+
+- Fix Stoplight errors when using `REDIS_NAMESPACE` (#34126 by @ClearlyClaire)
+
+## [4.3.5] - 2025-03-10
+
+### Changed
+
+- Change hashtag suggestion to prefer personal history capitalization (#34070 by @ClearlyClaire)
+
+### Fixed
+
+- Fix processing errors for some HEIF images from iOS 18 (#34086 by @renchap)
+- Fix streaming server not filtering unknown-language posts from public timelines (#33774 by @ClearlyClaire)
+- Fix preview cards under Content Warnings not being shown in detailed statuses (#34068 by @ClearlyClaire)
+- Fix username and display name being hidden on narrow screens in moderation interface (#33064 by @ClearlyClaire)
+
+## [4.3.4] - 2025-02-27
+
+### Security
+
+- Update dependencies
+- Change HTML sanitization to remove unusable and unused `embed` tag (#34021 by @ClearlyClaire, [GHSA-mq2m-hr29-8gqf](https://github.com/mastodon/mastodon/security/advisories/GHSA-mq2m-hr29-8gqf))
+- Fix rate-limit on sign-up email verification ([GHSA-v39f-c9jj-8w7h](https://github.com/mastodon/mastodon/security/advisories/GHSA-v39f-c9jj-8w7h))
+- Fix improper disclosure of domain blocks to unverified users ([GHSA-94h4-fj37-c825](https://github.com/mastodon/mastodon/security/advisories/GHSA-94h4-fj37-c825))
+
+### Changed
+
+- Change preview cards to be shown when Content Warnings are expanded (#33827 by @ClearlyClaire)
+- Change warnings against changing encryption secrets to be even more noticeable (#33631 by @ClearlyClaire)
+- Change `mastodon:setup` to prevent overwriting already-configured servers (#33603, #33616, and #33684 by @ClearlyClaire and @mjankowski)
+- Change notifications from moderators to not be filtered (#32974 and #33654 by @ClearlyClaire and @mjankowski)
+
+### Fixed
+
+- Fix `GET /api/v2/notifications/:id` and `POST /api/v2/notifications/:id/dismiss` for ungrouped notifications (#33990 by @ClearlyClaire)
+- Fix issue with some versions of libvips on some systems (#33853 by @kleisauke)
+- Fix handling of duplicate mentions in incoming status `Update` (#33911 by @ClearlyClaire)
+- Fix inefficiencies in timeline generation (#33839 and #33842 by @ClearlyClaire)
+- Fix emoji rewrite adding unnecessary curft to the DOM for most emoji (#33818 by @ClearlyClaire)
+- Fix `tootctl feeds build` not building list timelines (#33783 by @ClearlyClaire)
+- Fix flaky test in `/api/v2/notifications` tests (#33773 by @ClearlyClaire)
+- Fix incorrect signature after HTTP redirect (#33757 and #33769 by @ClearlyClaire)
+- Fix polls not being validated on edition (#33755 by @ClearlyClaire)
+- Fix media preview height in compose form when 3 or more images are attached (#33571 by @ClearlyClaire)
+- Fix preview card sizing in “Author attribution” in profile settings (#33482 by @ClearlyClaire)
+- Fix processing of incoming notifications for unfilterable types (#33429 by @ClearlyClaire)
+- Fix featured tags for remote accounts not being kept up to date (#33372, #33406, and #33425 by @ClearlyClaire and @mjankowski)
+- Fix notification polling showing a loading bar in web UI (#32960 by @Gargron)
+- Fix accounts table long display name (#29316 by @WebCoder49)
+- Fix exclusive lists interfering with notifications (#28162 by @ShadowJonathan)
+
 ## [4.3.3] - 2025-01-16
 
 ### Security

Dockerfile

@@ -96,6 +96,9 @@ RUN \
 # Set /opt/mastodon as working directory
 WORKDIR /opt/mastodon
 
+# Add backport repository for some specific packages where we need the latest version
+RUN echo 'deb http://deb.debian.org/debian bookworm-backports main' >> /etc/apt/sources.list
+
 # hadolint ignore=DL3008,DL3005
 RUN \
   # Mount Apt cache and lib directories from Docker buildx caches
@@ -165,7 +168,7 @@ RUN \
   libexif-dev \
   libexpat1-dev \
   libgirepository1.0-dev \
-  libheif-dev \
+  libheif-dev/bookworm-backports \
   libimagequant-dev \
   libjpeg62-turbo-dev \
   liblcms2-dev \
@@ -348,7 +351,7 @@ RUN \
   # libvips components
   libcgif0 \
   libexif12 \
-  libheif1 \
+  libheif1/bookworm-backports \
   libimagequant0 \
   libjpeg62-turbo \
   liblcms2-2 \

Gemfile.lock

@@ -416,7 +416,7 @@ GEM
     mutex_m (0.3.0)
     net-http (0.6.0)
       uri
-    net-imap (0.5.5)
+    net-imap (0.5.6)
       date
       net-protocol
     net-ldap (0.19.0)
@@ -424,10 +424,10 @@ GEM
       net-protocol
     net-protocol (0.2.2)
       timeout
-    net-smtp (0.5.0)
+    net-smtp (0.5.1)
       net-protocol
     nio4r (2.7.4)
-    nokogiri (1.18.2)
+    nokogiri (1.18.3)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     oj (3.16.9)
@@ -444,9 +444,9 @@ GEM
     omniauth-rails_csrf_protection (1.0.2)
       actionpack (>= 4.2)
       omniauth (~> 2.0)
-    omniauth-saml (2.2.1)
+    omniauth-saml (2.2.3)
       omniauth (~> 2.1)
-      ruby-saml (~> 1.17)
+      ruby-saml (~> 1.18)
     omniauth_openid_connect (0.6.1)
       omniauth (>= 1.9, < 3)
       openid_connect (~> 1.1)
@@ -597,7 +597,7 @@ GEM
       activesupport (>= 3.0.0)
     raabro (1.4.0)
     racc (1.8.1)
-    rack (2.2.10)
+    rack (2.2.13)
     rack-attack (6.7.0)
       rack (>= 1.0, < 4)
     rack-cors (2.0.2)
@@ -745,10 +745,10 @@ GEM
       rubocop-rspec (~> 3, >= 3.0.1)
     ruby-prof (1.7.1)
     ruby-progressbar (1.13.0)
-    ruby-saml (1.17.0)
+    ruby-saml (1.18.0)
       nokogiri (>= 1.13.10)
       rexml
-    ruby-vips (2.2.2)
+    ruby-vips (2.2.3)
       ffi (~> 1.12)
       logger
     rubyzip (2.4.1)
@@ -845,7 +845,7 @@ GEM
       unf_ext
     unf_ext (0.0.9.1)
     unicode-display_width (2.6.0)
-    uri (1.0.2)
+    uri (1.0.3)
     useragent (0.16.11)
     validate_email (0.1.6)
       activemodel (>= 3.0)

app/controllers/admin/ng_words/keywords_controller.rb

@@ -21,6 +21,10 @@ module Admin
       false
     end
 
+    def avoid_save?
+      true
+    end
+
     private
 
     def after_update_redirect_path

app/controllers/admin/ng_words_controller.rb

@@ -13,6 +13,12 @@ module Admin
 
       return unless validate
 
+      if avoid_save?
+        flash[:notice] = I18n.t('generic.changes_saved_msg')
+        redirect_to after_update_redirect_path
+        return
+      end
+
       @admin_settings = Form::AdminSettings.new(settings_params)
 
       if @admin_settings.save
@@ -33,6 +39,10 @@ module Admin
       admin_ng_words_path
     end
 
+    def avoid_save?
+      false
+    end
+
     private
 
     def settings_params
@@ -40,7 +50,7 @@ module Admin
     end
 
     def settings_params_test
-      params.require(:form_admin_settings)[:ng_words_test]
+      params.expect(form_admin_settings: [ng_words_test: [keywords: [], regexps: [], strangers: [], temporary_ids: []]])['ng_words_test']
     end
   end
 end

app/controllers/api/v1/accounts/credentials_controller.rb

@@ -14,7 +14,7 @@ class Api::V1::Accounts::CredentialsController < Api::BaseController
     @account = current_account
     UpdateAccountService.new.call(@account, account_params, raise_error: true)
     current_user.update(user_params) if user_params
-    ActivityPub::UpdateDistributionWorker.perform_async(@account.id)
+    ActivityPub::UpdateDistributionWorker.perform_in(ActivityPub::UpdateDistributionWorker::DEBOUNCE_DELAY, @account.id)
     render json: @account, serializer: REST::CredentialAccountSerializer
   rescue ActiveRecord::RecordInvalid => e
     render json: ValidationErrorFormatter.new(e).as_json, status: 422

app/controllers/api/v1/instances/domain_blocks_controller.rb

@@ -31,7 +31,7 @@ class Api::V1::Instances::DomainBlocksController < Api::V1::Instances::BaseContr
   end
 
   def show_domain_blocks_to_user?
-    Setting.show_domain_blocks == 'users' && user_signed_in?
+    Setting.show_domain_blocks == 'users' && user_signed_in? && current_user.functional_or_moved?
   end
 
   def set_domain_blocks
@@ -47,6 +47,6 @@ class Api::V1::Instances::DomainBlocksController < Api::V1::Instances::BaseContr
   end
 
   def show_rationale_for_user?
-    Setting.show_domain_blocks_rationale == 'users' && user_signed_in?
+    Setting.show_domain_blocks_rationale == 'users' && user_signed_in? && current_user.functional_or_moved?
   end
 end

app/controllers/api/v1/profile/avatars_controller.rb

@@ -7,7 +7,7 @@ class Api::V1::Profile::AvatarsController < Api::BaseController
   def destroy
     @account = current_account
     UpdateAccountService.new.call(@account, { avatar: nil }, raise_error: true)
-    ActivityPub::UpdateDistributionWorker.perform_async(@account.id)
+    ActivityPub::UpdateDistributionWorker.perform_in(ActivityPub::UpdateDistributionWorker::DEBOUNCE_DELAY, @account.id)
     render json: @account, serializer: REST::CredentialAccountSerializer
   end
 end

app/controllers/api/v1/profile/headers_controller.rb

@@ -7,7 +7,7 @@ class Api::V1::Profile::HeadersController < Api::BaseController
   def destroy
     @account = current_account
     UpdateAccountService.new.call(@account, { header: nil }, raise_error: true)
-    ActivityPub::UpdateDistributionWorker.perform_async(@account.id)
+    ActivityPub::UpdateDistributionWorker.perform_in(ActivityPub::UpdateDistributionWorker::DEBOUNCE_DELAY, @account.id)
     render json: @account, serializer: REST::CredentialAccountSerializer
   end
 end

app/controllers/api/v2/notifications_controller.rb

@@ -46,7 +46,7 @@ class Api::V2::NotificationsController < Api::BaseController
   end
 
   def show
-    @notification = current_account.notifications.without_suspended.find_by!(group_key: params[:group_key])
+    @notification = current_account.notifications.without_suspended.by_group_key(params[:group_key]).take!
     presenter = GroupedNotificationsPresenter.new(NotificationGroup.from_notifications([@notification]))
     render json: presenter, serializer: REST::DedupNotificationGroupSerializer
   end
@@ -57,7 +57,7 @@ class Api::V2::NotificationsController < Api::BaseController
   end
 
   def dismiss
-    current_account.notifications.where(group_key: params[:group_key]).destroy_all
+    current_account.notifications.by_group_key(params[:group_key]).destroy_all
     render_empty
   end
 

app/controllers/backups_controller.rb

@@ -9,13 +9,15 @@ class BackupsController < ApplicationController
   before_action :authenticate_user!
   before_action :set_backup
 
+  BACKUP_LINK_TIMEOUT = 1.hour.freeze
+
   def download
     case Paperclip::Attachment.default_options[:storage]
     when :s3, :azure
-      redirect_to @backup.dump.expiring_url(10), allow_other_host: true
+      redirect_to @backup.dump.expiring_url(BACKUP_LINK_TIMEOUT.to_i), allow_other_host: true
     when :fog
       if Paperclip::Attachment.default_options.dig(:fog_credentials, :openstack_temp_url_key).present?
-        redirect_to @backup.dump.expiring_url(Time.now.utc + 10), allow_other_host: true
+        redirect_to @backup.dump.expiring_url(BACKUP_LINK_TIMEOUT.from_now), allow_other_host: true
       else
         redirect_to full_asset_url(@backup.dump.url), allow_other_host: true
       end

app/controllers/settings/pictures_controller.rb

@@ -8,7 +8,7 @@ module Settings
     def destroy
       if valid_picture?
         if UpdateAccountService.new.call(@account, { @picture => nil, "#{@picture}_remote_url" => '' })
-          ActivityPub::UpdateDistributionWorker.perform_async(@account.id)
+          ActivityPub::UpdateDistributionWorker.perform_in(ActivityPub::UpdateDistributionWorker::DEBOUNCE_DELAY, @account.id)
           redirect_to settings_profile_path, notice: I18n.t('generic.changes_saved_msg'), status: 303
         else
           redirect_to settings_profile_path

app/controllers/settings/privacy_controller.rb

@@ -8,7 +8,7 @@ class Settings::PrivacyController < Settings::BaseController
   def update
     if UpdateAccountService.new.call(@account, account_params.except(:settings))
       current_user.update!(settings_attributes: account_params[:settings])
-      ActivityPub::UpdateDistributionWorker.perform_async(@account.id)
+      ActivityPub::UpdateDistributionWorker.perform_in(ActivityPub::UpdateDistributionWorker::DEBOUNCE_DELAY, @account.id)
       redirect_to settings_privacy_path, notice: I18n.t('generic.changes_saved_msg')
     else
       render :show

app/controllers/settings/privacy_extra_controller.rb

@@ -8,7 +8,7 @@ class Settings::PrivacyExtraController < Settings::BaseController
   def update
     if UpdateAccountService.new.call(@account, account_params.except(:settings))
       current_user.update!(settings_attributes: account_params[:settings])
-      ActivityPub::UpdateDistributionWorker.perform_async(@account.id)
+      ActivityPub::UpdateDistributionWorker.perform_in(ActivityPub::UpdateDistributionWorker::DEBOUNCE_DELAY, @account.id)
       redirect_to settings_privacy_extra_path, notice: I18n.t('generic.changes_saved_msg')
     else
       render :show

app/controllers/settings/profiles_controller.rb

@@ -9,7 +9,7 @@ class Settings::ProfilesController < Settings::BaseController
 
   def update
     if UpdateAccountService.new.call(@account, account_params)
-      ActivityPub::UpdateDistributionWorker.perform_async(@account.id)
+      ActivityPub::UpdateDistributionWorker.perform_in(ActivityPub::UpdateDistributionWorker::DEBOUNCE_DELAY, @account.id)
       redirect_to settings_profile_path, notice: I18n.t('generic.changes_saved_msg')
     else
       @account.build_fields

app/controllers/settings/verifications_controller.rb

@@ -8,7 +8,7 @@ class Settings::VerificationsController < Settings::BaseController
 
   def update
     if UpdateAccountService.new.call(@account, account_params)
-      ActivityPub::UpdateDistributionWorker.perform_async(@account.id)
+      ActivityPub::UpdateDistributionWorker.perform_in(ActivityPub::UpdateDistributionWorker::DEBOUNCE_DELAY, @account.id)
       redirect_to settings_verification_path, notice: I18n.t('generic.changes_saved_msg')
     else
       render :show

app/controllers/system_css_controller.rb

@@ -1,16 +1,8 @@
 # frozen_string_literal: true
 
 class SystemCssController < ActionController::Base # rubocop:disable Rails/ApplicationController
-  before_action :set_user_roles
-
   def show
     expires_in 3.minutes, public: true
     render content_type: 'text/css'
   end
-
-  private
-
-  def set_user_roles
-    @user_roles = UserRole.providing_styles
-  end
 end

app/helpers/admin/trends/statuses_helper.rb

@@ -2,11 +2,18 @@
 
 module Admin::Trends::StatusesHelper
   def one_line_preview(status)
-    text = if status.local?
+    text = begin
-             status.text.split("\n").first
+      if status.local?
-           else
+        status.text.split("\n").first
-             Nokogiri::HTML5(status.text).css('html > body > *').first&.text
+      else
-           end
+        Nokogiri::HTML5(status.text).css('html > body > *').first&.text
+      end
+    rescue ArgumentError
+      # This can happen if one of the Nokogumbo limits is encountered
+      # Unfortunately, it does not use a more precise error class
+      # nor allows more graceful handling
+      ''
+    end
 
     return '' if text.blank?
 

app/javascript/mastodon/features/bookmarked_statuses/index.jsx

@@ -101,6 +101,7 @@ class Bookmarks extends ImmutablePureComponent {
           onLoadMore={this.handleLoadMore}
           emptyMessage={emptyMessage}
           bindToDocument={!multiColumn}
+          timelineId='bookmarks'
         />
 
         <Helmet>

app/javascript/mastodon/features/emoji/__tests__/emoji-test.js

@@ -22,23 +22,23 @@ describe('emoji', () => {
 
     it('does unicode', () => {
       expect(emojify('\uD83D\uDC69\u200D\uD83D\uDC69\u200D\uD83D\uDC66\u200D\uD83D\uDC66')).toEqual(
-        '<picture><img draggable="false" class="emojione" alt="👩‍👩‍👦‍👦" title=":woman-woman-boy-boy:" src="/emoji/1f469-200d-1f469-200d-1f466-200d-1f466.svg"></picture>');
+        '<img draggable="false" class="emojione" alt="👩‍👩‍👦‍👦" title=":woman-woman-boy-boy:" src="/emoji/1f469-200d-1f469-200d-1f466-200d-1f466.svg">');
       expect(emojify('👨‍👩‍👧‍👧')).toEqual(
-        '<picture><img draggable="false" class="emojione" alt="👨‍👩‍👧‍👧" title=":man-woman-girl-girl:" src="/emoji/1f468-200d-1f469-200d-1f467-200d-1f467.svg"></picture>');
+        '<img draggable="false" class="emojione" alt="👨‍👩‍👧‍👧" title=":man-woman-girl-girl:" src="/emoji/1f468-200d-1f469-200d-1f467-200d-1f467.svg">');
-      expect(emojify('👩‍👩‍👦')).toEqual('<picture><img draggable="false" class="emojione" alt="👩‍👩‍👦" title=":woman-woman-boy:" src="/emoji/1f469-200d-1f469-200d-1f466.svg"></picture>');
+      expect(emojify('👩‍👩‍👦')).toEqual('<img draggable="false" class="emojione" alt="👩‍👩‍👦" title=":woman-woman-boy:" src="/emoji/1f469-200d-1f469-200d-1f466.svg">');
       expect(emojify('\u2757')).toEqual(
-        '<picture><img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"></picture>');
+        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg">');
     });
 
     it('does multiple unicode', () => {
       expect(emojify('\u2757 #\uFE0F\u20E3')).toEqual(
-        '<picture><img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"></picture> <picture><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"></picture>');
+        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg">');
       expect(emojify('\u2757#\uFE0F\u20E3')).toEqual(
-        '<picture><img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"></picture><picture><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"></picture>');
+        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg">');
       expect(emojify('\u2757 #\uFE0F\u20E3 \u2757')).toEqual(
-        '<picture><img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"></picture> <picture><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"></picture> <picture><img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"></picture>');
+        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"> <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg">');
       expect(emojify('foo \u2757 #\uFE0F\u20E3 bar')).toEqual(
-        'foo <picture><img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"></picture> <picture><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"></picture> bar');
+        'foo <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"> bar');
     });
 
     it('ignores unicode inside of tags', () => {
@@ -46,16 +46,16 @@ describe('emoji', () => {
     });
 
     it('does multiple emoji properly (issue 5188)', () => {
-      expect(emojify('👌🌈💕')).toEqual('<picture><img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg"></picture><picture><img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg"></picture><picture><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg"></picture>');
+      expect(emojify('👌🌈💕')).toEqual('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg"><img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg"><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg">');
-      expect(emojify('👌 🌈 💕')).toEqual('<picture><img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg"></picture> <picture><img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg"></picture> <picture><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg"></picture>');
+      expect(emojify('👌 🌈 💕')).toEqual('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg"> <img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg"> <img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg">');
     });
 
     it('does an emoji that has no shortcode', () => {
-      expect(emojify('👁‍🗨')).toEqual('<picture><img draggable="false" class="emojione" alt="👁‍🗨" title="" src="/emoji/1f441-200d-1f5e8.svg"></picture>');
+      expect(emojify('👁‍🗨')).toEqual('<img draggable="false" class="emojione" alt="👁‍🗨" title="" src="/emoji/1f441-200d-1f5e8.svg">');
     });
 
     it('does an emoji whose filename is irregular', () => {
-      expect(emojify('↙️')).toEqual('<picture><img draggable="false" class="emojione" alt="↙️" title=":arrow_lower_left:" src="/emoji/2199.svg"></picture>');
+      expect(emojify('↙️')).toEqual('<img draggable="false" class="emojione" alt="↙️" title=":arrow_lower_left:" src="/emoji/2199.svg">');
     });
 
     it('avoid emojifying on invisible text', () => {
@@ -67,11 +67,11 @@ describe('emoji', () => {
 
     it('avoid emojifying on invisible text with nested tags', () => {
       expect(emojify('<span class="invisible">😄<span class="foo">bar</span>😴</span>😇'))
-        .toEqual('<span class="invisible">😄<span class="foo">bar</span>😴</span><picture><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg"></picture>');
+        .toEqual('<span class="invisible">😄<span class="foo">bar</span>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg">');
       expect(emojify('<span class="invisible">😄<span class="invisible">😕</span>😴</span>😇'))
-        .toEqual('<span class="invisible">😄<span class="invisible">😕</span>😴</span><picture><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg"></picture>');
+        .toEqual('<span class="invisible">😄<span class="invisible">😕</span>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg">');
       expect(emojify('<span class="invisible">😄<br>😴</span>😇'))
-        .toEqual('<span class="invisible">😄<br>😴</span><picture><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg"></picture>');
+        .toEqual('<span class="invisible">😄<br>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg">');
     });
 
     it('does not emojify emojis with textual presentation VS15 character', () => {
@@ -81,17 +81,17 @@ describe('emoji', () => {
 
     it('does a simple emoji properly', () => {
       expect(emojify('♀♂'))
-        .toEqual('<picture><img draggable="false" class="emojione" alt="♀" title=":female_sign:" src="/emoji/2640.svg"></picture><picture><img draggable="false" class="emojione" alt="♂" title=":male_sign:" src="/emoji/2642.svg"></picture>');
+        .toEqual('<img draggable="false" class="emojione" alt="♀" title=":female_sign:" src="/emoji/2640.svg"><img draggable="false" class="emojione" alt="♂" title=":male_sign:" src="/emoji/2642.svg">');
     });
 
     it('does an emoji containing ZWJ properly', () => {
       expect(emojify('💂‍♀️💂‍♂️'))
-        .toEqual('<picture><img draggable="false" class="emojione" alt="💂\u200D♀️" title=":female-guard:" src="/emoji/1f482-200d-2640-fe0f_border.svg"></picture><picture><img draggable="false" class="emojione" alt="💂\u200D♂️" title=":male-guard:" src="/emoji/1f482-200d-2642-fe0f_border.svg"></picture>');
+        .toEqual('<img draggable="false" class="emojione" alt="💂\u200D♀️" title=":female-guard:" src="/emoji/1f482-200d-2640-fe0f_border.svg"><img draggable="false" class="emojione" alt="💂\u200D♂️" title=":male-guard:" src="/emoji/1f482-200d-2642-fe0f_border.svg">');
     });
 
     it('keeps ordering as expected (issue fixed by PR 20677)', () => {
       expect(emojify('<p>💕 <a class="hashtag" href="https://example.com/tags/foo" rel="nofollow noopener" target="_blank">#<span>foo</span></a> test: foo.</p>'))
-        .toEqual('<p><picture><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg"></picture> <a class="hashtag" href="https://example.com/tags/foo" rel="nofollow noopener" target="_blank">#<span>foo</span></a> test: foo.</p>');
+        .toEqual('<p><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg"> <a class="hashtag" href="https://example.com/tags/foo" rel="nofollow noopener" target="_blank">#<span>foo</span></a> test: foo.</p>');
     });
   });
 });

app/javascript/mastodon/features/emoji/emoji.js

@@ -97,30 +97,30 @@ const emojifyTextNode = (node, customEmojis) => {
       const { filename, shortCode } = unicodeMapping[unicode_emoji];
       const title = shortCode ? `:${shortCode}:` : '';
 
-      replacement = document.createElement('picture');
-
       const isSystemTheme = !!document.body?.classList.contains('theme-system');
 
-      if(isSystemTheme) {
+      const theme = (isSystemTheme || document.body?.classList.contains('theme-mastodon-light')) ? 'light' : 'dark';
-        let source = document.createElement('source');
-        source.setAttribute('media', '(prefers-color-scheme: dark)');
-        source.setAttribute('srcset', `${assetHost}/emoji/${emojiFilename(filename, "dark")}.svg`);
-        replacement.appendChild(source);
-      }
 
-      let img = document.createElement('img');
+      const imageFilename = emojiFilename(filename, theme);
+
+      const img = document.createElement('img');
       img.setAttribute('draggable', 'false');
       img.setAttribute('class', 'emojione');
       img.setAttribute('alt', unicode_emoji);
       img.setAttribute('title', title);
+      img.setAttribute('src', `${assetHost}/emoji/${imageFilename}.svg`);
 
-      let theme = "light";
+      if (isSystemTheme && imageFilename !== emojiFilename(filename, 'dark')) {
+        replacement = document.createElement('picture');
 
-      if(!isSystemTheme && !document.body?.classList.contains('theme-mastodon-light'))
+        const source = document.createElement('source');
-        theme = "dark";
+        source.setAttribute('media', '(prefers-color-scheme: dark)');
-
+        source.setAttribute('srcset', `${assetHost}/emoji/${emojiFilename(filename, 'dark')}.svg`);
-      img.setAttribute('src', `${assetHost}/emoji/${emojiFilename(filename, theme)}.svg`);
+        replacement.appendChild(source);
-      replacement.appendChild(img);
+        replacement.appendChild(img);
+      } else {
+        replacement = img;
+      }
     }
 
     // Add the processed-up-to-now string and the emoji replacement
@@ -135,7 +135,7 @@ const emojifyTextNode = (node, customEmojis) => {
 };
 
 const emojifyNode = (node, customEmojis) => {
-  for (const child of node.childNodes) {
+  for (const child of Array.from(node.childNodes)) {
     switch(child.nodeType) {
     case Node.TEXT_NODE:
       emojifyTextNode(child, customEmojis);

app/javascript/mastodon/features/favourited_statuses/index.jsx

@@ -101,6 +101,7 @@ class Favourites extends ImmutablePureComponent {
           onLoadMore={this.handleLoadMore}
           emptyMessage={emptyMessage}
           bindToDocument={!multiColumn}
+          timelineId='favourites'
         />
 
         <Helmet>

app/javascript/mastodon/features/status/components/detailed_status.tsx

@@ -15,6 +15,7 @@ import AlternateEmailIcon from '@/material-icons/400-24px/alternate_email.svg?re
 import { AnimatedNumber } from 'mastodon/components/animated_number';
 import { ContentWarning } from 'mastodon/components/content_warning';
 import EditedTimestamp from 'mastodon/components/edited_timestamp';
+import { FilterWarning } from 'mastodon/components/filter_warning';
 import type { StatusLike } from 'mastodon/components/hashtag_bar';
 import { getHashtagBarForStatus } from 'mastodon/components/hashtag_bar';
 import { Icon } from 'mastodon/components/icon';
@@ -80,6 +81,7 @@ export const DetailedStatus: React.FC<{
 }) => {
   const properStatus = status?.get('reblog') ?? status;
   const [height, setHeight] = useState(0);
+  const [showDespiteFilter, setShowDespiteFilter] = useState(false);
   const nodeRef = useRef<HTMLDivElement>();
 
   const handleOpenVideo = useCallback(
@@ -92,6 +94,10 @@ export const DetailedStatus: React.FC<{
     [onOpenVideo, status],
   );
 
+  const handleFilterToggle = useCallback(() => {
+    setShowDespiteFilter(!showDespiteFilter);
+  }, [showDespiteFilter, setShowDespiteFilter]);
+
   const handleExpandedToggle = useCallback(() => {
     if (onToggleHidden) onToggleHidden(status);
   }, [onToggleHidden, status]);
@@ -237,7 +243,7 @@ export const DetailedStatus: React.FC<{
       <Card
         sensitive={status.get('sensitive') && !status.get('spoiler_text')}
         onOpenMedia={onOpenMedia}
-        card={status.get('card', null)}
+        card={status.get('card')}
       />
     );
   }
@@ -369,8 +375,12 @@ export const DetailedStatus: React.FC<{
   const { statusContentProps, hashtagBar } = getHashtagBarForStatus(
     status as StatusLike,
   );
+
+  const matchedFilters = status.get('matched_filters');
+
   const expanded =
-    !status.get('hidden') || status.get('spoiler_text').length === 0;
+    (!matchedFilters || showDespiteFilter) &&
+    (!status.get('hidden') || status.get('spoiler_text').length === 0);
 
   const quote = !muted && status.get('quote_id') && (
     <>
@@ -418,17 +428,26 @@ export const DetailedStatus: React.FC<{
           )}
         </Link>
 
-        {status.get('spoiler_text').length > 0 && (
+        {matchedFilters && (
-          <ContentWarning
+          <FilterWarning
-            text={
+            title={matchedFilters.join(', ')}
-              status.getIn(['translation', 'spoilerHtml']) ||
+            expanded={showDespiteFilter}
-              status.get('spoilerHtml')
+            onClick={handleFilterToggle}
-            }
-            expanded={expanded}
-            onClick={handleExpandedToggle}
           />
         )}
 
+        {status.get('spoiler_text').length > 0 &&
+          (!matchedFilters || showDespiteFilter) && (
+            <ContentWarning
+              text={
+                status.getIn(['translation', 'spoilerHtml']) ||
+                status.get('spoilerHtml')
+              }
+              expanded={expanded}
+              onClick={handleExpandedToggle}
+            />
+          )}
+
         {expanded && (
           <>
             <StatusContent

app/javascript/mastodon/features/status/index.jsx

@@ -147,7 +147,7 @@ const makeMapStateToProps = () => {
   });
 
   const mapStateToProps = (state, props) => {
-    const status = getStatus(state, { id: props.params.statusId });
+    const status = getStatus(state, { id: props.params.statusId, contextType: 'detailed' });
 
     let ancestorsIds   = ImmutableList();
     let descendantsIds = ImmutableList();

app/javascript/mastodon/reducers/compose.js

@@ -342,12 +342,26 @@ const expiresInFromExpiresAt = expires_at => {
 
 const mergeLocalHashtagResults = (suggestions, prefix, tagHistory) => {
   prefix = prefix.toLowerCase();
+
   if (suggestions.length < 4) {
     const localTags = tagHistory.filter(tag => tag.toLowerCase().startsWith(prefix) && !suggestions.some(suggestion => suggestion.type === 'hashtag' && suggestion.name.toLowerCase() === tag.toLowerCase()));
-    return suggestions.concat(localTags.slice(0, 4 - suggestions.length).toJS().map(tag => ({ type: 'hashtag', name: tag })));
+    suggestions = suggestions.concat(localTags.slice(0, 4 - suggestions.length).toJS().map(tag => ({ type: 'hashtag', name: tag })));
-  } else {
-    return suggestions;
   }
+
+  // Prefer capitalization from personal history, unless personal history is all lower-case
+  const fixSuggestionCapitalization = (suggestion) => {
+    if (suggestion.type !== 'hashtag')
+      return suggestion;
+
+    const tagFromHistory = tagHistory.find((tag) => tag.localeCompare(suggestion.name, undefined, { sensitivity: 'accent' }) === 0);
+
+    if (!tagFromHistory || tagFromHistory.toLowerCase() === tagFromHistory)
+      return suggestion;
+
+    return { ...suggestion, name: tagFromHistory };
+  };
+
+  return suggestions.map(fixSuggestionCapitalization);
 };
 
 const normalizeSuggestions = (state, { accounts, emojis, tags, token }) => {

app/javascript/mastodon/selectors/index.js

@@ -17,9 +17,10 @@ export const makeGetStatus = () => {
       (state, { id }) => state.getIn(['accounts', state.getIn(['statuses', id, 'account'])]),
       (state, { id }) => state.getIn(['accounts', state.getIn(['statuses', state.getIn(['statuses', id, 'reblog']), 'account'])]),
       getFilters,
+      (_, { contextType }) => ['detailed', 'bookmarks', 'favourites'].includes(contextType),
     ],
 
-    (statusBase, statusReblog, statusQuote, statusReblogQuote, accountBase, accountReblog, filters) => {
+    (statusBase, statusReblog, statusQuote, statusReblogQuote, accountBase, accountReblog, filters, warnInsteadOfHide) => {
       if (!statusBase || statusBase.get('isLoading')) {
         return null;
       }
@@ -46,7 +47,7 @@ export const makeGetStatus = () => {
           }
         }
 
-        if (filterResults.some((result) => filters.getIn([result.get('filter'), 'filter_action']) === 'hide')) {
+        if (!warnInsteadOfHide && filterResults.some((result) => filters.getIn([result.get('filter'), 'filter_action']) === 'hide')) {
           return null;
         }
         filterResults = filterResults.filter(result => filters.has(result.get('filter')));

app/javascript/mastodon/utils/filters.ts

@@ -7,6 +7,11 @@ export const toServerSideType = (columnType: string) => {
     case 'account':
     case 'explore':
       return columnType;
+    case 'detailed':
+      return 'thread';
+    case 'bookmarks':
+    case 'favourites':
+      return 'home';
     default:
       if (columnType.includes('list:') || columnType.includes('antenna:')) {
         return 'home';

app/lib/account_reach_finder.rb

@@ -10,7 +10,7 @@ class AccountReachFinder
   end
 
   def inboxes
-    (followers_inboxes + reporters_inboxes + recently_mentioned_inboxes + relay_inboxes).uniq
+    (followers_inboxes + reporters_inboxes + recently_mentioned_inboxes + recently_followed_inboxes + recently_requested_inboxes + relay_inboxes).uniq
   end
 
   private
@@ -31,13 +31,32 @@ class AccountReachFinder
       .take(RECENT_LIMIT)
   end
 
+  def recently_followed_inboxes
+    @account
+      .following
+      .where(follows: { created_at: recent_date_cutoff... })
+      .inboxes
+      .take(RECENT_LIMIT)
+  end
+
+  def recently_requested_inboxes
+    Account
+      .where(id: @account.follow_requests.where({ created_at: recent_date_cutoff... }).select(:target_account_id))
+      .inboxes
+      .take(RECENT_LIMIT)
+  end
+
   def relay_inboxes
     Relay.enabled.pluck(:inbox_url)
   end
 
   def oldest_status_id
     Mastodon::Snowflake
-      .id_at(STATUS_SINCE.ago, with_random: false)
+      .id_at(recent_date_cutoff, with_random: false)
+  end
+
+  def recent_date_cutoff
+    @account.suspended? && @account.suspension_origin_local? ? @account.suspended_at - STATUS_SINCE : STATUS_SINCE.ago
   end
 
   def recent_statuses

app/lib/emoji_formatter.rb

@@ -24,7 +24,15 @@ class EmojiFormatter
   def to_s
     return html if custom_emojis.empty? || html.blank?
 
-    tree = Nokogiri::HTML5.fragment(html)
+    begin
+      tree = Nokogiri::HTML5.fragment(html)
+    rescue ArgumentError
+      # This can happen if one of the Nokogumbo limits is encountered
+      # Unfortunately, it does not use a more precise error class
+      # nor allows more graceful handling
+      return ''
+    end
+
     tree.xpath('./text()|.//text()[not(ancestor[@class="invisible"])]').to_a.each do |node|
       i                     = -1
       inside_shortname      = false

app/lib/feed_manager.rb

@@ -42,7 +42,7 @@ class FeedManager
     when :home
       filter_from_home(status, receiver.id, build_crutches(receiver.id, [status]), :home)
     when :list
-      (filter_from_list?(status, receiver) ? :filter : nil) || filter_from_home(status, receiver.account_id, build_crutches(receiver.account_id, [status]), :list, stl_home: stl_home)
+      (filter_from_list?(status, receiver) ? :filter : nil) || filter_from_home(status, receiver.account_id, build_crutches(receiver.account_id, [status], list: receiver), :list, stl_home: stl_home)
     when :mentions
       filter_from_mentions?(status, receiver.id) ? :filter : nil
     when :tags
@@ -136,7 +136,7 @@ class FeedManager
 
     timeline_key = key(:home, into_account.id)
     aggregate    = into_account.user&.aggregates_reblogs?
-    query        = from_account.statuses.list_eligible_visibility.includes(:preloadable_poll, :media_attachments, reblog: :account).limit(FeedManager::MAX_ITEMS / 4)
+    query        = from_account.statuses.list_eligible_visibility.includes(reblog: :account).limit(FeedManager::MAX_ITEMS / 4)
 
     if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4
       oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true).first.last.to_i
@@ -164,7 +164,7 @@ class FeedManager
 
     timeline_key = key(:list, list.id)
     aggregate    = list.account.user&.aggregates_reblogs?
-    query        = from_account.statuses.list_eligible_visibility.includes(:preloadable_poll, :media_attachments, reblog: :account).limit(FeedManager::MAX_ITEMS / 4)
+    query        = from_account.statuses.list_eligible_visibility.includes(reblog: :account).limit(FeedManager::MAX_ITEMS / 4)
 
     if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4
       oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true).first.last.to_i
@@ -172,10 +172,10 @@ class FeedManager
     end
 
     statuses = query.to_a
-    crutches = build_crutches(list.account_id, statuses)
+    crutches = build_crutches(list.account_id, statuses, list: list)
 
     statuses.each do |status|
-      next if filter_from_home(status, list.account_id, crutches) || filter_from_list?(status, list)
+      next if filter_from_home(status, list.account_id, crutches, :list)
 
       add_to_feed(:list, list.id, status, aggregate_reblogs: aggregate)
     end
@@ -309,23 +309,32 @@ class FeedManager
     limit        = FeedManager::MAX_ITEMS / 2
     aggregate    = account.user&.aggregates_reblogs?
     timeline_key = key(:home, account.id)
+    over_limit = false
 
     account.statuses.limit(limit).each do |status|
       add_to_feed(:home, account.id, status, aggregate_reblogs: aggregate)
     end
 
     account.following.includes(:account_stat).reorder(nil).find_each do |target_account|
-      if redis.zcard(timeline_key) >= limit
+      query = target_account.statuses.list_eligible_visibility.includes(reblog: :account).limit(limit)
+
+      over_limit ||= redis.zcard(timeline_key) >= limit
+      if over_limit
         oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true).first.last.to_i
-        last_status_score = Mastodon::Snowflake.id_at(target_account.last_status_at)
+        last_status_score = Mastodon::Snowflake.id_at(target_account.last_status_at, with_random: false)
 
         # If the feed is full and this account has not posted more recently
         # than the last item on the feed, then we can skip the whole account
         # because none of its statuses would stay on the feed anyway
         next if last_status_score < oldest_home_score
+
+        # No need to get older statuses
+        query = query.where(id: oldest_home_score...)
       end
 
-      statuses = target_account.statuses.list_eligible_visibility.includes(:preloadable_poll, :media_attachments, :account, reblog: :account).limit(limit)
+      statuses = query.to_a
+      next if statuses.empty?
+
       crutches = build_crutches(account.id, statuses)
 
       statuses.each do |status|
@@ -345,23 +354,32 @@ class FeedManager
     limit        = FeedManager::MAX_ITEMS / 2
     aggregate    = list.account.user&.aggregates_reblogs?
     timeline_key = key(:list, list.id)
+    over_limit = false
 
     list.active_accounts.includes(:account_stat).reorder(nil).find_each do |target_account|
-      if redis.zcard(timeline_key) >= limit
+      query = target_account.statuses.list_eligible_visibility.includes(reblog: :account).limit(limit)
+
+      over_limit ||= redis.zcard(timeline_key) >= limit
+      if over_limit
         oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true).first.last.to_i
-        last_status_score = Mastodon::Snowflake.id_at(target_account.last_status_at)
+        last_status_score = Mastodon::Snowflake.id_at(target_account.last_status_at, with_random: false)
 
         # If the feed is full and this account has not posted more recently
         # than the last item on the feed, then we can skip the whole account
         # because none of its statuses would stay on the feed anyway
         next if last_status_score < oldest_home_score
+
+        # No need to get older statuses
+        query = query.where(id: oldest_home_score...)
       end
 
-      statuses = target_account.statuses.list_eligible_visibility.includes(:preloadable_poll, :media_attachments, :account, reblog: :account).limit(limit)
+      statuses = query.to_a
-      crutches = build_crutches(list.account_id, statuses)
+      next if statuses.empty?
+
+      crutches = build_crutches(list.account_id, statuses, list: list)
 
       statuses.each do |status|
-        next if filter_from_home(status, list.account_id, crutches) || filter_from_list?(status, list)
+        next if filter_from_home(status, list.account_id, crutches, :list)
 
         add_to_feed(:list, list.id, status, aggregate_reblogs: aggregate)
       end
@@ -632,8 +650,9 @@ class FeedManager
   # are going to be checked by the filtering methods
   # @param [Integer] receiver_id
   # @param [Array<Status>] statuses
+  # @param [List] list
   # @return [Hash]
-  def build_crutches(receiver_id, statuses) # rubocop:disable Metrics/AbcSize
+  def build_crutches(receiver_id, statuses, list: nil)
     crutches = {}
 
     crutches[:active_mentions] = crutches_active_mentions(statuses)
@@ -650,25 +669,43 @@ class FeedManager
       arr
     end
 
-    lists = List.where(account_id: receiver_id, exclusive: true)
+    crutches[:following]            = crutches_following(receiver_id, statuses, list)
-    antennas = Antenna.where(list: lists, insert_feeds: true)
-
-    replied_accounts  = statuses.filter_map(&:in_reply_to_account_id)
-    replied_accounts += statuses.filter { |status| status.limited_visibility? && status.thread.present? }.map { |status| status.thread.account_id }
-
-    crutches[:following]            = Follow.where(account_id: receiver_id, target_account_id: replied_accounts).pluck(:target_account_id).index_with(true)
     crutches[:languages]            = Follow.where(account_id: receiver_id, target_account_id: statuses.map(&:account_id)).pluck(:target_account_id, :languages).to_h
     crutches[:hiding_reblogs]       = Follow.where(account_id: receiver_id, target_account_id: statuses.filter_map { |s| s.account_id if s.reblog? }, show_reblogs: false).pluck(:target_account_id).index_with(true)
     crutches[:blocking]             = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).index_with(true)
     crutches[:muting]               = Mute.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).index_with(true)
     crutches[:domain_blocking]      = AccountDomainBlock.where(account_id: receiver_id, domain: statuses.flat_map { |s| [s.account.domain, s.reblog&.account&.domain] }.compact).pluck(:domain).index_with(true)
     crutches[:blocked_by]           = Block.where(target_account_id: receiver_id, account_id: statuses.map { |s| [s.account_id, s.reblog&.account_id] }.flatten.compact).pluck(:account_id).index_with(true)
-    crutches[:exclusive_list_users] = ListAccount.where(list: lists, account_id: statuses.map(&:account_id)).pluck(:account_id).index_with(true)
+    crutches[:exclusive_list_users] = crutches_exclusive_list_users(receiver_id, statuses) if list.blank?
-    crutches[:exclusive_antenna_users] = AntennaAccount.where(antenna: antennas, account_id: statuses.map(&:account_id)).pluck(:account_id).index_with(true)
+    crutches[:exclusive_antenna_users] = crutches_exclusive_antenna_users(receiver_id, statuses)
 
     crutches
   end
 
+  def crutches_exclusive_list_users(recipient_id, statuses)
+    lists = List.where(account_id: recipient_id, exclusive: true)
+    ListAccount.where(list: lists, account_id: statuses.map(&:account_id)).pluck(:account_id).index_with(true)
+  end
+
+  def crutches_exclusive_antenna_users(recipient_id, statuses)
+    lists = List.where(account_id: recipient_id, exclusive: true)
+    antennas = Antenna.where(list: lists, insert_feeds: true)
+    AntennaAccount.where(antenna: antennas, account_id: statuses.map(&:account_id)).pluck(:account_id).index_with(true)
+  end
+
+  def crutches_following(recipient_id, statuses, list)
+    if list.blank? || list.show_followed?
+      replied_accounts  = statuses.filter_map(&:in_reply_to_account_id)
+      replied_accounts += statuses.filter { |status| status.limited_visibility? && status.thread.present? }.map { |status| status.thread.account_id }
+
+      Follow.where(account_id: recipient_id, target_account_id: replied_accounts).pluck(:target_account_id).index_with(true)
+    elsif list.show_list?
+      ListAccount.where(list_id: list.id, account_id: statuses.filter_map(&:in_reply_to_account_id)).pluck(:account_id).index_with(true)
+    else
+      {}
+    end
+  end
+
   def crutches_active_mentions(statuses)
     Mention
       .active

app/lib/plain_text_formatter.rb

@@ -16,7 +16,15 @@ class PlainTextFormatter
     if local?
       text
     else
-      node = Nokogiri::HTML5.fragment(insert_newlines)
+      begin
+        node = Nokogiri::HTML5.fragment(insert_newlines)
+      rescue ArgumentError
+        # This can happen if one of the Nokogumbo limits is encountered
+        # Unfortunately, it does not use a more precise error class
+        # nor allows more graceful handling
+        return ''
+      end
+
       # Elements that are entirely removed with our Sanitize config
       node.xpath('.//iframe|.//math|.//noembed|.//noframes|.//noscript|.//plaintext|.//script|.//style|.//svg|.//xmp').remove
       node.text.chomp

app/models/account.rb

@@ -160,7 +160,7 @@ class Account < ApplicationRecord
   scope :not_excluded_by_account, ->(account) { where.not(id: account.excluded_from_timeline_account_ids) }
   scope :not_domain_blocked_by_account, ->(account) { where(arel_table[:domain].eq(nil).or(arel_table[:domain].not_in(account.excluded_from_timeline_domains))) }
   scope :dormant, -> { joins(:account_stat).merge(AccountStat.without_recent_activity) }
-  scope :with_username, ->(value) { where arel_table[:username].lower.eq(value.to_s.downcase) }
+  scope :with_username, ->(value) { value.is_a?(Array) ? where(arel_table[:username].lower.in(value.map { |x| x.to_s.downcase })) : where(arel_table[:username].lower.eq(value.to_s.downcase)) }
   scope :with_domain, ->(value) { where arel_table[:domain].lower.eq(value&.to_s&.downcase) }
   scope :without_memorial, -> { where(memorial: false) }
   scope :duplicate_uris, -> { select(:uri, Arel.star.count).group(:uri).having(Arel.star.count.gt(1)) }

app/models/account/field.rb

@@ -73,7 +73,14 @@ class Account::Field < ActiveModelSerializers::Model
   end
 
   def extract_url_from_html
-    doc = Nokogiri::HTML5.fragment(value)
+    begin
+      doc = Nokogiri::HTML5.fragment(value)
+    rescue ArgumentError
+      # This can happen if one of the Nokogumbo limits is encountered
+      # Unfortunately, it does not use a more precise error class
+      # nor allows more graceful handling
+      return
+    end
 
     return if doc.nil?
     return if doc.children.size != 1

app/models/concerns/notification/groups.rb

@@ -7,6 +7,10 @@ module Notification::Groups
   GROUPABLE_NOTIFICATION_TYPES = %i(favourite reblog follow emoji_reaction).freeze
   MAXIMUM_GROUP_SPAN_HOURS = 12
 
+  included do
+    scope :by_group_key, ->(group_key) { group_key&.start_with?('ungrouped-') ? where(id: group_key.delete_prefix('ungrouped-')) : where(group_key: group_key) }
+  end
+
   def set_group_key!
     return if filtered? || GROUPABLE_NOTIFICATION_TYPES.exclude?(type)
 

app/models/media_attachment.rb

@@ -425,8 +425,10 @@ class MediaAttachment < ApplicationRecord
 
     @paths_to_cache_bust = MediaAttachment.attachment_definitions.keys.flat_map do |attachment_name|
       attachment = public_send(attachment_name)
+      next if attachment.blank?
+
       styles = DEFAULT_STYLES | attachment.styles.keys
-      styles.map { |style| attachment.path(style) }
+      styles.map { |style| attachment.url(style) }
     end.compact
   rescue => e
     # We really don't want any error here preventing media deletion

app/services/activitypub/process_status_update_service.rb

@@ -296,7 +296,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
       nil
     end
 
-    @status.mentions.upsert_all(currently_mentioned_account_ids.map { |id| { account_id: id, silent: false } }, unique_by: %w(status_id account_id))
+    @status.mentions.upsert_all(currently_mentioned_account_ids.uniq.map { |id| { account_id: id, silent: false } }, unique_by: %w(status_id account_id))
 
     # If previous mentions are no longer contained in the text, convert them
     # to silent mentions, since withdrawing access from someone who already

app/services/activitypub/synchronize_followers_service.rb

@@ -4,32 +4,46 @@ class ActivityPub::SynchronizeFollowersService < BaseService
   include JsonLdHelper
   include Payloadable
 
+  MAX_COLLECTION_PAGES = 10
+
   def call(account, partial_collection_url)
     @account = account
+    @expected_followers_ids = []
 
-    items = collection_items(partial_collection_url)
+    return unless process_collection!(partial_collection_url)
-    return if items.nil?
-
-    # There could be unresolved accounts (hence the call to .compact) but this
-    # should never happen in practice, since in almost all cases we keep an
-    # Account record, and should we not do that, we should have sent a Delete.
-    # In any case there is not much we can do if that occurs.
-    @expected_followers = items.filter_map { |uri| ActivityPub::TagManager.instance.uri_to_resource(uri, Account) }
 
     remove_unexpected_local_followers!
-    handle_unexpected_outgoing_follows!
   end
 
   private
 
+  def process_page!(items)
+    page_expected_followers = extract_local_followers(items)
+    @expected_followers_ids.concat(page_expected_followers.pluck(:id))
+
+    handle_unexpected_outgoing_follows!(page_expected_followers)
+  end
+
+  def extract_local_followers(items)
+    # There could be unresolved accounts (hence the call to .filter_map) but this
+    # should never happen in practice, since in almost all cases we keep an
+    # Account record, and should we not do that, we should have sent a Delete.
+    # In any case there is not much we can do if that occurs.
+
+    # TODO: this will need changes when switching to numeric IDs
+
+    usernames = items.filter_map { |uri| ActivityPub::TagManager.instance.uri_to_local_id(uri, :username)&.downcase }
+    Account.local.with_username(usernames)
+  end
+
   def remove_unexpected_local_followers!
-    @account.followers.local.where.not(id: @expected_followers.map(&:id)).reorder(nil).find_each do |unexpected_follower|
+    @account.followers.local.where.not(id: @expected_followers_ids).reorder(nil).find_each do |unexpected_follower|
       UnfollowService.new.call(unexpected_follower, @account)
     end
   end
 
-  def handle_unexpected_outgoing_follows!
+  def handle_unexpected_outgoing_follows!(expected_followers)
-    @expected_followers.each do |expected_follower|
+    expected_followers.each do |expected_follower|
       next if expected_follower.following?(@account)
 
       if expected_follower.requested?(@account)
@@ -50,18 +64,33 @@ class ActivityPub::SynchronizeFollowersService < BaseService
     Oj.dump(serialize_payload(follow, ActivityPub::UndoFollowSerializer))
   end
 
-  def collection_items(collection_or_uri)
+  # Only returns true if the whole collection has been processed
-    collection = fetch_collection(collection_or_uri)
+  def process_collection!(collection_uri, max_pages: MAX_COLLECTION_PAGES)
-    return unless collection.is_a?(Hash)
+    collection = fetch_collection(collection_uri)
+    return false unless collection.is_a?(Hash)
 
     collection = fetch_collection(collection['first']) if collection['first'].present?
-    return unless collection.is_a?(Hash)
 
+    while collection.is_a?(Hash)
+      process_page!(as_array(collection_page_items(collection)))
+
+      max_pages -= 1
+
+      return true if collection['next'].blank? # We reached the end of the collection
+      return false if max_pages <= 0 # We reached our pages limit
+
+      collection = fetch_collection(collection['next'])
+    end
+
+    false
+  end
+
+  def collection_page_items(collection)
     case collection['type']
     when 'Collection', 'CollectionPage'
-      as_array(collection['items'])
+      collection['items']
     when 'OrderedCollection', 'OrderedCollectionPage'
-      as_array(collection['orderedItems'])
+      collection['orderedItems']
     end
   end
 

app/services/suspend_account_service.rb

@@ -95,7 +95,7 @@ class SuspendAccountService < BaseService
             end
           end
 
-          CacheBusterWorker.perform_async(attachment.path(style)) if Rails.configuration.x.cache_buster_enabled
+          CacheBusterWorker.perform_async(attachment.url(style)) if Rails.configuration.x.cache_buster_enabled
         end
       end
     end

app/services/unsuspend_account_service.rb

@@ -91,7 +91,7 @@ class UnsuspendAccountService < BaseService
             end
           end
 
-          CacheBusterWorker.perform_async(attachment.path(style)) if Rails.configuration.x.cache_buster_enabled
+          CacheBusterWorker.perform_async(attachment.url(style)) if Rails.configuration.x.cache_buster_enabled
         end
       end
     end

app/views/custom_css/show_system.css.erb

@@ -1,6 +0,0 @@
-<%- @user_roles.each do |role| %>
-.user-role-<%= role.id %> {
-  --user-role-accent: <%= role.color %>;
-}
-
-<%- end %>

app/views/system_css/show.css.erb

@@ -1,6 +0,0 @@
-<%- @user_roles.each do |role| %>
-.user-role-<%= role.id %> {
-  --user-role-accent: <%= role.color %>;
-}
-
-<%- end %>

app/workers/activitypub/update_distribution_worker.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class ActivityPub::UpdateDistributionWorker < ActivityPub::RawDistributionWorker
+  DEBOUNCE_DELAY = 5.seconds
+
   sidekiq_options queue: 'push', lock: :until_executed, lock_ttl: 1.day.to_i
 
   # Distribute an profile update to servers that might have a copy

config/initializers/rack_attack.rb

@@ -122,7 +122,7 @@ class Rack::Attack
   end
 
   throttle('throttle_email_confirmations/ip', limit: 25, period: 5.minutes) do |req|
-    req.throttleable_remote_ip if req.post? && (req.path_matches?('/auth/confirmation') || req.path == '/api/v1/emails/confirmations')
+    req.throttleable_remote_ip if (req.post? && (req.path_matches?('/auth/confirmation') || req.path == '/api/v1/emails/confirmations')) || ((req.put? || req.patch?) && req.path_matches?('/auth/setup'))
   end
 
   throttle('throttle_email_confirmations/email', limit: 5, period: 30.minutes) do |req|
@@ -133,6 +133,14 @@ class Rack::Attack
     end
   end
 
+  throttle('throttle_auth_setup/email', limit: 5, period: 10.minutes) do |req|
+    req.params.dig('user', 'email').presence if (req.put? || req.patch?) && req.path_matches?('/auth/setup')
+  end
+
+  throttle('throttle_auth_setup/account', limit: 5, period: 10.minutes) do |req|
+    req.warden_user_id if (req.put? || req.patch?) && req.path_matches?('/auth/setup')
+  end
+
   throttle('throttle_login_attempts/ip', limit: 25, period: 5.minutes) do |req|
     req.throttleable_remote_ip if req.post? && req.path_matches?('/auth/sign_in')
   end

docker-compose.yml

@@ -59,7 +59,7 @@ services:
   web:
     # You can uncomment the following line if you want to not use the prebuilt image, for example if you have local code changes
     build: .
-    image: kmyblue:17.0-dev
+    image: kmyblue:17.4
     restart: always
     env_file: .env.production
     command: bundle exec puma -C config/puma.rb
@@ -83,7 +83,7 @@ services:
     build:
       dockerfile: ./streaming/Dockerfile
       context: .
-    image: kmyblue-streaming:17.0-dev
+    image: kmyblue-streaming:17.4
     restart: always
     env_file: .env.production
     command: node ./streaming/index.js
@@ -101,7 +101,7 @@ services:
 
   sidekiq:
     build: .
-    image: kmyblue:17.0-dev
+    image: kmyblue:17.4
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq

lib/mastodon/version.rb

@@ -13,13 +13,13 @@ module Mastodon
     end
 
     def kmyblue_minor
-      0
+      4
     end
 
     def kmyblue_flag
       # 'LTS'
-      'dev'
+      # 'dev'
-      # nil
+      nil
     end
 
     def major
@@ -35,7 +35,7 @@ module Mastodon
     end
 
     def default_prerelease
-      'alpha.2'
+      'alpha.4'
     end
 
     def prerelease

lib/paperclip/vips_lazy_thumbnail.rb

@@ -123,7 +123,14 @@ module Paperclip
     end
 
     def needs_convert?
-      needs_different_geometry? || needs_different_format? || needs_metadata_stripping?
+      strip_animations? || needs_different_geometry? || needs_different_format? || needs_metadata_stripping?
+    end
+
+    def strip_animations?
+      # Detecting whether the source image is animated across all our supported
+      # input file formats is not trivial, and converting unconditionally is just
+      # as simple for now
+      options[:style] == :static
     end
 
     def needs_different_geometry?

lib/redis/namespace_extensions.rb

@@ -5,6 +5,10 @@ class Redis
     def exists?(...)
       call_with_namespace('exists?', ...)
     end
+
+    def with
+      yield self
+    end
   end
 end
 

lib/sanitize_ext/sanitize_config.rb

@@ -155,18 +155,16 @@ class Sanitize
     )
 
     MASTODON_OEMBED = freeze_config(
-      elements: %w(audio embed iframe source video),
+      elements: %w(audio iframe source video),
 
       attributes: {
         'audio' => %w(controls),
-        'embed' => %w(height src type width),
         'iframe' => %w(allowfullscreen frameborder height scrolling src width),
         'source' => %w(src type),
         'video' => %w(controls height loop width),
       },
 
       protocols: {
-        'embed' => { 'src' => HTTP_PROTOCOLS },
         'iframe' => { 'src' => HTTP_PROTOCOLS },
         'source' => { 'src' => HTTP_PROTOCOLS },
       },

spec/controllers/settings/privacy_controller_spec.rb

@@ -31,7 +31,7 @@ RSpec.describe Settings::PrivacyController do
   describe 'PUT #update' do
     context 'when update succeeds' do
       before do
-        allow(ActivityPub::UpdateDistributionWorker).to receive(:perform_async)
+        allow(ActivityPub::UpdateDistributionWorker).to receive(:perform_in)
       end
 
       it 'updates the user profile' do
@@ -44,14 +44,14 @@ RSpec.describe Settings::PrivacyController do
           .to redirect_to(settings_privacy_path)
 
         expect(ActivityPub::UpdateDistributionWorker)
-          .to have_received(:perform_async).with(account.id)
+          .to have_received(:perform_in).with(ActivityPub::UpdateDistributionWorker::DEBOUNCE_DELAY, account.id)
       end
     end
 
     context 'when update fails' do
       before do
         allow(UpdateAccountService).to receive(:new).and_return(failing_update_service)
-        allow(ActivityPub::UpdateDistributionWorker).to receive(:perform_async)
+        allow(ActivityPub::UpdateDistributionWorker).to receive(:perform_in)
       end
 
       it 'updates the user profile' do
@@ -61,7 +61,7 @@ RSpec.describe Settings::PrivacyController do
           .to render_template(:show)
 
         expect(ActivityPub::UpdateDistributionWorker)
-          .to_not have_received(:perform_async)
+          .to_not have_received(:perform_in)
       end
 
       private

spec/lib/account_reach_finder_spec.rb

@@ -13,13 +13,28 @@ RSpec.describe AccountReachFinder do
   let(:ap_mentioned_example_com) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-3', domain: 'example.com') }
   let(:ap_mentioned_example_org) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.org/inbox-4', domain: 'example.org') }
 
+  let(:ap_followed_example_com) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-5', domain: 'example.com') }
+  let(:ap_followed_example_org) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-6', domain: 'example.org') }
+
+  let(:ap_requested_example_com) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-7', domain: 'example.com') }
+  let(:ap_requested_example_org) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-8', domain: 'example.org') }
+
   let(:unrelated_account) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/unrelated-inbox', domain: 'example.com') }
+  let(:old_followed_account) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/old-followed-inbox', domain: 'example.com') }
 
   before do
+    travel_to(2.months.ago) { account.follow!(old_followed_account) }
+
     ap_follower_example_com.follow!(account)
     ap_follower_example_org.follow!(account)
     ap_follower_with_shared.follow!(account)
 
+    account.follow!(ap_followed_example_com)
+    account.follow!(ap_followed_example_org)
+
+    account.request_follow!(ap_requested_example_com)
+    account.request_follow!(ap_requested_example_org)
+
     Fabricate(:status, account: account).tap do |status|
       status.mentions << Mention.new(account: ap_follower_example_com)
       status.mentions << Mention.new(account: ap_mentioned_with_shared)
@@ -44,7 +59,10 @@ RSpec.describe AccountReachFinder do
       expect(subject)
         .to include(*follower_inbox_urls)
         .and include(*mentioned_account_inbox_urls)
+        .and include(*recently_followed_inbox_urls)
+        .and include(*recently_requested_inbox_urls)
         .and not_include(unrelated_account.preferred_inbox_url)
+        .and not_include(old_followed_account.preferred_inbox_url)
     end
 
     def follower_inbox_urls
@@ -56,5 +74,15 @@ RSpec.describe AccountReachFinder do
       [ap_mentioned_with_shared, ap_mentioned_example_com, ap_mentioned_example_org]
         .map(&:preferred_inbox_url)
     end
+
+    def recently_followed_inbox_urls
+      [ap_followed_example_com, ap_followed_example_org]
+        .map(&:preferred_inbox_url)
+    end
+
+    def recently_requested_inbox_urls
+      [ap_requested_example_com, ap_requested_example_org]
+        .map(&:preferred_inbox_url)
+    end
   end
 end

spec/lib/activitypub/activity/create_spec.rb

@@ -48,10 +48,16 @@ RSpec.describe ActivityPub::Activity::Create do
         content: '@bob lorem ipsum',
         published: 1.hour.ago.utc.iso8601,
         updated: 1.hour.ago.utc.iso8601,
-        tag: {
+        tag: [
-          type: 'Mention',
+          {
-          href: ActivityPub::TagManager.instance.uri_for(follower),
+            type: 'Mention',
-        },
+            href: ActivityPub::TagManager.instance.uri_for(follower),
+          },
+          {
+            type: 'Mention',
+            href: ActivityPub::TagManager.instance.uri_for(follower),
+          },
+        ],
       }
     end
 

spec/lib/feed_manager_spec.rb

@@ -233,6 +233,28 @@ RSpec.describe FeedManager do
       end
     end
 
+    context 'with list feed' do
+      let(:list) { Fabricate(:list, account: bob) }
+
+      before do
+        bob.follow!(alice)
+        list.list_accounts.create!(account: alice)
+      end
+
+      it "returns false for followee's status" do
+        status = Fabricate(:status, text: 'Hello world', account: alice)
+
+        expect(subject.filter?(:list, status, list)).to be false
+      end
+
+      it 'returns false for reblog by followee' do
+        status = Fabricate(:status, text: 'Hello world', account: jeff)
+        reblog = Fabricate(:status, reblog: status, account: alice)
+
+        expect(subject.filter?(:list, reblog, list)).to be false
+      end
+    end
+
     context 'with mentions feed' do
       it 'returns true for status that mentions blocked account' do
         bob.block!(jeff)

spec/models/media_attachment_spec.rb

@@ -295,12 +295,21 @@ RSpec.describe MediaAttachment, :attachment_processing do
     end
 
     it 'queues CacheBusterWorker jobs' do
-      original_path = media.file.path(:original)
+      original_url = media.file.url(:original)
-      small_path = media.file.path(:small)
+      small_url = media.file.url(:small)
 
       expect { media.destroy }
-        .to enqueue_sidekiq_job(CacheBusterWorker).with(original_path)
+        .to enqueue_sidekiq_job(CacheBusterWorker).with(original_url)
-        .and enqueue_sidekiq_job(CacheBusterWorker).with(small_path)
+        .and enqueue_sidekiq_job(CacheBusterWorker).with(small_url)
+    end
+
+    context 'with a missing remote attachment' do
+      let(:media) { Fabricate(:media_attachment, remote_url: 'https://example.com/foo.png', file: nil) }
+
+      it 'does not queue CacheBusterWorker jobs' do
+        expect { media.destroy }
+          .to_not enqueue_sidekiq_job(CacheBusterWorker)
+      end
     end
   end
 

spec/requests/api/v1/accounts/credentials_spec.rb

@@ -53,8 +53,6 @@ RSpec.describe 'credentials API' do
       patch '/api/v1/accounts/update_credentials', headers: headers, params: params
     end
 
-    before { allow(ActivityPub::UpdateDistributionWorker).to receive(:perform_async) }
-
     let(:params) do
       {
         avatar: fixture_file_upload('avatar.gif', 'image/gif'),
@@ -113,7 +111,7 @@ RSpec.describe 'credentials API' do
       })
 
       expect(ActivityPub::UpdateDistributionWorker)
-        .to have_received(:perform_async).with(user.account_id)
+        .to have_enqueued_sidekiq_job(user.account_id)
     end
 
     def expect_account_updates

spec/requests/api/v1/instances/domain_blocks_spec.rb

@@ -4,14 +4,15 @@ require 'rails_helper'
 
 RSpec.describe 'Domain Blocks' do
   let(:user)    { Fabricate(:user) }
-  let(:token)   { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) }
+  let(:token)   { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes).token }
   let(:scopes)  { 'read' }
-  let(:headers) { { Authorization: "Bearer #{token.token}" } }
+  let(:headers) { { Authorization: "Bearer #{token}" } }
 
   describe 'GET /api/v1/instance/domain_blocks' do
-    before do
+    let(:user) { Fabricate(:user) }
-      Fabricate(:domain_block)
+    let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id).token }
-    end
+
+    before { Fabricate(:domain_block) }
 
     context 'with domain blocks set to all' do
       before { Setting.show_domain_blocks = 'all' }
@@ -45,11 +46,95 @@ RSpec.describe 'Domain Blocks' do
     context 'with domain blocks set to users' do
       before { Setting.show_domain_blocks = 'users' }
 
-      it 'returns http not found' do
+      context 'without authentication token' do
-        get api_v1_instance_domain_blocks_path
+        it 'returns http not found' do
+          get api_v1_instance_domain_blocks_path
 
-        expect(response)
+          expect(response)
-          .to have_http_status(404)
+            .to have_http_status(404)
+        end
+      end
+
+      context 'with authentication token' do
+        context 'with unapproved user' do
+          before { user.update(approved: false) }
+
+          it 'returns http not found' do
+            get api_v1_instance_domain_blocks_path, headers: { 'Authorization' => "Bearer #{token}" }
+
+            expect(response)
+              .to have_http_status(404)
+          end
+        end
+
+        context 'with unconfirmed user' do
+          before { user.update(confirmed_at: nil) }
+
+          it 'returns http not found' do
+            get api_v1_instance_domain_blocks_path, headers: { 'Authorization' => "Bearer #{token}" }
+
+            expect(response)
+              .to have_http_status(404)
+          end
+        end
+
+        context 'with disabled user' do
+          before { user.update(disabled: true) }
+
+          it 'returns http not found' do
+            get api_v1_instance_domain_blocks_path, headers: { 'Authorization' => "Bearer #{token}" }
+
+            expect(response)
+              .to have_http_status(404)
+          end
+        end
+
+        context 'with suspended user' do
+          before { user.account.update(suspended_at: Time.zone.now) }
+
+          it 'returns http not found' do
+            get api_v1_instance_domain_blocks_path, headers: { 'Authorization' => "Bearer #{token}" }
+
+            expect(response)
+              .to have_http_status(403)
+          end
+        end
+
+        context 'with moved user' do
+          before { user.account.update(moved_to_account_id: Fabricate(:account).id) }
+
+          it 'returns http success' do
+            get api_v1_instance_domain_blocks_path, headers: { 'Authorization' => "Bearer #{token}" }
+
+            expect(response)
+              .to have_http_status(200)
+
+            expect(response.content_type)
+              .to start_with('application/json')
+
+            expect(response.parsed_body)
+              .to be_present
+              .and(be_an(Array))
+              .and(have_attributes(size: 1))
+          end
+        end
+
+        context 'with normal user' do
+          it 'returns http success' do
+            get api_v1_instance_domain_blocks_path, headers: { 'Authorization' => "Bearer #{token}" }
+
+            expect(response)
+              .to have_http_status(200)
+
+            expect(response.content_type)
+              .to start_with('application/json')
+
+            expect(response.parsed_body)
+              .to be_present
+              .and(be_an(Array))
+              .and(have_attributes(size: 1))
+          end
+        end
       end
     end
 

spec/requests/api/v1/profiles_spec.rb

@@ -15,10 +15,6 @@ RSpec.describe 'Deleting profile images' do
   let(:headers) { { 'Authorization' => "Bearer #{token.token}" } }
 
   describe 'DELETE /api/v1/profile' do
-    before do
-      allow(ActivityPub::UpdateDistributionWorker).to receive(:perform_async)
-    end
-
     context 'when deleting an avatar' do
       context 'with wrong scope' do
         before do
@@ -38,7 +34,8 @@ RSpec.describe 'Deleting profile images' do
         account.reload
         expect(account.avatar).to_not exist
         expect(account.header).to exist
-        expect(ActivityPub::UpdateDistributionWorker).to have_received(:perform_async).with(account.id)
+        expect(ActivityPub::UpdateDistributionWorker)
+          .to have_enqueued_sidekiq_job(account.id)
       end
     end
 
@@ -61,7 +58,8 @@ RSpec.describe 'Deleting profile images' do
         account.reload
         expect(account.avatar).to exist
         expect(account.header).to_not exist
-        expect(ActivityPub::UpdateDistributionWorker).to have_received(:perform_async).with(account.id)
+        expect(ActivityPub::UpdateDistributionWorker)
+          .to have_enqueued_sidekiq_job(account.id)
       end
     end
   end

spec/requests/api/v2/notifications_spec.rb

@@ -170,7 +170,7 @@ RSpec.describe 'Notifications' do
       end
 
       context 'with min_id param' do
-        let(:params) { { min_id: user.account.notifications.reload.first.id - 1 } }
+        let(:params) { { min_id: user.account.notifications.order(id: :asc).first.id - 1 } }
 
         it 'returns a notification group covering all notifications' do
           subject
@@ -365,6 +365,18 @@ RSpec.describe 'Notifications' do
         .to start_with('application/json')
     end
 
+    context 'with an ungrouped notification' do
+      let(:notification) { Fabricate(:notification, account: user.account, type: :favourite) }
+
+      it 'returns http success' do
+        get "/api/v2/notifications/ungrouped-#{notification.id}", headers: headers
+
+        expect(response).to have_http_status(200)
+        expect(response.content_type)
+          .to start_with('application/json')
+      end
+    end
+
     context 'when notification belongs to someone else' do
       let(:notification) { Fabricate(:notification, group_key: 'foobar') }
 
@@ -396,6 +408,19 @@ RSpec.describe 'Notifications' do
       expect { notification.reload }.to raise_error(ActiveRecord::RecordNotFound)
     end
 
+    context 'with an ungrouped notification' do
+      let(:notification) { Fabricate(:notification, account: user.account, type: :favourite) }
+
+      it 'destroys the notification' do
+        post "/api/v2/notifications/ungrouped-#{notification.id}/dismiss", headers: headers
+
+        expect(response).to have_http_status(200)
+        expect(response.content_type)
+          .to start_with('application/json')
+        expect { notification.reload }.to raise_error(ActiveRecord::RecordNotFound)
+      end
+    end
+
     context 'when notification belongs to someone else' do
       let(:notification) { Fabricate(:notification, group_key: 'foobar') }
 

spec/requests/auth/setup_spec.rb

@@ -24,15 +24,4 @@ RSpec.describe 'Auth Setup' do
       end
     end
   end
-
-  describe 'PUT /auth/setup' do
-    before { sign_in Fabricate(:user, confirmed_at: nil) }
-
-    it 'gracefully handles invalid nested params' do
-      put '/auth/setup?user=invalid'
-
-      expect(response)
-        .to have_http_status(400)
-    end
-  end
 end

spec/services/activitypub/process_status_update_service_spec.rb

@@ -12,6 +12,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService do
     [
       { type: 'Hashtag', name: 'hoge' },
       { type: 'Mention', href: ActivityPub::TagManager.instance.uri_for(alice) },
+      { type: 'Mention', href: ActivityPub::TagManager.instance.uri_for(alice) },
       { type: 'Mention', href: bogus_mention },
     ]
   end

spec/services/activitypub/synchronize_followers_service_spec.rb

@@ -10,7 +10,7 @@ RSpec.describe ActivityPub::SynchronizeFollowersService do
   let(:bob)            { Fabricate(:account, username: 'bob') }
   let(:eve)            { Fabricate(:account, username: 'eve') }
   let(:mallory)        { Fabricate(:account, username: 'mallory') }
-  let(:collection_uri) { 'http://example.com/partial-followers' }
+  let(:collection_uri) { 'https://example.com/partial-followers' }
 
   let(:items) do
     [alice, eve, mallory].map do |account|
@@ -27,14 +27,14 @@ RSpec.describe ActivityPub::SynchronizeFollowersService do
     }.with_indifferent_access
   end
 
+  before do
+    alice.follow!(actor)
+    bob.follow!(actor)
+    mallory.request_follow!(actor)
+  end
+
   shared_examples 'synchronizes followers' do
     before do
-      alice.follow!(actor)
-      bob.follow!(actor)
-      mallory.request_follow!(actor)
-
-      allow(ActivityPub::DeliveryWorker).to receive(:perform_async)
-
       subject.call(actor, collection_uri)
     end
 
@@ -46,7 +46,7 @@ RSpec.describe ActivityPub::SynchronizeFollowersService do
       expect(mallory)
         .to be_following(actor) # Convert follow request to follow when accepted
       expect(ActivityPub::DeliveryWorker)
-        .to have_received(:perform_async).with(anything, eve.id, actor.inbox_url) # Send Undo Follow to actor
+        .to have_enqueued_sidekiq_job(anything, eve.id, actor.inbox_url) # Send Undo Follow to actor
     end
   end
 
@@ -76,7 +76,7 @@ RSpec.describe ActivityPub::SynchronizeFollowersService do
       it_behaves_like 'synchronizes followers'
     end
 
-    context 'when the endpoint is a paginated Collection of actor URIs' do
+    context 'when the endpoint is a single-page paginated Collection of actor URIs' do
       let(:payload) do
         {
           '@context': 'https://www.w3.org/ns/activitystreams',
@@ -96,5 +96,106 @@ RSpec.describe ActivityPub::SynchronizeFollowersService do
 
       it_behaves_like 'synchronizes followers'
     end
+
+    context 'when the endpoint is a paginated Collection of actor URIs split across multiple pages' do
+      before do
+        stub_request(:get, 'https://example.com/partial-followers')
+          .to_return(status: 200, headers: { 'Content-Type': 'application/activity+json' }, body: Oj.dump({
+            '@context': 'https://www.w3.org/ns/activitystreams',
+            type: 'Collection',
+            id: 'https://example.com/partial-followers',
+            first: 'https://example.com/partial-followers/1',
+          }))
+
+        stub_request(:get, 'https://example.com/partial-followers/1')
+          .to_return(status: 200, headers: { 'Content-Type': 'application/activity+json' }, body: Oj.dump({
+            '@context': 'https://www.w3.org/ns/activitystreams',
+            type: 'CollectionPage',
+            id: 'https://example.com/partial-followers/1',
+            partOf: 'https://example.com/partial-followers',
+            next: 'https://example.com/partial-followers/2',
+            items: [alice, eve].map { |account| ActivityPub::TagManager.instance.uri_for(account) },
+          }))
+
+        stub_request(:get, 'https://example.com/partial-followers/2')
+          .to_return(status: 200, headers: { 'Content-Type': 'application/activity+json' }, body: Oj.dump({
+            '@context': 'https://www.w3.org/ns/activitystreams',
+            type: 'CollectionPage',
+            id: 'https://example.com/partial-followers/2',
+            partOf: 'https://example.com/partial-followers',
+            items: ActivityPub::TagManager.instance.uri_for(mallory),
+          }))
+      end
+
+      it_behaves_like 'synchronizes followers'
+    end
+
+    context 'when the endpoint is a paginated Collection of actor URIs split across, but one page errors out' do
+      before do
+        stub_request(:get, 'https://example.com/partial-followers')
+          .to_return(status: 200, headers: { 'Content-Type': 'application/activity+json' }, body: Oj.dump({
+            '@context': 'https://www.w3.org/ns/activitystreams',
+            type: 'Collection',
+            id: 'https://example.com/partial-followers',
+            first: 'https://example.com/partial-followers/1',
+          }))
+
+        stub_request(:get, 'https://example.com/partial-followers/1')
+          .to_return(status: 200, headers: { 'Content-Type': 'application/activity+json' }, body: Oj.dump({
+            '@context': 'https://www.w3.org/ns/activitystreams',
+            type: 'CollectionPage',
+            id: 'https://example.com/partial-followers/1',
+            partOf: 'https://example.com/partial-followers',
+            next: 'https://example.com/partial-followers/2',
+            items: [mallory].map { |account| ActivityPub::TagManager.instance.uri_for(account) },
+          }))
+
+        stub_request(:get, 'https://example.com/partial-followers/2')
+          .to_return(status: 404)
+      end
+
+      it 'confirms pending follow request but does not remove extra followers' do
+        previous_follower_ids = actor.followers.pluck(:id)
+
+        subject.call(actor, collection_uri)
+
+        expect(previous_follower_ids - actor.followers.reload.pluck(:id))
+          .to be_empty
+        expect(mallory)
+          .to be_following(actor)
+      end
+    end
+
+    context 'when the endpoint is a paginated Collection of actor URIs with more pages than we allow' do
+      let(:payload) do
+        {
+          '@context': 'https://www.w3.org/ns/activitystreams',
+          type: 'Collection',
+          id: collection_uri,
+          first: {
+            type: 'CollectionPage',
+            partOf: collection_uri,
+            items: items,
+            next: "#{collection_uri}/page2",
+          },
+        }.with_indifferent_access
+      end
+
+      before do
+        stub_const('ActivityPub::SynchronizeFollowersService::MAX_COLLECTION_PAGES', 1)
+        stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' })
+      end
+
+      it 'confirms pending follow request but does not remove extra followers' do
+        previous_follower_ids = actor.followers.pluck(:id)
+
+        subject.call(actor, collection_uri)
+
+        expect(previous_follower_ids - actor.followers.reload.pluck(:id))
+          .to be_empty
+        expect(mallory)
+          .to be_following(actor)
+      end
+    end
   end
 end

spec/services/suspend_account_service_spec.rb

@@ -2,7 +2,7 @@
 
 require 'rails_helper'
 
-RSpec.describe SuspendAccountService, :inline_jobs do
+RSpec.describe SuspendAccountService do
   shared_examples 'common behavior' do
     subject { described_class.new.call(account) }
 
@@ -11,6 +11,7 @@ RSpec.describe SuspendAccountService, :inline_jobs do
 
     before do
       allow(FeedManager.instance).to receive_messages(unmerge_from_home: nil, unmerge_from_list: nil)
+      allow(Rails.configuration.x).to receive(:cache_buster_enabled).and_return(true)
 
       local_follower.follow!(account)
       list.accounts << account
@@ -23,6 +24,7 @@ RSpec.describe SuspendAccountService, :inline_jobs do
     it 'unmerges from feeds of local followers and changes file mode and preserves suspended flag' do
       expect { subject }
         .to change_file_mode
+        .and enqueue_sidekiq_job(CacheBusterWorker).with(account.media_attachments.first.file.url(:original))
         .and not_change_suspended_flag
       expect(FeedManager.instance).to have_received(:unmerge_from_home).with(account, local_follower)
       expect(FeedManager.instance).to have_received(:unmerge_from_list).with(account, list)
@@ -38,17 +40,12 @@ RSpec.describe SuspendAccountService, :inline_jobs do
   end
 
   describe 'suspending a local account' do
-    def match_update_actor_request(req, account)
+    def match_update_actor_request(json, account)
-      json = JSON.parse(req.body)
+      json = JSON.parse(json)
       actor_id = ActivityPub::TagManager.instance.uri_for(account)
       json['type'] == 'Update' && json['actor'] == actor_id && json['object']['id'] == actor_id && json['object']['suspended']
     end
 
-    before do
-      stub_request(:post, 'https://alice.com/inbox').to_return(status: 201)
-      stub_request(:post, 'https://bob.com/inbox').to_return(status: 201)
-    end
-
     include_examples 'common behavior' do
       let!(:account)         { Fabricate(:account) }
       let!(:remote_follower) { Fabricate(:account, uri: 'https://alice.com', inbox_url: 'https://alice.com/inbox', protocol: :activitypub, domain: 'alice.com') }
@@ -61,22 +58,20 @@ RSpec.describe SuspendAccountService, :inline_jobs do
 
       it 'sends an Update actor activity to followers and reporters' do
         subject
-        expect(a_request(:post, remote_follower.inbox_url).with { |req| match_update_actor_request(req, account) }).to have_been_made.once
+
-        expect(a_request(:post, remote_reporter.inbox_url).with { |req| match_update_actor_request(req, account) }).to have_been_made.once
+        expect(ActivityPub::DeliveryWorker)
+          .to have_enqueued_sidekiq_job(satisfying { |json| match_update_actor_request(json, account) }, account.id, remote_follower.inbox_url).once
+          .and have_enqueued_sidekiq_job(satisfying { |json| match_update_actor_request(json, account) }, account.id, remote_reporter.inbox_url).once
       end
     end
   end
 
   describe 'suspending a remote account' do
-    def match_reject_follow_request(req, account, followee)
+    def match_reject_follow_request(json, account, followee)
-      json = JSON.parse(req.body)
+      json = JSON.parse(json)
       json['type'] == 'Reject' && json['actor'] == ActivityPub::TagManager.instance.uri_for(followee) && json['object']['actor'] == account.uri
     end
 
-    before do
-      stub_request(:post, 'https://bob.com/inbox').to_return(status: 201)
-    end
-
     include_examples 'common behavior' do
       let!(:account)        { Fabricate(:account, domain: 'bob.com', uri: 'https://bob.com', inbox_url: 'https://bob.com/inbox', protocol: :activitypub) }
       let!(:local_followee) { Fabricate(:account) }
@@ -88,7 +83,8 @@ RSpec.describe SuspendAccountService, :inline_jobs do
       it 'sends a Reject Follow activity', :aggregate_failures do
         subject
 
-        expect(a_request(:post, account.inbox_url).with { |req| match_reject_follow_request(req, account, local_followee) }).to have_been_made.once
+        expect(ActivityPub::DeliveryWorker)
+          .to have_enqueued_sidekiq_job(satisfying { |json| match_reject_follow_request(json, account, local_followee) }, local_followee.id, account.inbox_url).once
       end
     end
   end

spec/system/settings/profiles_spec.rb

@@ -7,7 +7,6 @@ RSpec.describe 'Settings profile page' do
   let(:account) { user.account }
 
   before do
-    allow(ActivityPub::UpdateDistributionWorker).to receive(:perform_async)
     sign_in user
   end
 
@@ -24,7 +23,7 @@ RSpec.describe 'Settings profile page' do
       .to change { account.reload.display_name }.to('New name')
       .and(change { account.reload.avatar.instance.avatar_file_name }.from(nil).to(be_present))
     expect(ActivityPub::UpdateDistributionWorker)
-      .to have_received(:perform_async).with(account.id)
+      .to have_enqueued_sidekiq_job(account.id)
   end
 
   def display_name_field

streaming/database.js

@@ -49,7 +49,7 @@ export function configFromEnv(env, environment) {
     if (typeof parsedUrl.password === 'string') baseConfig.password = parsedUrl.password;
     if (typeof parsedUrl.host === 'string') baseConfig.host = parsedUrl.host;
     if (typeof parsedUrl.user === 'string') baseConfig.user = parsedUrl.user;
-    if (typeof parsedUrl.port === 'string') {
+    if (typeof parsedUrl.port === 'string' && parsedUrl.port) {
       const parsedPort = parseInt(parsedUrl.port, 10);
       if (isNaN(parsedPort)) {
         throw new Error('Invalid port specified in DATABASE_URL environment variable');

streaming/index.js

@@ -689,7 +689,7 @@ const startServer = async () => {
       // filtering of statuses:
 
       // Filter based on language:
-      if (Array.isArray(req.chosenLanguages) && payload.language !== null && req.chosenLanguages.indexOf(payload.language) === -1) {
+      if (Array.isArray(req.chosenLanguages) && req.chosenLanguages.indexOf(payload.language) === -1) {
         log.debug(`Message ${payload.id} filtered by language (${payload.language})`);
         return;
       }