Merge pull request #991 from kmycode/kb-draft-17.3

Release: 17.3

CHANGELOG.md

@@ -2,6 +2,65 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.3.6] - 2025-03-13
+
+### Security
+
+- Update dependency `omniauth-saml`
+- Update dependency `rack`
+
+### Fixed
+
+- Fix Stoplight errors when using `REDIS_NAMESPACE` (#34126 by @ClearlyClaire)
+
+## [4.3.5] - 2025-03-10
+
+### Changed
+
+- Change hashtag suggestion to prefer personal history capitalization (#34070 by @ClearlyClaire)
+
+### Fixed
+
+- Fix processing errors for some HEIF images from iOS 18 (#34086 by @renchap)
+- Fix streaming server not filtering unknown-language posts from public timelines (#33774 by @ClearlyClaire)
+- Fix preview cards under Content Warnings not being shown in detailed statuses (#34068 by @ClearlyClaire)
+- Fix username and display name being hidden on narrow screens in moderation interface (#33064 by @ClearlyClaire)
+
+## [4.3.4] - 2025-02-27
+
+### Security
+
+- Update dependencies
+- Change HTML sanitization to remove unusable and unused `embed` tag (#34021 by @ClearlyClaire, [GHSA-mq2m-hr29-8gqf](https://github.com/mastodon/mastodon/security/advisories/GHSA-mq2m-hr29-8gqf))
+- Fix rate-limit on sign-up email verification ([GHSA-v39f-c9jj-8w7h](https://github.com/mastodon/mastodon/security/advisories/GHSA-v39f-c9jj-8w7h))
+- Fix improper disclosure of domain blocks to unverified users ([GHSA-94h4-fj37-c825](https://github.com/mastodon/mastodon/security/advisories/GHSA-94h4-fj37-c825))
+
+### Changed
+
+- Change preview cards to be shown when Content Warnings are expanded (#33827 by @ClearlyClaire)
+- Change warnings against changing encryption secrets to be even more noticeable (#33631 by @ClearlyClaire)
+- Change `mastodon:setup` to prevent overwriting already-configured servers (#33603, #33616, and #33684 by @ClearlyClaire and @mjankowski)
+- Change notifications from moderators to not be filtered (#32974 and #33654 by @ClearlyClaire and @mjankowski)
+
+### Fixed
+
+- Fix `GET /api/v2/notifications/:id` and `POST /api/v2/notifications/:id/dismiss` for ungrouped notifications (#33990 by @ClearlyClaire)
+- Fix issue with some versions of libvips on some systems (#33853 by @kleisauke)
+- Fix handling of duplicate mentions in incoming status `Update` (#33911 by @ClearlyClaire)
+- Fix inefficiencies in timeline generation (#33839 and #33842 by @ClearlyClaire)
+- Fix emoji rewrite adding unnecessary curft to the DOM for most emoji (#33818 by @ClearlyClaire)
+- Fix `tootctl feeds build` not building list timelines (#33783 by @ClearlyClaire)
+- Fix flaky test in `/api/v2/notifications` tests (#33773 by @ClearlyClaire)
+- Fix incorrect signature after HTTP redirect (#33757 and #33769 by @ClearlyClaire)
+- Fix polls not being validated on edition (#33755 by @ClearlyClaire)
+- Fix media preview height in compose form when 3 or more images are attached (#33571 by @ClearlyClaire)
+- Fix preview card sizing in “Author attribution” in profile settings (#33482 by @ClearlyClaire)
+- Fix processing of incoming notifications for unfilterable types (#33429 by @ClearlyClaire)
+- Fix featured tags for remote accounts not being kept up to date (#33372, #33406, and #33425 by @ClearlyClaire and @mjankowski)
+- Fix notification polling showing a loading bar in web UI (#32960 by @Gargron)
+- Fix accounts table long display name (#29316 by @WebCoder49)
+- Fix exclusive lists interfering with notifications (#28162 by @ShadowJonathan)
+
 ## [4.3.3] - 2025-01-16
 
 ### Security

Dockerfile

@@ -96,6 +96,9 @@ RUN \
 # Set /opt/mastodon as working directory
 WORKDIR /opt/mastodon
 
+# Add backport repository for some specific packages where we need the latest version
+RUN echo 'deb http://deb.debian.org/debian bookworm-backports main' >> /etc/apt/sources.list
+
 # hadolint ignore=DL3008,DL3005
 RUN \
   # Mount Apt cache and lib directories from Docker buildx caches
@@ -165,7 +168,7 @@ RUN \
   libexif-dev \
   libexpat1-dev \
   libgirepository1.0-dev \
-  libheif-dev \
+  libheif-dev/bookworm-backports \
   libimagequant-dev \
   libjpeg62-turbo-dev \
   liblcms2-dev \
@@ -348,7 +351,7 @@ RUN \
   # libvips components
   libcgif0 \
   libexif12 \
-  libheif1 \
+  libheif1/bookworm-backports \
   libimagequant0 \
   libjpeg62-turbo \
   liblcms2-2 \

Gemfile.lock

@@ -416,7 +416,7 @@ GEM
     mutex_m (0.3.0)
     net-http (0.6.0)
       uri
-    net-imap (0.5.5)
+    net-imap (0.5.6)
       date
       net-protocol
     net-ldap (0.19.0)
@@ -424,10 +424,10 @@ GEM
       net-protocol
     net-protocol (0.2.2)
       timeout
-    net-smtp (0.5.0)
+    net-smtp (0.5.1)
       net-protocol
     nio4r (2.7.4)
-    nokogiri (1.18.2)
+    nokogiri (1.18.3)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     oj (3.16.9)
@@ -444,9 +444,9 @@ GEM
     omniauth-rails_csrf_protection (1.0.2)
       actionpack (>= 4.2)
       omniauth (~> 2.0)
-    omniauth-saml (2.2.1)
+    omniauth-saml (2.2.3)
       omniauth (~> 2.1)
-      ruby-saml (~> 1.17)
+      ruby-saml (~> 1.18)
     omniauth_openid_connect (0.6.1)
       omniauth (>= 1.9, < 3)
       openid_connect (~> 1.1)
@@ -597,7 +597,7 @@ GEM
       activesupport (>= 3.0.0)
     raabro (1.4.0)
     racc (1.8.1)
-    rack (2.2.10)
+    rack (2.2.13)
     rack-attack (6.7.0)
       rack (>= 1.0, < 4)
     rack-cors (2.0.2)
@@ -745,10 +745,10 @@ GEM
       rubocop-rspec (~> 3, >= 3.0.1)
     ruby-prof (1.7.1)
     ruby-progressbar (1.13.0)
-    ruby-saml (1.17.0)
+    ruby-saml (1.18.0)
       nokogiri (>= 1.13.10)
       rexml
-    ruby-vips (2.2.2)
+    ruby-vips (2.2.3)
       ffi (~> 1.12)
       logger
     rubyzip (2.4.1)
@@ -845,7 +845,7 @@ GEM
       unf_ext
     unf_ext (0.0.9.1)
     unicode-display_width (2.6.0)
-    uri (1.0.2)
+    uri (1.0.3)
     useragent (0.16.11)
     validate_email (0.1.6)
       activemodel (>= 3.0)

app/controllers/admin/ng_words/keywords_controller.rb

@@ -21,6 +21,10 @@ module Admin
       false
     end
 
+    def avoid_save?
+      true
+    end
+
     private
 
     def after_update_redirect_path

app/controllers/admin/ng_words_controller.rb

@@ -13,6 +13,12 @@ module Admin
 
       return unless validate
 
+      if avoid_save?
+        flash[:notice] = I18n.t('generic.changes_saved_msg')
+        redirect_to after_update_redirect_path
+        return
+      end
+
       @admin_settings = Form::AdminSettings.new(settings_params)
 
       if @admin_settings.save
@@ -33,6 +39,10 @@ module Admin
       admin_ng_words_path
     end
 
+    def avoid_save?
+      false
+    end
+
     private
 
     def settings_params
@@ -40,7 +50,7 @@ module Admin
     end
 
     def settings_params_test
-      params.require(:form_admin_settings)[:ng_words_test]
+      params.expect(form_admin_settings: [ng_words_test: [keywords: [], regexps: [], strangers: [], temporary_ids: []]])['ng_words_test']
     end
   end
 end

app/controllers/api/v1/instances/domain_blocks_controller.rb

@@ -31,7 +31,7 @@ class Api::V1::Instances::DomainBlocksController < Api::V1::Instances::BaseContr
   end
 
   def show_domain_blocks_to_user?
-    Setting.show_domain_blocks == 'users' && user_signed_in?
+    Setting.show_domain_blocks == 'users' && user_signed_in? && current_user.functional_or_moved?
   end
 
   def set_domain_blocks
@@ -47,6 +47,6 @@ class Api::V1::Instances::DomainBlocksController < Api::V1::Instances::BaseContr
   end
 
   def show_rationale_for_user?
-    Setting.show_domain_blocks_rationale == 'users' && user_signed_in?
+    Setting.show_domain_blocks_rationale == 'users' && user_signed_in? && current_user.functional_or_moved?
   end
 end

app/controllers/api/v2/notifications_controller.rb

@@ -46,7 +46,7 @@ class Api::V2::NotificationsController < Api::BaseController
   end
 
   def show
-    @notification = current_account.notifications.without_suspended.find_by!(group_key: params[:group_key])
+    @notification = current_account.notifications.without_suspended.by_group_key(params[:group_key]).take!
     presenter = GroupedNotificationsPresenter.new(NotificationGroup.from_notifications([@notification]))
     render json: presenter, serializer: REST::DedupNotificationGroupSerializer
   end
@@ -57,7 +57,7 @@ class Api::V2::NotificationsController < Api::BaseController
   end
 
   def dismiss
-    current_account.notifications.where(group_key: params[:group_key]).destroy_all
+    current_account.notifications.by_group_key(params[:group_key]).destroy_all
     render_empty
   end
 

app/controllers/system_css_controller.rb

@@ -1,16 +1,8 @@
 # frozen_string_literal: true
 
 class SystemCssController < ActionController::Base # rubocop:disable Rails/ApplicationController
-  before_action :set_user_roles
-
   def show
     expires_in 3.minutes, public: true
     render content_type: 'text/css'
   end
-
-  private
-
-  def set_user_roles
-    @user_roles = UserRole.providing_styles
-  end
 end

app/javascript/mastodon/features/emoji/__tests__/emoji-test.js

@@ -22,23 +22,23 @@ describe('emoji', () => {
 
     it('does unicode', () => {
       expect(emojify('\uD83D\uDC69\u200D\uD83D\uDC69\u200D\uD83D\uDC66\u200D\uD83D\uDC66')).toEqual(
-        '<picture><img draggable="false" class="emojione" alt="👩‍👩‍👦‍👦" title=":woman-woman-boy-boy:" src="/emoji/1f469-200d-1f469-200d-1f466-200d-1f466.svg"></picture>');
+        '<img draggable="false" class="emojione" alt="👩‍👩‍👦‍👦" title=":woman-woman-boy-boy:" src="/emoji/1f469-200d-1f469-200d-1f466-200d-1f466.svg">');
       expect(emojify('👨‍👩‍👧‍👧')).toEqual(
-        '<picture><img draggable="false" class="emojione" alt="👨‍👩‍👧‍👧" title=":man-woman-girl-girl:" src="/emoji/1f468-200d-1f469-200d-1f467-200d-1f467.svg"></picture>');
-      expect(emojify('👩‍👩‍👦')).toEqual('<picture><img draggable="false" class="emojione" alt="👩‍👩‍👦" title=":woman-woman-boy:" src="/emoji/1f469-200d-1f469-200d-1f466.svg"></picture>');
+        '<img draggable="false" class="emojione" alt="👨‍👩‍👧‍👧" title=":man-woman-girl-girl:" src="/emoji/1f468-200d-1f469-200d-1f467-200d-1f467.svg">');
+      expect(emojify('👩‍👩‍👦')).toEqual('<img draggable="false" class="emojione" alt="👩‍👩‍👦" title=":woman-woman-boy:" src="/emoji/1f469-200d-1f469-200d-1f466.svg">');
       expect(emojify('\u2757')).toEqual(
-        '<picture><img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"></picture>');
+        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg">');
     });
 
     it('does multiple unicode', () => {
       expect(emojify('\u2757 #\uFE0F\u20E3')).toEqual(
-        '<picture><img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"></picture> <picture><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"></picture>');
+        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg">');
       expect(emojify('\u2757#\uFE0F\u20E3')).toEqual(
-        '<picture><img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"></picture><picture><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"></picture>');
+        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg">');
       expect(emojify('\u2757 #\uFE0F\u20E3 \u2757')).toEqual(
-        '<picture><img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"></picture> <picture><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"></picture> <picture><img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"></picture>');
+        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"> <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg">');
       expect(emojify('foo \u2757 #\uFE0F\u20E3 bar')).toEqual(
-        'foo <picture><img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"></picture> <picture><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"></picture> bar');
+        'foo <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"> bar');
     });
 
     it('ignores unicode inside of tags', () => {
@@ -46,16 +46,16 @@ describe('emoji', () => {
     });
 
     it('does multiple emoji properly (issue 5188)', () => {
-      expect(emojify('👌🌈💕')).toEqual('<picture><img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg"></picture><picture><img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg"></picture><picture><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg"></picture>');
-      expect(emojify('👌 🌈 💕')).toEqual('<picture><img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg"></picture> <picture><img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg"></picture> <picture><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg"></picture>');
+      expect(emojify('👌🌈💕')).toEqual('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg"><img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg"><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg">');
+      expect(emojify('👌 🌈 💕')).toEqual('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg"> <img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg"> <img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg">');
     });
 
     it('does an emoji that has no shortcode', () => {
-      expect(emojify('👁‍🗨')).toEqual('<picture><img draggable="false" class="emojione" alt="👁‍🗨" title="" src="/emoji/1f441-200d-1f5e8.svg"></picture>');
+      expect(emojify('👁‍🗨')).toEqual('<img draggable="false" class="emojione" alt="👁‍🗨" title="" src="/emoji/1f441-200d-1f5e8.svg">');
     });
 
     it('does an emoji whose filename is irregular', () => {
-      expect(emojify('↙️')).toEqual('<picture><img draggable="false" class="emojione" alt="↙️" title=":arrow_lower_left:" src="/emoji/2199.svg"></picture>');
+      expect(emojify('↙️')).toEqual('<img draggable="false" class="emojione" alt="↙️" title=":arrow_lower_left:" src="/emoji/2199.svg">');
     });
 
     it('avoid emojifying on invisible text', () => {
@@ -67,11 +67,11 @@ describe('emoji', () => {
 
     it('avoid emojifying on invisible text with nested tags', () => {
       expect(emojify('<span class="invisible">😄<span class="foo">bar</span>😴</span>😇'))
-        .toEqual('<span class="invisible">😄<span class="foo">bar</span>😴</span><picture><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg"></picture>');
+        .toEqual('<span class="invisible">😄<span class="foo">bar</span>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg">');
       expect(emojify('<span class="invisible">😄<span class="invisible">😕</span>😴</span>😇'))
-        .toEqual('<span class="invisible">😄<span class="invisible">😕</span>😴</span><picture><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg"></picture>');
+        .toEqual('<span class="invisible">😄<span class="invisible">😕</span>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg">');
       expect(emojify('<span class="invisible">😄<br>😴</span>😇'))
-        .toEqual('<span class="invisible">😄<br>😴</span><picture><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg"></picture>');
+        .toEqual('<span class="invisible">😄<br>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg">');
     });
 
     it('does not emojify emojis with textual presentation VS15 character', () => {
@@ -81,17 +81,17 @@ describe('emoji', () => {
 
     it('does a simple emoji properly', () => {
       expect(emojify('♀♂'))
-        .toEqual('<picture><img draggable="false" class="emojione" alt="♀" title=":female_sign:" src="/emoji/2640.svg"></picture><picture><img draggable="false" class="emojione" alt="♂" title=":male_sign:" src="/emoji/2642.svg"></picture>');
+        .toEqual('<img draggable="false" class="emojione" alt="♀" title=":female_sign:" src="/emoji/2640.svg"><img draggable="false" class="emojione" alt="♂" title=":male_sign:" src="/emoji/2642.svg">');
     });
 
     it('does an emoji containing ZWJ properly', () => {
       expect(emojify('💂‍♀️💂‍♂️'))
-        .toEqual('<picture><img draggable="false" class="emojione" alt="💂\u200D♀️" title=":female-guard:" src="/emoji/1f482-200d-2640-fe0f_border.svg"></picture><picture><img draggable="false" class="emojione" alt="💂\u200D♂️" title=":male-guard:" src="/emoji/1f482-200d-2642-fe0f_border.svg"></picture>');
+        .toEqual('<img draggable="false" class="emojione" alt="💂\u200D♀️" title=":female-guard:" src="/emoji/1f482-200d-2640-fe0f_border.svg"><img draggable="false" class="emojione" alt="💂\u200D♂️" title=":male-guard:" src="/emoji/1f482-200d-2642-fe0f_border.svg">');
     });
 
     it('keeps ordering as expected (issue fixed by PR 20677)', () => {
       expect(emojify('<p>💕 <a class="hashtag" href="https://example.com/tags/foo" rel="nofollow noopener" target="_blank">#<span>foo</span></a> test: foo.</p>'))
-        .toEqual('<p><picture><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg"></picture> <a class="hashtag" href="https://example.com/tags/foo" rel="nofollow noopener" target="_blank">#<span>foo</span></a> test: foo.</p>');
+        .toEqual('<p><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg"> <a class="hashtag" href="https://example.com/tags/foo" rel="nofollow noopener" target="_blank">#<span>foo</span></a> test: foo.</p>');
     });
   });
 });

app/javascript/mastodon/features/emoji/emoji.js

@@ -97,30 +97,30 @@ const emojifyTextNode = (node, customEmojis) => {
       const { filename, shortCode } = unicodeMapping[unicode_emoji];
       const title = shortCode ? `:${shortCode}:` : '';
 
-      replacement = document.createElement('picture');
-
       const isSystemTheme = !!document.body?.classList.contains('theme-system');
 
-      if(isSystemTheme) {
-        let source = document.createElement('source');
-        source.setAttribute('media', '(prefers-color-scheme: dark)');
-        source.setAttribute('srcset', `${assetHost}/emoji/${emojiFilename(filename, "dark")}.svg`);
-        replacement.appendChild(source);
-      }
+      const theme = (isSystemTheme || document.body?.classList.contains('theme-mastodon-light')) ? 'light' : 'dark';
 
-      let img = document.createElement('img');
+      const imageFilename = emojiFilename(filename, theme);
+
+      const img = document.createElement('img');
       img.setAttribute('draggable', 'false');
       img.setAttribute('class', 'emojione');
       img.setAttribute('alt', unicode_emoji);
       img.setAttribute('title', title);
+      img.setAttribute('src', `${assetHost}/emoji/${imageFilename}.svg`);
 
-      let theme = "light";
+      if (isSystemTheme && imageFilename !== emojiFilename(filename, 'dark')) {
+        replacement = document.createElement('picture');
 
-      if(!isSystemTheme && !document.body?.classList.contains('theme-mastodon-light'))
-        theme = "dark";
-
-      img.setAttribute('src', `${assetHost}/emoji/${emojiFilename(filename, theme)}.svg`);
-      replacement.appendChild(img);
+        const source = document.createElement('source');
+        source.setAttribute('media', '(prefers-color-scheme: dark)');
+        source.setAttribute('srcset', `${assetHost}/emoji/${emojiFilename(filename, 'dark')}.svg`);
+        replacement.appendChild(source);
+        replacement.appendChild(img);
+      } else {
+        replacement = img;
+      }
     }
 
     // Add the processed-up-to-now string and the emoji replacement
@@ -135,7 +135,7 @@ const emojifyTextNode = (node, customEmojis) => {
 };
 
 const emojifyNode = (node, customEmojis) => {
-  for (const child of node.childNodes) {
+  for (const child of Array.from(node.childNodes)) {
     switch(child.nodeType) {
     case Node.TEXT_NODE:
       emojifyTextNode(child, customEmojis);

app/javascript/mastodon/features/status/components/detailed_status.tsx

@@ -237,7 +237,7 @@ export const DetailedStatus: React.FC<{
       <Card
         sensitive={status.get('sensitive') && !status.get('spoiler_text')}
         onOpenMedia={onOpenMedia}
-        card={status.get('card', null)}
+        card={status.get('card')}
       />
     );
   }

app/javascript/mastodon/reducers/compose.js

@@ -342,12 +342,26 @@ const expiresInFromExpiresAt = expires_at => {
 
 const mergeLocalHashtagResults = (suggestions, prefix, tagHistory) => {
   prefix = prefix.toLowerCase();
+
   if (suggestions.length < 4) {
     const localTags = tagHistory.filter(tag => tag.toLowerCase().startsWith(prefix) && !suggestions.some(suggestion => suggestion.type === 'hashtag' && suggestion.name.toLowerCase() === tag.toLowerCase()));
-    return suggestions.concat(localTags.slice(0, 4 - suggestions.length).toJS().map(tag => ({ type: 'hashtag', name: tag })));
-  } else {
-    return suggestions;
+    suggestions = suggestions.concat(localTags.slice(0, 4 - suggestions.length).toJS().map(tag => ({ type: 'hashtag', name: tag })));
   }
+
+  // Prefer capitalization from personal history, unless personal history is all lower-case
+  const fixSuggestionCapitalization = (suggestion) => {
+    if (suggestion.type !== 'hashtag')
+      return suggestion;
+
+    const tagFromHistory = tagHistory.find((tag) => tag.localeCompare(suggestion.name, undefined, { sensitivity: 'accent' }) === 0);
+
+    if (!tagFromHistory || tagFromHistory.toLowerCase() === tagFromHistory)
+      return suggestion;
+
+    return { ...suggestion, name: tagFromHistory };
+  };
+
+  return suggestions.map(fixSuggestionCapitalization);
 };
 
 const normalizeSuggestions = (state, { accounts, emojis, tags, token }) => {

app/lib/feed_manager.rb

@@ -42,7 +42,7 @@ class FeedManager
     when :home
       filter_from_home(status, receiver.id, build_crutches(receiver.id, [status]), :home)
     when :list
-      (filter_from_list?(status, receiver) ? :filter : nil) || filter_from_home(status, receiver.account_id, build_crutches(receiver.account_id, [status]), :list, stl_home: stl_home)
+      (filter_from_list?(status, receiver) ? :filter : nil) || filter_from_home(status, receiver.account_id, build_crutches(receiver.account_id, [status], list: receiver), :list, stl_home: stl_home)
     when :mentions
       filter_from_mentions?(status, receiver.id) ? :filter : nil
     when :tags
@@ -136,7 +136,7 @@ class FeedManager
 
     timeline_key = key(:home, into_account.id)
     aggregate    = into_account.user&.aggregates_reblogs?
-    query        = from_account.statuses.list_eligible_visibility.includes(:preloadable_poll, :media_attachments, reblog: :account).limit(FeedManager::MAX_ITEMS / 4)
+    query        = from_account.statuses.list_eligible_visibility.includes(reblog: :account).limit(FeedManager::MAX_ITEMS / 4)
 
     if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4
       oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true).first.last.to_i
@@ -164,7 +164,7 @@ class FeedManager
 
     timeline_key = key(:list, list.id)
     aggregate    = list.account.user&.aggregates_reblogs?
-    query        = from_account.statuses.list_eligible_visibility.includes(:preloadable_poll, :media_attachments, reblog: :account).limit(FeedManager::MAX_ITEMS / 4)
+    query        = from_account.statuses.list_eligible_visibility.includes(reblog: :account).limit(FeedManager::MAX_ITEMS / 4)
 
     if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4
       oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true).first.last.to_i
@@ -172,10 +172,10 @@ class FeedManager
     end
 
     statuses = query.to_a
-    crutches = build_crutches(list.account_id, statuses)
+    crutches = build_crutches(list.account_id, statuses, list: list)
 
     statuses.each do |status|
-      next if filter_from_home(status, list.account_id, crutches) || filter_from_list?(status, list)
+      next if filter_from_home(status, list.account_id, crutches, :list)
 
       add_to_feed(:list, list.id, status, aggregate_reblogs: aggregate)
     end
@@ -309,23 +309,32 @@ class FeedManager
     limit        = FeedManager::MAX_ITEMS / 2
     aggregate    = account.user&.aggregates_reblogs?
     timeline_key = key(:home, account.id)
+    over_limit = false
 
     account.statuses.limit(limit).each do |status|
       add_to_feed(:home, account.id, status, aggregate_reblogs: aggregate)
     end
 
     account.following.includes(:account_stat).reorder(nil).find_each do |target_account|
-      if redis.zcard(timeline_key) >= limit
+      query = target_account.statuses.list_eligible_visibility.includes(reblog: :account).limit(limit)
+
+      over_limit ||= redis.zcard(timeline_key) >= limit
+      if over_limit
         oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true).first.last.to_i
-        last_status_score = Mastodon::Snowflake.id_at(target_account.last_status_at)
+        last_status_score = Mastodon::Snowflake.id_at(target_account.last_status_at, with_random: false)
 
         # If the feed is full and this account has not posted more recently
         # than the last item on the feed, then we can skip the whole account
         # because none of its statuses would stay on the feed anyway
         next if last_status_score < oldest_home_score
+
+        # No need to get older statuses
+        query = query.where(id: oldest_home_score...)
       end
 
-      statuses = target_account.statuses.list_eligible_visibility.includes(:preloadable_poll, :media_attachments, :account, reblog: :account).limit(limit)
+      statuses = query.to_a
+      next if statuses.empty?
+
       crutches = build_crutches(account.id, statuses)
 
       statuses.each do |status|
@@ -345,23 +354,32 @@ class FeedManager
     limit        = FeedManager::MAX_ITEMS / 2
     aggregate    = list.account.user&.aggregates_reblogs?
     timeline_key = key(:list, list.id)
+    over_limit = false
 
     list.active_accounts.includes(:account_stat).reorder(nil).find_each do |target_account|
-      if redis.zcard(timeline_key) >= limit
+      query = target_account.statuses.list_eligible_visibility.includes(reblog: :account).limit(limit)
+
+      over_limit ||= redis.zcard(timeline_key) >= limit
+      if over_limit
         oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true).first.last.to_i
-        last_status_score = Mastodon::Snowflake.id_at(target_account.last_status_at)
+        last_status_score = Mastodon::Snowflake.id_at(target_account.last_status_at, with_random: false)
 
         # If the feed is full and this account has not posted more recently
         # than the last item on the feed, then we can skip the whole account
         # because none of its statuses would stay on the feed anyway
         next if last_status_score < oldest_home_score
+
+        # No need to get older statuses
+        query = query.where(id: oldest_home_score...)
       end
 
-      statuses = target_account.statuses.list_eligible_visibility.includes(:preloadable_poll, :media_attachments, :account, reblog: :account).limit(limit)
-      crutches = build_crutches(list.account_id, statuses)
+      statuses = query.to_a
+      next if statuses.empty?
+
+      crutches = build_crutches(list.account_id, statuses, list: list)
 
       statuses.each do |status|
-        next if filter_from_home(status, list.account_id, crutches) || filter_from_list?(status, list)
+        next if filter_from_home(status, list.account_id, crutches, :list)
 
         add_to_feed(:list, list.id, status, aggregate_reblogs: aggregate)
       end
@@ -632,8 +650,9 @@ class FeedManager
   # are going to be checked by the filtering methods
   # @param [Integer] receiver_id
   # @param [Array<Status>] statuses
+  # @param [List] list
   # @return [Hash]
-  def build_crutches(receiver_id, statuses) # rubocop:disable Metrics/AbcSize
+  def build_crutches(receiver_id, statuses, list: nil)
     crutches = {}
 
     crutches[:active_mentions] = crutches_active_mentions(statuses)
@@ -650,25 +669,43 @@ class FeedManager
       arr
     end
 
-    lists = List.where(account_id: receiver_id, exclusive: true)
-    antennas = Antenna.where(list: lists, insert_feeds: true)
-
-    replied_accounts  = statuses.filter_map(&:in_reply_to_account_id)
-    replied_accounts += statuses.filter { |status| status.limited_visibility? && status.thread.present? }.map { |status| status.thread.account_id }
-
-    crutches[:following]            = Follow.where(account_id: receiver_id, target_account_id: replied_accounts).pluck(:target_account_id).index_with(true)
+    crutches[:following]            = crutches_following(receiver_id, statuses, list)
     crutches[:languages]            = Follow.where(account_id: receiver_id, target_account_id: statuses.map(&:account_id)).pluck(:target_account_id, :languages).to_h
     crutches[:hiding_reblogs]       = Follow.where(account_id: receiver_id, target_account_id: statuses.filter_map { |s| s.account_id if s.reblog? }, show_reblogs: false).pluck(:target_account_id).index_with(true)
     crutches[:blocking]             = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).index_with(true)
     crutches[:muting]               = Mute.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).index_with(true)
     crutches[:domain_blocking]      = AccountDomainBlock.where(account_id: receiver_id, domain: statuses.flat_map { |s| [s.account.domain, s.reblog&.account&.domain] }.compact).pluck(:domain).index_with(true)
     crutches[:blocked_by]           = Block.where(target_account_id: receiver_id, account_id: statuses.map { |s| [s.account_id, s.reblog&.account_id] }.flatten.compact).pluck(:account_id).index_with(true)
-    crutches[:exclusive_list_users] = ListAccount.where(list: lists, account_id: statuses.map(&:account_id)).pluck(:account_id).index_with(true)
-    crutches[:exclusive_antenna_users] = AntennaAccount.where(antenna: antennas, account_id: statuses.map(&:account_id)).pluck(:account_id).index_with(true)
+    crutches[:exclusive_list_users] = crutches_exclusive_list_users(receiver_id, statuses) if list.blank?
+    crutches[:exclusive_antenna_users] = crutches_exclusive_antenna_users(receiver_id, statuses)
 
     crutches
   end
 
+  def crutches_exclusive_list_users(recipient_id, statuses)
+    lists = List.where(account_id: recipient_id, exclusive: true)
+    ListAccount.where(list: lists, account_id: statuses.map(&:account_id)).pluck(:account_id).index_with(true)
+  end
+
+  def crutches_exclusive_antenna_users(recipient_id, statuses)
+    lists = List.where(account_id: recipient_id, exclusive: true)
+    antennas = Antenna.where(list: lists, insert_feeds: true)
+    AntennaAccount.where(antenna: antennas, account_id: statuses.map(&:account_id)).pluck(:account_id).index_with(true)
+  end
+
+  def crutches_following(recipient_id, statuses, list)
+    if list.blank? || list.show_followed?
+      replied_accounts  = statuses.filter_map(&:in_reply_to_account_id)
+      replied_accounts += statuses.filter { |status| status.limited_visibility? && status.thread.present? }.map { |status| status.thread.account_id }
+
+      Follow.where(account_id: recipient_id, target_account_id: replied_accounts).pluck(:target_account_id).index_with(true)
+    elsif list.show_list?
+      ListAccount.where(list_id: list.id, account_id: statuses.filter_map(&:in_reply_to_account_id)).pluck(:account_id).index_with(true)
+    else
+      {}
+    end
+  end
+
   def crutches_active_mentions(statuses)
     Mention
       .active

app/models/concerns/notification/groups.rb

@@ -7,6 +7,10 @@ module Notification::Groups
   GROUPABLE_NOTIFICATION_TYPES = %i(favourite reblog follow emoji_reaction).freeze
   MAXIMUM_GROUP_SPAN_HOURS = 12
 
+  included do
+    scope :by_group_key, ->(group_key) { group_key&.start_with?('ungrouped-') ? where(id: group_key.delete_prefix('ungrouped-')) : where(group_key: group_key) }
+  end
+
   def set_group_key!
     return if filtered? || GROUPABLE_NOTIFICATION_TYPES.exclude?(type)
 

app/services/activitypub/process_status_update_service.rb

@@ -296,7 +296,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
       nil
     end
 
-    @status.mentions.upsert_all(currently_mentioned_account_ids.map { |id| { account_id: id, silent: false } }, unique_by: %w(status_id account_id))
+    @status.mentions.upsert_all(currently_mentioned_account_ids.uniq.map { |id| { account_id: id, silent: false } }, unique_by: %w(status_id account_id))
 
     # If previous mentions are no longer contained in the text, convert them
     # to silent mentions, since withdrawing access from someone who already

app/views/custom_css/show_system.css.erb

@@ -1,6 +0,0 @@
-<%- @user_roles.each do |role| %>
-.user-role-<%= role.id %> {
-  --user-role-accent: <%= role.color %>;
-}
-
-<%- end %>

app/views/system_css/show.css.erb

@@ -1,6 +0,0 @@
-<%- @user_roles.each do |role| %>
-.user-role-<%= role.id %> {
-  --user-role-accent: <%= role.color %>;
-}
-
-<%- end %>

config/initializers/rack_attack.rb

@@ -122,7 +122,7 @@ class Rack::Attack
   end
 
   throttle('throttle_email_confirmations/ip', limit: 25, period: 5.minutes) do |req|
-    req.throttleable_remote_ip if req.post? && (req.path_matches?('/auth/confirmation') || req.path == '/api/v1/emails/confirmations')
+    req.throttleable_remote_ip if (req.post? && (req.path_matches?('/auth/confirmation') || req.path == '/api/v1/emails/confirmations')) || ((req.put? || req.patch?) && req.path_matches?('/auth/setup'))
   end
 
   throttle('throttle_email_confirmations/email', limit: 5, period: 30.minutes) do |req|
@@ -133,6 +133,14 @@ class Rack::Attack
     end
   end
 
+  throttle('throttle_auth_setup/email', limit: 5, period: 10.minutes) do |req|
+    req.params.dig('user', 'email').presence if (req.put? || req.patch?) && req.path_matches?('/auth/setup')
+  end
+
+  throttle('throttle_auth_setup/account', limit: 5, period: 10.minutes) do |req|
+    req.warden_user_id if (req.put? || req.patch?) && req.path_matches?('/auth/setup')
+  end
+
   throttle('throttle_login_attempts/ip', limit: 25, period: 5.minutes) do |req|
     req.throttleable_remote_ip if req.post? && req.path_matches?('/auth/sign_in')
   end

docker-compose.yml

@@ -59,7 +59,7 @@ services:
   web:
     # You can uncomment the following line if you want to not use the prebuilt image, for example if you have local code changes
     build: .
-    image: kmyblue:17.0-dev
+    image: kmyblue:17.3
     restart: always
     env_file: .env.production
     command: bundle exec puma -C config/puma.rb
@@ -83,7 +83,7 @@ services:
     build:
       dockerfile: ./streaming/Dockerfile
       context: .
-    image: kmyblue-streaming:17.0-dev
+    image: kmyblue-streaming:17.3
     restart: always
     env_file: .env.production
     command: node ./streaming/index.js
@@ -101,7 +101,7 @@ services:
 
   sidekiq:
     build: .
-    image: kmyblue:17.0-dev
+    image: kmyblue:17.3
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq

lib/mastodon/version.rb

@@ -13,13 +13,13 @@ module Mastodon
     end
 
     def kmyblue_minor
-      0
+      3
     end
 
     def kmyblue_flag
       # 'LTS'
-      'dev'
-      # nil
+      # 'dev'
+      nil
     end
 
     def major
@@ -35,7 +35,7 @@ module Mastodon
     end
 
     def default_prerelease
-      'alpha.2'
+      'alpha.4'
     end
 
     def prerelease

lib/redis/namespace_extensions.rb

@@ -5,6 +5,10 @@ class Redis
     def exists?(...)
       call_with_namespace('exists?', ...)
     end
+
+    def with
+      yield self
+    end
   end
 end
 

lib/sanitize_ext/sanitize_config.rb

@@ -155,18 +155,16 @@ class Sanitize
     )
 
     MASTODON_OEMBED = freeze_config(
-      elements: %w(audio embed iframe source video),
+      elements: %w(audio iframe source video),
 
       attributes: {
         'audio' => %w(controls),
-        'embed' => %w(height src type width),
         'iframe' => %w(allowfullscreen frameborder height scrolling src width),
         'source' => %w(src type),
         'video' => %w(controls height loop width),
       },
 
       protocols: {
-        'embed' => { 'src' => HTTP_PROTOCOLS },
         'iframe' => { 'src' => HTTP_PROTOCOLS },
         'source' => { 'src' => HTTP_PROTOCOLS },
       },

spec/lib/activitypub/activity/create_spec.rb

@@ -48,10 +48,16 @@ RSpec.describe ActivityPub::Activity::Create do
         content: '@bob lorem ipsum',
         published: 1.hour.ago.utc.iso8601,
         updated: 1.hour.ago.utc.iso8601,
-        tag: {
-          type: 'Mention',
-          href: ActivityPub::TagManager.instance.uri_for(follower),
-        },
+        tag: [
+          {
+            type: 'Mention',
+            href: ActivityPub::TagManager.instance.uri_for(follower),
+          },
+          {
+            type: 'Mention',
+            href: ActivityPub::TagManager.instance.uri_for(follower),
+          },
+        ],
       }
     end
 

spec/lib/feed_manager_spec.rb

@@ -233,6 +233,28 @@ RSpec.describe FeedManager do
       end
     end
 
+    context 'with list feed' do
+      let(:list) { Fabricate(:list, account: bob) }
+
+      before do
+        bob.follow!(alice)
+        list.list_accounts.create!(account: alice)
+      end
+
+      it "returns false for followee's status" do
+        status = Fabricate(:status, text: 'Hello world', account: alice)
+
+        expect(subject.filter?(:list, status, list)).to be false
+      end
+
+      it 'returns false for reblog by followee' do
+        status = Fabricate(:status, text: 'Hello world', account: jeff)
+        reblog = Fabricate(:status, reblog: status, account: alice)
+
+        expect(subject.filter?(:list, reblog, list)).to be false
+      end
+    end
+
     context 'with mentions feed' do
       it 'returns true for status that mentions blocked account' do
         bob.block!(jeff)

spec/requests/api/v1/instances/domain_blocks_spec.rb

@@ -4,14 +4,15 @@ require 'rails_helper'
 
 RSpec.describe 'Domain Blocks' do
   let(:user)    { Fabricate(:user) }
-  let(:token)   { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) }
+  let(:token)   { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes).token }
   let(:scopes)  { 'read' }
-  let(:headers) { { Authorization: "Bearer #{token.token}" } }
+  let(:headers) { { Authorization: "Bearer #{token}" } }
 
   describe 'GET /api/v1/instance/domain_blocks' do
-    before do
-      Fabricate(:domain_block)
-    end
+    let(:user) { Fabricate(:user) }
+    let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id).token }
+
+    before { Fabricate(:domain_block) }
 
     context 'with domain blocks set to all' do
       before { Setting.show_domain_blocks = 'all' }
@@ -45,11 +46,95 @@ RSpec.describe 'Domain Blocks' do
     context 'with domain blocks set to users' do
       before { Setting.show_domain_blocks = 'users' }
 
-      it 'returns http not found' do
-        get api_v1_instance_domain_blocks_path
+      context 'without authentication token' do
+        it 'returns http not found' do
+          get api_v1_instance_domain_blocks_path
 
-        expect(response)
-          .to have_http_status(404)
+          expect(response)
+            .to have_http_status(404)
+        end
+      end
+
+      context 'with authentication token' do
+        context 'with unapproved user' do
+          before { user.update(approved: false) }
+
+          it 'returns http not found' do
+            get api_v1_instance_domain_blocks_path, headers: { 'Authorization' => "Bearer #{token}" }
+
+            expect(response)
+              .to have_http_status(404)
+          end
+        end
+
+        context 'with unconfirmed user' do
+          before { user.update(confirmed_at: nil) }
+
+          it 'returns http not found' do
+            get api_v1_instance_domain_blocks_path, headers: { 'Authorization' => "Bearer #{token}" }
+
+            expect(response)
+              .to have_http_status(404)
+          end
+        end
+
+        context 'with disabled user' do
+          before { user.update(disabled: true) }
+
+          it 'returns http not found' do
+            get api_v1_instance_domain_blocks_path, headers: { 'Authorization' => "Bearer #{token}" }
+
+            expect(response)
+              .to have_http_status(404)
+          end
+        end
+
+        context 'with suspended user' do
+          before { user.account.update(suspended_at: Time.zone.now) }
+
+          it 'returns http not found' do
+            get api_v1_instance_domain_blocks_path, headers: { 'Authorization' => "Bearer #{token}" }
+
+            expect(response)
+              .to have_http_status(403)
+          end
+        end
+
+        context 'with moved user' do
+          before { user.account.update(moved_to_account_id: Fabricate(:account).id) }
+
+          it 'returns http success' do
+            get api_v1_instance_domain_blocks_path, headers: { 'Authorization' => "Bearer #{token}" }
+
+            expect(response)
+              .to have_http_status(200)
+
+            expect(response.content_type)
+              .to start_with('application/json')
+
+            expect(response.parsed_body)
+              .to be_present
+              .and(be_an(Array))
+              .and(have_attributes(size: 1))
+          end
+        end
+
+        context 'with normal user' do
+          it 'returns http success' do
+            get api_v1_instance_domain_blocks_path, headers: { 'Authorization' => "Bearer #{token}" }
+
+            expect(response)
+              .to have_http_status(200)
+
+            expect(response.content_type)
+              .to start_with('application/json')
+
+            expect(response.parsed_body)
+              .to be_present
+              .and(be_an(Array))
+              .and(have_attributes(size: 1))
+          end
+        end
       end
     end
 

spec/requests/api/v2/notifications_spec.rb

@@ -365,6 +365,18 @@ RSpec.describe 'Notifications' do
         .to start_with('application/json')
     end
 
+    context 'with an ungrouped notification' do
+      let(:notification) { Fabricate(:notification, account: user.account, type: :favourite) }
+
+      it 'returns http success' do
+        get "/api/v2/notifications/ungrouped-#{notification.id}", headers: headers
+
+        expect(response).to have_http_status(200)
+        expect(response.content_type)
+          .to start_with('application/json')
+      end
+    end
+
     context 'when notification belongs to someone else' do
       let(:notification) { Fabricate(:notification, group_key: 'foobar') }
 
@@ -396,6 +408,19 @@ RSpec.describe 'Notifications' do
       expect { notification.reload }.to raise_error(ActiveRecord::RecordNotFound)
     end
 
+    context 'with an ungrouped notification' do
+      let(:notification) { Fabricate(:notification, account: user.account, type: :favourite) }
+
+      it 'destroys the notification' do
+        post "/api/v2/notifications/ungrouped-#{notification.id}/dismiss", headers: headers
+
+        expect(response).to have_http_status(200)
+        expect(response.content_type)
+          .to start_with('application/json')
+        expect { notification.reload }.to raise_error(ActiveRecord::RecordNotFound)
+      end
+    end
+
     context 'when notification belongs to someone else' do
       let(:notification) { Fabricate(:notification, group_key: 'foobar') }
 

spec/requests/auth/setup_spec.rb

@@ -24,15 +24,4 @@ RSpec.describe 'Auth Setup' do
       end
     end
   end
-
-  describe 'PUT /auth/setup' do
-    before { sign_in Fabricate(:user, confirmed_at: nil) }
-
-    it 'gracefully handles invalid nested params' do
-      put '/auth/setup?user=invalid'
-
-      expect(response)
-        .to have_http_status(400)
-    end
-  end
 end

spec/services/activitypub/process_status_update_service_spec.rb

@@ -12,6 +12,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService do
     [
       { type: 'Hashtag', name: 'hoge' },
       { type: 'Mention', href: ActivityPub::TagManager.instance.uri_for(alice) },
+      { type: 'Mention', href: ActivityPub::TagManager.instance.uri_for(alice) },
       { type: 'Mention', href: bogus_mention },
     ]
   end

streaming/index.js

@@ -689,7 +689,7 @@ const startServer = async () => {
       // filtering of statuses:
 
       // Filter based on language:
-      if (Array.isArray(req.chosenLanguages) && payload.language !== null && req.chosenLanguages.indexOf(payload.language) === -1) {
+      if (Array.isArray(req.chosenLanguages) && req.chosenLanguages.indexOf(payload.language) === -1) {
         log.debug(`Message ${payload.id} filtered by language (${payload.language})`);
         return;
       }