Merge pull request #346 from kmycode/kb-draft-9.4

Release: 9.4

CHANGELOG.md

@@ -2,6 +2,33 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.2.2] - 2023-12-04
+
+### Changed
+
+- Change dismissed banners to be stored server-side ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27055))
+- Change GIF max matrix size error to explicitly mention GIF files ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27927))
+- Change `Follow` activities delivery to bypass availability check ([ShadowJonathan](https://github.com/mastodon/mastodon/pull/27586))
+- Change single-column navigation notice to be displayed outside of the logo container ([renchap](https://github.com/mastodon/mastodon/pull/27462), [renchap](https://github.com/mastodon/mastodon/pull/27476))
+- Change Content-Security-Policy to be tighter on media paths ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26889))
+- Change post language code to include country code when relevant ([gunchleoc](https://github.com/mastodon/mastodon/pull/27099), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/27207))
+
+### Fixed
+
+- Fix upper border radius of onboarding columns ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27890))
+- Fix incoming status creation date not being restricted to standard ISO8601 ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27655), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/28081))
+- Fix some posts from threads received out-of-order sometimes not being inserted into timelines ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27653))
+- Fix posts from force-sensitized accounts being able to trend ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27620))
+- Fix error when trying to delete already-deleted file with OpenStack Swift ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27569))
+- Fix batch attachment deletion when using OpenStack Swift ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27554))
+- Fix processing LDSigned activities from actors with unknown public keys ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27474))
+- Fix error and incorrect URLs in `/api/v1/accounts/:id/featured_tags` for remote accounts ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27459))
+- Fix report processing notice not mentioning the report number when performing a custom action ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27442))
+- Fix handling of `inLanguage` attribute in preview card processing ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27423))
+- Fix own posts being removed from home timeline when unfollowing a used hashtag ([kmycode](https://github.com/mastodon/mastodon/pull/27391))
+- Fix some link anchors being recognized as hashtags ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27271), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/27584))
+- Fix format-dependent redirects being cached regardless of requested format ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27634))
+
 ## [4.2.1] - 2023-10-10
 
 ### Added

Gemfile.lock

@@ -373,16 +373,16 @@ GEM
       rdoc
       reline (>= 0.3.8)
     jmespath (1.6.2)
-    json (2.6.3)
-    json-canonicalization (0.3.2)
+    json (2.7.0)
+    json-canonicalization (1.0.0)
     json-jwt (1.15.3)
       activesupport (>= 4.2)
       aes_key_wrap
       bindata
       httpclient
-    json-ld (3.3.0)
+    json-ld (3.3.1)
       htmlentities (~> 4.3)
-      json-canonicalization (~> 0.3, >= 0.3.2)
+      json-canonicalization (~> 1.0)
       link_header (~> 0.0, >= 0.0.8)
       multi_json (~> 1.15)
       rack (>= 2.2, < 4)

app/chewy/accounts_index.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class AccountsIndex < Chewy::Index
+  include DatetimeClampingConcern
+
   DEVELOPMENT_SETTINGS = {
     filter: {
       english_stop: {
@@ -153,7 +155,7 @@ class AccountsIndex < Chewy::Index
     field(:following_count, type: 'long', value: ->(account) { account.public_following_count })
     field(:followers_count, type: 'long', value: ->(account) { account.public_followers_count })
     field(:properties, type: 'keyword', value: ->(account) { account.searchable_properties })
-    field(:last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at })
+    field(:last_status_at, type: 'date', value: ->(account) { clamp_date(account.last_status_at || account.created_at) })
     field(:domain, type: 'keyword', value: ->(account) { account.domain || '' })
     field(:display_name, type: 'text', analyzer: 'verbatim') { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
     field(:username, type: 'text', analyzer: 'verbatim', value: ->(account) { [account.username, account.domain].compact.join('@') }) { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }

app/chewy/concerns/datetime_clamping_concern.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module DatetimeClampingConcern
+  extend ActiveSupport::Concern
+
+  MIN_ISO8601_DATETIME = '0000-01-01T00:00:00Z'.to_datetime.freeze
+  MAX_ISO8601_DATETIME = '9999-12-31T23:59:59Z'.to_datetime.freeze
+
+  class_methods do
+    def clamp_date(datetime)
+      datetime.clamp(MIN_ISO8601_DATETIME, MAX_ISO8601_DATETIME)
+    end
+  end
+end

app/chewy/public_statuses_index.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class PublicStatusesIndex < Chewy::Index
+  include DatetimeClampingConcern
+
   DEVELOPMENT_SETTINGS = {
     filter: {
       english_stop: {
@@ -154,6 +156,6 @@ class PublicStatusesIndex < Chewy::Index
     field(:language, type: 'keyword')
     field(:domain, type: 'keyword', value: ->(status) { status.account.domain || '' })
     field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties })
-    field(:created_at, type: 'date')
+    field(:created_at, type: 'date', value: ->(status) { clamp_date(status.created_at) })
   end
 end

app/chewy/statuses_index.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class StatusesIndex < Chewy::Index
+  include DatetimeClampingConcern
+
   DEVELOPMENT_SETTINGS = {
     filter: {
       english_stop: {
@@ -184,6 +186,6 @@ class StatusesIndex < Chewy::Index
     field(:language, type: 'keyword')
     field(:domain, type: 'keyword', value: ->(status) { status.account.domain || '' })
     field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties })
-    field(:created_at, type: 'date')
+    field(:created_at, type: 'date', value: ->(status) { clamp_date(status.created_at) })
   end
 end

app/chewy/tags_index.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class TagsIndex < Chewy::Index
+  include DatetimeClampingConcern
+
   settings index: index_preset(refresh_interval: '30s'), analysis: {
     analyzer: {
       content: {
@@ -42,6 +44,6 @@ class TagsIndex < Chewy::Index
     field(:name, type: 'text', analyzer: 'content', value: :display_name) { field(:edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content') }
     field(:reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? })
     field(:usage, type: 'long', value: ->(tag, crutches) { tag.history.aggregate(crutches.time_period).accounts })
-    field(:last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at })
+    field(:last_status_at, type: 'date', value: ->(tag) { clamp_date(tag.last_status_at || tag.created_at) })
   end
 end

app/controllers/concerns/cache_concern.rb

@@ -172,6 +172,16 @@ module CacheConcern
   def render_with_cache(**options)
     raise ArgumentError, 'Only JSON render calls are supported' unless options.key?(:json) || block_given?
 
+    if options.delete(:cancel_cache)
+      if block_given?
+        options[:json] = yield
+      elsif options[:json].is_a?(Symbol)
+        options[:json] = send(options[:json])
+      end
+
+      return render(options)
+    end
+
     key        = options.delete(:key) || [[params[:controller], params[:action]].join('/'), options[:json].respond_to?(:cache_key) ? options[:json].cache_key : nil, options[:fields].nil? ? nil : options[:fields].join(',')].compact.join(':')
     expires_in = options.delete(:expires_in) || 3.minutes
     body       = Rails.cache.read(key, raw: true)

app/controllers/statuses_controller.rb

@@ -29,15 +29,15 @@ class StatusesController < ApplicationController
       end
 
       format.json do
-        expires_in 3.minutes, public: true if @status.distributable? && public_fetch_mode?
-        render_with_cache json: @status, content_type: 'application/activity+json', serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter
+        expires_in 3.minutes, public: true if @status.distributable? && public_fetch_mode? && !misskey_software?
+        render_with_cache json: @status, content_type: 'application/activity+json', serializer: status_activity_serializer, adapter: ActivityPub::Adapter, cancel_cache: misskey_software?
       end
     end
   end
 
   def activity
-    expires_in 3.minutes, public: @status.distributable? && public_fetch_mode?
-    render_with_cache json: ActivityPub::ActivityPresenter.from_status(@status), content_type: 'application/activity+json', serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter
+    expires_in 3.minutes, public: @status.distributable? && public_fetch_mode? && !misskey_software?
+    render_with_cache json: ActivityPub::ActivityPresenter.from_status(@status, for_misskey: misskey_software?), content_type: 'application/activity+json', serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter, cancel_cache: misskey_software?
   end
 
   def embed
@@ -71,6 +71,29 @@ class StatusesController < ApplicationController
     not_found
   end
 
+  def misskey_software?
+    return @misskey_software if defined?(@misskey_software)
+
+    @misskey_software = false
+
+    return false if !@status.local? || signed_request_account&.domain.blank?
+
+    info = InstanceInfo.find_by(domain: signed_request_account.domain)
+    return false if info.nil?
+
+    @misskey_software = %w(misskey calckey cherrypick sharkey).include?(info.software) &&
+                        ((@status.public_unlisted_visibility? && @status.account.user&.setting_reject_public_unlisted_subscription) ||
+                         (@status.unlisted_visibility? && @status.account.user&.setting_reject_unlisted_subscription))
+  end
+
+  def status_activity_serializer
+    if misskey_software?
+      ActivityPub::NoteForMisskeySerializer
+    else
+      ActivityPub::NoteSerializer
+    end
+  end
+
   def redirect_to_original
     redirect_to(ActivityPub::TagManager.instance.url_for(@status.reblog), allow_other_host: true) if @status.reblog?
   end

app/javascript/mastodon/components/dismissable_banner.tsx

@@ -1,11 +1,18 @@
+/* eslint-disable @typescript-eslint/no-unsafe-call,
+                  @typescript-eslint/no-unsafe-return,
+                  @typescript-eslint/no-unsafe-assignment,
+                  @typescript-eslint/no-unsafe-member-access
+                  -- the settings store is not yet typed */
 import type { PropsWithChildren } from 'react';
-import { useCallback, useState } from 'react';
+import { useCallback, useState, useEffect } from 'react';
 
 import { defineMessages, useIntl } from 'react-intl';
 
 import { ReactComponent as CloseIcon } from '@material-symbols/svg-600/outlined/close.svg';
 
+import { changeSetting } from 'mastodon/actions/settings';
 import { bannerSettings } from 'mastodon/settings';
+import { useAppSelector, useAppDispatch } from 'mastodon/store';
 
 import { IconButton } from './icon_button';
 
@@ -21,13 +28,25 @@ export const DismissableBanner: React.FC<PropsWithChildren<Props>> = ({
   id,
   children,
 }) => {
-  const [visible, setVisible] = useState(!bannerSettings.get(id));
+  const dismissed = useAppSelector((state) =>
+    state.settings.getIn(['dismissed_banners', id], false),
+  );
+  const dispatch = useAppDispatch();
+
+  const [visible, setVisible] = useState(!bannerSettings.get(id) && !dismissed);
   const intl = useIntl();
 
   const handleDismiss = useCallback(() => {
     setVisible(false);
     bannerSettings.set(id, true);
-  }, [id]);
+    dispatch(changeSetting(['dismissed_banners', id], true));
+  }, [id, dispatch]);
+
+  useEffect(() => {
+    if (!visible && !dismissed) {
+      dispatch(changeSetting(['dismissed_banners', id], true));
+    }
+  }, [id, dispatch, visible, dismissed]);
 
   if (!visible) {
     return null;

app/javascript/mastodon/reducers/notifications.js

@@ -284,7 +284,7 @@ export default function notifications(state = initialState, action) {
   case blockAccountSuccess.type:
     return filterNotifications(state, [action.payload.relationship.id]);
   case muteAccountSuccess.type:
-    return action.relationship.muting_notifications ? filterNotifications(state, [action.payload.relationship.id]) : state;
+    return action.payload.relationship.muting_notifications ? filterNotifications(state, [action.payload.relationship.id]) : state;
   case blockDomainSuccess.type:
     return filterNotifications(state, action.payload.accounts);
   case authorizeFollowRequestSuccess.type:

app/javascript/mastodon/reducers/settings.js

@@ -113,6 +113,15 @@ const initialState = ImmutableMap({
       body: '',
     }),
   }),
+
+  dismissed_banners: ImmutableMap({
+    'public_timeline': false,
+    'community_timeline': false,
+    'home.explore_prompt': false,
+    'explore/links': false,
+    'explore/statuses': false,
+    'explore/tags': false,
+  }),
 });
 
 const defaultColumns = fromJS([

app/javascript/styles/mastodon/components.scss

@@ -2371,8 +2371,7 @@ $ui-header-height: 55px;
 
   > .scrollable {
     background: $ui-base-color;
-    border-bottom-left-radius: 4px;
-    border-bottom-right-radius: 4px;
+    border-radius: 0 0 4px 4px;
   }
 }
 

app/lib/activitypub/activity/create.rb

@@ -505,7 +505,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     url = quote
 
     if url.present?
-      ResolveURLService.new.call(url, on_behalf_of: @account, local_only: true).present?
+      ActivityPub::TagManager.instance.uri_to_resource(url, Status)&.local?
     else
       false
     end

app/lib/activitypub/activity/follow.rb

@@ -103,7 +103,7 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity
     info = instance_info
     return false if info.nil?
 
-    %w(misskey calckey firefish meisskey cherrypick).include?(info.software)
+    %w(misskey calckey firefish meisskey cherrypick sharkey).include?(info.software)
   end
 
   def instance_info

app/lib/activitypub/tag_manager.rb

@@ -208,7 +208,7 @@ class ActivityPub::TagManager
     uri_to_resource(uri, Account)
   end
 
-  def uri_to_resource(uri, klass)
+  def uri_to_resource(uri, klass, url: false)
     return if uri.nil?
 
     if local_uri?(uri)
@@ -221,7 +221,9 @@ class ActivityPub::TagManager
     elsif OStatus::TagManager.instance.local_id?(uri)
       klass.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, klass.to_s))
     else
-      klass.find_by(uri: uri.split('#').first)
+      resource   = klass.find_by(uri: uri.split('#').first)
+      resource ||= klass.where('uri != url').find_by(url: uri.split('#').first) if url
+      resource
     end
   rescue ActiveRecord::RecordNotFound
     nil

app/lib/status_reach_finder.rb

@@ -216,7 +216,7 @@ class StatusReachFinder
     return [] if status.public_searchability?
     return [] unless (status.public_unlisted_visibility? && status.account.user&.setting_reject_public_unlisted_subscription) || (status.unlisted_visibility? && status.account.user&.setting_reject_unlisted_subscription)
 
-    from_info = InstanceInfo.where(software: %w(misskey calckey cherrypick)).pluck(:domain)
+    from_info = InstanceInfo.where(software: %w(misskey calckey cherrypick sharkey)).pluck(:domain)
     from_domain_block = DomainBlock.where(detect_invalid_subscription: true).pluck(:domain)
     (from_info + from_domain_block).uniq
   end

app/models/concerns/attachmentable.rb

@@ -52,9 +52,13 @@ module Attachmentable
     return if attachment.blank? || !/image.*/.match?(attachment.content_type) || attachment.queued_for_write[:original].blank?
 
     width, height = FastImage.size(attachment.queued_for_write[:original].path)
-    matrix_limit  = attachment.content_type == 'image/gif' ? GIF_MATRIX_LIMIT : MAX_MATRIX_LIMIT
+    return unless width.present? && height.present?
 
-    raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported" if width.present? && height.present? && (width * height > matrix_limit)
+    if attachment.content_type == 'image/gif' && width * height > GIF_MATRIX_LIMIT
+      raise Mastodon::DimensionsValidationError, "#{width}x#{height} GIF files are not supported"
+    elsif width * height > MAX_MATRIX_LIMIT
+      raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported"
+    end
   end
 
   def appropriate_extension(attachment)

app/models/instance_info.rb

@@ -21,6 +21,7 @@ class InstanceInfo < ApplicationRecord
     calckey
     cherrypick
     meisskey
+    sharkey
     firefish
     renedon
     fedibird
@@ -39,6 +40,7 @@ class InstanceInfo < ApplicationRecord
     return false if info.nil?
 
     return true if EMOJI_REACTION_AVAILABLE_SOFTWARES.include?(info['software'])
+    return false if info.data['metadata'].nil? || !info.data['metadata'].is_a?(Hash)
 
     features = info.data.dig('metadata', 'features')
     return false if features.nil? || !features.is_a?(Array)

app/models/status.rb

@@ -83,7 +83,7 @@ class Status < ApplicationRecord
   has_many :mentions, dependent: :destroy, inverse_of: :status
   has_many :mentioned_accounts, through: :mentions, source: :account, class_name: 'Account'
   has_many :active_mentions, -> { active }, class_name: 'Mention', inverse_of: :status
-  has_many :media_attachments, dependent: :nullify
+  has_many :media_attachments, -> { order('id asc') }, dependent: :nullify, inverse_of: false
   has_many :reference_objects, class_name: 'StatusReference', inverse_of: :status, dependent: :destroy
   has_many :references, through: :reference_objects, class_name: 'Status', source: :target_status
   has_many :referenced_by_status_objects, foreign_key: 'target_status_id', class_name: 'StatusReference', inverse_of: :target_status, dependent: :destroy
@@ -405,6 +405,8 @@ class Status < ApplicationRecord
         end
 
         public_emoji_reactions
+      else
+        emoji_reactions
       end
     end
   end

app/services/activitypub/process_account_service.rb

@@ -46,7 +46,6 @@ class ActivityPub::ProcessAccountService < BaseService
         end
 
         create_account
-        fetch_instance_info
       end
 
       update_account
@@ -66,6 +65,8 @@ class ActivityPub::ProcessAccountService < BaseService
       check_links! if @account.fields.any?(&:requires_verification?)
     end
 
+    fetch_instance_info
+
     @account
   rescue Oj::ParseError
     nil
@@ -210,7 +211,7 @@ class ActivityPub::ProcessAccountService < BaseService
   end
 
   def fetch_instance_info
-    ActivityPub::FetchInstanceInfoWorker.perform_async(@account.domain) unless InstanceInfo.exists?(domain: @account.domain)
+    ActivityPub::FetchInstanceInfoWorker.perform_async(@account.domain) unless Rails.cache.exist?("fetch_instance_info:#{@account.domain}", expires_in: 1.day)
   end
 
   def actor_type

app/services/process_references_service.rb

@@ -108,7 +108,16 @@ class ProcessReferencesService < BaseService
   end
 
   def url_to_status(url)
-    ResolveURLService.new.call(url, on_behalf_of: @status.account, fetch_remote: @fetch_remote && @no_fetch_urls.exclude?(url))
+    status   = ActivityPub::TagManager.instance.uri_to_resource(url, Status, url: true)
+    status ||= ResolveURLService.new.call(url, on_behalf_of: @status.account) if @fetch_remote && @no_fetch_urls.exclude?(url)
+    referrable?(status) ? status : nil
+  end
+
+  def referrable?(target_status)
+    return false if target_status.nil?
+    return @referrable if defined?(@referrable)
+
+    @referrable = StatusPolicy.new(@status.account, target_status).show?
   end
 
   def quote_status_ids
@@ -116,7 +125,7 @@ class ProcessReferencesService < BaseService
   end
 
   def quotable?(target_status)
-    target_status.account.allow_quote? && (!@status.local? || StatusPolicy.new(@status.account, target_status).quote?)
+    target_status.account.allow_quote? && StatusPolicy.new(@status.account, target_status).quote?
   end
 
   def add_references

app/services/resolve_url_service.rb

@@ -6,16 +6,15 @@ class ResolveURLService < BaseService
 
   USERNAME_STATUS_RE = %r{/@(?<username>#{Account::USERNAME_RE})/(?<status_id>[0-9]+)\Z}
 
-  def call(url, on_behalf_of: nil, fetch_remote: true, local_only: false)
+  def call(url, on_behalf_of: nil)
     @url          = url
     @on_behalf_of = on_behalf_of
-    @fetch_remote = fetch_remote
 
     if local_url?
       process_local_url
-    elsif !local_only && fetch_remote && !fetched_resource.nil?
+    elsif !fetched_resource.nil?
       process_url
-    elsif !local_only
+    else
       process_url_from_db
     end
   end
@@ -38,7 +37,7 @@ class ResolveURLService < BaseService
       return account unless account.nil?
     end
 
-    return unless @on_behalf_of.present? && (!@fetch_remote || [401, 403, 404].include?(fetch_resource_service.response_code))
+    return unless @on_behalf_of.present? && [401, 403, 404].include?(fetch_resource_service.response_code)
 
     # It may happen that the resource is a private toot, and thus not fetchable,
     # but we can return the toot if we already know about it.

app/services/un_emoji_react_service.rb

@@ -42,8 +42,8 @@ class UnEmojiReactService < BaseService
   end
 
   def write_stream(emoji_reaction)
-    emoji_group = @status.emoji_reactions_grouped_by_name
-                         .find { |reaction_group| reaction_group['name'] == emoji_reaction.name && (!reaction_group.key?(:domain) || reaction_group['domain'] == emoji_reaction.custom_emoji&.domain) }
+    emoji_group = @status.emoji_reactions_grouped_by_name(@account)
+                         .find { |reaction_group| reaction_group['name'] == emoji_reaction.name }
     if emoji_group
       emoji_group['status_id'] = @status.id.to_s
     else
@@ -51,6 +51,7 @@ class UnEmojiReactService < BaseService
       emoji_group = { 'name' => emoji_reaction.name, 'count' => 0, 'account_ids' => [], 'status_id' => @status.id.to_s }
       emoji_group['domain'] = emoji_reaction.custom_emoji.domain if emoji_reaction.custom_emoji
     end
+
     DeliveryEmojiReactionWorker.perform_async(render_emoji_reaction(emoji_group), @status.id, emoji_reaction.account_id)
   end
 

app/views/admin/custom_emojis/edit.html.haml

@@ -24,7 +24,7 @@
     %h4= t('admin.custom_emojis.edit.label')
 
     .fields-group
-      = f.select :category_id, options_from_collection_for_select(CustomEmojiCategory.all, 'id', 'name'), prompt: t('admin.custom_emojis.assign_category'), class: 'select optional', 'aria-label': t('admin.custom_emojis.assign_category')
+      = f.input :category_id, collection: CustomEmojiCategory.all, label_method: ->(item) { item.name }, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li', include_blank: t('admin.custom_emojis.assign_category'), wrapper: :with_label, label: false, hint: false
 
     .fields-group
       = f.input :visible_in_picker, as: :boolean, wrapper: :with_label, label: t('admin.custom_emojis.visible_in_picker')

app/views/admin/custom_emojis/new.html.haml

@@ -11,7 +11,7 @@
     = f.input :image, wrapper: :with_label, input_html: { accept: CustomEmoji::IMAGE_MIME_TYPES.join(' ') }, hint: t('admin.custom_emojis.image_hint', size: number_to_human_size(CustomEmoji::LIMIT))
 
   .fields-group
-    = f.select :category_id, options_from_collection_for_select(CustomEmojiCategory.all, 'id', 'name'), prompt: t('admin.custom_emojis.assign_category'), class: 'select optional', 'aria-label': t('admin.custom_emojis.assign_category')
+    = f.input :category_id, collection: CustomEmojiCategory.all, label_method: ->(item) { item.name }, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li', include_blank: t('admin.custom_emojis.assign_category'), wrapper: :with_label, label: false, hint: false
 
   .fields-group
     = f.input :visible_in_picker, as: :boolean, wrapper: :with_label, label: t('admin.custom_emojis.visible_in_picker')

app/workers/activitypub/fetch_instance_info_worker.rb

@@ -8,28 +8,32 @@ class ActivityPub::FetchInstanceInfoWorker
 
   sidekiq_options queue: 'push', retry: 2
 
-  class Error < StandardError; end
-  class RequestError < Error; end
-  class DeadError < Error; end
-
   SUPPORTED_NOTEINFO_RELS = ['http://nodeinfo.diaspora.software/ns/schema/2.0', 'http://nodeinfo.diaspora.software/ns/schema/2.1'].freeze
 
   def perform(domain)
     @instance = Instance.find_by(domain: domain)
     return if !@instance || @instance.unavailable_domain.present?
 
-    with_redis_lock("instance_info:#{domain}") do
-      link = nodeinfo_link
-      return if link.nil?
-
-      update_info!(link)
+    Rails.cache.fetch("fetch_instance_info:#{@instance.domain}", expires_in: 1.day, race_condition_ttl: 1.hour) do
+      fetch!
     end
-  rescue ActivityPub::FetchInstanceInfoWorker::DeadError
+
     true
   end
 
   private
 
+  def fetch!
+    link = nodeinfo_link
+    return if link.nil?
+
+    update_info!(link)
+
+    true
+  rescue Mastodon::UnexpectedResponseError
+    true
+  end
+
   def nodeinfo_link
     nodeinfo = fetch_json("https://#{@instance.domain}/.well-known/nodeinfo")
     return nil if nodeinfo.nil? || !nodeinfo.key?('links')
@@ -63,15 +67,9 @@ class ActivityPub::FetchInstanceInfoWorker
 
   def fetch_json(url)
     build_request(url).perform do |response|
-      if [200, 203].include?(response.code)
       raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response)
 
       body_to_json(response.body_with_limit)
-      elsif [400, 401, 403, 404, 410].include?(response.code)
-        raise ActivityPub::FetchInstanceInfoWorker::DeadError, "Request for #{@instance.domain} returned HTTP #{response.code}"
-      else
-        raise ActivityPub::FetchInstanceInfoWorker::RequestError, "Request for #{@instance.domain} returned HTTP #{response.code}"
-      end
     end
   end
 

app/workers/scheduler/update_instance_info_scheduler.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-class Scheduler::UpdateInstanceInfoScheduler
-  include Sidekiq::Worker
-
-  sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
-
-  def perform
-    Instance.select(:domain).reorder(nil).find_in_batches do |instances|
-      ActivityPub::FetchInstanceInfoWorker.push_bulk(instances) do |instance|
-        [instance.domain]
-      end
-    end
-  end
-end

config/sidekiq.yml

@@ -63,10 +63,6 @@
       interval: 30 seconds
       class: Scheduler::SidekiqHealthScheduler
       queue: scheduler
-    update_instance_info_scheduler:
-      cron: '0 0 * * *'
-      class: Scheduler::UpdateInstanceInfoScheduler
-      queue: scheduler
     software_update_check_scheduler:
       interval: 30 minutes
       class: Scheduler::SoftwareUpdateCheckScheduler

db/migrate/20231105225839_add_master_settings_to_accounts.rb

@@ -13,18 +13,8 @@ class AddMasterSettingsToAccounts < ActiveRecord::Migration[7.1]
     safety_assured do
       add_column :accounts, :master_settings, :jsonb
 
-      if Rails.env.test?
-        Account.transaction do
-          Account.find_in_batches do |accounts|
-            accounts.each do |account|
-              account.update(master_settings: { 'subscription_policy' => account.dissubscribable ? 'block' : 'allow' })
-            end
-          end
-        end
-      else
-        Account.where(dissubscribable: true).update_all(master_settings: { 'subscription_policy' => 'block' }) # rubocop:disable Rails/SkipsModelValidations
-        Account.where(dissubscribable: false).update_all(master_settings: { 'subscription_policy' => 'allow' }) # rubocop:disable Rails/SkipsModelValidations
-      end
+      ActiveRecord::Base.connection.execute("UPDATE accounts SET master_settings = json_build_object('subscription_policy', 'block') WHERE accounts.dissubscribable IS TRUE")
+      ActiveRecord::Base.connection.execute("UPDATE accounts SET master_settings = json_build_object('subscription_policy', 'allow') WHERE accounts.dissubscribable IS FALSE")
 
       remove_column :accounts, :dissubscribable
     end
@@ -34,18 +24,8 @@ class AddMasterSettingsToAccounts < ActiveRecord::Migration[7.1]
     safety_assured do
       add_column_with_default :accounts, :dissubscribable, :boolean, default: false, allow_null: false
 
-      if Rails.env.test?
-        Account.transaction do
-          Account.find_in_batches do |accounts|
-            accounts.each do |account|
-              account.update(dissubscribable: account.master_settings.present? && account.master_settings['subscription_policy'] != 'allow')
-            end
-          end
-        end
-      else
-        Account.where(master_settings: { subscription_policy: 'block' }).update_all(dissubscribable: true) # rubocop:disable Rails/SkipsModelValidations
-        Account.where(master_settings: { subscription_policy: 'allow' }).update_all(dissubscribable: false) # rubocop:disable Rails/SkipsModelValidations
-      end
+      ActiveRecord::Base.connection.execute("UPDATE accounts SET dissubscribable = TRUE WHERE master_settings ->> 'subscription_policy' = 'block'")
+      ActiveRecord::Base.connection.execute("UPDATE accounts SET dissubscribable = FALSE WHERE master_settings ->> 'subscription_policy' = 'allow'")
 
       remove_column :accounts, :master_settings
     end

db/migrate/20231212225737_improve_index_for_public_timeline_speed.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class ImproveIndexForPublicTimelineSpeed < ActiveRecord::Migration[7.0]
+  disable_ddl_transaction!
+
+  def up
+    add_index :statuses, [:id, :account_id], name: :index_statuses_local_20231213, algorithm: :concurrently, order: { id: :desc }, where: '(local OR (uri IS NULL)) AND deleted_at IS NULL AND visibility IN (0, 10, 11) AND reblog_of_id IS NULL AND ((NOT reply) OR (in_reply_to_account_id = account_id))'
+    add_index :statuses, [:id, :account_id], name: :index_statuses_public_20231213, algorithm: :concurrently, order: { id: :desc }, where: 'deleted_at IS NULL AND visibility IN (0, 10, 11) AND reblog_of_id IS NULL AND ((NOT reply) OR (in_reply_to_account_id = account_id))'
+    remove_index :statuses, name: :index_statuses_local_20190824
+    remove_index :statuses, name: :index_statuses_public_20200119
+  end
+
+  def down
+    add_index :statuses, [:id, :account_id], name: :index_statuses_local_20190824, algorithm: :concurrently, order: { id: :desc }, where: '(local OR (uri IS NULL)) AND deleted_at IS NULL AND visibility = 0 AND reblog_of_id IS NULL AND ((NOT reply) OR (in_reply_to_account_id = account_id))'
+    add_index :statuses, [:id, :account_id], name: :index_statuses_public_20200119, algorithm: :concurrently, order: { id: :desc }, where: 'deleted_at IS NULL AND visibility = 0 AND reblog_of_id IS NULL AND ((NOT reply) OR (in_reply_to_account_id = account_id))'
+    remove_index :statuses, name: :index_statuses_local_20231213
+    remove_index :statuses, name: :index_statuses_public_20231213
+  end
+end

db/migrate/20231214225249_index_to_statuses_url.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class IndexToStatusesURL < ActiveRecord::Migration[7.1]
+  disable_ddl_transaction!
+
+  def up
+    add_index :statuses, :url, name: :index_statuses_on_url, algorithm: :concurrently, opclass: :text_pattern_ops, where: 'url IS NOT NULL AND url <> uri'
+  end
+
+  def down
+    remove_index :statuses, name: :index_statuses_on_url
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.1].define(version: 2023_11_05_225839) do
+ActiveRecord::Schema[7.1].define(version: 2023_12_14_225249) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -1235,13 +1235,14 @@ ActiveRecord::Schema[7.1].define(version: 2023_11_05_225839) do
     t.index ["account_id", "reblog_of_id", "deleted_at", "searchability"], name: "index_statuses_for_get_following_accounts_to_search", where: "((deleted_at IS NULL) AND (reblog_of_id IS NULL) AND (searchability = ANY (ARRAY[0, 10, 1])))"
     t.index ["account_id"], name: "index_statuses_on_account_id"
     t.index ["deleted_at"], name: "index_statuses_on_deleted_at", where: "(deleted_at IS NOT NULL)"
-    t.index ["id", "account_id"], name: "index_statuses_local_20190824", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
-    t.index ["id", "account_id"], name: "index_statuses_public_20200119", order: { id: :desc }, where: "((deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
+    t.index ["id", "account_id"], name: "index_statuses_local_20231213", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = ANY (ARRAY[0, 10, 11])) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
+    t.index ["id", "account_id"], name: "index_statuses_public_20231213", order: { id: :desc }, where: "((deleted_at IS NULL) AND (visibility = ANY (ARRAY[0, 10, 11])) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
     t.index ["in_reply_to_account_id"], name: "index_statuses_on_in_reply_to_account_id", where: "(in_reply_to_account_id IS NOT NULL)"
     t.index ["in_reply_to_id"], name: "index_statuses_on_in_reply_to_id", where: "(in_reply_to_id IS NOT NULL)"
     t.index ["quote_of_id", "account_id"], name: "index_statuses_on_quote_of_id_and_account_id"
     t.index ["reblog_of_id", "account_id"], name: "index_statuses_on_reblog_of_id_and_account_id"
     t.index ["uri"], name: "index_statuses_on_uri", unique: true, opclass: :text_pattern_ops, where: "(uri IS NOT NULL)"
+    t.index ["url"], name: "index_statuses_on_url", opclass: :text_pattern_ops, where: "((url IS NOT NULL) AND ((url)::text <> (uri)::text))"
   end
 
   create_table "statuses_tags", primary_key: ["tag_id", "status_id"], force: :cascade do |t|

docker-compose.yml

@@ -56,7 +56,7 @@ services:
 
   web:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.1
+    image: ghcr.io/mastodon/mastodon:v4.2.2
     restart: always
     env_file: .env.production
     command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@@ -77,7 +77,7 @@ services:
 
   streaming:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.1
+    image: ghcr.io/mastodon/mastodon:v4.2.2
     restart: always
     env_file: .env.production
     command: node ./streaming
@@ -95,7 +95,7 @@ services:
 
   sidekiq:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.1
+    image: ghcr.io/mastodon/mastodon:v4.2.2
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq

lib/mastodon/version.rb

@@ -9,7 +9,7 @@ module Mastodon
     end
 
     def kmyblue_minor
-      0
+      4
     end
 
     def kmyblue_flag

spec/lib/activitypub/activity/update_spec.rb

@@ -55,6 +55,7 @@ RSpec.describe ActivityPub::Activity::Update do
         stub_request(:get, actor_json[:following]).to_return(status: 404)
         stub_request(:get, actor_json[:featured]).to_return(status: 404)
         stub_request(:get, actor_json[:featuredTags]).to_return(status: 404)
+        stub_request(:get, 'https://example.com/.well-known/nodeinfo').to_return(status: 404)
 
         subject.perform
       end

spec/models/status_spec.rb

@@ -463,7 +463,7 @@ RSpec.describe Status do
   describe '.emoji_reaction_availables_map' do
     subject { described_class.emoji_reaction_availables_map(domains) }
 
-    let(:domains) { %w(features_available.com features_unavailable.com features_invalid.com features_nil.com no_info.com mastodon.com misskey.com) }
+    let(:domains) { %w(features_available.com features_unavailable.com features_invalid.com features_nil.com no_info.com mastodon.com misskey.com old_mastodon.com) }
 
     before do
       Fabricate(:instance_info, domain: 'features_available.com', software: 'mastodon', data: { metadata: { features: ['emoji_reaction'] } })
@@ -472,6 +472,7 @@ RSpec.describe Status do
       Fabricate(:instance_info, domain: 'features_nil.com', software: 'mastodon', data: { metadata: { features: nil } })
       Fabricate(:instance_info, domain: 'mastodon.com', software: 'mastodon')
       Fabricate(:instance_info, domain: 'misskey.com', software: 'misskey')
+      Fabricate(:instance_info, domain: 'old_mastodon.com', software: 'mastodon', data: { metadata: [] })
     end
 
     it 'availables if features contains emoji_reaction' do
@@ -497,6 +498,10 @@ RSpec.describe Status do
     it 'availables if misskey server' do
       expect(subject['misskey.com']).to be true
     end
+
+    it 'unavailables if old mastodon server' do
+      expect(subject['old_mastodon.com']).to be false
+    end
   end
 
   describe '.tagged_with' do

spec/services/activitypub/process_account_service_spec.rb

@@ -5,6 +5,10 @@ require 'rails_helper'
 RSpec.describe ActivityPub::ProcessAccountService, type: :service do
   subject { described_class.new }
 
+  before do
+    stub_request(:get, 'https://example.com/.well-known/nodeinfo').to_return(status: 404)
+  end
+
   context 'with searchability' do
     subject { described_class.new.call('alice', 'example.com', payload) }
 
@@ -252,6 +256,32 @@ RSpec.describe ActivityPub::ProcessAccountService, type: :service do
     end
   end
 
+  context 'with other settings' do
+    let(:payload) do
+      {
+        id: 'https://foo.test',
+        type: 'Actor',
+        inbox: 'https://foo.test/inbox',
+        otherSetting: [
+          { type: 'PropertyValue', name: 'Pronouns', value: 'They/them' },
+          { type: 'PropertyValue', name: 'Occupation', value: 'Unit test' },
+        ],
+      }.with_indifferent_access
+    end
+
+    before do
+      stub_request(:get, 'https://example.com/.well-known/nodeinfo').to_return(body: '{}')
+    end
+
+    it 'parses out of attachment' do
+      account = subject.call('alice', 'example.com', payload)
+      expect(account.settings).to be_a Hash
+      expect(account.settings.size).to eq 2
+      expect(account.settings['Pronouns']).to eq 'They/them'
+      expect(account.settings['Occupation']).to eq 'Unit test'
+    end
+  end
+
   context 'when account is not suspended' do
     subject { described_class.new.call('alice', 'example.com', payload) }
 

spec/services/process_references_service_spec.rb

@@ -241,6 +241,76 @@ RSpec.describe ProcessReferencesService, type: :service do
         end
       end
     end
+
+    context 'when already fetched remote post' do
+      let(:account) { Fabricate(:account, followers_url: 'http://example.com/followers', domain: 'example.com', uri: 'https://example.com/actor') }
+      let!(:remote_status) { Fabricate(:status, account: account, uri: 'https://example.com/test_post', url: 'https://example.com/test_post', text: 'Lorem ipsum') }
+      let(:object_json) do
+        {
+          id: 'https://example.com/test_post',
+          to: ActivityPub::TagManager::COLLECTIONS[:public],
+          '@context': ActivityPub::TagManager::CONTEXT,
+          type: 'Note',
+          actor: account.uri,
+          attributedTo: account.uri,
+          content: 'Lorem ipsum',
+        }
+      end
+      let(:text) { 'BT:https://example.com/test_post' }
+
+      shared_examples 'reference once' do |uri, url|
+        it 'reference it' do
+          expect(subject.size).to eq 1
+          expect(subject[0][1]).to eq 'BT'
+
+          status = Status.find_by(id: subject[0][0])
+          expect(status).to_not be_nil
+          expect(status.id).to eq remote_status.id
+          expect(status.uri).to eq uri
+          expect(status.url).to eq url
+        end
+      end
+
+      before do
+        stub_request(:get, 'https://example.com/test_post').to_return(status: 200, body: Oj.dump(object_json), headers: { 'Content-Type' => 'application/activity+json' })
+      end
+
+      it_behaves_like 'reference once', 'https://example.com/test_post', 'https://example.com/test_post'
+
+      context 'when uri and url is difference and url is not accessable' do
+        let(:remote_status) { Fabricate(:status, account: account, uri: 'https://example.com/test_post', url: 'https://example.com/test_post_ohagi', text: 'Lorem ipsum') }
+        let(:text) { 'BT:https://example.com/test_post_ohagi' }
+        let(:object_json) do
+          {
+            id: 'https://example.com/test_post',
+            url: 'https://example.com/test_post_ohagi',
+            to: ActivityPub::TagManager::COLLECTIONS[:public],
+            '@context': ActivityPub::TagManager::CONTEXT,
+            type: 'Note',
+            actor: account.uri,
+            attributedTo: account.uri,
+            content: 'Lorem ipsum',
+          }
+        end
+
+        before do
+          stub_request(:get, 'https://example.com/test_post_ohagi').to_return(status: 404)
+        end
+
+        it_behaves_like 'reference once', 'https://example.com/test_post', 'https://example.com/test_post_ohagi'
+
+        it 'do not request to uri' do
+          subject
+          expect(a_request(:get, 'https://example.com/test_post_ohagi')).to_not have_been_made
+        end
+
+        context 'when url and uri is specified at the same time' do
+          let(:text) { 'BT:https://example.com/test_post_ohagi BT:https://example.com/test_post' }
+
+          it_behaves_like 'reference once', 'https://example.com/test_post', 'https://example.com/test_post_ohagi'
+        end
+      end
+    end
   end
 
   describe 'editing new status' do

spec/workers/activitypub/fetch_instance_info_worker_spec.rb

@@ -67,9 +67,22 @@ describe ActivityPub::FetchInstanceInfoWorker do
       Instance.refresh
     end
 
+    it 'does not update immediately' do
+      stub_request(:get, 'https://example.com/nodeinfo/2.0').to_return(status: 200, body: nodeinfo_json)
+      subject.perform('example.com')
+      stub_request(:get, 'https://example.com/nodeinfo/2.0').to_return(status: 200, body: new_nodeinfo_json)
+      subject.perform('example.com')
+
+      info = InstanceInfo.find_by(domain: 'example.com')
+      expect(info).to_not be_nil
+      expect(info.software).to eq 'mastodon'
+      expect(info.version).to eq '4.2.0-beta1'
+    end
+
     it 'performs a mastodon instance' do
       stub_request(:get, 'https://example.com/nodeinfo/2.0').to_return(status: 200, body: nodeinfo_json)
       subject.perform('example.com')
+      Rails.cache.delete('fetch_instance_info:example.com')
       stub_request(:get, 'https://example.com/nodeinfo/2.0').to_return(status: 200, body: new_nodeinfo_json)
       subject.perform('example.com')
 
@@ -93,5 +106,12 @@ describe ActivityPub::FetchInstanceInfoWorker do
       info = InstanceInfo.find_by(domain: 'example.com')
       expect(info).to be_nil
     end
+
+    it 'does not fetch again immediately' do
+      expect(subject.perform('example.com')).to be true
+      expect(subject.perform('example.com')).to be true
+
+      expect(a_request(:get, 'https://example.com/.well-known/nodeinfo')).to have_been_made.once
+    end
   end
 end

spec/workers/scheduler/update_instance_info_scheduler_spec.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe Scheduler::UpdateInstanceInfoScheduler do
-  let(:worker) { described_class.new }
-
-  before do
-    stub_request(:get, 'https://example.com/.well-known/nodeinfo').to_return(status: 200, body: '{}')
-    Fabricate(:account, domain: 'example.com')
-    Instance.refresh
-  end
-
-  describe 'perform' do
-    it 'runs without error' do
-      expect { worker.perform }.to_not raise_error
-    end
-  end
-end