Merge commit 'eaa1f9e450' into kb_migration

CHANGELOG.md

@@ -2,6 +2,54 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.1.3] - 2023-07-06
+
+### Added
+
+- Add fallback redirection when getting a webfinger query `LOCAL_DOMAIN@LOCAL_DOMAIN` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23600))
+
+### Changed
+
+- Change OpenGraph-based embeds to allow fullscreen ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25058))
+- Change AccessTokensVacuum to also delete expired tokens ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24868))
+- Change profile updates to be sent to recently-mentioned servers ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24852))
+- Change automatic post deletion thresholds and load detection ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24614))
+- Change `/api/v1/statuses/:id/history` to always return at least one item ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25510))
+- Change auto-linking to allow carets in URL query params ([renchap](https://github.com/mastodon/mastodon/pull/25216))
+
+### Removed
+
+- Remove invalid `X-Frame-Options: ALLOWALL` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25070))
+
+### Fixed
+
+- Fix wrong view being displayed when a webhook fails validation ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25464))
+- Fix soft-deleted post cleanup scheduler overwhelming the streaming server ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25519))
+- Fix incorrect pagination headers in `/api/v2/admin/accounts` ([danielmbrasil](https://github.com/mastodon/mastodon/pull/25477))
+- Fix multiple inefficiencies in automatic post cleanup worker ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24607), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24785), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24840))
+- Fix performance of streaming by parsing message JSON once ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25278), [ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25361))
+- Fix CSP headers when `S3_ALIAS_HOST` includes a path component ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25273))
+- Fix `tootctl accounts approve --number N` not aproving N earliest registrations ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24605))
+- Fix reports not being closed when performing batch suspensions ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24988))
+- Fix being able to vote on your own polls ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25015))
+- Fix race condition when reblogging a status ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25016))
+- Fix “Authorized applications” inefficiently and incorrectly getting last use date ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25060))
+- Fix “Authorized applications” crashing when listing apps with certain admin API scopes ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25713))
+- Fix multiple N+1s in ConversationsController ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25134), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25399), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25499))
+- Fix user archive takeouts when using OpenStack Swift ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24431))
+- Fix searching for remote content by URL not working under certain conditions ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25637))
+- Fix inefficiencies in indexing content for search ([VyrCossont](https://github.com/mastodon/mastodon/pull/24285), [VyrCossont](https://github.com/mastodon/mastodon/pull/24342))
+
+### Security
+
+- Add finer permission requirements for managing webhooks ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25463))
+- Update dependencies
+- Add hardening headers for user-uploaded files ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25756))
+- Fix verified links possibly hiding important parts of the URL (CVE-2023-36462)
+- Fix timeout handling of outbound HTTP requests (CVE-2023-36461)
+- Fix arbitrary file creation through media processing (CVE-2023-36460)
+- Fix possible XSS in preview cards (CVE-2023-36459)
+
 ## [4.1.2] - 2023-04-04
 
 ### Fixed

app/controllers/admin/reports/actions_controller.rb

@@ -12,7 +12,7 @@ class Admin::Reports::ActionsController < Admin::BaseController
     authorize @report, :show?
 
     case action_from_button
-    when 'delete', 'mark_as_sensitive'
+    when 'delete', 'mark_as_sensitive', 'force_cw'
       status_batch_action = Admin::StatusBatchAction.new(
         type: action_from_button,
         status_ids: @report.status_ids,
@@ -52,6 +52,8 @@ class Admin::Reports::ActionsController < Admin::BaseController
       'delete'
     elsif params[:mark_as_sensitive]
       'mark_as_sensitive'
+    elsif params[:force_cw]
+      'force_cw'
     elsif params[:silence]
       'silence'
     elsif params[:suspend]

app/controllers/api/v1/statuses/referred_by_statuses_controller.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+class Api::V1::Statuses::ReferredByStatusesController < Api::BaseController
+  include Authorization
+
+  before_action -> { authorize_if_got_token! :read, :'read:accounts' }
+  before_action :set_status
+  after_action :insert_pagination_headers
+
+  def index
+    @statuses = load_statuses
+    render json: @statuses, each_serializer: REST::StatusSerializer, relationships: StatusRelationshipsPresenter.new(@statuses, current_user&.account_id)
+  end
+
+  private
+
+  def load_statuses
+    cached_references
+  end
+
+  def cached_references
+    cache_collection(results, Status)
+  end
+
+  def results
+    @results ||= Status.where(id: @status.referenced_by_status_objects.select(:status_id), visibility: [:public, :public_unlisted, :unlisted, :login]).paginate_by_max_id(
+      limit_param(DEFAULT_STATUSES_LIMIT),
+      params[:max_id],
+      params[:since_id]
+    )
+  end
+
+  def insert_pagination_headers
+    set_pagination_headers(next_path, prev_path)
+  end
+
+  def next_path
+    api_v1_status_referred_by_index_url pagination_params(max_id: pagination_max_id) if records_continue?
+  end
+
+  def prev_path
+    api_v1_status_referred_by_index_url pagination_params(min_id: pagination_since_id) unless results.empty?
+  end
+
+  def pagination_max_id
+    results.last.id
+  end
+
+  def pagination_since_id
+    results.first.id
+  end
+
+  def records_continue?
+    results.size == limit_param(DEFAULT_STATUSES_LIMIT)
+  end
+
+  def set_status
+    @status = Status.find(params[:status_id])
+    authorize @status, :show?
+  rescue Mastodon::NotPermittedError
+    not_found
+  end
+
+  def pagination_params(core_params)
+    params.slice(:limit).permit(:limit).merge(core_params)
+  end
+end

app/controllers/api/v1/statuses_controller.rb

@@ -44,11 +44,13 @@ class Api::V1::StatusesController < Api::BaseController
 
     ancestors_results   = @status.in_reply_to_id.nil? ? [] : @status.ancestors(ancestors_limit, current_account)
     descendants_results = @status.descendants(descendants_limit, current_account, descendants_depth_limit)
+    references_results  = @status.references
     loaded_ancestors    = cache_collection(ancestors_results, Status)
     loaded_descendants  = cache_collection(descendants_results, Status)
+    loaded_references   = cache_collection(references_results, Status)
 
-    @context = Context.new(ancestors: loaded_ancestors, descendants: loaded_descendants)
+    @context = Context.new(ancestors: loaded_ancestors, descendants: loaded_descendants, references: loaded_references)
-    statuses = [@status] + @context.ancestors + @context.descendants
+    statuses = [@status] + @context.ancestors + @context.descendants + @context.references
 
     render json: @context, serializer: REST::ContextSerializer, relationships: StatusRelationshipsPresenter.new(statuses, current_user&.account_id)
   end

app/helpers/formatting_helper.rb

@@ -58,6 +58,10 @@ module FormattingHelper
   end
 
   def account_field_value_format(field, with_rel_me: true)
+    if field.verified? && !field.account.local?
+      TextFormatter.shortened_link(field.value_for_verification)
+    else
       html_aware_format(field.value, field.account.local?, markdown: false, with_rel_me: with_rel_me, with_domains: true, multiline: false)
     end
+  end
 end

app/javascript/mastodon/actions/compose.js

@@ -62,6 +62,7 @@ export const COMPOSE_LANGUAGE_CHANGE     = 'COMPOSE_LANGUAGE_CHANGE';
 
 export const COMPOSE_EMOJI_INSERT = 'COMPOSE_EMOJI_INSERT';
 export const COMPOSE_EXPIRATION_INSERT = 'COMPOSE_EXPIRATION_INSERT';
+export const COMPOSE_REFERENCE_INSERT = 'COMPOSE_REFERENCE_INSERT';
 
 export const COMPOSE_UPLOAD_CHANGE_REQUEST     = 'COMPOSE_UPLOAD_UPDATE_REQUEST';
 export const COMPOSE_UPLOAD_CHANGE_SUCCESS     = 'COMPOSE_UPLOAD_UPDATE_SUCCESS';
@@ -770,6 +771,14 @@ export function insertExpirationCompose(position, data) {
   };
 }
 
+export function insertReferenceCompose(position, url) {
+  return {
+    type: COMPOSE_REFERENCE_INSERT,
+    position,
+    url,
+  };
+}
+
 export function changeComposing(value) {
   return {
     type: COMPOSE_COMPOSING_CHANGE,

app/javascript/mastodon/actions/interactions.js

@@ -1,6 +1,6 @@
 import api from '../api';
 
-import { importFetchedAccounts, importFetchedStatus } from './importer';
+import { importFetchedAccounts, importFetchedStatus, importFetchedStatuses } from './importer';
 
 export const REBLOG_REQUEST = 'REBLOG_REQUEST';
 export const REBLOG_SUCCESS = 'REBLOG_SUCCESS';
@@ -34,6 +34,10 @@ export const FAVOURITES_FETCH_REQUEST = 'FAVOURITES_FETCH_REQUEST';
 export const FAVOURITES_FETCH_SUCCESS = 'FAVOURITES_FETCH_SUCCESS';
 export const FAVOURITES_FETCH_FAIL    = 'FAVOURITES_FETCH_FAIL';
 
+export const STATUS_REFERENCES_FETCH_REQUEST = 'STATUS_REFERENCES_FETCH_REQUEST';
+export const STATUS_REFERENCES_FETCH_SUCCESS = 'STATUS_REFERENCES_FETCH_SUCCESS';
+export const STATUS_REFERENCES_FETCH_FAIL    = 'STATUS_REFERENCES_FETCH_FAIL';
+
 export const EMOJI_REACTIONS_FETCH_REQUEST = 'EMOJI_REACTIONS_FETCH_REQUEST';
 export const EMOJI_REACTIONS_FETCH_SUCCESS = 'EMOJI_REACTIONS_FETCH_SUCCESS';
 export const EMOJI_REACTIONS_FETCH_FAIL    = 'EMOJI_REACTIONS_FETCH_FAIL';
@@ -470,6 +474,41 @@ export function fetchEmojiReactionsFail(id, error) {
   };
 }
 
+export function fetchStatusReferences(id) {
+  return (dispatch, getState) => {
+    dispatch(fetchStatusReferencesRequest(id));
+
+    api(getState).get(`/api/v1/statuses/${id}/referred_by`).then(response => {
+      dispatch(importFetchedStatuses(response.data));
+      dispatch(fetchStatusReferencesSuccess(id, response.data));
+    }).catch(error => {
+      dispatch(fetchStatusReferencesFail(id, error));
+    });
+  };
+}
+
+export function fetchStatusReferencesRequest(id) {
+  return {
+    type: STATUS_REFERENCES_FETCH_REQUEST,
+    id,
+  };
+}
+
+export function fetchStatusReferencesSuccess(id, statuses) {
+  return {
+    type: STATUS_REFERENCES_FETCH_SUCCESS,
+    id,
+    statuses,
+  };
+}
+
+export function fetchStatusReferencesFail(id, error) {
+  return {
+    type: STATUS_REFERENCES_FETCH_FAIL,
+    error,
+  };
+}
+
 export function pin(status) {
   return (dispatch, getState) => {
     dispatch(pinRequest(status));

app/javascript/mastodon/actions/notifications.js

@@ -143,6 +143,7 @@ const excludeTypesFromFilter = filter => {
     'favourite',
     'emoji_reaction',
     'reblog',
+    'status_reference',
     'mention',
     'poll',
     'status',

app/javascript/mastodon/actions/statuses.js

@@ -181,8 +181,8 @@ export function fetchContext(id) {
     dispatch(fetchContextRequest(id));
 
     api(getState).get(`/api/v1/statuses/${id}/context`).then(response => {
-      dispatch(importFetchedStatuses(response.data.ancestors.concat(response.data.descendants)));
+      dispatch(importFetchedStatuses(response.data.ancestors.concat(response.data.descendants).concat(response.data.references)));
-      dispatch(fetchContextSuccess(id, response.data.ancestors, response.data.descendants));
+      dispatch(fetchContextSuccess(id, response.data.ancestors, response.data.descendants, response.data.references));
 
     }).catch(error => {
       if (error.response && error.response.status === 404) {
@@ -201,12 +201,13 @@ export function fetchContextRequest(id) {
   };
 }
 
-export function fetchContextSuccess(id, ancestors, descendants) {
+export function fetchContextSuccess(id, ancestors, descendants, references) {
   return {
     type: CONTEXT_FETCH_SUCCESS,
     id,
     ancestors,
     descendants,
+    references,
     statuses: ancestors.concat(descendants),
   };
 }

app/javascript/mastodon/components/status_action_bar.jsx

@@ -49,6 +49,7 @@ const messages = defineMessages({
   admin_status: { id: 'status.admin_status', defaultMessage: 'Open this post in the moderation interface' },
   admin_domain: { id: 'status.admin_domain', defaultMessage: 'Open moderation interface for {domain}' },
   copy: { id: 'status.copy', defaultMessage: 'Copy link to post' },
+  reference: { id: 'status.reference', defaultMessage: 'Add reference' },
   hide: { id: 'status.hide', defaultMessage: 'Hide post' },
   blockDomain: { id: 'account.block_domain', defaultMessage: 'Block domain {domain}' },
   unblockDomain: { id: 'account.unblock_domain', defaultMessage: 'Unblock domain {domain}' },
@@ -251,6 +252,10 @@ class StatusActionBar extends ImmutablePureComponent {
     navigator.clipboard.writeText(url);
   };
 
+  handleReference = () => {
+    this.props.onReference(this.props.status);
+  };
+
   handleHideClick = () => {
     this.props.onFilter();
   };
@@ -289,6 +294,11 @@ class StatusActionBar extends ImmutablePureComponent {
     menu.push(null);
 
     menu.push({ text: intl.formatMessage(status.get('reblogged') ? messages.cancelReblog : messages.reblog), action: this.handleReblogForceModalClick });
+
+    if (publicStatus) {
+      menu.push({ text: intl.formatMessage(messages.reference), action: this.handleReference });
+    }
+    
     menu.push({ text: intl.formatMessage(status.get('bookmarked') ? messages.removeBookmark : messages.bookmark), action: this.handleBookmarkClick });
 
     if (writtenByMe && pinnableStatus) {

app/javascript/mastodon/containers/status_container.jsx

@@ -13,6 +13,7 @@ import {
   replyCompose,
   mentionCompose,
   directCompose,
+  insertReferenceCompose,
 } from '../actions/compose';
 import {
   blockDomain,
@@ -192,6 +193,10 @@ const mapDispatchToProps = (dispatch, { intl, contextType }) => ({
     });
   },
 
+  onReference (status) {
+    dispatch(insertReferenceCompose(0, status.get('url')));
+  },
+
   onTranslate (status) {
     if (status.get('translation')) {
       dispatch(undoStatusTranslation(status.get('id'), status.get('poll')));

app/javascript/mastodon/features/account_timeline/containers/header_container.jsx

@@ -165,8 +165,11 @@ const mapDispatchToProps = (dispatch, { intl }) => ({
   },
 
   onAddToAntenna (account) {
-    dispatch(openModal('ANTENNA_ADDER', {
+    dispatch(openModal({
+      modalType: 'ANTENNA_ADDER',
+      modalProps: {
         accountId: account.get('id'),
+      },
     }));
   },
 

app/javascript/mastodon/features/notifications/components/column_settings.jsx

@@ -152,6 +152,17 @@ export default class ColumnSettings extends PureComponent {
           </div>
         </div>
 
+        <div role='group' aria-labelledby='notifications-status_reference'>
+          <span id='notifications-status_reference' className='column-settings__section'><FormattedMessage id='notifications.column_settings.status_reference' defaultMessage='References:' /></span>
+
+          <div className='column-settings__row'>
+            <SettingToggle disabled={browserPermission === 'denied'} prefix='notifications_desktop' settings={settings} settingPath={['alerts', 'status_reference']} onChange={onChange} label={alertStr} />
+            {showPushSettings && <SettingToggle prefix='notifications_push' settings={pushSettings} settingPath={['alerts', 'status_reference']} onChange={this.onPushChange} label={pushStr} />}
+            <SettingToggle prefix='notifications' settings={settings} settingPath={['shows', 'status_reference']} onChange={onChange} label={showStr} />
+            <SettingToggle prefix='notifications' settings={settings} settingPath={['sounds', 'status_reference']} onChange={onChange} label={soundStr} />
+          </div>
+        </div>
+
         <div role='group' aria-labelledby='notifications-poll'>
           <span id='notifications-poll' className='column-settings__section'><FormattedMessage id='notifications.column_settings.poll' defaultMessage='Poll results:' /></span>
 

app/javascript/mastodon/features/notifications/components/filter_bar.jsx

@@ -10,6 +10,7 @@ const tooltips = defineMessages({
   favourites: { id: 'notifications.filter.favourites', defaultMessage: 'Favourites' },
   emojiReactions: { id: 'notifications.filter.emoji_reactions', defaultMessage: 'Stamps' },
   boosts: { id: 'notifications.filter.boosts', defaultMessage: 'Boosts' },
+  status_references: { id: 'notifications.filter.status_references', defaultMessage: 'Status references' },
   polls: { id: 'notifications.filter.polls', defaultMessage: 'Poll results' },
   follows: { id: 'notifications.filter.follows', defaultMessage: 'Follows' },
   statuses: { id: 'notifications.filter.statuses', defaultMessage: 'Updates from people you follow' },
@@ -90,6 +91,13 @@ class FilterBar extends PureComponent {
         >
           <Icon id='retweet' fixedWidth />
         </button>
+        <button
+          className={selectedFilter === 'status_reference' ? 'active' : ''}
+          onClick={this.onClick('status_reference')}
+          title={intl.formatMessage(tooltips.status_references)}
+        >
+          <Icon id='link' fixedWidth />
+        </button>
         <button
           className={selectedFilter === 'poll' ? 'active' : ''}
           onClick={this.onClick('poll')}

app/javascript/mastodon/features/notifications/components/notification.jsx

@@ -28,6 +28,7 @@ const messages = defineMessages({
   poll: { id: 'notification.poll', defaultMessage: 'A poll you have voted in has ended' },
   reblog: { id: 'notification.reblog', defaultMessage: '{name} boosted your status' },
   status: { id: 'notification.status', defaultMessage: '{name} just posted' },
+  statusReference: { id: 'notification.status_reference', defaultMessage: '{name} refered' },
   update: { id: 'notification.update', defaultMessage: '{name} edited a post' },
   adminSignUp: { id: 'notification.admin.sign_up', defaultMessage: '{name} signed up' },
   adminReport: { id: 'notification.admin.report', defaultMessage: '{name} reported {target}' },
@@ -288,6 +289,40 @@ class Notification extends ImmutablePureComponent {
     );
   }
 
+  renderStatusReference (notification, link) {
+    const { intl, unread } = this.props;
+
+    return (
+      <HotKeys handlers={this.getHandlers()}>
+        <div className={classNames('notification notification-status_reference focusable', { unread })} tabIndex={0} aria-label={notificationForScreenReader(intl, intl.formatMessage(messages.statusReference, { name: notification.getIn(['account', 'acct']) }), notification.get('created_at'))}>
+          <div className='notification__message'>
+            <div className='notification__favourite-icon-wrapper'>
+              <Icon id='link' fixedWidth />
+            </div>
+
+            <span title={notification.get('created_at')}>
+              <FormattedMessage id='notification.status_reference' defaultMessage='{name} referenced your status' values={{ name: link }} />
+            </span>
+          </div>
+
+          <StatusContainer
+            id={notification.get('status')}
+            withDismiss
+            hidden={this.props.hidden}
+            onMoveDown={this.handleMoveDown}
+            onMoveUp={this.handleMoveUp}
+            contextType='notifications'
+            getScrollPosition={this.props.getScrollPosition}
+            updateScrollBottom={this.props.updateScrollBottom}
+            cachedMediaWidth={this.props.cachedMediaWidth}
+            cacheMediaWidth={this.props.cacheMediaWidth}
+            unread={this.props.unread}
+          />
+        </div>
+      </HotKeys>
+    );
+  }
+
   renderStatus (notification, link) {
     const { intl, unread, status } = this.props;
 
@@ -479,6 +514,8 @@ class Notification extends ImmutablePureComponent {
       return this.renderEmojiReaction(notification, link);
     case 'reblog':
       return this.renderReblog(notification, link);
+    case 'status_reference':
+      return this.renderStatusReference(notification, link);
     case 'status':
       return this.renderStatus(notification, link);
     case 'update':

app/javascript/mastodon/features/status/components/action_bar.jsx

@@ -42,6 +42,7 @@ const messages = defineMessages({
   admin_status: { id: 'status.admin_status', defaultMessage: 'Open this post in the moderation interface' },
   admin_domain: { id: 'status.admin_domain', defaultMessage: 'Open moderation interface for {domain}' },
   copy: { id: 'status.copy', defaultMessage: 'Copy link to post' },
+  reference: { id: 'status.reference', defaultMessage: 'Add reference' },
   blockDomain: { id: 'account.block_domain', defaultMessage: 'Block domain {domain}' },
   unblockDomain: { id: 'account.unblock_domain', defaultMessage: 'Unblock domain {domain}' },
   unmute: { id: 'account.unmute', defaultMessage: 'Unmute @{name}' },
@@ -69,6 +70,7 @@ class ActionBar extends PureComponent {
     onReblogForceModal: PropTypes.func.isRequired,
     onFavourite: PropTypes.func.isRequired,
     onEmojiReact: PropTypes.func.isRequired,
+    onReference: PropTypes.func.isRequired,
     onBookmark: PropTypes.func.isRequired,
     onDelete: PropTypes.func.isRequired,
     onEdit: PropTypes.func.isRequired,
@@ -190,6 +192,10 @@ class ActionBar extends PureComponent {
     navigator.clipboard.writeText(url);
   };
 
+  handleReference = () => {
+    this.props.onReference(this.props.status);
+  };
+
   handleEmojiPick = (data) => {
     this.props.onEmojiReact(this.props.status, data);
   };
@@ -227,6 +233,11 @@ class ActionBar extends PureComponent {
       
       menu.push(null);
       menu.push({ text: intl.formatMessage(messages.reblog), action: this.handleReblogForceModalClick });
+
+      if (publicStatus) {
+        menu.push({ text: intl.formatMessage(messages.reference), action: this.handleReference });
+      }
+
       menu.push(null);
     }
 

app/javascript/mastodon/features/status/components/detailed_status.jsx

@@ -137,6 +137,7 @@ class DetailedStatus extends ImmutablePureComponent {
     let reblogIcon = 'retweet';
     let favouriteLink = '';
     let emojiReactionsLink = '';
+    let statusReferencesLink = '';
     let edited = '';
 
     if (this.props.measureHeight) {
@@ -310,6 +311,26 @@ class DetailedStatus extends ImmutablePureComponent {
       );
     }
 
+    if (this.context.router) {
+      statusReferencesLink = (
+        <Link to={`/@${status.getIn(['account', 'acct'])}/${status.get('id')}/references`} className='detailed-status__link'>
+          <Icon id='link' />
+          <span className='detailed-status__favorites'>
+            <AnimatedNumber value={status.get('status_referred_by_count')} />
+          </span>
+        </Link>
+      );
+    } else {
+      statusReferencesLink = (
+        <a href={`/interact/${status.get('id')}?type=references`} className='detailed-status__link' onClick={this.handleModalLink}>
+          <Icon id='link' />
+          <span className='detailed-status__favorites'>
+            <AnimatedNumber value={status.get('status_referred_by_count')} />
+          </span>
+        </a>
+      );
+    }
+
     if (status.get('edited_at')) {
       edited = (
         <>
@@ -347,7 +368,7 @@ class DetailedStatus extends ImmutablePureComponent {
           <div className='detailed-status__meta'>
             <a className='detailed-status__datetime' href={`/@${status.getIn(['account', 'acct'])}/${status.get('id')}`} target='_blank' rel='noopener noreferrer'>
               <FormattedDate value={new Date(status.get('created_at'))} hour12={false} year='numeric' month='short' day='2-digit' hour='2-digit' minute='2-digit' />
-            </a>{edited}{visibilityLink}{searchabilityLink}{applicationLink}{reblogLink} · {favouriteLink} · {emojiReactionsLink}
+            </a>{edited}{visibilityLink}{searchabilityLink}{applicationLink}{reblogLink} · {favouriteLink} · {emojiReactionsLink} - {statusReferencesLink}
           </div>
         </div>
       </div>

app/javascript/mastodon/features/status/index.jsx

@@ -28,6 +28,7 @@ import {
   replyCompose,
   mentionCompose,
   directCompose,
+  insertReferenceCompose,
 } from '../../actions/compose';
 import {
   blockDomain,
@@ -88,6 +89,12 @@ const makeMapStateToProps = () => {
   const getStatus = makeGetStatus();
   const getPictureInPicture = makeGetPictureInPicture();
 
+  const getReferenceIds = createSelector([
+    (state, { id }) => state.getIn(['contexts', 'references', id]),
+  ], (references) => {
+    return references;
+  });
+
   const getAncestorsIds = createSelector([
     (_, { id }) => id,
     state => state.getIn(['contexts', 'inReplyTos']),
@@ -147,10 +154,12 @@ const makeMapStateToProps = () => {
 
     let ancestorsIds   = Immutable.List();
     let descendantsIds = Immutable.List();
+    let referenceIds   = Immutable.List();
 
     if (status) {
       ancestorsIds   = getAncestorsIds(state, { id: status.get('in_reply_to_id') });
       descendantsIds = getDescendantsIds(state, { id: status.get('id') });
+      referenceIds   = getReferenceIds(state, { id: status.get('id') });
     }
 
     return {
@@ -158,6 +167,7 @@ const makeMapStateToProps = () => {
       status,
       ancestorsIds,
       descendantsIds,
+      referenceIds,
       askReplyConfirmation: state.getIn(['compose', 'text']).trim().length !== 0,
       domain: state.getIn(['meta', 'domain']),
       pictureInPicture: getPictureInPicture(state, { id: props.params.statusId }),
@@ -200,6 +210,7 @@ class Status extends ImmutablePureComponent {
     isLoading: PropTypes.bool,
     ancestorsIds: ImmutablePropTypes.list,
     descendantsIds: ImmutablePropTypes.list,
+    referenceIds: ImmutablePropTypes.list,
     intl: PropTypes.object.isRequired,
     askReplyConfirmation: PropTypes.bool,
     multiColumn: PropTypes.bool,
@@ -356,6 +367,10 @@ class Status extends ImmutablePureComponent {
     this.handleReblogClick(status, e, true);
   };
 
+  handleReference = (status) => {
+    this.props.dispatch(insertReferenceCompose(0, status.get('url')));
+  };
+
   handleBookmarkClick = (status) => {
     if (status.get('bookmarked')) {
       this.props.dispatch(unbookmark(status));
@@ -442,8 +457,8 @@ class Status extends ImmutablePureComponent {
   };
 
   handleToggleAll = () => {
-    const { status, ancestorsIds, descendantsIds } = this.props;
+    const { status, ancestorsIds, descendantsIds, referenceIds } = this.props;
-    const statusIds = [status.get('id')].concat(ancestorsIds.toJS(), descendantsIds.toJS());
+    const statusIds = [status.get('id')].concat(ancestorsIds.toJS(), descendantsIds.toJS(), referenceIds.toJS());
 
     if (status.get('hidden')) {
       this.props.dispatch(revealStatus(statusIds));
@@ -636,8 +651,8 @@ class Status extends ImmutablePureComponent {
   };
 
   render () {
-    let ancestors, descendants;
+    let ancestors, descendants, references;
-    const { isLoading, status, ancestorsIds, descendantsIds, intl, domain, multiColumn, pictureInPicture } = this.props;
+    const { isLoading, status, ancestorsIds, descendantsIds, referenceIds, intl, domain, multiColumn, pictureInPicture } = this.props;
     const { fullscreen } = this.state;
 
     if (isLoading) {
@@ -654,6 +669,10 @@ class Status extends ImmutablePureComponent {
       );
     }
 
+    if (referenceIds && referenceIds.size > 0) {
+      references = <>{this.renderChildren(referenceIds, true)}</>;
+    }
+
     if (ancestorsIds && ancestorsIds.size > 0) {
       ancestors = <>{this.renderChildren(ancestorsIds, true)}</>;
     }
@@ -690,6 +709,7 @@ class Status extends ImmutablePureComponent {
 
         <ScrollContainer scrollKey='thread'>
           <div className={classNames('scrollable', { fullscreen })} ref={this.setRef}>
+            {references}
             {ancestors}
 
             <HotKeys handlers={handlers}>
@@ -717,6 +737,7 @@ class Status extends ImmutablePureComponent {
                   onEmojiReact={this.handleEmojiReact}
                   onReblog={this.handleReblogClick}
                   onReblogForceModal={this.handleReblogForceModalClick}
+                  onReference={this.handleReference}
                   onBookmark={this.handleBookmarkClick}
                   onDelete={this.handleDeleteClick}
                   onEdit={this.handleEditClick}

app/javascript/mastodon/features/status_references/index.jsx

@@ -0,0 +1,95 @@
+import PropTypes from 'prop-types';
+
+import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
+
+import { Helmet } from 'react-helmet';
+
+import ImmutablePropTypes from 'react-immutable-proptypes';
+import ImmutablePureComponent from 'react-immutable-pure-component';
+import { connect } from 'react-redux';
+
+import { fetchStatusReferences } from 'mastodon/actions/interactions';
+import ColumnHeader from 'mastodon/components/column_header';
+import { Icon }  from 'mastodon/components/icon';
+import LoadingIndicator from 'mastodon/components/loading_indicator';
+import ScrollableList from 'mastodon/components/scrollable_list';
+import StatusContainer from 'mastodon/containers/status_container';
+import Column from 'mastodon/features/ui/components/column';
+
+const messages = defineMessages({
+  refresh: { id: 'refresh', defaultMessage: 'Refresh' },
+});
+
+const mapStateToProps = (state, props) => ({
+  accountIds: state.getIn(['user_lists', 'referred_by', props.params.statusId]),
+});
+
+class StatusReferences extends ImmutablePureComponent {
+
+  static propTypes = {
+    params: PropTypes.object.isRequired,
+    dispatch: PropTypes.func.isRequired,
+    accountIds: ImmutablePropTypes.list,
+    multiColumn: PropTypes.bool,
+    intl: PropTypes.object.isRequired,
+  };
+
+  UNSAFE_componentWillMount () {
+    if (!this.props.accountIds) {
+      this.props.dispatch(fetchStatusReferences(this.props.params.statusId));
+    }
+  }
+
+  UNSAFE_componentWillReceiveProps (nextProps) {
+    if (nextProps.params.statusId !== this.props.params.statusId && nextProps.params.statusId) {
+      this.props.dispatch(fetchStatusReferences(nextProps.params.statusId));
+    }
+  }
+
+  handleRefresh = () => {
+    this.props.dispatch(fetchStatusReferences(this.props.params.statusId));
+  };
+
+  render () {
+    const { intl, accountIds, multiColumn } = this.props;
+
+    if (!accountIds) {
+      return (
+        <Column>
+          <LoadingIndicator />
+        </Column>
+      );
+    }
+
+    const emptyMessage = <FormattedMessage id='empty_column.status_references' defaultMessage='No one has referred this post yet. When someone does, they will show up here.' />;
+
+    return (
+      <Column bindToDocument={!multiColumn}>
+        <ColumnHeader
+          showBackButton
+          multiColumn={multiColumn}
+          extraButton={(
+            <button type='button' className='column-header__button' title={intl.formatMessage(messages.refresh)} aria-label={intl.formatMessage(messages.refresh)} onClick={this.handleRefresh}><Icon id='refresh' /></button>
+          )}
+        />
+
+        <ScrollableList
+          scrollKey='references'
+          emptyMessage={emptyMessage}
+          bindToDocument={!multiColumn}
+        >
+          {accountIds.map(id =>
+            <StatusContainer key={id} id={id} withNote={false} />,
+          )}
+        </ScrollableList>
+
+        <Helmet>
+          <meta name='robots' content='noindex' />
+        </Helmet>
+      </Column>
+    );
+  }
+
+}
+
+export default connect(mapStateToProps)(injectIntl(StatusReferences));

app/javascript/mastodon/features/ui/index.jsx

@@ -45,6 +45,7 @@ import {
   Reblogs,
   Favourites,
   EmojiReactions,
+  StatusReferences,
   DirectTimeline,
   HashtagTimeline,
   Notifications,
@@ -223,6 +224,7 @@ class SwitchingColumnsArea extends PureComponent {
           <WrappedRoute path='/@:acct/:statusId/reblogs' component={Reblogs} content={children} />
           <WrappedRoute path='/@:acct/:statusId/favourites' component={Favourites} content={children} />
           <WrappedRoute path='/@:acct/:statusId/emoji_reactions' component={EmojiReactions} content={children} />
+          <WrappedRoute path='/@:acct/:statusId/references' component={StatusReferences} content={children} />
 
           {/* Legacy routes, cannot be easily factored with other routes because they share a param name */}
           <WrappedRoute path='/timelines/tag/:id' component={HashtagTimeline} content={children} />
@@ -231,6 +233,7 @@ class SwitchingColumnsArea extends PureComponent {
           <WrappedRoute path='/statuses/:statusId/reblogs' component={Reblogs} content={children} />
           <WrappedRoute path='/statuses/:statusId/favourites' component={Favourites} content={children} />
           <WrappedRoute path='/statuses/:statusId/emoji_reactions' component={EmojiReactions} content={children} />
+          <WrappedRoute path='/statuses/:statusId/references' component={StatusReferences} content={children} />
 
           <WrappedRoute path='/follow_requests' component={FollowRequests} content={children} />
           <WrappedRoute path='/blocks' component={Blocks} content={children} />

app/javascript/mastodon/features/ui/util/async-components.js

@@ -86,6 +86,10 @@ export function EmojiReactions () {
   return import(/* webpackChunkName: "features/emoji_reactions" */'../../emoji_reactions');
 }
 
+export function StatusReferences () {
+  return import(/* webpackChunkName: "features/status_references" */'../../status_references');
+}
+
 export function FollowRequests () {
   return import(/* webpackChunkName: "features/follow_requests" */'../../follow_requests');
 }

app/javascript/mastodon/locales/en.json

@@ -427,6 +427,7 @@
   "notification.poll": "A poll you have voted in has ended",
   "notification.reblog": "{name} boosted your post",
   "notification.status": "{name} just posted",
+  "notification.status_reference": "{name} refered your post",
   "notification.update": "{name} edited a post",
   "notifications.clear": "Clear notifications",
   "notifications.clear_confirmation": "Are you sure you want to permanently clear all your notifications?",

app/javascript/mastodon/locales/ja.json

@@ -415,6 +415,7 @@
   "notification.poll": "アンケートが終了しました",
   "notification.reblog": "{name}さんがあなたの投稿をブーストしました",
   "notification.status": "{name}さんが投稿しました",
+  "notification.status_reference": "{name}さんがあなたの投稿を参照しました",
   "notification.update": "{name}さんが投稿を編集しました",
   "notifications.clear": "通知を消去",
   "notifications.clear_confirmation": "本当に通知を消去しますか？",

app/javascript/mastodon/reducers/compose.js

@@ -36,6 +36,7 @@ import {
   COMPOSE_COMPOSING_CHANGE,
   COMPOSE_EMOJI_INSERT,
   COMPOSE_EXPIRATION_INSERT,
+  COMPOSE_REFERENCE_INSERT,
   COMPOSE_UPLOAD_CHANGE_REQUEST,
   COMPOSE_UPLOAD_CHANGE_SUCCESS,
   COMPOSE_UPLOAD_CHANGE_FAIL,
@@ -238,6 +239,41 @@ const insertExpiration = (state, position, data) => {
   });
 };
 
+const insertReference = (state, url) => {
+  const oldText = state.get('text');
+
+  if (oldText.indexOf(`BT ${url}`) >= 0) {
+    return state;
+  }
+
+  let newLine = '\n\n';
+  if (oldText.length === 0) newLine = '';
+  else if (oldText[oldText.length - 1] === '\n') {
+    if (oldText.length === 1 || oldText[oldText.length - 2] === '\n') {
+      newLine = '';
+    } else {
+      newLine = '\n';
+    }
+  }
+
+  if (oldText.length > 0) {
+    const lastLine = oldText.slice(oldText.lastIndexOf('\n') + 1, oldText.length - 1);
+    if (lastLine.startsWith('BT ')) {
+      newLine = '\n';
+    }
+  }
+
+  const referenceText = `${newLine}BT ${url}`;
+  const text = `${oldText}${referenceText}`;
+
+  return state.merge({
+    text,
+    focusDate: new Date(),
+    caretPosition: text.length - referenceText.length,
+    idempotencyKey: uuid(),
+  });
+};
+
 const privacyPreference = (a, b) => {
   const order = ['public', 'public_unlisted', 'unlisted', 'login', 'private', 'direct'];
   return order[Math.max(order.indexOf(a), order.indexOf(b), 0)];
@@ -476,6 +512,8 @@ export default function compose(state = initialState, action) {
     return insertEmoji(state, action.position, action.emoji, action.needsSpace);
   case COMPOSE_EXPIRATION_INSERT:
     return insertExpiration(state, action.position, action.data);
+  case COMPOSE_REFERENCE_INSERT:
+    return insertReference(state, action.url);
   case COMPOSE_UPLOAD_CHANGE_SUCCESS:
     return state
       .set('is_changing_upload', false)

app/javascript/mastodon/reducers/contexts.js

@@ -11,11 +11,13 @@ import { compareId } from '../compare_id';
 const initialState = ImmutableMap({
   inReplyTos: ImmutableMap(),
   replies: ImmutableMap(),
+  references: ImmutableMap(),
 });
 
-const normalizeContext = (immutableState, id, ancestors, descendants) => immutableState.withMutations(state => {
+const normalizeContext = (immutableState, id, ancestors, descendants, references) => immutableState.withMutations(state => {
   state.update('inReplyTos', immutableAncestors => immutableAncestors.withMutations(inReplyTos => {
     state.update('replies', immutableDescendants => immutableDescendants.withMutations(replies => {
+      state.update('references', immutableReferences => immutableReferences.withMutations(referencePosts => {
         function addReply({ id, in_reply_to_id }) {
           if (in_reply_to_id && !inReplyTos.has(id)) {
 
@@ -38,6 +40,9 @@ const normalizeContext = (immutableState, id, ancestors, descendants) => immutab
         }
 
         descendants.forEach(addReply);
+
+        referencePosts.set(id, ImmutableList(references.map((r) => r.id)));
+      }));
     }));
   }));
 });
@@ -96,7 +101,7 @@ export default function replies(state = initialState, action) {
   case ACCOUNT_MUTE_SUCCESS:
     return filterContexts(state, action.relationship, action.statuses);
   case CONTEXT_FETCH_SUCCESS:
-    return normalizeContext(state, action.id, action.ancestors, action.descendants);
+    return normalizeContext(state, action.id, action.ancestors, action.descendants, action.references);
   case TIMELINE_DELETE:
     return deleteFromContexts(state, [action.id]);
   case TIMELINE_UPDATE:

app/javascript/mastodon/reducers/user_lists.js

@@ -48,6 +48,7 @@ import {
   REBLOGS_FETCH_SUCCESS,
   FAVOURITES_FETCH_SUCCESS,
   EMOJI_REACTIONS_FETCH_SUCCESS,
+  STATUS_REFERENCES_FETCH_SUCCESS,
 } from '../actions/interactions';
 import {
   MUTES_FETCH_REQUEST,
@@ -75,6 +76,7 @@ const initialState = ImmutableMap({
   reblogged_by: initialListState,
   favourited_by: initialListState,
   emoji_reactioned_by: initialListState,
+  referred_by: initialListState,
   follow_requests: initialListState,
   blocks: initialListState,
   mutes: initialListState,
@@ -141,6 +143,8 @@ export default function userLists(state = initialState, action) {
     return state.setIn(['favourited_by', action.id], ImmutableList(action.accounts.map(item => item.id)));
   case EMOJI_REACTIONS_FETCH_SUCCESS:
     return state.setIn(['emoji_reactioned_by', action.id], ImmutableList(action.accounts));
+  case STATUS_REFERENCES_FETCH_SUCCESS:
+    return state.setIn(['referred_by', action.id], ImmutableList(action.statuses.map(item => item.id)));
   case NOTIFICATIONS_UPDATE:
     return action.notification.type === 'follow_request' ? normalizeFollowRequest(state, action.notification) : state;
   case FOLLOW_REQUESTS_FETCH_SUCCESS:

app/javascript/styles/contrast/variables.scss

@@ -6,6 +6,9 @@ $classic-primary-color: #9baec8;
 $classic-secondary-color: #d9e1e8;
 $classic-highlight-color: #6364ff;
 
+$emoji-reaction-color: #42485a !default;
+$emoji-reaction-selected-color: #617ed5 !default;
+
 $ui-base-color: $classic-base-color !default;
 $ui-primary-color: $classic-primary-color !default;
 $ui-secondary-color: $classic-secondary-color !default;

app/javascript/styles/mastodon-light/variables.scss

@@ -13,6 +13,9 @@ $blurple-300: #858afa; // Faded Blue
 $grey-600: #4e4c5a; // Trout
 $grey-100: #dadaf3; // Topaz
 
+$emoji-reaction-color: #dfe5f5 !default;
+$emoji-reaction-selected-color: #9ac1f2 !default;
+
 // Differences
 $success-green: lighten(#3c754d, 8%);
 

app/javascript/styles/mastodon/components.scss

@@ -1326,7 +1326,7 @@ body > [data-popper-placement] {
   margin: 12px 0 2px 4px;
 
   .emoji-reactions-bar__button {
-    background: lighten($ui-base-color, 12%);
+    background: $emoji-reaction-color;
     border: 0;
     cursor: pointer;
     display: flex;
@@ -1335,7 +1335,7 @@ body > [data-popper-placement] {
     height: 24px;
 
     &.toggled {
-      background: darken($ui-primary-color, 16%);
+      background: $emoji-reaction-selected-color;
     }
 
     > .emoji {
@@ -2128,7 +2128,7 @@ a.account__display-name {
     font: inherit;
     display: block;
     width: 100%;
-    padding: 10px 14px;
+    padding: 8px 10px;
     border: 0;
     margin: 0;
     background: transparent;

app/javascript/styles/mastodon/variables.scss

@@ -23,6 +23,10 @@ $classic-primary-color: #9baec8; // Echo Blue
 $classic-secondary-color: #d9e1e8; // Pattens Blue
 $classic-highlight-color: #6364ff; // Brand purple
 
+// Values for kmyblue original functions
+$emoji-reaction-color: #42485a !default;
+$emoji-reaction-selected-color: #617ed5 !default;
+
 // Variables for defaults in UI
 $base-shadow-color: $black !default;
 $base-overlay-background: $black !default;

app/lib/activitypub/activity/create.rb

@@ -97,6 +97,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     fetch_replies(@status)
     distribute
     forward_for_reply
+    process_references!
     join_group!
   end
 
@@ -471,6 +472,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     retry
   end
 
+  def process_references!
+    references = @json['references'].nil? ? [] : ActivityPub::FetchReferencesService(@json['references'])
+    ProcessReferencesWorker.perform_async(@status.id, [], references)
+  end
+
   def join_group!
     GroupReblogService.new.call(@status)
   end

app/lib/request.rb

@@ -7,11 +7,48 @@ require 'resolv'
 # Monkey-patch the HTTP.rb timeout class to avoid using a timeout block
 # around the Socket#open method, since we use our own timeout blocks inside
 # that method
+#
+# Also changes how the read timeout behaves so that it is cumulative (closer
+# to HTTP::Timeout::Global, but still having distinct timeouts for other
+# operation types)
 class HTTP::Timeout::PerOperation
   def connect(socket_class, host, port, nodelay = false)
     @socket = socket_class.open(host, port)
     @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay
   end
+
+  # Reset deadline when the connection is re-used for different requests
+  def reset_counter
+    @deadline = nil
+  end
+
+  # Read data from the socket
+  def readpartial(size, buffer = nil)
+    @deadline ||= Process.clock_gettime(Process::CLOCK_MONOTONIC) + @read_timeout
+
+    timeout = false
+    loop do
+      result = @socket.read_nonblock(size, buffer, exception: false)
+
+      return :eof if result.nil?
+
+      remaining_time = @deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      raise HTTP::TimeoutError, "Read timed out after #{@read_timeout} seconds" if timeout || remaining_time <= 0
+      return result if result != :wait_readable
+
+      # marking the socket for timeout. Why is this not being raised immediately?
+      # it seems there is some race-condition on the network level between calling
+      # #read_nonblock and #wait_readable, in which #read_nonblock signalizes waiting
+      # for reads, and when waiting for x seconds, it returns nil suddenly without completing
+      # the x seconds. In a normal case this would be a timeout on wait/read, but it can
+      # also mean that the socket has been closed by the server. Therefore we "mark" the
+      # socket for timeout and try to read more bytes. If it returns :eof, it's all good, no
+      # timeout. Else, the first timeout was a proper timeout.
+      # This hack has to be done because io/wait#wait_readable doesn't provide a value for when
+      # the socket is closed by the server, and HTTP::Parser doesn't provide the limit for the chunks.
+      timeout = true unless @socket.to_io.wait_readable(remaining_time)
+    end
+  end
 end
 
 class Request

app/lib/text_formatter.rb

@@ -53,6 +53,26 @@ class TextFormatter
     html.html_safe # rubocop:disable Rails/OutputSafety
   end
 
+  class << self
+    include ERB::Util
+
+    def shortened_link(url, rel_me: false)
+      url = Addressable::URI.parse(url).to_s
+      rel = rel_me ? (DEFAULT_REL + %w(me)) : DEFAULT_REL
+
+      prefix      = url.match(URL_PREFIX_REGEX).to_s
+      display_url = url[prefix.length, 30]
+      suffix      = url[prefix.length + 30..-1]
+      cutoff      = url[prefix.length..-1].length > 30
+
+      <<~HTML.squish
+        <a href="#{h(url)}" target="_blank" rel="#{rel.join(' ')}" translate="no"><span class="invisible">#{h(prefix)}</span><span class="#{cutoff ? 'ellipsis' : ''}">#{h(display_url)}</span><span class="invisible">#{h(suffix)}</span></a>
+      HTML
+    rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
+      h(url)
+    end
+  end
+
   private
 
   def rewrite
@@ -75,19 +95,7 @@ class TextFormatter
   end
 
   def link_to_url(entity)
-    url = Addressable::URI.parse(entity[:url]).to_s
+    TextFormatter.shortened_link(entity[:url], rel_me: with_rel_me?)
-    rel = with_rel_me? ? (DEFAULT_REL + %w(me)) : DEFAULT_REL
-
-    prefix      = url.match(URL_PREFIX_REGEX).to_s
-    display_url = url[prefix.length, 30]
-    suffix      = url[prefix.length + 30..-1]
-    cutoff      = url[prefix.length..-1].length > 30
-
-    <<~HTML.squish
-      <a href="#{h(url)}" target="_blank" rel="#{rel.join(' ')}" translate="no"><span class="invisible">#{h(prefix)}</span><span class="#{cutoff ? 'ellipsis' : ''}">#{h(display_url)}</span><span class="invisible">#{h(suffix)}</span></a>
-    HTML
-  rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
-    h(entity[:url])
   end
 
   def link_to_hashtag(entity)

app/models/account_warning.rb

@@ -20,6 +20,7 @@ class AccountWarning < ApplicationRecord
   enum action: {
     none: 0,
     disable: 1_000,
+    force_cw: 1_200,
     mark_statuses_as_sensitive: 1_250,
     delete_statuses: 1_500,
     sensitive: 2_000,

app/models/admin/status_batch_action.rb

@@ -33,6 +33,8 @@ class Admin::StatusBatchAction
       handle_delete!
     when 'mark_as_sensitive'
       handle_mark_as_sensitive!
+    when 'force_cw'
+      handle_force_cw!
     when 'report'
       handle_report!
     when 'remove_from_report'
@@ -104,6 +106,42 @@ class Admin::StatusBatchAction
     UserMailer.warning(target_account.user, @warning).deliver_later! if warnable?
   end
 
+  def handle_force_cw!
+    representative_account = Account.representative
+
+    # Can't use a transaction here because UpdateStatusService queues
+    # Sidekiq jobs
+    statuses.find_each do |status|
+      authorize([:admin, status], :update?)
+
+      status_text = status.text
+      status_text = "#{status.spoiler_text}\n\n#{status_text}" if status.spoiler_text
+
+      if target_account.local?
+        UpdateStatusService.new.call(status, representative_account.id, spoiler_text: 'CW', text: status_text)
+      else
+        status.update(spoiler_text: 'CW', text: status_text)
+      end
+
+      log_action(:update, status)
+
+      if with_report?
+        report.resolve!(current_account)
+        log_action(:resolve, report)
+      end
+    end
+
+    @warning = target_account.strikes.create!(
+      action: :force_cw,
+      account: current_account,
+      report: report,
+      status_ids: status_ids,
+      text: text
+    )
+
+    UserMailer.warning(target_account.user, @warning).deliver_later! if warnable?
+  end
+
   def handle_report!
     @report = Report.new(report_params) unless with_report?
     @report.status_ids = (@report.status_ids + allowed_status_ids).uniq

app/models/concerns/attachmentable.rb

@@ -22,15 +22,14 @@ module Attachmentable
 
   included do
     def self.has_attached_file(name, options = {}) # rubocop:disable Naming/PredicateName
-      options = { validate_media_type: false }.merge(options)
       super(name, options)
-      send(:"before_#{name}_post_process") do
+
+      send(:"before_#{name}_validate") do
         attachment = send(name)
         check_image_dimension(attachment)
         set_file_content_type(attachment)
         obfuscate_file_name(attachment)
         set_file_extension(attachment)
-        Paperclip::Validators::MediaTypeSpoofDetectionValidator.new(attributes: [name]).validate(self)
       end
     end
   end

app/models/context.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Context < ActiveModelSerializers::Model
-  attributes :ancestors, :descendants
+  attributes :ancestors, :descendants, :references
 end

app/models/notification.rb

@@ -26,6 +26,7 @@ class Notification < ApplicationRecord
     'FollowRequest' => :follow_request,
     'Favourite' => :favourite,
     'EmojiReaction' => :emoji_reaction,
+    'StatusReference' => :status_reference,
     'Poll' => :poll,
   }.freeze
 
@@ -33,6 +34,7 @@ class Notification < ApplicationRecord
     mention
     status
     reblog
+    status_reference
     follow
     follow_request
     favourite
@@ -47,6 +49,7 @@ class Notification < ApplicationRecord
   TARGET_STATUS_INCLUDES_BY_TYPE = {
     status: :status,
     reblog: [status: :reblog],
+    status_reference: [status_reference: :status],
     mention: [mention: :status],
     favourite: [favourite: :status],
     emoji_reaction: [emoji_reaction: :status],
@@ -67,6 +70,7 @@ class Notification < ApplicationRecord
     belongs_to :follow_request, inverse_of: :notification
     belongs_to :favourite, inverse_of: :notification
     belongs_to :emoji_reaction, inverse_of: :notification
+    belongs_to :status_reference, inverse_of: :notification
     belongs_to :poll, inverse_of: false
     belongs_to :report, inverse_of: false
   end
@@ -85,6 +89,8 @@ class Notification < ApplicationRecord
       status
     when :reblog
       status&.reblog
+    when :status_reference
+      status_reference&.status
     when :favourite
       favourite&.status
     when :emoji_reaction, :reaction
@@ -136,6 +142,8 @@ class Notification < ApplicationRecord
           notification.status = cached_status
         when :reblog
           notification.status.reblog = cached_status
+        when :status_reference
+          notification.status_reference.status = cached_status
         when :favourite
           notification.favourite.status = cached_status
         when :emoji_reaction, :reaction
@@ -162,7 +170,7 @@ class Notification < ApplicationRecord
     case activity_type
     when 'Status', 'Follow', 'Favourite', 'EmojiReaction', 'EmojiReact', 'FollowRequest', 'Poll', 'Report'
       self.from_account_id = activity&.account_id
-    when 'Mention'
+    when 'Mention', 'StatusReference'
       self.from_account_id = activity&.status&.account_id
     when 'Account'
       self.from_account_id = activity&.id

app/models/status.rb

@@ -75,6 +75,10 @@ class Status < ApplicationRecord
   has_many :mentioned_accounts, through: :mentions, source: :account, class_name: 'Account'
   has_many :active_mentions, -> { active }, class_name: 'Mention', inverse_of: :status
   has_many :media_attachments, dependent: :nullify
+  has_many :reference_objects, class_name: 'StatusReference', inverse_of: :status, dependent: :destroy
+  has_many :references, through: :reference_objects, class_name: 'Status', source: :target_status
+  has_many :referenced_by_status_objects, foreign_key: 'target_status_id', class_name: 'StatusReference', inverse_of: :target_status, dependent: :destroy
+  has_many :referenced_by_statuses, through: :referenced_by_status_objects, class_name: 'Status', source: :status
 
   has_and_belongs_to_many :tags
   has_and_belongs_to_many :preview_cards
@@ -332,6 +336,10 @@ class Status < ApplicationRecord
     status_stat&.emoji_reaction_accounts_count || 0
   end
 
+  def status_referred_by_count
+    status_stat&.status_referred_by_count || 0
+  end
+
   def increment_count!(key)
     update_status_stat!(key => public_send(key) + 1)
   end
@@ -340,6 +348,10 @@ class Status < ApplicationRecord
     update_status_stat!(key => [public_send(key) - 1, 0].max)
   end
 
+  def add_status_referred_by_count!(diff)
+    update_status_stat!(status_referred_by_count: [public_send(:status_referred_by_count) + diff, 0].max)
+  end
+
   def emoji_reactions_grouped_by_name(account = nil)
     (Oj.load(status_stat&.emoji_reactions || '', mode: :strict) || []).tap do |emoji_reactions|
       if account.present?

app/models/status_reference.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: status_references
+#
+#  id               :bigint(8)        not null, primary key
+#  status_id        :bigint(8)        not null
+#  target_status_id :bigint(8)        not null
+#  created_at       :datetime         not null
+#  updated_at       :datetime         not null
+#
+
+class StatusReference < ApplicationRecord
+  belongs_to :status
+  belongs_to :target_status, class_name: 'Status'
+
+  has_one :notification, as: :activity, dependent: :destroy
+
+  validate :validate_status_visibilities
+
+  def validate_status_visibilities
+    raise Mastodon::ValidationError, I18n.t('status_references.errors.invalid_status_visibilities') if [:public, :public_unlisted, :unlisted, :login].exclude?(target_status.visibility.to_sym)
+  end
+end

app/models/status_stat.rb

@@ -15,6 +15,7 @@
 #  emoji_reactions_count         :integer          default(0), not null
 #  test                          :integer          default(0), not null
 #  emoji_reaction_accounts_count :integer          default(0), not null
+#  status_referred_by_count      :integer          default(0), not null
 #
 
 class StatusStat < ApplicationRecord
@@ -46,6 +47,10 @@ class StatusStat < ApplicationRecord
     [attributes['emoji_reaction_accounts_count'], 0].max
   end
 
+  def status_referred_by_count
+    [attributes['status_referred_by_count'] || 0, 0].max
+  end
+
   private
 
   def reset_parent_cache

app/serializers/rest/context_serializer.rb

@@ -3,4 +3,5 @@
 class REST::ContextSerializer < ActiveModel::Serializer
   has_many :ancestors,   serializer: REST::StatusSerializer
   has_many :descendants, serializer: REST::StatusSerializer
+  has_many :references, serializer: REST::StatusSerializer
 end

app/serializers/rest/instance_serializer.rb

@@ -117,6 +117,7 @@ class REST::InstanceSerializer < ActiveModel::Serializer
       :kmyblue_markdown,
       :kmyblue_reaction_deck,
       :kmyblue_visibility_login,
+      :status_reference,
     ]
 
     capabilities << :profile_search unless Chewy.enabled?

app/serializers/rest/notification_serializer.rb

@@ -13,7 +13,7 @@ class REST::NotificationSerializer < ActiveModel::Serializer
   end
 
   def status_type?
-    [:favourite, :emoji_reaction, :reaction, :reblog, :status, :mention, :poll, :update].include?(object.type)
+    [:favourite, :emoji_reaction, :reaction, :reblog, :status_reference, :status, :mention, :poll, :update].include?(object.type)
   end
 
   def report_type?

app/serializers/rest/preview_card_serializer.rb

@@ -11,4 +11,8 @@ class REST::PreviewCardSerializer < ActiveModel::Serializer
   def image
     object.image? ? full_asset_url(object.image.url(:original)) : nil
   end
+
+  def html
+    Sanitize.fragment(object.html, Sanitize::Config::MASTODON_OEMBED)
+  end
 end

app/serializers/rest/status_serializer.rb

@@ -6,6 +6,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
   attributes :id, :created_at, :in_reply_to_id, :in_reply_to_account_id,
              :sensitive, :spoiler_text, :visibility, :visibility_ex, :language,
              :uri, :url, :replies_count, :reblogs_count, :searchability, :markdown,
+             :status_reference_ids, :status_references_count, :status_referred_by_count,
              :favourites_count, :emoji_reactions, :emoji_reactions_count, :reactions, :edited_at
 
   attribute :favourited, if: :current_user?
@@ -92,6 +93,14 @@ class REST::StatusSerializer < ActiveModel::Serializer
     ActivityPub::TagManager.instance.url_for(object)
   end
 
+  def status_reference_ids
+    @status_reference_ids = object.reference_objects.pluck(:target_status_id)
+  end
+
+  def status_references_count
+    status_reference_ids.size
+  end
+
   def favourited
     if instance_options && instance_options[:relationships]
       instance_options[:relationships].favourites_map[object.id] || false

app/serializers/rest/v1/instance_serializer.rb

@@ -126,6 +126,7 @@ class REST::V1::InstanceSerializer < ActiveModel::Serializer
       :kmyblue_markdown,
       :kmyblue_reaction_deck,
       :kmyblue_visibility_login,
+      :status_reference,
     ]
 
     capabilities << :profile_search unless Chewy.enabled?

app/services/activitypub/fetch_references_service.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+class ActivityPub::FetchReferencesService < BaseService
+  include JsonLdHelper
+
+  def call(status, collection_or_uri)
+    @account = status.account
+
+    collection_items(collection_or_uri)&.map { |item| value_or_id(item) }
+  end
+
+  private
+
+  def collection_items(collection_or_uri)
+    collection = fetch_collection(collection_or_uri)
+    return unless collection.is_a?(Hash) && collection['first'].present?
+
+    all_items = []
+    collection = fetch_collection(collection['first'])
+
+    while collection.is_a?(Hash)
+      items = begin
+        case collection['type']
+        when 'Collection', 'CollectionPage'
+          collection['items']
+        when 'OrderedCollection', 'OrderedCollectionPage'
+          collection['orderedItems']
+        end
+      end
+
+      break if items.blank?
+
+      all_items.concat(items)
+
+      break if all_items.size >= StatusReferenceValidator::LIMIT
+
+      collection = collection['next'].present? ? fetch_collection(collection['next']) : nil
+    end
+
+    all_items
+  end
+
+  def fetch_collection(collection_or_uri)
+    return collection_or_uri if collection_or_uri.is_a?(Hash)
+    return if invalid_origin?(collection_or_uri)
+
+    fetch_resource_without_id_validation(collection_or_uri, nil, true)
+  end
+end

app/services/approve_appeal_service.rb

@@ -29,6 +29,8 @@ class ApproveAppealService < BaseService
       undo_delete_statuses!
     when 'mark_statuses_as_sensitive'
       undo_mark_statuses_as_sensitive!
+    when 'force_cw'
+      undo_force_cw!
     when 'sensitive'
       undo_sensitive!
     when 'silence'
@@ -58,6 +60,13 @@ class ApproveAppealService < BaseService
     end
   end
 
+  def undo_force_cw!
+    representative_account = Account.representative
+    @strike.statuses.includes(:media_attachments).each do |status|
+      UpdateStatusService.new.call(status, representative_account.id, spoiler_text: '')
+    end
+  end
+
   def undo_sensitive!
     target_account.unsensitize!
   end

app/services/post_status_service.rb

@@ -34,6 +34,7 @@ class PostStatusService < BaseService
   # @option [String] :idempotency Optional idempotency key
   # @option [Boolean] :with_rate_limit
   # @option [Enumerable] :allowed_mentions Optional array of expected mentioned account IDs, raises `UnexpectedMentionsError` if unexpected accounts end up in mentions
+  # @option [Enumerable] :status_reference_ids Optional array
   # @return [Status]
   def call(account, options = {})
     @account     = account
@@ -78,6 +79,7 @@ class PostStatusService < BaseService
     @markdown     = @options[:markdown] || false
     @scheduled_at = @options[:scheduled_at]&.to_datetime
     @scheduled_at = nil if scheduled_in_the_past?
+    @reference_ids = (@options[:status_reference_ids] || []).map(&:to_i).filter(&:positive?)
   rescue ArgumentError
     raise ActiveRecord::RecordInvalid
   end
@@ -146,6 +148,7 @@ class PostStatusService < BaseService
 
   def postprocess_status!
     process_hashtags_service.call(@status)
+    ProcessReferencesWorker.perform_async(@status.id, @reference_ids, [])
     Trends.tags.register(@status)
     LinkCrawlWorker.perform_async(@status.id)
     DistributionWorker.perform_async(@status.id)
@@ -221,6 +224,7 @@ class PostStatusService < BaseService
       media_attachments: @media || [],
       ordered_media_attachment_ids: (@options[:media_ids] || []).map(&:to_i) & @media.map(&:id),
       thread: @in_reply_to,
+      status_reference_ids: @status_reference_ids,
       poll_attributes: poll_attributes,
       sensitive: @sensitive,
       spoiler_text: @options[:spoiler_text] || '',

app/services/process_references_service.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+class ProcessReferencesService < BaseService
+  include Payloadable
+
+  DOMAIN = ENV['WEB_DOMAIN'] || ENV.fetch('LOCAL_DOMAIN', nil)
+  REFURL_EXP = /(RT|QT|BT|RN|RE)((:|;)?\s+|:|;)(#{URI::DEFAULT_PARSER.make_regexp(%w(http https))})/
+
+  def call(status, reference_parameters, save_records: true, urls: nil)
+    @status = status
+    @reference_parameters = reference_parameters || []
+    @save_records = save_records
+    @urls = urls || []
+
+    old_references
+
+    return unless added_references.size.positive? || removed_references.size.positive?
+
+    StatusReference.transaction do
+      remove_old_references
+      add_references
+
+      @status.save! if @save_records
+
+      create_notifications!
+    end
+  end
+
+  private
+
+  def references
+    @references = @reference_parameters + scan_text!
+  end
+
+  def old_references
+    @old_references = @status.references.pluck(:id)
+  end
+
+  def added_references
+    (references - old_references).uniq
+  end
+
+  def removed_references
+    (old_references - references).uniq
+  end
+
+  def scan_text!
+    text = @status.account.local? ? @status.text : @status.text.gsub(%r{</?[^>]*>}, '')
+    @scan_text = fetch_statuses!(text.scan(REFURL_EXP).pluck(3).uniq).map(&:id).uniq.filter { |status_id| !status_id.zero? }
+  end
+
+  def fetch_statuses!(urls)
+    (urls + @urls)
+      .map { |url| ResolveURLService.new.call(url) }
+      .filter { |status| status }
+  end
+
+  def add_references
+    return if added_references.empty?
+
+    @added_objects = []
+
+    statuses = Status.where(id: added_references)
+    statuses.each do |status|
+      @added_objects << @status.reference_objects.new(target_status: status)
+      status.increment_count!(:status_referred_by_count)
+    end
+  end
+
+  def create_notifications!
+    local_reference_objects = @added_objects.filter { |ref| ref.target_status.account.local? }
+    return if local_reference_objects.empty?
+
+    LocalNotificationWorker.push_bulk(local_reference_objects) do |ref|
+      [ref.target_status.account_id, ref.id, 'StatusReference', 'status_reference']
+    end
+  end
+
+  def remove_old_references
+    return if removed_references.empty?
+
+    statuses = Status.where(id: removed_references)
+    @status.reference_objects.where(target_status: statuses).destroy_all
+    statuses.each do |status|
+      status.decrement_count!(:status_referred_by_count)
+    end
+  end
+end

app/services/remove_status_service.rb

@@ -46,6 +46,7 @@ class RemoveStatusService < BaseService
         remove_from_public
         remove_from_media if @status.with_media?
         remove_media
+        decrement_references
       end
 
       @status.destroy! if permanently?
@@ -123,6 +124,12 @@ class RemoveStatusService < BaseService
     end
   end
 
+  def decrement_references
+    @status.references.each do |ref|
+      ref.decrement_count!(:status_referred_by_count)
+    end
+  end
+
   def remove_from_hashtags
     @account.featured_tags.where(tag_id: @status.tags.map(&:id)).each do |featured_tag|
       featured_tag.decrement(@status.id)

app/services/update_status_service.rb

@@ -142,6 +142,7 @@ class UpdateStatusService < BaseService
   def update_metadata!
     ProcessHashtagsService.new.call(@status)
     ProcessMentionsService.new.call(@status)
+    ProcessReferencesWorker.perform_async(@status.id, (@options[:status_reference_ids] || []).map(&:to_i).filter(&:positive?), [])
   end
 
   def broadcast_updates!

app/views/admin/reports/_actions.html.haml

@@ -11,6 +11,11 @@
           = button_tag t('admin.reports.mark_as_sensitive'), name: :mark_as_sensitive, class: 'button'
         .report-actions__item__description
           = t('admin.reports.actions.mark_as_sensitive_description_html')
+    .report-actions__item
+      .report-actions__item__button
+        = button_tag t('admin.reports.force_cw'), name: :force_cw, class: 'button'
+      .report-actions__item__description
+        = t('admin.reports.actions.force_cw_description_html')
     .report-actions__item
       .report-actions__item__button
         = button_tag t('admin.reports.delete_and_resolve'), name: :delete, class: 'button button--destructive'

app/views/antennas/_antenna.html.haml

@@ -60,7 +60,7 @@
                 = t('antennas.index.tags', count: antenna.antenna_tags.size)
               .permissions-list__item__text__type
                 - tags = antenna.antenna_tags.map { |tag| tag.tag.name }
-                - tags = keywords.take(5) + ['…'] if tags.size > 5 # TODO
+                - tags = tags.take(5) + ['…'] if tags.size > 5 # TODO
                 = tags.join(', ')
 
   .announcements-list__item__action-bar

app/workers/process_references_worker.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class ProcessReferencesWorker
+  include Sidekiq::Worker
+
+  def perform(status_id, ids, urls)
+    ProcessReferencesService.new.call(Status.find(status_id), ids || [], urls: urls || [])
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end

config/application.rb

@@ -28,6 +28,7 @@ require_relative '../lib/paperclip/url_generator_extensions'
 require_relative '../lib/paperclip/attachment_extensions'
 require_relative '../lib/paperclip/lazy_thumbnail'
 require_relative '../lib/paperclip/gif_transcoder'
+require_relative '../lib/paperclip/media_type_spoof_detector_extensions'
 require_relative '../lib/paperclip/transcoder'
 require_relative '../lib/paperclip/type_corrector'
 require_relative '../lib/paperclip/response_with_limit_adapter'

config/imagemagick/policy.xml

@@ -0,0 +1,27 @@
+<policymap>
+  <!-- Set some basic system resource limits -->
+  <policy domain="resource" name="time" value="60" />
+
+  <policy domain="module" rights="none" pattern="URL" />
+
+  <policy domain="filter" rights="none" pattern="*" />
+
+  <!--
+    Ideally, we would restrict ImageMagick to only accessing its own
+    disk-backed pixel cache as well as Mastodon-created Tempfiles.
+
+    However, those paths depend on the operating system and environment
+    variables, so they can only be known at runtime.
+
+    Furthermore, those paths are not necessarily shared across Mastodon
+    processes, so even creating a policy.xml at runtime is impractical.
+
+    For the time being, only disable indirect reads.
+  -->
+  <policy domain="path" rights="none" pattern="@*" />
+
+  <!-- Disallow any coder by default, and only enable ones required by Mastodon -->
+  <policy domain="coder" rights="none" pattern="*" />
+  <policy domain="coder" rights="read | write" pattern="{PNG,JPEG,GIF,HEIC,WEBP}" />
+  <policy domain="coder" rights="write" pattern="{HISTOGRAM,RGB,INFO}" />
+</policymap>

config/initializers/paperclip.rb

@@ -153,3 +153,10 @@ unless defined?(Seahorse)
     end
   end
 end
+
+# Set our ImageMagick security policy, but allow admins to override it
+ENV['MAGICK_CONFIGURE_PATH'] = begin
+  imagemagick_config_paths = ENV.fetch('MAGICK_CONFIGURE_PATH', '').split(File::PATH_SEPARATOR)
+  imagemagick_config_paths << Rails.root.join('config', 'imagemagick').expand_path.to_s
+  imagemagick_config_paths.join(File::PATH_SEPARATOR)
+end

config/locales/en.yml

@@ -1856,6 +1856,7 @@ en:
       explanation:
         delete_statuses: Some of your posts have been found to violate one or more community guidelines and have been subsequently removed by the moderators of %{instance}.
         disable: You can no longer use your account, but your profile and other data remains intact. You can request a backup of your data, change account settings or delete your account.
+        force_cw: Some of your posts have been added warning (CW) by the moderators of %{instance}. This means that people will need to tap the text in the posts before a preview is displayed. You can add posts warning yourself when posting in the future.
         mark_statuses_as_sensitive: Some of your posts have been marked as sensitive by the moderators of %{instance}. This means that people will need to tap the media in the posts before a preview is displayed. You can mark media as sensitive yourself when posting in the future.
         sensitive: From now on, all your uploaded media files will be marked as sensitive and hidden behind a click-through warning.
         silence: You can still use your account but only people who are already following you will see your posts on this server, and you may be excluded from various discovery features. However, others may still manually follow you.
@@ -1865,6 +1866,7 @@ en:
       subject:
         delete_statuses: Your posts on %{acct} have been removed
         disable: Your account %{acct} has been frozen
+        force_cw: Your posts on %{acct} have been added warning
         mark_statuses_as_sensitive: Your posts on %{acct} have been marked as sensitive
         none: Warning for %{acct}
         sensitive: Your posts on %{acct} will be marked as sensitive from now on
@@ -1873,6 +1875,7 @@ en:
       title:
         delete_statuses: Posts removed
         disable: Account frozen
+        force_cw: Posts added warning
         mark_statuses_as_sensitive: Posts marked as sensitive
         none: Warning
         sensitive: Account marked as sensitive

config/locales/ja.yml

@@ -1768,6 +1768,7 @@ ja:
       explanation:
         delete_statuses: あなたの投稿のいくつかは、1つ以上のコミュニティガイドラインに違反していることが判明し、%{instance}のモデレータによって削除されました。
         disable: アカウントは使用できませんが、プロフィールやその他のデータはそのまま残ります。 データのバックアップをリクエストしたり、アカウント設定を変更したり、アカウントを削除したりできます。
+        force_cw: あなたのいくつかの投稿は、%{instance}のモデレータによって閲覧注意としてマークされています。これは、投稿本文が表示される前にユーザが投稿内のボタンをタップする必要があることを意味します。あなたは将来投稿する際に自分自身で文章に警告を記述することができます。
         mark_statuses_as_sensitive: あなたのいくつかの投稿は、%{instance}のモデレータによって閲覧注意としてマークされています。これは、プレビューが表示される前にユーザが投稿内のメディアをタップする必要があることを意味します。あなたは将来投稿する際に自分自身でメディアを閲覧注意としてマークすることができます。
         sensitive: 今後、アップロードされたすべてのメディアファイルは閲覧注意としてマークされ、クリック解除式の警告で覆われるようになります。
         silence: アカウントが制限されています。このサーバーでは既にフォローしている人だけがあなたの投稿を見ることができます。 様々な発見機能から除外されるかもしれません。他の人があなたを手動でフォローすることは可能です。
@@ -1777,6 +1778,7 @@ ja:
       subject:
         delete_statuses: "%{acct}さんの投稿が削除されました"
         disable: あなたのアカウント %{acct}は凍結されました
+        force_cw: あなたの%{acct}の投稿はCWとして警告文が追加されました
         mark_statuses_as_sensitive: あなたの%{acct}の投稿は閲覧注意としてマークされました
         none: "%{acct}に対する警告"
         sensitive: あなたの%{acct}の投稿はこれから閲覧注意としてマークされます
@@ -1785,6 +1787,7 @@ ja:
       title:
         delete_statuses: 投稿が削除されました
         disable: アカウントが凍結されました
+        force_cw: 閲覧注意として警告が追加された投稿
         mark_statuses_as_sensitive: 閲覧注意としてマークされた投稿
         none: 警告
         sensitive: 閲覧注意としてマークされたアカウント

config/routes/api.rb

@@ -12,6 +12,7 @@ namespace :api, format: false do
         resources :favourited_by, controller: :favourited_by_accounts, only: :index
         resources :emoji_reactioned_by, controller: :emoji_reactioned_by_accounts, only: :index
         resources :emoji_reactioned_by_slim, controller: :emoji_reactioned_by_accounts_slim, only: :index
+        resources :referred_by, controller: :referred_by_statuses, only: :index
         resource :reblog, only: :create
         post :unreblog, to: 'reblogs#destroy'
 

db/migrate/20230705232953_create_status_references.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class CreateStatusReferences < ActiveRecord::Migration[6.1]
+  def change
+    create_table :status_references do |t|
+      t.belongs_to :status, null: false, foreign_key: { on_delete: :cascade }
+      t.belongs_to :target_status, null: false, foreign_key: { on_delete: :cascade, to_table: :statuses }
+      t.datetime :created_at, null: false
+      t.datetime :updated_at, null: false
+    end
+  end
+end

db/migrate/20230706031715_add_status_referred_by_count_to_status_stats.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class AddStatusReferredByCountToStatusStats < ActiveRecord::Migration[6.1]
+  def up
+    safety_assured do
+      add_column :status_stats, :status_referred_by_count, :integer, null: false, default: 0
+    end
+  end
+
+  def down
+    remove_column :status_stats, :status_referred_by_count
+  end
+end

db/schema.rb

@@ -12,7 +12,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2023_07_02_131023) do
+ActiveRecord::Schema.define(version: 2023_07_06_031715) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1043,6 +1043,15 @@ ActiveRecord::Schema.define(version: 2023_07_02_131023) do
     t.index ["status_id"], name: "index_status_pins_on_status_id"
   end
 
+  create_table "status_references", force: :cascade do |t|
+    t.bigint "status_id", null: false
+    t.bigint "target_status_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["status_id"], name: "index_status_references_on_status_id"
+    t.index ["target_status_id"], name: "index_status_references_on_target_status_id"
+  end
+
   create_table "status_stats", force: :cascade do |t|
     t.bigint "status_id", null: false
     t.bigint "replies_count", default: 0, null: false
@@ -1054,6 +1063,7 @@ ActiveRecord::Schema.define(version: 2023_07_02_131023) do
     t.integer "emoji_reactions_count", default: 0, null: false
     t.integer "test", default: 0, null: false
     t.integer "emoji_reaction_accounts_count", default: 0, null: false
+    t.integer "status_referred_by_count", default: 0, null: false
     t.index ["status_id"], name: "index_status_stats_on_status_id", unique: true
   end
 
@@ -1377,6 +1387,8 @@ ActiveRecord::Schema.define(version: 2023_07_02_131023) do
   add_foreign_key "status_edits", "statuses", on_delete: :cascade
   add_foreign_key "status_pins", "accounts", name: "fk_d4cb435b62", on_delete: :cascade
   add_foreign_key "status_pins", "statuses", on_delete: :cascade
+  add_foreign_key "status_references", "statuses", column: "target_status_id", on_delete: :cascade
+  add_foreign_key "status_references", "statuses", on_delete: :cascade
   add_foreign_key "status_stats", "statuses", on_delete: :cascade
   add_foreign_key "status_trends", "accounts", on_delete: :cascade
   add_foreign_key "status_trends", "statuses", on_delete: :cascade

dist/nginx.conf

@@ -109,6 +109,8 @@ server {
   location ~ ^/system/ {
     add_header Cache-Control "public, max-age=2419200, immutable";
     add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
+    add_header X-Content-Type-Options nosniff;
+    add_header Content-Security-Policy "default-src 'none'; form-action 'none'";
     try_files $uri =404;
   }
 

lib/mastodon/version.rb

@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      2
+      3
     end
 
     def flags

lib/paperclip/media_type_spoof_detector_extensions.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Paperclip
+  module MediaTypeSpoofDetectorExtensions
+    def calculated_content_type
+      return @calculated_content_type if defined?(@calculated_content_type)
+
+      @calculated_content_type = type_from_file_command.chomp
+
+      # The `file` command fails to recognize some MP3 files as such
+      @calculated_content_type = type_from_marcel if @calculated_content_type == 'application/octet-stream' && type_from_marcel == 'audio/mpeg'
+      @calculated_content_type
+    end
+
+    def type_from_marcel
+      @type_from_marcel ||= Marcel::MimeType.for Pathname.new(@file.path),
+                                                 name: @file.path
+    end
+  end
+end
+
+Paperclip::MediaTypeSpoofDetector.prepend(Paperclip::MediaTypeSpoofDetectorExtensions)

lib/paperclip/transcoder.rb

@@ -19,10 +19,7 @@ module Paperclip
     def make
       metadata = VideoMetadataExtractor.new(@file.path)
 
-      unless metadata.valid?
+      raise Paperclip::Error, "Error while transcoding #{@file.path}: unsupported file" unless metadata.valid?
-        Paperclip.log("Unsupported file #{@file.path}")
-        return File.open(@file.path)
-      end
 
       update_attachment_type(metadata)
       update_options_from_metadata(metadata)

lib/public_file_server_middleware.rb

@@ -32,6 +32,11 @@ class PublicFileServerMiddleware
       end
     end
 
+    # Override the default CSP header set by the CSP middleware
+    headers['Content-Security-Policy'] = "default-src 'none'; form-action 'none'" if request_path.start_with?(paperclip_root_url)
+
+    headers['X-Content-Type-Options'] = 'nosniff'
+
     [status, headers, response]
   end
 

lib/sanitize_ext/sanitize_config.rb

@@ -115,26 +115,26 @@ class Sanitize
       ]
     )
 
-    MASTODON_OEMBED ||= freeze_config merge(
+    MASTODON_OEMBED ||= freeze_config(
-      RELAXED,
+      elements: %w(audio embed iframe source video),
-      elements: RELAXED[:elements] + %w(audio embed iframe source video),
 
-      attributes: merge(
+      attributes: {
-        RELAXED[:attributes],
         'audio' => %w(controls),
         'embed' => %w(height src type width),
         'iframe' => %w(allowfullscreen frameborder height scrolling src width),
         'source' => %w(src type),
         'video' => %w(controls height loop width),
-        'div' => [:data]
+      },
-      ),
 
-      protocols: merge(
+      protocols: {
-        RELAXED[:protocols],
         'embed' => { 'src' => HTTP_PROTOCOLS },
         'iframe' => { 'src' => HTTP_PROTOCOLS },
-        'source' => { 'src' => HTTP_PROTOCOLS }
+        'source' => { 'src' => HTTP_PROTOCOLS },
-      )
+      },
+
+      add_attributes: {
+        'iframe' => { 'sandbox' => 'allow-scripts allow-same-origin allow-popups allow-popups-to-escape-sandbox allow-forms' },
+      }
     )
   end
 end

spec/models/media_attachment_spec.rb

@@ -152,6 +152,26 @@ RSpec.describe MediaAttachment, paperclip_processing: true do
     end
   end
 
+  describe 'mp3 with large cover art' do
+    let(:media) { described_class.create(account: Fabricate(:account), file: attachment_fixture('boop.mp3')) }
+
+    it 'detects it as an audio file' do
+      expect(media.type).to eq 'audio'
+    end
+
+    it 'sets meta for the duration' do
+      expect(media.file.meta['original']['duration']).to be_within(0.05).of(0.235102)
+    end
+
+    it 'extracts thumbnail' do
+      expect(media.thumbnail.present?).to be true
+    end
+
+    it 'gives the file a random name' do
+      expect(media.file_file_name).to_not eq 'boop.mp3'
+    end
+  end
+
   describe 'jpeg' do
     let(:media) { described_class.create(account: Fabricate(:account), file: attachment_fixture('attachment.jpg')) }