Merge commit '5a0483ed21' into kb_migration

app/controllers/activitypub/references_controller.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+class ActivityPub::ReferencesController < ActivityPub::BaseController
+  include SignatureVerification
+  include Authorization
+  include AccountOwnedConcern
+
+  REFERENCES_LIMIT = 5
+
+  before_action :require_signature!, if: :authorized_fetch_mode?
+  before_action :set_status
+
+  def index
+    expires_in 0, public: public_fetch_mode?
+    render json: references_collection_presenter, serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json', skip_activities: true
+  end
+
+  private
+
+  def pundit_user
+    signed_request_account
+  end
+
+  def set_status
+    @status = @account.statuses.find(params[:status_id])
+    authorize @status, :show?
+  rescue Mastodon::NotPermittedError
+    not_found
+  end
+
+  def load_statuses
+    cached_references
+  end
+
+  def cached_references
+    cache_collection(Status.where(id: results).reorder(:id), Status)
+  end
+
+  def results
+    @results ||= begin
+      references = @status.reference_objects.order(target_status_id: :asc)
+      references = references.where('target_status_id > ?', page_params[:min_id]) if page_params[:min_id].present?
+      references = references.limit(limit_param(REFERENCES_LIMIT))
+      references.pluck(:target_status_id)
+    end
+  end
+
+  def pagination_min_id
+    results.last
+  end
+
+  def records_continue?
+    results.size == limit_param(REFERENCES_LIMIT)
+  end
+
+  def references_collection_presenter
+    page = ActivityPub::CollectionPresenter.new(
+      id: ActivityPub::TagManager.instance.references_uri_for(@status, page_params),
+      type: :unordered,
+      part_of: ActivityPub::TagManager.instance.references_uri_for(@status),
+      items: load_statuses.map(&:uri),
+      next: next_page
+    )
+
+    return page if page_requested?
+
+    ActivityPub::CollectionPresenter.new(
+      type: :unordered,
+      id: ActivityPub::TagManager.instance.references_uri_for(@status),
+      first: page
+    )
+  end
+
+  def page_requested?
+    truthy_param?(:page)
+  end
+
+  def next_page
+    return unless records_continue?
+
+    ActivityPub::TagManager.instance.references_uri_for(@status, page_params.merge(min_id: pagination_min_id))
+  end
+
+  def page_params
+    params_slice(:min_id, :limit).merge(page: true)
+  end
+end

app/controllers/api/v1/filters_controller.rb

@@ -52,11 +52,11 @@ class Api::V1::FiltersController < Api::BaseController
   end
 
   def resource_params
-    params.permit(:phrase, :expires_in, :irreversible, :whole_word, context: [])
+    params.permit(:phrase, :expires_in, :irreversible, :exclude_follows, :exclude_localusers, :whole_word, context: [])
   end
 
   def filter_params
-    resource_params.slice(:phrase, :expires_in, :irreversible, :context)
+    resource_params.slice(:phrase, :expires_in, :irreversible, :exclude_follows, :exclude_localusers, :context)
   end
 
   def keyword_params

app/controllers/api/v2/filters_controller.rb

@@ -43,6 +43,6 @@ class Api::V2::FiltersController < Api::BaseController
   end
 
   def resource_params
-    params.permit(:title, :expires_in, :filter_action, context: [], keywords_attributes: [:id, :keyword, :whole_word, :_destroy])
+    params.permit(:title, :expires_in, :filter_action, :exclude_follows, :exclude_localusers, context: [], keywords_attributes: [:id, :keyword, :whole_word, :_destroy])
   end
 end

app/controllers/filters_controller.rb

@@ -49,7 +49,7 @@ class FiltersController < ApplicationController
   end
 
   def resource_params
-    params.require(:custom_filter).permit(:title, :expires_in, :filter_action, context: [], keywords_attributes: [:id, :keyword, :whole_word, :_destroy])
+    params.require(:custom_filter).permit(:title, :expires_in, :filter_action, :exclude_follows, :exclude_localusers, context: [], keywords_attributes: [:id, :keyword, :whole_word, :_destroy])
   end
 
   def set_body_classes

app/helpers/context_helper.rb

@@ -25,6 +25,7 @@ module ContextHelper
     searchable_by: { 'fedibird' => 'http://fedibird.com/ns#', 'searchableBy' => { '@id' => 'fedibird:searchableBy', '@type' => '@id' } },
     subscribable_by: { 'kmyblue' => 'http://kmy.blue/ns#', 'subscribableBy' => { '@id' => 'kmyblue:subscribableBy', '@type' => '@id' } },
     other_setting: { 'fedibird' => 'http://fedibird.com/ns#', 'otherSetting' => 'fedibird:otherSetting' },
+    references: { 'fedibird' => 'http://fedibird.com/ns#', 'references' => { '@id' => 'fedibird:references', '@type' => '@id' } },
     olm: { 'toot' => 'http://joinmastodon.org/ns#', 'Device' => 'toot:Device', 'Ed25519Signature' => 'toot:Ed25519Signature', 'Ed25519Key' => 'toot:Ed25519Key', 'Curve25519Key' => 'toot:Curve25519Key', 'EncryptedMessage' => 'toot:EncryptedMessage', 'publicKeyBase64' => 'toot:publicKeyBase64', 'deviceId' => 'toot:deviceId', 'claim' => { '@type' => '@id', '@id' => 'toot:claim' }, 'fingerprintKey' => { '@type' => '@id', '@id' => 'toot:fingerprintKey' }, 'identityKey' => { '@type' => '@id', '@id' => 'toot:identityKey' }, 'devices' => { '@type' => '@id', '@id' => 'toot:devices' }, 'messageFranking' => 'toot:messageFranking', 'messageType' => 'toot:messageType', 'cipherText' => 'toot:cipherText' },
     suspended: { 'toot' => 'http://joinmastodon.org/ns#', 'suspended' => 'toot:suspended' },
   }.freeze

app/lib/activitypub/activity/create.rb

@@ -473,7 +473,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def process_references!
-    references = @json['references'].nil? ? [] : ActivityPub::FetchReferencesService(@json['references'])
+    references = @object['references'].nil? ? [] : ActivityPub::FetchReferencesService.new.call(@status, @object['references'])
+    quote = @object['quote'] || @object['quoteUrl'] || @object['quoteURL'] || @object['_misskey_quote']
+    references << quote if quote
     ProcessReferencesWorker.perform_async(@status.id, [], references)
   end
 

app/lib/activitypub/tag_manager.rb

@@ -78,6 +78,12 @@ class ActivityPub::TagManager
     account_status_replies_url(target.account, target, page_params)
   end
 
+  def references_uri_for(target, page_params = nil)
+    raise ArgumentError, 'target must be a local activity' unless %i(note comment activity).include?(target.object_type) && target.local?
+
+    account_status_references_url(target.account, target, page_params)
+  end
+
   def followers_uri_for(target)
     target.local? ? account_followers_url(target) : target.followers_url.presence
   end

app/models/concerns/account_interactions.rb

@@ -265,7 +265,7 @@ module AccountInteractions
 
   def status_matches_filters(status)
     active_filters = CustomFilter.cached_filters_for(id)
-    CustomFilter.apply_cached_filters(active_filters, status)
+    CustomFilter.apply_cached_filters(active_filters, status, following?(status.account))
   end
 
   def followers_for_local_distribution

app/models/custom_filter.rb

@@ -12,6 +12,8 @@
 #  created_at         :datetime         not null
 #  updated_at         :datetime         not null
 #  action             :integer          default("warn"), not null
+#  exclude_follows    :boolean          default(FALSE), not null
+#  exclude_localusers :boolean          default(FALSE), not null
 #
 
 class CustomFilter < ApplicationRecord
@@ -94,8 +96,11 @@ class CustomFilter < ApplicationRecord
     active_filters.select { |custom_filter, _| !custom_filter.expired? }
   end
 
-  def self.apply_cached_filters(cached_filters, status)
+  def self.apply_cached_filters(cached_filters, status, following)
     cached_filters.filter_map do |filter, rules|
+      next if filter.exclude_follows && following
+      next if filter.exclude_localusers && status.account.local?
+
       match = rules[:keywords].match(status.proper.searchable_text) if rules[:keywords].present?
       keyword_matches = [match.to_s] unless match.nil?
 

app/models/custom_filter_keyword.rb

@@ -7,7 +7,7 @@
 #  id               :bigint(8)        not null, primary key
 #  custom_filter_id :bigint(8)        not null
 #  keyword          :text             default(""), not null
-#  whole_word       :boolean          default(TRUE), not null
+#  whole_word       :boolean          default(FALSE), not null
 #  created_at       :datetime         not null
 #  updated_at       :datetime         not null
 #

app/models/user_settings.rb

@@ -47,8 +47,8 @@ class UserSettings
     setting :reduce_motion, default: false
     setting :expand_content_warnings, default: false
     setting :display_media, default: 'default', in: %w(default show_all hide_all)
-    setting :display_media_expand, default: false
+    setting :display_media_expand, default: true
-    setting :auto_play, default: false
+    setting :auto_play, default: true
   end
 
   namespace :notification_emails do

app/presenters/status_relationships_presenter.rb

@@ -7,6 +7,8 @@ class StatusRelationshipsPresenter
               :bookmarks_map, :filters_map, :emoji_reactions_map
 
   def initialize(statuses, current_account_id = nil, **options)
+    @current_account_id = current_account_id
+
     if current_account_id.nil?
       @reblogs_map         = {}
       @favourites_map      = {}
@@ -37,7 +39,7 @@ class StatusRelationshipsPresenter
     active_filters = CustomFilter.cached_filters_for(current_account_id)
 
     @filters_map = statuses.each_with_object({}) do |status, h|
-      filter_matches = CustomFilter.apply_cached_filters(active_filters, status)
+      filter_matches = CustomFilter.apply_cached_filters(active_filters, status, following?(status.account_id))
 
       unless filter_matches.empty?
         h[status.id] = filter_matches
@@ -45,4 +47,14 @@ class StatusRelationshipsPresenter
       end
     end
   end
+
+  def following?(other_account_id)
+    return false if @current_account_id.nil?
+
+    @account ||= Account.find(@current_account_id)
+    return false unless @account
+
+    @following_map ||= @account.following.pluck(:id)
+    @following_map.include?(other_account_id)
+  end
 end

app/serializers/activitypub/note_serializer.rb

@@ -3,13 +3,13 @@
 class ActivityPub::NoteSerializer < ActivityPub::Serializer
   include FormattingHelper
 
-  context_extensions :atom_uri, :conversation, :sensitive, :voters_count, :searchable_by
+  context_extensions :atom_uri, :conversation, :sensitive, :voters_count, :searchable_by, :references
 
   attributes :id, :type, :summary,
              :in_reply_to, :published, :url,
              :attributed_to, :to, :cc, :sensitive,
              :atom_uri, :in_reply_to_atom_uri,
-             :conversation, :searchable_by
+             :conversation, :searchable_by, :references
 
   attribute :content
   attribute :content_map, if: :language?
@@ -64,6 +64,10 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
     )
   end
 
+  def references
+    ActivityPub::TagManager.instance.references_uri_for(object)
+  end
+
   def language?
     object.language.present?
   end

app/serializers/rest/filter_serializer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class REST::FilterSerializer < ActiveModel::Serializer
-  attributes :id, :title, :context, :expires_at, :filter_action
+  attributes :id, :title, :exclude_follows, :exclude_localusers, :context, :expires_at, :filter_action
   has_many :keywords, serializer: REST::FilterKeywordSerializer, if: :rules_requested?
   has_many :statuses, serializer: REST::FilterStatusSerializer, if: :rules_requested?
 

app/serializers/rest/v1/filter_serializer.rb

@@ -2,7 +2,7 @@
 
 class REST::V1::FilterSerializer < ActiveModel::Serializer
   attributes :id, :phrase, :context, :whole_word, :expires_at,
-             :irreversible
+             :irreversible, :exclude_follows, :exclude_localusers
 
   delegate :context, :expires_at, to: :custom_filter
 
@@ -18,6 +18,10 @@ class REST::V1::FilterSerializer < ActiveModel::Serializer
     custom_filter.irreversible?
   end
 
+  delegate :exclude_follows, to: :custom_filter
+
+  delegate :exclude_localusers, to: :custom_filter
+
   private
 
   def custom_filter

app/services/activitypub/fetch_references_service.rb

@@ -13,13 +13,11 @@ class ActivityPub::FetchReferencesService < BaseService
 
   def collection_items(collection_or_uri)
     collection = fetch_collection(collection_or_uri)
-    return unless collection.is_a?(Hash) && collection['first'].present?
+    return unless collection.is_a?(Hash)
 
-    all_items = []
+    collection = fetch_collection(collection['first']) if collection['first'].present?
-    collection = fetch_collection(collection['first'])
+    return unless collection.is_a?(Hash)
 
-    while collection.is_a?(Hash)
-      items = begin
     case collection['type']
     when 'Collection', 'CollectionPage'
       collection['items']
@@ -28,21 +26,10 @@ class ActivityPub::FetchReferencesService < BaseService
     end
   end
 
-      break if items.blank?
-
-      all_items.concat(items)
-
-      break if all_items.size >= StatusReferenceValidator::LIMIT
-
-      collection = collection['next'].present? ? fetch_collection(collection['next']) : nil
-    end
-
-    all_items
-  end
-
   def fetch_collection(collection_or_uri)
     return collection_or_uri if collection_or_uri.is_a?(Hash)
-    return if invalid_origin?(collection_or_uri)
+    return if unsupported_uri_scheme?(collection_or_uri)
+    return if ActivityPub::TagManager.instance.local_uri?(collection_or_uri)
 
     fetch_resource_without_id_validation(collection_or_uri, nil, true)
   end

app/services/activitypub/process_status_update_service.rb

@@ -45,6 +45,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
         create_edits!
       end
 
+      update_references!
       download_media_files!
       queue_poll_notifications!
 
@@ -240,6 +241,13 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
     end
   end
 
+  def update_references!
+    references = @json['references'].nil? ? [] : ActivityPub::FetchReferencesService.new.call(@status, @json['references'])
+    quote = @json['quote'] || @json['quoteUrl'] || @json['quoteURL'] || @json['_misskey_quote']
+    references << quote if quote
+    ProcessReferencesWorker.perform_async(@status.id, [], references)
+  end
+
   def expected_type?
     equals_or_includes_any?(@json['type'], %w(Note Question))
   end

app/services/post_status_service.rb

@@ -76,6 +76,7 @@ class PostStatusService < BaseService
     @visibility   = :unlisted if (@visibility&.to_sym == :public || @visibility&.to_sym == :public_unlisted || @visibility&.to_sym == :login) && @account.silenced?
     @visibility   = :public_unlisted if @visibility&.to_sym == :public && !@options[:force_visibility] && !@options[:application]&.superapp && @account.user&.setting_public_post_to_unlisted
     @searchability = searchability
+    @searchability = :private if @account.silenced? && @searchability&.to_sym == :public
     @markdown     = @options[:markdown] || false
     @scheduled_at = @options[:scheduled_at]&.to_datetime
     @scheduled_at = nil if scheduled_in_the_past?

app/services/process_references_service.rb

@@ -68,6 +68,8 @@ class ProcessReferencesService < BaseService
   end
 
   def create_notifications!
+    return if @added_objects.blank?
+
     local_reference_objects = @added_objects.filter { |ref| ref.target_status.account.local? }
     return if local_reference_objects.empty?
 
@@ -80,6 +82,7 @@ class ProcessReferencesService < BaseService
     return if removed_references.empty?
 
     statuses = Status.where(id: removed_references)
+
     @status.reference_objects.where(target_status: statuses).destroy_all
     statuses.each do |status|
       status.decrement_count!(:status_referred_by_count)

app/views/filters/_filter_fields.html.haml

@@ -12,6 +12,10 @@
 .fields-group
   = f.input :filter_action, as: :radio_buttons, collection: %i(warn hide), include_blank: false, wrapper: :with_block_label, label_method: ->(action) { safe_join([t("simple_form.labels.filters.actions.#{action}"), content_tag(:span, t("simple_form.hints.filters.actions.#{action}"), class: 'hint')]) }, hint: t('simple_form.hints.filters.action'), required: true
 
+.fields-group
+  = f.input :exclude_follows, wrapper: :with_label, kmyblue: true, label: t('simple_form.labels.filters.options.exclude_follows')
+  = f.input :exclude_localusers, wrapper: :with_label, kmyblue: true, label: t('simple_form.labels.filters.options.exclude_localusers')
+
 %hr.spacer/
 
 - unless f.object.statuses.empty?

config/locales/simple_form.en.yml

@@ -263,6 +263,9 @@ en:
         actions:
           hide: Hide completely
           warn: Hide with a warning
+        options:
+          exclude_follows: Exclude following users
+          exclude_localusers: Exclude local users
       form_admin_settings:
         activity_api_enabled: Publish aggregate statistics about user activity in the API
         backups_retention_period: User archive retention period

config/locales/simple_form.ja.yml

@@ -267,6 +267,9 @@ ja:
         actions:
           hide: 完全に隠す
           warn: 警告付きで隠す
+        options:
+          exclude_follows: フォロー中のユーザーをフィルターの対象にしない
+          exclude_localusers: ローカルユーザーをフィルターの対象にしない
       form_admin_settings:
         activity_api_enabled: APIでユーザーアクティビティに関する集計統計を公開する
         backups_retention_period: ユーザーアーカイブの保持期間

config/routes.rb

@@ -103,6 +103,7 @@ Rails.application.routes.draw do
       end
 
       resources :replies, only: [:index], module: :activitypub
+      resources :references, only: [:index], module: :activitypub
     end
 
     resources :followers, only: [:index], controller: :follower_accounts

db/migrate/20230714004824_add_exclude_options_to_filters.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class AddExcludeOptionsToFilters < ActiveRecord::Migration[6.1]
+  def change
+    safety_assured do
+      add_column :custom_filters, :exclude_follows, :boolean, null: false, default: false
+      add_column :custom_filters, :exclude_localusers, :boolean, null: false, default: false
+      change_column_default :custom_filter_keywords, :whole_word, from: true, to: false
+    end
+  end
+end

db/schema.rb

@@ -12,7 +12,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2023_07_06_031715) do
+ActiveRecord::Schema.define(version: 2023_07_14_004824) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -448,7 +448,7 @@ ActiveRecord::Schema.define(version: 2023_07_06_031715) do
   create_table "custom_filter_keywords", force: :cascade do |t|
     t.bigint "custom_filter_id", null: false
     t.text "keyword", default: "", null: false
-    t.boolean "whole_word", default: true, null: false
+    t.boolean "whole_word", default: false, null: false
     t.datetime "created_at", precision: 6, null: false
     t.datetime "updated_at", precision: 6, null: false
     t.index ["custom_filter_id"], name: "index_custom_filter_keywords_on_custom_filter_id"
@@ -471,6 +471,8 @@ ActiveRecord::Schema.define(version: 2023_07_06_031715) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.integer "action", default: 0, null: false
+    t.boolean "exclude_follows", default: false, null: false
+    t.boolean "exclude_localusers", default: false, null: false
     t.index ["account_id"], name: "index_custom_filters_on_account_id"
   end
 

streaming/index.js

@@ -680,7 +680,13 @@ const startServer = async () => {
         }
 
         if (!unpackedPayload.filtered && !req.cachedFilters) {
-          queries.push(client.query('SELECT filter.id AS id, filter.phrase AS title, filter.context AS context, filter.expires_at AS expires_at, filter.action AS filter_action, keyword.keyword AS keyword, keyword.whole_word AS whole_word FROM custom_filter_keywords keyword JOIN custom_filters filter ON keyword.custom_filter_id = filter.id WHERE filter.account_id = $1 AND (filter.expires_at IS NULL OR filter.expires_at > NOW())', [req.accountId]));
+          queries.push(client.query('SELECT filter.id AS id, filter.phrase AS title, filter.context AS context, filter.expires_at AS expires_at, filter.action AS filter_action, keyword.keyword AS keyword, keyword.whole_word AS whole_word, filter.exclude_follows AS exclude_follows, filter.exclude_localusers AS exclude_localusers FROM custom_filter_keywords keyword JOIN custom_filters filter ON keyword.custom_filter_id = filter.id WHERE filter.account_id = $1 AND (filter.expires_at IS NULL OR filter.expires_at > NOW())', [req.accountId]));
+        }
+        if (!unpackedPayload.filtered) {
+          queries.push(client.query(`SELECT accounts.domain AS domain
+                                     FROM follows
+                                     JOIN accounts ON follows.target_account_id = accounts.id
+                                     WHERE (account_id = $1 AND target_account_id = $2)`, [req.accountId, unpackedPayload.account.id]));
         }
 
         Promise.all(queries).then(values => {
@@ -690,6 +696,8 @@ const startServer = async () => {
             return;
           }
 
+          const following = !unpackedPayload.filtered && values[values.length - 1].rows.length > 0;
+
           if (!unpackedPayload.filtered && !req.cachedFilters) {
             const filterRows = values[accountDomain ? 2 : 1].rows;
 
@@ -706,6 +714,8 @@ const startServer = async () => {
                     context: row.context,
                     expires_at: row.expires_at,
                     filter_action: ['warn', 'hide'][row.filter_action],
+                    excludeFollows: row.exclude_follows,
+                    excludeLocalusers: row.exclude_localusers,
                   },
                 };
               }
@@ -741,7 +751,7 @@ const startServer = async () => {
             const now = new Date();
             payload.filtered = [];
             Object.values(req.cachedFilters).forEach((cachedFilter) => {
-              if ((cachedFilter.expires_at === null || cachedFilter.expires_at > now)) {
+              if ((cachedFilter.expires_at === null || cachedFilter.expires_at > now) && (!cachedFilter.repr.excludeFollows || !following) && (!cachedFilter.repr.excludeLocalusers || accountDomain)) {
                 const keyword_matches = searchIndex.match(cachedFilter.regexp);
                 if (keyword_matches) {
                   payload.filtered.push({