Add: #545 NGワード指定で実際に除外された投稿のログ (#561)

app/controllers/admin/ngword_histories_controller.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Admin
+  class NgwordHistoriesController < BaseController
+    before_action :set_histories
+
+    PER_PAGE = 20
+
+    def index
+      authorize :ng_words, :show?
+    end
+
+    private
+
+    def set_histories
+      @histories = NgwordHistory.order(id: :desc).page(params[:page]).per(PER_PAGE)
+    end
+  end
+end

app/lib/activitypub/activity/create.rb

@@ -143,9 +143,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def valid_status?
-    valid = !Admin::NgWord.reject?("#{@params[:spoiler_text]}\n#{@params[:text]}") && !Admin::NgWord.hashtag_reject?(@tags.size)
+    valid = !Admin::NgWord.reject?("#{@params[:spoiler_text]}\n#{@params[:text]}", uri: @params[:uri], target_type: :status, public: @status_parser.distributable_visibility?) && !Admin::NgWord.hashtag_reject?(@tags.size)
 
-    valid = !Admin::NgWord.stranger_mention_reject?("#{@params[:spoiler_text]}\n#{@params[:text]}") if valid && mention_to_local_but_not_followed?
+    valid = !Admin::NgWord.stranger_mention_reject?("#{@params[:spoiler_text]}\n#{@params[:text]}", uri: @params[:uri], target_type: :status, public: @status_parser.distributable_visibility?) if valid && mention_to_local_but_not_followed?
 
     valid
   end

app/lib/activitypub/parser/status_parser.rb

@@ -92,6 +92,10 @@ class ActivityPub::Parser::StatusParser
     end
   end
 
+  def distributable_visibility?
+    %i(public public_unlisted unlisted login).include?(visibility)
+  end
+
   def searchability
     from_audience = searchability_from_audience
     return from_audience if from_audience

app/models/admin/ng_word.rb

@@ -2,8 +2,16 @@
 
 class Admin::NgWord
   class << self
-    def reject?(text)
+    def reject?(text, **options)
-      ng_words.any? { |word| include?(text, word) }
+      hit_word = ng_words.detect { |word| include?(text, word) ? word : nil }
+      record!(:ng_words, text, hit_word, options) if hit_word.present?
+      hit_word.present?
+    end
+
+    def stranger_mention_reject?(text, **options)
+      hit_word = ng_words_for_stranger_mention.detect { |word| include?(text, word) ? word : nil }
+      record!(:ng_words_for_stranger_mention, text, hit_word, options) if hit_word.present?
+      hit_word.present?
     end
 
     def reject_with_custom_words?(text, custom_ng_words)
@@ -18,10 +26,6 @@ class Admin::NgWord
       hashtag_reject?(Extractor.extract_hashtags(text)&.size || 0)
     end
 
-    def stranger_mention_reject?(text)
-      ng_words_for_stranger_mention.any? { |word| include?(text, word) }
-    end
-
     private
 
     def include?(text, word)
@@ -44,5 +48,18 @@ class Admin::NgWord
       value = Setting.post_hash_tags_max
       value.is_a?(Integer) && value.positive? ? value : 0
     end
+
+    def record!(type, text, keyword, options)
+      return unless options[:uri] && options[:target_type]
+      return if options.key?(:public) && !options.delete(:public)
+
+      return if NgwordHistory.where('created_at > ?', 1.day.ago).exists?(uri: options[:uri], keyword: options[:keyword])
+
+      NgwordHistory.create(options.merge({
+        reason: type,
+        text: text,
+        keyword: keyword,
+      }))
+    end
   end
 end

app/models/ngword_history.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: ngword_histories
+#
+#  id          :bigint(8)        not null, primary key
+#  uri         :string           not null
+#  target_type :integer          not null
+#  reason      :integer          not null
+#  text        :string           not null
+#  keyword     :string           not null
+#  created_at  :datetime         not null
+#  updated_at  :datetime         not null
+#
+class NgwordHistory < ApplicationRecord
+  include Paginable
+
+  enum target_type: { status: 0, account_note: 1, account_name: 2 }, _suffix: :blocked
+  enum reason: { ng_words: 0, ng_words_for_stranger_mention: 1 }, _prefix: :within
+end

app/models/user.rb

@@ -14,7 +14,6 @@
 #  sign_in_count             :integer          default(0), not null
 #  current_sign_in_at        :datetime
 #  last_sign_in_at           :datetime
-#  admin                     :boolean          default(FALSE), not null
 #  confirmation_token        :string
 #  confirmed_at              :datetime
 #  confirmation_sent_at      :datetime
@@ -29,7 +28,6 @@
 #  otp_backup_codes          :string           is an Array
 #  account_id                :bigint(8)        not null
 #  disabled                  :boolean          default(FALSE), not null
-#  moderator                 :boolean          default(FALSE), not null
 #  invite_id                 :bigint(8)
 #  chosen_languages          :string           is an Array
 #  created_by_application_id :bigint(8)

app/services/activitypub/process_account_service.rb

@@ -133,7 +133,8 @@ class ActivityPub::ProcessAccountService < BaseService
   def valid_account?
     display_name = @json['name'] || ''
     note = @json['summary'] || ''
-    !Admin::NgWord.reject?(display_name) && !Admin::NgWord.reject?(note)
+    !Admin::NgWord.reject?(display_name, uri: @uri, target_type: :account_name) &&
+      !Admin::NgWord.reject?(note, uri: @uri, target_type: :account_note)
   end
 
   def set_fetchable_key!

app/services/activitypub/process_status_update_service.rb

@@ -161,11 +161,11 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
   end
 
   def valid_status?
-    !Admin::NgWord.reject?("#{@status_parser.spoiler_text}\n#{@status_parser.text}") && !Admin::NgWord.hashtag_reject?(@raw_tags.size)
+    !Admin::NgWord.reject?("#{@status_parser.spoiler_text}\n#{@status_parser.text}", uri: @status.uri, target_type: :status) && !Admin::NgWord.hashtag_reject?(@raw_tags.size)
   end
 
   def validate_status_mentions!
-    raise AbortError if mention_to_stranger? && Admin::NgWord.stranger_mention_reject?("#{@status.spoiler_text}\n#{@status.text}")
+    raise AbortError if mention_to_stranger? && Admin::NgWord.stranger_mention_reject?("#{@status.spoiler_text}\n#{@status.text}", uri: @status.uri, target_type: :status)
   end
 
   def mention_to_stranger?

app/views/admin/ng_words/show.html.haml

@@ -7,14 +7,18 @@
 = simple_form_for @admin_settings, url: admin_ng_words_path, html: { method: :post } do |f|
   = render 'shared/error_messages', object: @admin_settings
 
+  %p.hint
+    = t 'admin.ng_words.history_hint'
+    = link_to t('admin.ngword_histories.title'), admin_ngword_histories_path
+
   .fields-group
-    = f.input :ng_words_for_stranger_mention, wrapper: :with_label, as: :text, input_html: { rows: 12 }, label: t('admin.ng_words.keywords_for_stranger_mention'), hint: t('admin.ng_words.keywords_for_stranger_mention_hint')
+    = f.input :ng_words_for_stranger_mention, wrapper: :with_label, as: :text, input_html: { rows: 10 }, label: t('admin.ng_words.keywords_for_stranger_mention'), hint: t('admin.ng_words.keywords_for_stranger_mention_hint')
 
   .fields-group
     = f.input :stranger_mention_from_local_ng, wrapper: :with_label, as: :boolean, label: t('admin.ng_words.stranger_mention_from_local_ng'), hint: t('admin.ng_words.stranger_mention_from_local_ng_hint')
 
   .fields-group
-    = f.input :ng_words, wrapper: :with_label, as: :text, input_html: { rows: 12 }, label: t('admin.ng_words.keywords'), hint: t('admin.ng_words.keywords_hint')
+    = f.input :ng_words, wrapper: :with_label, as: :text, input_html: { rows: 10 }, label: t('admin.ng_words.keywords'), hint: t('admin.ng_words.keywords_hint')
 
   .fields-group
     = f.input :post_hash_tags_max, wrapper: :with_label, as: :integer, label: t('admin.ng_words.post_hash_tags_max')

app/views/admin/ngword_histories/_history.html.haml

@@ -0,0 +1,30 @@
+.batch-table__row
+  %label.batch-table__row__select.batch-checkbox
+    -# = f.check_box :history_ids, { multiple: true, include_hidden: false }, history.id
+  .batch-table__row__content
+    .status__content><
+      = html_aware_format(history.text, false)
+
+    .detailed-status__meta
+      %span.negative-hint= history.keyword
+      ·
+      - if history.within_ng_words?
+        = t('admin.ng_words.keywords')
+      - elsif history.within_ng_words_for_stranger_mention?
+        = t('admin.ng_words.keywords_for_stranger_mention')
+
+      %br/
+
+      %time.formatted{ datetime: history.created_at.iso8601, title: l(history.created_at) }= l(history.created_at)
+      ·
+      - if history.account_note_blocked?
+        = t('admin.ngword_history.target_types.account_note')
+      - elsif history.account_name_blocked?
+        = t('admin.ngword_history.target_types.account_name')
+      - elsif history.status_blocked?
+        = t('admin.ngword_history.target_types.status')
+      ·
+      = history.uri
+      -# if history.application
+        = history.application.name
+        ·

app/views/admin/ngword_histories/index.html.haml

@@ -0,0 +1,22 @@
+- content_for :page_title do
+  = t('admin.ngword_histories.title')
+
+.filters
+  .back-link
+    = link_to admin_ng_words_path do
+      = fa_icon 'chevron-left fw'
+      = t('admin.ngword_histories.back_to_ng_words')
+
+%hr.spacer/
+
+.batch-table
+  .batch-table__toolbar
+    %label.batch-table__toolbar__select.batch-checkbox-all
+      = check_box_tag :batch_checkbox_all, nil, false
+  .batch-table__body
+    - if @histories.empty?
+      = nothing_here 'nothing-here--under-tabs'
+    - else
+      = render partial: 'admin/ngword_histories/history', collection: @histories
+
+= paginate @histories

config/locales/en.yml

@@ -642,6 +642,7 @@ en:
       title: Media attachments
     ng_words:
       hide_local_users_for_anonymous: Hide timeline local user posts from anonymous
+      history_hint: We recommend that you regularly check your NG words to make sure that you have not specified the NG words incorrectly.
       keywords: Reject keywords
       keywords_for_stranger_mention: Reject keywords when mention/reply from strangers
       keywords_for_stranger_mention_hint: Currently this words are checked posts from other servers only.
@@ -651,6 +652,13 @@ en:
       stranger_mention_from_local_ng_hint: サーバーの登録が承認制でない場合、あなたのサーバーにもスパムが入り込む可能性があります
       test_error: Testing is returned any errors
       title: NG words and against spams
+    ngword_histories:
+      back_to_ng_words: NG words and against spams
+      target_types:
+        account_name: Account name
+        account_note: Account note
+        status: Post
+      title: NG words history
     relationships:
       title: "%{acct}'s relationships"
     relays:

config/locales/ja.yml

@@ -635,6 +635,7 @@ ja:
       title: 投稿された画像
     ng_words:
       hide_local_users_for_anonymous: ログインしていない状態でローカルユーザーの投稿をタイムラインから取得できないようにする
+      history_hint: 設定されたNGワードによって実際に拒否された投稿などは、履歴より確認できます。NGワードの指定に誤りがないか定期的に確認することをおすすめします。
       keywords: 投稿できないキーワード
       keywords_for_stranger_mention: フォローしていないアカウントへのメンションで利用できないキーワード
       keywords_for_stranger_mention_hint: フォローしていないアカウントへのメンションにのみ適用されます。現状は外部サーバーから来た投稿のみに適用されます
@@ -644,6 +645,13 @@ ja:
       stranger_mention_from_local_ng_hint: サーバーの登録が承認制でない場合、あなたのサーバーにもスパムが入り込む可能性があります
       test_error: NGワードのテストに失敗しました。正規表現のミスが含まれているかもしれません
       title: NGワードとスパム
+    ngword_histories:
+      back_to_ng_words: NGワードとスパム
+      target_types:
+        account_name: アカウントの名前
+        account_note: アカウントの説明文
+        status: 投稿
+      title: NGワード検出履歴
     relationships:
       title: "%{acct} さんのフォロー・フォロワー"
     relays:

config/navigation.rb

@@ -49,7 +49,7 @@ SimpleNavigation::Configuration.run do |navigation|
     n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), nil, if: -> { current_user.can?(:manage_reports, :view_audit_log, :manage_users, :manage_invites, :manage_taxonomies, :manage_federation, :manage_blocks, :manage_ng_words, :manage_sensitive_words) && !self_destruct } do |s|
       s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_path, highlights_on: %r{/admin/reports}, if: -> { current_user.can?(:manage_reports) }
       s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_path(origin: 'local'), highlights_on: %r{/admin/accounts|/admin/pending_accounts|/admin/disputes|/admin/users}, if: -> { current_user.can?(:manage_users) }
-      s.item :ng_words, safe_join([fa_icon('list fw'), t('admin.ng_words.title')]), admin_ng_words_path, highlights_on: %r{/admin/ng_words}, if: -> { current_user.can?(:manage_ng_words) }
+      s.item :ng_words, safe_join([fa_icon('list fw'), t('admin.ng_words.title')]), admin_ng_words_path, highlights_on: %r{/admin/(ng_words|ngword_histories)}, if: -> { current_user.can?(:manage_ng_words) }
       s.item :sensitive_words, safe_join([fa_icon('list fw'), t('admin.sensitive_words.title')]), admin_sensitive_words_path, highlights_on: %r{/admin/sensitive_words}, if: -> { current_user.can?(:manage_sensitive_words) }
       s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path, if: -> { current_user.can?(:manage_invites) }
       s.item :follow_recommendations, safe_join([fa_icon('user-plus fw'), t('admin.follow_recommendations.title')]), admin_follow_recommendations_path, highlights_on: %r{/admin/follow_recommendations}, if: -> { current_user.can?(:manage_taxonomies) }

config/routes/admin.rb

@@ -33,6 +33,7 @@ namespace :admin do
   resources :action_logs, only: [:index]
   resources :warning_presets, except: [:new, :show]
   resource :ng_words, only: [:show, :create]
+  resources :ngword_histories, only: [:index]
   resource :sensitive_words, only: [:show, :create]
   resource :special_instances, only: [:show, :create]
 

db/migrate/20240216042730_create_ngword_histories.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class CreateNgwordHistories < ActiveRecord::Migration[7.1]
+  def change
+    create_table :ngword_histories do |t|
+      t.string :uri, null: false
+      t.integer :target_type, null: false
+      t.integer :reason, null: false
+      t.string :text, null: false
+      t.string :keyword, null: false
+
+      t.timestamps
+    end
+
+    add_index :ngword_histories, [:uri, :keyword, :created_at], unique: false
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.1].define(version: 2024_02_12_230358) do
+ActiveRecord::Schema[7.1].define(version: 2024_02_16_042730) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -868,6 +868,17 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_12_230358) do
     t.index ["target_account_id"], name: "index_mutes_on_target_account_id"
   end
 
+  create_table "ngword_histories", force: :cascade do |t|
+    t.string "uri", null: false
+    t.integer "target_type", null: false
+    t.integer "reason", null: false
+    t.string "text", null: false
+    t.string "keyword", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["uri", "keyword", "created_at"], name: "index_ngword_histories_on_uri_and_keyword_and_created_at"
+  end
+
   create_table "notifications", force: :cascade do |t|
     t.bigint "activity_id", null: false
     t.string "activity_type", null: false

spec/lib/activitypub/activity/create_spec.rb

@@ -1781,6 +1781,11 @@ RSpec.describe ActivityPub::Activity::Create do
           it 'creates status' do
             expect(sender.statuses.first).to_not be_nil
           end
+
+          it 'does not record history' do
+            history = NgwordHistory.find_by(uri: object_json[:id])
+            expect(history).to be_nil
+          end
         end
 
         context 'when hit ng words' do
@@ -1789,6 +1794,34 @@ RSpec.describe ActivityPub::Activity::Create do
           it 'creates status' do
             expect(sender.statuses.first).to be_nil
           end
+
+          it 'records history' do
+            history = NgwordHistory.find_by(uri: object_json[:id])
+            expect(history).to_not be_nil
+            expect(history.status_blocked?).to be true
+            expect(history.within_ng_words?).to be true
+            expect(history.keyword).to eq ng_words
+          end
+        end
+
+        context 'when hit ng words but does not public visibility' do
+          let(:content) { 'hello, world!' }
+          let(:object_json) do
+            {
+              id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join,
+              type: 'Note',
+              content: content,
+            }
+          end
+
+          it 'creates status' do
+            expect(sender.statuses.first).to be_nil
+          end
+
+          it 'records history' do
+            history = NgwordHistory.find_by(uri: object_json[:id])
+            expect(history).to be_nil
+          end
         end
 
         context 'when mention from tags' do
@@ -1799,6 +1832,7 @@ RSpec.describe ActivityPub::Activity::Create do
               id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join,
               type: 'Note',
               content: content,
+              to: 'https://www.w3.org/ns/activitystreams#Public',
               tag: [
                 {
                   type: 'Mention',
@@ -1814,6 +1848,11 @@ RSpec.describe ActivityPub::Activity::Create do
             it 'creates status' do
               expect(sender.statuses.first).to_not be_nil
             end
+
+            it 'does not record history' do
+              history = NgwordHistory.find_by(uri: object_json[:id])
+              expect(history).to be_nil
+            end
           end
 
           context 'with using ng words for stranger' do
@@ -1822,6 +1861,14 @@ RSpec.describe ActivityPub::Activity::Create do
             it 'creates status' do
               expect(sender.statuses.first).to be_nil
             end
+
+            it 'records history' do
+              history = NgwordHistory.find_by(uri: object_json[:id])
+              expect(history).to_not be_nil
+              expect(history.status_blocked?).to be true
+              expect(history.within_ng_words_for_stranger_mention?).to be true
+              expect(history.keyword).to eq ng_words_for_stranger_mention
+            end
           end
 
           context 'with using ng words for stranger but receiver is following him' do
@@ -1836,6 +1883,11 @@ RSpec.describe ActivityPub::Activity::Create do
             it 'creates status' do
               expect(sender.statuses.first).to_not be_nil
             end
+
+            it 'does not record history' do
+              history = NgwordHistory.find_by(uri: object_json[:id])
+              expect(history).to be_nil
+            end
           end
 
           context 'with using ng words for stranger but multiple receivers are partically following him' do
@@ -1847,6 +1899,7 @@ RSpec.describe ActivityPub::Activity::Create do
                 id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join,
                 type: 'Note',
                 content: content,
+                to: 'https://www.w3.org/ns/activitystreams#Public',
                 tag: [
                   {
                     type: 'Mention',
@@ -1868,6 +1921,14 @@ RSpec.describe ActivityPub::Activity::Create do
             it 'creates status' do
               expect(sender.statuses.first).to be_nil
             end
+
+            it 'records history' do
+              history = NgwordHistory.find_by(uri: object_json[:id])
+              expect(history).to_not be_nil
+              expect(history.status_blocked?).to be true
+              expect(history.within_ng_words_for_stranger_mention?).to be true
+              expect(history.keyword).to eq ng_words_for_stranger_mention
+            end
           end
         end
 
@@ -1880,6 +1941,7 @@ RSpec.describe ActivityPub::Activity::Create do
               id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join,
               type: 'Note',
               content: 'ohagi peers',
+              to: 'https://www.w3.org/ns/activitystreams#Public',
               inReplyTo: ActivityPub::TagManager.instance.uri_for(original_status),
             }
           end
@@ -1888,6 +1950,14 @@ RSpec.describe ActivityPub::Activity::Create do
             it 'creates status' do
               expect(sender.statuses.first).to be_nil
             end
+
+            it 'records history' do
+              history = NgwordHistory.find_by(uri: object_json[:id])
+              expect(history).to_not be_nil
+              expect(history.status_blocked?).to be true
+              expect(history.within_ng_words_for_stranger_mention?).to be true
+              expect(history.keyword).to eq ng_words_for_stranger_mention
+            end
           end
 
           context 'with following' do
@@ -1901,6 +1971,11 @@ RSpec.describe ActivityPub::Activity::Create do
             it 'creates status' do
               expect(sender.statuses.first).to_not be_nil
             end
+
+            it 'does not record history' do
+              history = NgwordHistory.find_by(uri: object_json[:id])
+              expect(history).to be_nil
+            end
           end
         end
       end

spec/services/activitypub/process_account_service_spec.rb

@@ -308,12 +308,21 @@ RSpec.describe ActivityPub::ProcessAccountService, type: :service do
       Setting.ng_words = ['Amazon']
       subject
       expect(account.reload.display_name).to eq 'Ohagi'
+
+      history = NgwordHistory.find_by(uri: payload[:id])
+      expect(history).to be_nil
     end
 
     it 'does not create account when ng word is set' do
       Setting.ng_words = ['Ohagi']
       subject
       expect(account.reload.display_name).to_not eq 'Ohagi'
+
+      history = NgwordHistory.find_by(uri: payload[:id])
+      expect(history).to_not be_nil
+      expect(history.account_name_blocked?).to be true
+      expect(history.within_ng_words?).to be true
+      expect(history.keyword).to eq 'Ohagi'
     end
   end