diff --git a/app/controllers/admin/ngword_histories_controller.rb b/app/controllers/admin/ngword_histories_controller.rb new file mode 100644 index 0000000000..90f13db2fe --- /dev/null +++ b/app/controllers/admin/ngword_histories_controller.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Admin + class NgwordHistoriesController < BaseController + before_action :set_histories + + PER_PAGE = 20 + + def index + authorize :ng_words, :show? + end + + private + + def set_histories + @histories = NgwordHistory.order(id: :desc).page(params[:page]).per(PER_PAGE) + end + end +end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 1ef7bd29bc..d3f9fea0be 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -143,9 +143,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def valid_status? - valid = !Admin::NgWord.reject?("#{@params[:spoiler_text]}\n#{@params[:text]}") && !Admin::NgWord.hashtag_reject?(@tags.size) + valid = !Admin::NgWord.reject?("#{@params[:spoiler_text]}\n#{@params[:text]}", uri: @params[:uri], target_type: :status, public: @status_parser.distributable_visibility?) && !Admin::NgWord.hashtag_reject?(@tags.size) - valid = !Admin::NgWord.stranger_mention_reject?("#{@params[:spoiler_text]}\n#{@params[:text]}") if valid && mention_to_local_but_not_followed? + valid = !Admin::NgWord.stranger_mention_reject?("#{@params[:spoiler_text]}\n#{@params[:text]}", uri: @params[:uri], target_type: :status, public: @status_parser.distributable_visibility?) if valid && mention_to_local_but_not_followed? valid end diff --git a/app/lib/activitypub/parser/status_parser.rb b/app/lib/activitypub/parser/status_parser.rb index 1d5b2f48c6..a77686dcb3 100644 --- a/app/lib/activitypub/parser/status_parser.rb +++ b/app/lib/activitypub/parser/status_parser.rb @@ -92,6 +92,10 @@ class ActivityPub::Parser::StatusParser end end + def distributable_visibility? + %i(public public_unlisted unlisted login).include?(visibility) + end + def searchability from_audience = searchability_from_audience return from_audience if from_audience diff --git a/app/models/admin/ng_word.rb b/app/models/admin/ng_word.rb index 801c12094e..5be03409d6 100644 --- a/app/models/admin/ng_word.rb +++ b/app/models/admin/ng_word.rb @@ -2,8 +2,16 @@ class Admin::NgWord class << self - def reject?(text) - ng_words.any? { |word| include?(text, word) } + def reject?(text, **options) + hit_word = ng_words.detect { |word| include?(text, word) ? word : nil } + record!(:ng_words, text, hit_word, options) if hit_word.present? + hit_word.present? + end + + def stranger_mention_reject?(text, **options) + hit_word = ng_words_for_stranger_mention.detect { |word| include?(text, word) ? word : nil } + record!(:ng_words_for_stranger_mention, text, hit_word, options) if hit_word.present? + hit_word.present? end def reject_with_custom_words?(text, custom_ng_words) @@ -18,10 +26,6 @@ class Admin::NgWord hashtag_reject?(Extractor.extract_hashtags(text)&.size || 0) end - def stranger_mention_reject?(text) - ng_words_for_stranger_mention.any? { |word| include?(text, word) } - end - private def include?(text, word) @@ -44,5 +48,18 @@ class Admin::NgWord value = Setting.post_hash_tags_max value.is_a?(Integer) && value.positive? ? value : 0 end + + def record!(type, text, keyword, options) + return unless options[:uri] && options[:target_type] + return if options.key?(:public) && !options.delete(:public) + + return if NgwordHistory.where('created_at > ?', 1.day.ago).exists?(uri: options[:uri], keyword: options[:keyword]) + + NgwordHistory.create(options.merge({ + reason: type, + text: text, + keyword: keyword, + })) + end end end diff --git a/app/models/ngword_history.rb b/app/models/ngword_history.rb new file mode 100644 index 0000000000..992d3c16fb --- /dev/null +++ b/app/models/ngword_history.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: ngword_histories +# +# id :bigint(8) not null, primary key +# uri :string not null +# target_type :integer not null +# reason :integer not null +# text :string not null +# keyword :string not null +# created_at :datetime not null +# updated_at :datetime not null +# +class NgwordHistory < ApplicationRecord + include Paginable + + enum target_type: { status: 0, account_note: 1, account_name: 2 }, _suffix: :blocked + enum reason: { ng_words: 0, ng_words_for_stranger_mention: 1 }, _prefix: :within +end diff --git a/app/models/user.rb b/app/models/user.rb index 3ed618ac5e..2d20ca0401 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -14,7 +14,6 @@ # sign_in_count :integer default(0), not null # current_sign_in_at :datetime # last_sign_in_at :datetime -# admin :boolean default(FALSE), not null # confirmation_token :string # confirmed_at :datetime # confirmation_sent_at :datetime @@ -29,7 +28,6 @@ # otp_backup_codes :string is an Array # account_id :bigint(8) not null # disabled :boolean default(FALSE), not null -# moderator :boolean default(FALSE), not null # invite_id :bigint(8) # chosen_languages :string is an Array # created_by_application_id :bigint(8) diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 697bca0543..5b1cfe2f59 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -133,7 +133,8 @@ class ActivityPub::ProcessAccountService < BaseService def valid_account? display_name = @json['name'] || '' note = @json['summary'] || '' - !Admin::NgWord.reject?(display_name) && !Admin::NgWord.reject?(note) + !Admin::NgWord.reject?(display_name, uri: @uri, target_type: :account_name) && + !Admin::NgWord.reject?(note, uri: @uri, target_type: :account_note) end def set_fetchable_key! diff --git a/app/services/activitypub/process_status_update_service.rb b/app/services/activitypub/process_status_update_service.rb index 358a51f6c3..296f87b9cf 100644 --- a/app/services/activitypub/process_status_update_service.rb +++ b/app/services/activitypub/process_status_update_service.rb @@ -161,11 +161,11 @@ class ActivityPub::ProcessStatusUpdateService < BaseService end def valid_status? - !Admin::NgWord.reject?("#{@status_parser.spoiler_text}\n#{@status_parser.text}") && !Admin::NgWord.hashtag_reject?(@raw_tags.size) + !Admin::NgWord.reject?("#{@status_parser.spoiler_text}\n#{@status_parser.text}", uri: @status.uri, target_type: :status) && !Admin::NgWord.hashtag_reject?(@raw_tags.size) end def validate_status_mentions! - raise AbortError if mention_to_stranger? && Admin::NgWord.stranger_mention_reject?("#{@status.spoiler_text}\n#{@status.text}") + raise AbortError if mention_to_stranger? && Admin::NgWord.stranger_mention_reject?("#{@status.spoiler_text}\n#{@status.text}", uri: @status.uri, target_type: :status) end def mention_to_stranger? diff --git a/app/views/admin/ng_words/show.html.haml b/app/views/admin/ng_words/show.html.haml index 2a81eec057..8b7c943f64 100644 --- a/app/views/admin/ng_words/show.html.haml +++ b/app/views/admin/ng_words/show.html.haml @@ -7,14 +7,18 @@ = simple_form_for @admin_settings, url: admin_ng_words_path, html: { method: :post } do |f| = render 'shared/error_messages', object: @admin_settings + %p.hint + = t 'admin.ng_words.history_hint' + = link_to t('admin.ngword_histories.title'), admin_ngword_histories_path + .fields-group - = f.input :ng_words_for_stranger_mention, wrapper: :with_label, as: :text, input_html: { rows: 12 }, label: t('admin.ng_words.keywords_for_stranger_mention'), hint: t('admin.ng_words.keywords_for_stranger_mention_hint') + = f.input :ng_words_for_stranger_mention, wrapper: :with_label, as: :text, input_html: { rows: 10 }, label: t('admin.ng_words.keywords_for_stranger_mention'), hint: t('admin.ng_words.keywords_for_stranger_mention_hint') .fields-group = f.input :stranger_mention_from_local_ng, wrapper: :with_label, as: :boolean, label: t('admin.ng_words.stranger_mention_from_local_ng'), hint: t('admin.ng_words.stranger_mention_from_local_ng_hint') .fields-group - = f.input :ng_words, wrapper: :with_label, as: :text, input_html: { rows: 12 }, label: t('admin.ng_words.keywords'), hint: t('admin.ng_words.keywords_hint') + = f.input :ng_words, wrapper: :with_label, as: :text, input_html: { rows: 10 }, label: t('admin.ng_words.keywords'), hint: t('admin.ng_words.keywords_hint') .fields-group = f.input :post_hash_tags_max, wrapper: :with_label, as: :integer, label: t('admin.ng_words.post_hash_tags_max') diff --git a/app/views/admin/ngword_histories/_history.html.haml b/app/views/admin/ngword_histories/_history.html.haml new file mode 100644 index 0000000000..f781924683 --- /dev/null +++ b/app/views/admin/ngword_histories/_history.html.haml @@ -0,0 +1,30 @@ +.batch-table__row + %label.batch-table__row__select.batch-checkbox + -# = f.check_box :history_ids, { multiple: true, include_hidden: false }, history.id + .batch-table__row__content + .status__content>< + = html_aware_format(history.text, false) + + .detailed-status__meta + %span.negative-hint= history.keyword + · + - if history.within_ng_words? + = t('admin.ng_words.keywords') + - elsif history.within_ng_words_for_stranger_mention? + = t('admin.ng_words.keywords_for_stranger_mention') + + %br/ + + %time.formatted{ datetime: history.created_at.iso8601, title: l(history.created_at) }= l(history.created_at) + · + - if history.account_note_blocked? + = t('admin.ngword_history.target_types.account_note') + - elsif history.account_name_blocked? + = t('admin.ngword_history.target_types.account_name') + - elsif history.status_blocked? + = t('admin.ngword_history.target_types.status') + · + = history.uri + -# if history.application + = history.application.name + · diff --git a/app/views/admin/ngword_histories/index.html.haml b/app/views/admin/ngword_histories/index.html.haml new file mode 100644 index 0000000000..27442441f4 --- /dev/null +++ b/app/views/admin/ngword_histories/index.html.haml @@ -0,0 +1,22 @@ +- content_for :page_title do + = t('admin.ngword_histories.title') + +.filters + .back-link + = link_to admin_ng_words_path do + = fa_icon 'chevron-left fw' + = t('admin.ngword_histories.back_to_ng_words') + +%hr.spacer/ + +.batch-table + .batch-table__toolbar + %label.batch-table__toolbar__select.batch-checkbox-all + = check_box_tag :batch_checkbox_all, nil, false + .batch-table__body + - if @histories.empty? + = nothing_here 'nothing-here--under-tabs' + - else + = render partial: 'admin/ngword_histories/history', collection: @histories + += paginate @histories diff --git a/config/locales/en.yml b/config/locales/en.yml index dde3e35fa0..a8b0c0ff06 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -642,6 +642,7 @@ en: title: Media attachments ng_words: hide_local_users_for_anonymous: Hide timeline local user posts from anonymous + history_hint: We recommend that you regularly check your NG words to make sure that you have not specified the NG words incorrectly. keywords: Reject keywords keywords_for_stranger_mention: Reject keywords when mention/reply from strangers keywords_for_stranger_mention_hint: Currently this words are checked posts from other servers only. @@ -651,6 +652,13 @@ en: stranger_mention_from_local_ng_hint: サーバーの登録が承認制でない場合、あなたのサーバーにもスパムが入り込む可能性があります test_error: Testing is returned any errors title: NG words and against spams + ngword_histories: + back_to_ng_words: NG words and against spams + target_types: + account_name: Account name + account_note: Account note + status: Post + title: NG words history relationships: title: "%{acct}'s relationships" relays: diff --git a/config/locales/ja.yml b/config/locales/ja.yml index 901bacac93..34d8a56300 100644 --- a/config/locales/ja.yml +++ b/config/locales/ja.yml @@ -635,6 +635,7 @@ ja: title: 投稿された画像 ng_words: hide_local_users_for_anonymous: ログインしていない状態でローカルユーザーの投稿をタイムラインから取得できないようにする + history_hint: 設定されたNGワードによって実際に拒否された投稿などは、履歴より確認できます。NGワードの指定に誤りがないか定期的に確認することをおすすめします。 keywords: 投稿できないキーワード keywords_for_stranger_mention: フォローしていないアカウントへのメンションで利用できないキーワード keywords_for_stranger_mention_hint: フォローしていないアカウントへのメンションにのみ適用されます。現状は外部サーバーから来た投稿のみに適用されます @@ -644,6 +645,13 @@ ja: stranger_mention_from_local_ng_hint: サーバーの登録が承認制でない場合、あなたのサーバーにもスパムが入り込む可能性があります test_error: NGワードのテストに失敗しました。正規表現のミスが含まれているかもしれません title: NGワードとスパム + ngword_histories: + back_to_ng_words: NGワードとスパム + target_types: + account_name: アカウントの名前 + account_note: アカウントの説明文 + status: 投稿 + title: NGワード検出履歴 relationships: title: "%{acct} さんのフォロー・フォロワー" relays: diff --git a/config/navigation.rb b/config/navigation.rb index 2fa9a6b4f9..00d048d039 100644 --- a/config/navigation.rb +++ b/config/navigation.rb @@ -49,7 +49,7 @@ SimpleNavigation::Configuration.run do |navigation| n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), nil, if: -> { current_user.can?(:manage_reports, :view_audit_log, :manage_users, :manage_invites, :manage_taxonomies, :manage_federation, :manage_blocks, :manage_ng_words, :manage_sensitive_words) && !self_destruct } do |s| s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_path, highlights_on: %r{/admin/reports}, if: -> { current_user.can?(:manage_reports) } s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_path(origin: 'local'), highlights_on: %r{/admin/accounts|/admin/pending_accounts|/admin/disputes|/admin/users}, if: -> { current_user.can?(:manage_users) } - s.item :ng_words, safe_join([fa_icon('list fw'), t('admin.ng_words.title')]), admin_ng_words_path, highlights_on: %r{/admin/ng_words}, if: -> { current_user.can?(:manage_ng_words) } + s.item :ng_words, safe_join([fa_icon('list fw'), t('admin.ng_words.title')]), admin_ng_words_path, highlights_on: %r{/admin/(ng_words|ngword_histories)}, if: -> { current_user.can?(:manage_ng_words) } s.item :sensitive_words, safe_join([fa_icon('list fw'), t('admin.sensitive_words.title')]), admin_sensitive_words_path, highlights_on: %r{/admin/sensitive_words}, if: -> { current_user.can?(:manage_sensitive_words) } s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path, if: -> { current_user.can?(:manage_invites) } s.item :follow_recommendations, safe_join([fa_icon('user-plus fw'), t('admin.follow_recommendations.title')]), admin_follow_recommendations_path, highlights_on: %r{/admin/follow_recommendations}, if: -> { current_user.can?(:manage_taxonomies) } diff --git a/config/routes/admin.rb b/config/routes/admin.rb index 48dd4395e5..6c53ca79e6 100644 --- a/config/routes/admin.rb +++ b/config/routes/admin.rb @@ -33,6 +33,7 @@ namespace :admin do resources :action_logs, only: [:index] resources :warning_presets, except: [:new, :show] resource :ng_words, only: [:show, :create] + resources :ngword_histories, only: [:index] resource :sensitive_words, only: [:show, :create] resource :special_instances, only: [:show, :create] diff --git a/db/migrate/20240216042730_create_ngword_histories.rb b/db/migrate/20240216042730_create_ngword_histories.rb new file mode 100644 index 0000000000..46cedbed73 --- /dev/null +++ b/db/migrate/20240216042730_create_ngword_histories.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class CreateNgwordHistories < ActiveRecord::Migration[7.1] + def change + create_table :ngword_histories do |t| + t.string :uri, null: false + t.integer :target_type, null: false + t.integer :reason, null: false + t.string :text, null: false + t.string :keyword, null: false + + t.timestamps + end + + add_index :ngword_histories, [:uri, :keyword, :created_at], unique: false + end +end diff --git a/db/schema.rb b/db/schema.rb index bbe9102c05..271b1f24be 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_02_12_230358) do +ActiveRecord::Schema[7.1].define(version: 2024_02_16_042730) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -868,6 +868,17 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_12_230358) do t.index ["target_account_id"], name: "index_mutes_on_target_account_id" end + create_table "ngword_histories", force: :cascade do |t| + t.string "uri", null: false + t.integer "target_type", null: false + t.integer "reason", null: false + t.string "text", null: false + t.string "keyword", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["uri", "keyword", "created_at"], name: "index_ngword_histories_on_uri_and_keyword_and_created_at" + end + create_table "notifications", force: :cascade do |t| t.bigint "activity_id", null: false t.string "activity_type", null: false diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb index 971c512be2..2966ae82b6 100644 --- a/spec/lib/activitypub/activity/create_spec.rb +++ b/spec/lib/activitypub/activity/create_spec.rb @@ -1781,6 +1781,11 @@ RSpec.describe ActivityPub::Activity::Create do it 'creates status' do expect(sender.statuses.first).to_not be_nil end + + it 'does not record history' do + history = NgwordHistory.find_by(uri: object_json[:id]) + expect(history).to be_nil + end end context 'when hit ng words' do @@ -1789,6 +1794,34 @@ RSpec.describe ActivityPub::Activity::Create do it 'creates status' do expect(sender.statuses.first).to be_nil end + + it 'records history' do + history = NgwordHistory.find_by(uri: object_json[:id]) + expect(history).to_not be_nil + expect(history.status_blocked?).to be true + expect(history.within_ng_words?).to be true + expect(history.keyword).to eq ng_words + end + end + + context 'when hit ng words but does not public visibility' do + let(:content) { 'hello, world!' } + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: content, + } + end + + it 'creates status' do + expect(sender.statuses.first).to be_nil + end + + it 'records history' do + history = NgwordHistory.find_by(uri: object_json[:id]) + expect(history).to be_nil + end end context 'when mention from tags' do @@ -1799,6 +1832,7 @@ RSpec.describe ActivityPub::Activity::Create do id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, type: 'Note', content: content, + to: 'https://www.w3.org/ns/activitystreams#Public', tag: [ { type: 'Mention', @@ -1814,6 +1848,11 @@ RSpec.describe ActivityPub::Activity::Create do it 'creates status' do expect(sender.statuses.first).to_not be_nil end + + it 'does not record history' do + history = NgwordHistory.find_by(uri: object_json[:id]) + expect(history).to be_nil + end end context 'with using ng words for stranger' do @@ -1822,6 +1861,14 @@ RSpec.describe ActivityPub::Activity::Create do it 'creates status' do expect(sender.statuses.first).to be_nil end + + it 'records history' do + history = NgwordHistory.find_by(uri: object_json[:id]) + expect(history).to_not be_nil + expect(history.status_blocked?).to be true + expect(history.within_ng_words_for_stranger_mention?).to be true + expect(history.keyword).to eq ng_words_for_stranger_mention + end end context 'with using ng words for stranger but receiver is following him' do @@ -1836,6 +1883,11 @@ RSpec.describe ActivityPub::Activity::Create do it 'creates status' do expect(sender.statuses.first).to_not be_nil end + + it 'does not record history' do + history = NgwordHistory.find_by(uri: object_json[:id]) + expect(history).to be_nil + end end context 'with using ng words for stranger but multiple receivers are partically following him' do @@ -1847,6 +1899,7 @@ RSpec.describe ActivityPub::Activity::Create do id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, type: 'Note', content: content, + to: 'https://www.w3.org/ns/activitystreams#Public', tag: [ { type: 'Mention', @@ -1868,6 +1921,14 @@ RSpec.describe ActivityPub::Activity::Create do it 'creates status' do expect(sender.statuses.first).to be_nil end + + it 'records history' do + history = NgwordHistory.find_by(uri: object_json[:id]) + expect(history).to_not be_nil + expect(history.status_blocked?).to be true + expect(history.within_ng_words_for_stranger_mention?).to be true + expect(history.keyword).to eq ng_words_for_stranger_mention + end end end @@ -1880,6 +1941,7 @@ RSpec.describe ActivityPub::Activity::Create do id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, type: 'Note', content: 'ohagi peers', + to: 'https://www.w3.org/ns/activitystreams#Public', inReplyTo: ActivityPub::TagManager.instance.uri_for(original_status), } end @@ -1888,6 +1950,14 @@ RSpec.describe ActivityPub::Activity::Create do it 'creates status' do expect(sender.statuses.first).to be_nil end + + it 'records history' do + history = NgwordHistory.find_by(uri: object_json[:id]) + expect(history).to_not be_nil + expect(history.status_blocked?).to be true + expect(history.within_ng_words_for_stranger_mention?).to be true + expect(history.keyword).to eq ng_words_for_stranger_mention + end end context 'with following' do @@ -1901,6 +1971,11 @@ RSpec.describe ActivityPub::Activity::Create do it 'creates status' do expect(sender.statuses.first).to_not be_nil end + + it 'does not record history' do + history = NgwordHistory.find_by(uri: object_json[:id]) + expect(history).to be_nil + end end end end diff --git a/spec/services/activitypub/process_account_service_spec.rb b/spec/services/activitypub/process_account_service_spec.rb index 5ff3e68137..2763805f6d 100644 --- a/spec/services/activitypub/process_account_service_spec.rb +++ b/spec/services/activitypub/process_account_service_spec.rb @@ -308,12 +308,21 @@ RSpec.describe ActivityPub::ProcessAccountService, type: :service do Setting.ng_words = ['Amazon'] subject expect(account.reload.display_name).to eq 'Ohagi' + + history = NgwordHistory.find_by(uri: payload[:id]) + expect(history).to be_nil end it 'does not create account when ng word is set' do Setting.ng_words = ['Ohagi'] subject expect(account.reload.display_name).to_not eq 'Ohagi' + + history = NgwordHistory.find_by(uri: payload[:id]) + expect(history).to_not be_nil + expect(history.account_name_blocked?).to be true + expect(history.within_ng_words?).to be true + expect(history.keyword).to eq 'Ohagi' end end