gitea/nas
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 17

Add: #545 NGワード指定で実際に除外された投稿のログ (#561)

Browse source
This commit is contained in:
KMY（雪あすか） 2024-02-16 17:52:59 +09:00 • committed by GitHub
parent 0ca2a73fd2
commit 7421c89431
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 262 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

19
app/controllers/admin/ngword_histories_controller.rb Normal file
View file

@ -0,0 +1,19 @@
# frozen_string_literal: true
module Admin
class NgwordHistoriesController < BaseController
before_action :set_histories
PER_PAGE = 20
def index
authorize :ng_words, :show?
end
private
def set_histories
@histories = NgwordHistory.order(id: :desc).page(params[:page]).per(PER_PAGE)
end
end
end

4
app/lib/activitypub/activity/create.rb
View file

@ -143,9 +143,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
end
def valid_status?
valid = !Admin::NgWord.reject?("#{@params[:spoiler_text]}\n#{@params[:text]}") && !Admin::NgWord.hashtag_reject?(@tags.size)
valid = !Admin::NgWord.reject?("#{@params[:spoiler_text]}\n#{@params[:text]}", uri: @params[:uri], target_type: :status, public: @status_parser.distributable_visibility?) && !Admin::NgWord.hashtag_reject?(@tags.size)
valid = !Admin::NgWord.stranger_mention_reject?("#{@params[:spoiler_text]}\n#{@params[:text]}") if valid && mention_to_local_but_not_followed?
valid = !Admin::NgWord.stranger_mention_reject?("#{@params[:spoiler_text]}\n#{@params[:text]}", uri: @params[:uri], target_type: :status, public: @status_parser.distributable_visibility?) if valid && mention_to_local_but_not_followed?
valid
end

4
app/lib/activitypub/parser/status_parser.rb
View file

@ -92,6 +92,10 @@ class ActivityPub::Parser::StatusParser
end
end
def distributable_visibility?
%i(public public_unlisted unlisted login).include?(visibility)
end
def searchability
from_audience = searchability_from_audience
return from_audience if from_audience

29
app/models/admin/ng_word.rb
View file

@ -2,8 +2,16 @@
class Admin::NgWord
class << self
def reject?(text)
ng_words.any? { |word| include?(text, word) }
def reject?(text, **options)
hit_word = ng_words.detect { |word| include?(text, word) ? word : nil }
record!(:ng_words, text, hit_word, options) if hit_word.present?
hit_word.present?
end
def stranger_mention_reject?(text, **options)
hit_word = ng_words_for_stranger_mention.detect { |word| include?(text, word) ? word : nil }
record!(:ng_words_for_stranger_mention, text, hit_word, options) if hit_word.present?
hit_word.present?
end
def reject_with_custom_words?(text, custom_ng_words)
@ -18,10 +26,6 @@ class Admin::NgWord
hashtag_reject?(Extractor.extract_hashtags(text)&.size || 0)
end
def stranger_mention_reject?(text)
ng_words_for_stranger_mention.any? { |word| include?(text, word) }
end
private
def include?(text, word)
@ -44,5 +48,18 @@ class Admin::NgWord
value = Setting.post_hash_tags_max
value.is_a?(Integer) && value.positive? ? value : 0
end
def record!(type, text, keyword, options)
return unless options[:uri] && options[:target_type]
return if options.key?(:public) && !options.delete(:public)
return if NgwordHistory.where('created_at > ?', 1.day.ago).exists?(uri: options[:uri], keyword: options[:keyword])
NgwordHistory.create(options.merge({
reason: type,
text: text,
keyword: keyword,
}))
end
end
end

21
app/models/ngword_history.rb Normal file
View file

@ -0,0 +1,21 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: ngword_histories
#
# id :bigint(8) not null, primary key
# uri :string not null
# target_type :integer not null
# reason :integer not null
# text :string not null
# keyword :string not null
# created_at :datetime not null
# updated_at :datetime not null
#
class NgwordHistory < ApplicationRecord
include Paginable
enum target_type: { status: 0, account_note: 1, account_name: 2 }, _suffix: :blocked
enum reason: { ng_words: 0, ng_words_for_stranger_mention: 1 }, _prefix: :within
end

2
app/models/user.rb
View file

@ -14,7 +14,6 @@
# sign_in_count :integer default(0), not null
# current_sign_in_at :datetime
# last_sign_in_at :datetime
# admin :boolean default(FALSE), not null
# confirmation_token :string
# confirmed_at :datetime
# confirmation_sent_at :datetime
@ -29,7 +28,6 @@
# otp_backup_codes :string is an Array
# account_id :bigint(8) not null
# disabled :boolean default(FALSE), not null
# moderator :boolean default(FALSE), not null
# invite_id :bigint(8)
# chosen_languages :string is an Array
# created_by_application_id :bigint(8)

3
app/services/activitypub/process_account_service.rb
View file

@ -133,7 +133,8 @@ class ActivityPub::ProcessAccountService < BaseService
def valid_account?
display_name = @json['name'] || ''
note = @json['summary'] || ''
!Admin::NgWord.reject?(display_name) && !Admin::NgWord.reject?(note)
!Admin::NgWord.reject?(display_name, uri: @uri, target_type: :account_name) &&
!Admin::NgWord.reject?(note, uri: @uri, target_type: :account_note)
end
def set_fetchable_key!

4
app/services/activitypub/process_status_update_service.rb
View file

@ -161,11 +161,11 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
end
def valid_status?
!Admin::NgWord.reject?("#{@status_parser.spoiler_text}\n#{@status_parser.text}") && !Admin::NgWord.hashtag_reject?(@raw_tags.size)
!Admin::NgWord.reject?("#{@status_parser.spoiler_text}\n#{@status_parser.text}", uri: @status.uri, target_type: :status) && !Admin::NgWord.hashtag_reject?(@raw_tags.size)
end
def validate_status_mentions!
raise AbortError if mention_to_stranger? && Admin::NgWord.stranger_mention_reject?("#{@status.spoiler_text}\n#{@status.text}")
raise AbortError if mention_to_stranger? && Admin::NgWord.stranger_mention_reject?("#{@status.spoiler_text}\n#{@status.text}", uri: @status.uri, target_type: :status)
end
def mention_to_stranger?

8
app/views/admin/ng_words/show.html.haml
View file

@ -7,14 +7,18 @@
= simple_form_for @admin_settings, url: admin_ng_words_path, html: { method: :post } do |f|
= render 'shared/error_messages', object: @admin_settings
%p.hint
= t 'admin.ng_words.history_hint'
= link_to t('admin.ngword_histories.title'), admin_ngword_histories_path
.fields-group
= f.input :ng_words_for_stranger_mention, wrapper: :with_label, as: :text, input_html: { rows: 12 }, label: t('admin.ng_words.keywords_for_stranger_mention'), hint: t('admin.ng_words.keywords_for_stranger_mention_hint')
= f.input :ng_words_for_stranger_mention, wrapper: :with_label, as: :text, input_html: { rows: 10 }, label: t('admin.ng_words.keywords_for_stranger_mention'), hint: t('admin.ng_words.keywords_for_stranger_mention_hint')
.fields-group
= f.input :stranger_mention_from_local_ng, wrapper: :with_label, as: :boolean, label: t('admin.ng_words.stranger_mention_from_local_ng'), hint: t('admin.ng_words.stranger_mention_from_local_ng_hint')
.fields-group
= f.input :ng_words, wrapper: :with_label, as: :text, input_html: { rows: 12 }, label: t('admin.ng_words.keywords'), hint: t('admin.ng_words.keywords_hint')
= f.input :ng_words, wrapper: :with_label, as: :text, input_html: { rows: 10 }, label: t('admin.ng_words.keywords'), hint: t('admin.ng_words.keywords_hint')
.fields-group
= f.input :post_hash_tags_max, wrapper: :with_label, as: :integer, label: t('admin.ng_words.post_hash_tags_max')

30
app/views/admin/ngword_histories/_history.html.haml Normal file
View file

@ -0,0 +1,30 @@
.batch-table__row
%label.batch-table__row__select.batch-checkbox
-# = f.check_box :history_ids, { multiple: true, include_hidden: false }, history.id
.batch-table__row__content
.status__content><
= html_aware_format(history.text, false)
.detailed-status__meta
%span.negative-hint= history.keyword
·
- if history.within_ng_words?
= t('admin.ng_words.keywords')
- elsif history.within_ng_words_for_stranger_mention?
= t('admin.ng_words.keywords_for_stranger_mention')
%br/
%time.formatted{ datetime: history.created_at.iso8601, title: l(history.created_at) }= l(history.created_at)
·
- if history.account_note_blocked?
= t('admin.ngword_history.target_types.account_note')
- elsif history.account_name_blocked?
= t('admin.ngword_history.target_types.account_name')
- elsif history.status_blocked?
= t('admin.ngword_history.target_types.status')
·
= history.uri
-# if history.application
= history.application.name
·

22
app/views/admin/ngword_histories/index.html.haml Normal file
View file

@ -0,0 +1,22 @@
- content_for :page_title do
= t('admin.ngword_histories.title')
.filters
.back-link
= link_to admin_ng_words_path do
= fa_icon 'chevron-left fw'
= t('admin.ngword_histories.back_to_ng_words')
%hr.spacer/
.batch-table
.batch-table__toolbar
%label.batch-table__toolbar__select.batch-checkbox-all
= check_box_tag :batch_checkbox_all, nil, false
.batch-table__body
- if @histories.empty?
= nothing_here 'nothing-here--under-tabs'
- else
= render partial: 'admin/ngword_histories/history', collection: @histories
= paginate @histories

8
config/locales/en.yml
View file

@ -642,6 +642,7 @@ en:
title: Media attachments
ng_words:
hide_local_users_for_anonymous: Hide timeline local user posts from anonymous
history_hint: We recommend that you regularly check your NG words to make sure that you have not specified the NG words incorrectly.
keywords: Reject keywords
keywords_for_stranger_mention: Reject keywords when mention/reply from strangers
keywords_for_stranger_mention_hint: Currently this words are checked posts from other servers only.
@ -651,6 +652,13 @@ en:
stranger_mention_from_local_ng_hint: サーバーの登録が承認制でない場合、あなたのサーバーにもスパムが入り込む可能性があります
test_error: Testing is returned any errors
title: NG words and against spams
ngword_histories:
back_to_ng_words: NG words and against spams
target_types:
account_name: Account name
account_note: Account note
status: Post
title: NG words history
relationships:
title: "%{acct}'s relationships"
relays:

8
config/locales/ja.yml
View file

@ -635,6 +635,7 @@ ja:
title: 投稿された画像
ng_words:
hide_local_users_for_anonymous: ログインしていない状態でローカルユーザーの投稿をタイムラインから取得できないようにする
history_hint: 設定されたNGワードによって実際に拒否された投稿などは、履歴より確認できます。NGワードの指定に誤りがないか定期的に確認することをおすすめします。
keywords: 投稿できないキーワード
keywords_for_stranger_mention: フォローしていないアカウントへのメンションで利用できないキーワード
keywords_for_stranger_mention_hint: フォローしていないアカウントへのメンションにのみ適用されます。現状は外部サーバーから来た投稿のみに適用されます
@ -644,6 +645,13 @@ ja:
stranger_mention_from_local_ng_hint: サーバーの登録が承認制でない場合、あなたのサーバーにもスパムが入り込む可能性があります
test_error: NGワードのテストに失敗しました。正規表現のミスが含まれているかもしれません
title: NGワードとスパム
ngword_histories:
back_to_ng_words: NGワードとスパム
target_types:
account_name: アカウントの名前
account_note: アカウントの説明文
status: 投稿
title: NGワード検出履歴
relationships:
title: "%{acct} さんのフォロー・フォロワー"
relays:

2
config/navigation.rb
View file

@ -49,7 +49,7 @@ SimpleNavigation::Configuration.run do |navigation|
n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), nil, if: -> { current_user.can?(:manage_reports, :view_audit_log, :manage_users, :manage_invites, :manage_taxonomies, :manage_federation, :manage_blocks, :manage_ng_words, :manage_sensitive_words) && !self_destruct } do |s|
s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_path, highlights_on: %r{/admin/reports}, if: -> { current_user.can?(:manage_reports) }
s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_path(origin: 'local'), highlights_on: %r{/admin/accounts|/admin/pending_accounts|/admin/disputes|/admin/users}, if: -> { current_user.can?(:manage_users) }
s.item :ng_words, safe_join([fa_icon('list fw'), t('admin.ng_words.title')]), admin_ng_words_path, highlights_on: %r{/admin/ng_words}, if: -> { current_user.can?(:manage_ng_words) }
s.item :ng_words, safe_join([fa_icon('list fw'), t('admin.ng_words.title')]), admin_ng_words_path, highlights_on: %r{/admin/(ng_words|ngword_histories)}, if: -> { current_user.can?(:manage_ng_words) }
s.item :sensitive_words, safe_join([fa_icon('list fw'), t('admin.sensitive_words.title')]), admin_sensitive_words_path, highlights_on: %r{/admin/sensitive_words}, if: -> { current_user.can?(:manage_sensitive_words) }
s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path, if: -> { current_user.can?(:manage_invites) }
s.item :follow_recommendations, safe_join([fa_icon('user-plus fw'), t('admin.follow_recommendations.title')]), admin_follow_recommendations_path, highlights_on: %r{/admin/follow_recommendations}, if: -> { current_user.can?(:manage_taxonomies) }

1
config/routes/admin.rb
View file

@ -33,6 +33,7 @@ namespace :admin do
resources :action_logs, only: [:index]
resources :warning_presets, except: [:new, :show]
resource :ng_words, only: [:show, :create]
resources :ngword_histories, only: [:index]
resource :sensitive_words, only: [:show, :create]
resource :special_instances, only: [:show, :create]

17
db/migrate/20240216042730_create_ngword_histories.rb Normal file
View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
class CreateNgwordHistories < ActiveRecord::Migration[7.1]
def change
create_table :ngword_histories do |t|
t.string :uri, null: false
t.integer :target_type, null: false
t.integer :reason, null: false
t.string :text, null: false
t.string :keyword, null: false
t.timestamps
end
add_index :ngword_histories, [:uri, :keyword, :created_at], unique: false
end
end

13
db/schema.rb
View file

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.1].define(version: 2024_02_12_230358) do
ActiveRecord::Schema[7.1].define(version: 2024_02_16_042730) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@ -868,6 +868,17 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_12_230358) do
t.index ["target_account_id"], name: "index_mutes_on_target_account_id"
end
create_table "ngword_histories", force: :cascade do |t|
t.string "uri", null: false
t.integer "target_type", null: false
t.integer "reason", null: false
t.string "text", null: false
t.string "keyword", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["uri", "keyword", "created_at"], name: "index_ngword_histories_on_uri_and_keyword_and_created_at"
end
create_table "notifications", force: :cascade do |t|
t.bigint "activity_id", null: false
t.string "activity_type", null: false

75
spec/lib/activitypub/activity/create_spec.rb
View file

@ -1781,6 +1781,11 @@ RSpec.describe ActivityPub::Activity::Create do
it 'creates status' do
expect(sender.statuses.first).to_not be_nil
end
it 'does not record history' do
history = NgwordHistory.find_by(uri: object_json[:id])
expect(history).to be_nil
end
end
context 'when hit ng words' do
@ -1789,6 +1794,34 @@ RSpec.describe ActivityPub::Activity::Create do
it 'creates status' do
expect(sender.statuses.first).to be_nil
end
it 'records history' do
history = NgwordHistory.find_by(uri: object_json[:id])
expect(history).to_not be_nil
expect(history.status_blocked?).to be true
expect(history.within_ng_words?).to be true
expect(history.keyword).to eq ng_words
end
end
context 'when hit ng words but does not public visibility' do
let(:content) { 'hello, world!' }
let(:object_json) do
{
id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join,
type: 'Note',
content: content,
}
end
it 'creates status' do
expect(sender.statuses.first).to be_nil
end
it 'records history' do
history = NgwordHistory.find_by(uri: object_json[:id])
expect(history).to be_nil
end
end
context 'when mention from tags' do
@ -1799,6 +1832,7 @@ RSpec.describe ActivityPub::Activity::Create do
id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join,
type: 'Note',
content: content,
to: 'https://www.w3.org/ns/activitystreams#Public',
tag: [
{
type: 'Mention',
@ -1814,6 +1848,11 @@ RSpec.describe ActivityPub::Activity::Create do
it 'creates status' do
expect(sender.statuses.first).to_not be_nil
end
it 'does not record history' do
history = NgwordHistory.find_by(uri: object_json[:id])
expect(history).to be_nil
end
end
context 'with using ng words for stranger' do
@ -1822,6 +1861,14 @@ RSpec.describe ActivityPub::Activity::Create do
it 'creates status' do
expect(sender.statuses.first).to be_nil
end
it 'records history' do
history = NgwordHistory.find_by(uri: object_json[:id])
expect(history).to_not be_nil
expect(history.status_blocked?).to be true
expect(history.within_ng_words_for_stranger_mention?).to be true
expect(history.keyword).to eq ng_words_for_stranger_mention
end
end
context 'with using ng words for stranger but receiver is following him' do
@ -1836,6 +1883,11 @@ RSpec.describe ActivityPub::Activity::Create do
it 'creates status' do
expect(sender.statuses.first).to_not be_nil
end
it 'does not record history' do
history = NgwordHistory.find_by(uri: object_json[:id])
expect(history).to be_nil
end
end
context 'with using ng words for stranger but multiple receivers are partically following him' do
@ -1847,6 +1899,7 @@ RSpec.describe ActivityPub::Activity::Create do
id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join,
type: 'Note',
content: content,
to: 'https://www.w3.org/ns/activitystreams#Public',
tag: [
{
type: 'Mention',
@ -1868,6 +1921,14 @@ RSpec.describe ActivityPub::Activity::Create do
it 'creates status' do
expect(sender.statuses.first).to be_nil
end
it 'records history' do
history = NgwordHistory.find_by(uri: object_json[:id])
expect(history).to_not be_nil
expect(history.status_blocked?).to be true
expect(history.within_ng_words_for_stranger_mention?).to be true
expect(history.keyword).to eq ng_words_for_stranger_mention
end
end
end
@ -1880,6 +1941,7 @@ RSpec.describe ActivityPub::Activity::Create do
id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join,
type: 'Note',
content: 'ohagi peers',
to: 'https://www.w3.org/ns/activitystreams#Public',
inReplyTo: ActivityPub::TagManager.instance.uri_for(original_status),
}
end
@ -1888,6 +1950,14 @@ RSpec.describe ActivityPub::Activity::Create do
it 'creates status' do
expect(sender.statuses.first).to be_nil
end
it 'records history' do
history = NgwordHistory.find_by(uri: object_json[:id])
expect(history).to_not be_nil
expect(history.status_blocked?).to be true
expect(history.within_ng_words_for_stranger_mention?).to be true
expect(history.keyword).to eq ng_words_for_stranger_mention
end
end
context 'with following' do
@ -1901,6 +1971,11 @@ RSpec.describe ActivityPub::Activity::Create do
it 'creates status' do
expect(sender.statuses.first).to_not be_nil
end
it 'does not record history' do
history = NgwordHistory.find_by(uri: object_json[:id])
expect(history).to be_nil
end
end
end
end

9
spec/services/activitypub/process_account_service_spec.rb
View file

@ -308,12 +308,21 @@ RSpec.describe ActivityPub::ProcessAccountService, type: :service do
Setting.ng_words = ['Amazon']
subject
expect(account.reload.display_name).to eq 'Ohagi'
history = NgwordHistory.find_by(uri: payload[:id])
expect(history).to be_nil
end
it 'does not create account when ng word is set' do
Setting.ng_words = ['Ohagi']
subject
expect(account.reload.display_name).to_not eq 'Ohagi'
history = NgwordHistory.find_by(uri: payload[:id])
expect(history).to_not be_nil
expect(history.account_name_blocked?).to be true
expect(history.within_ng_words?).to be true
expect(history.keyword).to eq 'Ohagi'
end
end