From 78a20bbc2d2a52b1a2dd0835b9932dda76e67dca Mon Sep 17 00:00:00 2001 From: KMY Date: Thu, 25 Apr 2024 18:31:41 +0900 Subject: [PATCH 01/16] Bump ruby version to 3.2.4 --- install/12.0/setup2.sh | 4 ++-- install/9.0/setup4.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/install/12.0/setup2.sh b/install/12.0/setup2.sh index 847cc62dd5..ef98eb8a99 100644 --- a/install/12.0/setup2.sh +++ b/install/12.0/setup2.sh @@ -6,8 +6,8 @@ Install Ruby EOF git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build -RUBY_CONFIGURE_OPTS=--with-jemalloc rbenv install 3.2.3 -rbenv global 3.2.3 +RUBY_CONFIGURE_OPTS=--with-jemalloc rbenv install 3.2.4 +rbenv global 3.2.4 cat << EOF diff --git a/install/9.0/setup4.sh b/install/9.0/setup4.sh index a278c21fed..2cc053882a 100644 --- a/install/9.0/setup4.sh +++ b/install/9.0/setup4.sh @@ -6,8 +6,8 @@ Install Ruby EOF git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build -RUBY_CONFIGURE_OPTS=--with-jemalloc rbenv install 3.2.3 -rbenv global 3.2.3 +RUBY_CONFIGURE_OPTS=--with-jemalloc rbenv install 3.2.4 +rbenv global 3.2.4 cat << EOF From 86adad74b46a1af86fee7936ae78015aacf37eca Mon Sep 17 00:00:00 2001 From: KMY Date: Thu, 25 Apr 2024 18:32:21 +0900 Subject: [PATCH 02/16] Bump version to 12.0 --- lib/mastodon/version.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index f9670f25f3..53a520b75a 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -14,8 +14,8 @@ module Mastodon def kmyblue_flag # 'LTS' - 'dev' - # nil + # 'dev' + nil end def major From 2432d870f5fff911d534d4a04929ecb1be1a73c8 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 29 May 2024 11:09:54 +0200 Subject: [PATCH 03/16] Update dependency rexml to 3.2.8 --- Gemfile.lock | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 620da84992..b4ac5e4f22 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -612,7 +612,8 @@ GEM responders (3.1.1) actionpack (>= 5.2) railties (>= 5.2) - rexml (3.2.6) + rexml (3.2.8) + strscan (>= 3.0.9) rotp (6.3.0) rouge (4.2.1) rpam2 (4.0.2) @@ -735,6 +736,7 @@ GEM stringio (3.1.0) strong_migrations (1.8.0) activerecord (>= 5.2) + strscan (3.0.9) swd (1.3.0) activesupport (>= 3) attr_required (>= 0.0.5) From 3807dd23524d2f27db6449768520b66f38d111cd Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 29 May 2024 10:15:06 +0200 Subject: [PATCH 04/16] Fix leaking Elasticsearch connections in Sidekiq processes (#30450) --- lib/mastodon/sidekiq_middleware.rb | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/lib/mastodon/sidekiq_middleware.rb b/lib/mastodon/sidekiq_middleware.rb index 3a747afb63..c5f4d8da35 100644 --- a/lib/mastodon/sidekiq_middleware.rb +++ b/lib/mastodon/sidekiq_middleware.rb @@ -8,6 +8,7 @@ class Mastodon::SidekiqMiddleware rescue Mastodon::HostValidationError # Do not retry rescue => e + clean_up_elasticsearch_connections! limit_backtrace_and_raise(e) ensure clean_up_sockets! @@ -25,6 +26,32 @@ class Mastodon::SidekiqMiddleware clean_up_statsd_socket! end + # This is a hack to immediately free up unused Elasticsearch connections. + # + # Indeed, Chewy creates one `Elasticsearch::Client` instance per thread, + # and each such client manages its long-lasting connection to + # Elasticsearch. + # + # As far as I know, neither `chewy`, `elasticsearch-transport` or even + # `faraday` provide a reliable way to immediately close a connection, and + # rely on the underlying object to be garbage-collected instead. + # + # Furthermore, `sidekiq` creates a new thread each time a job throws an + # exception, meaning that each failure will create a new connection, and + # the old one will only be closed on full garbage collection. + def clean_up_elasticsearch_connections! + return unless Chewy.enabled? && Chewy.current[:chewy_client].present? + + Chewy.client.transport.transport.connections.each do |connection| + # NOTE: This bit of code is tailored for the HTTPClient Faraday adapter + connection.connection.app.instance_variable_get(:@client)&.reset_all + end + + Chewy.current.delete(:chewy_client) + rescue + nil + end + def clean_up_redis_socket! RedisConfiguration.pool.checkin if Thread.current[:redis] Thread.current[:redis] = nil From 5ba5aa5c5cbd6afa26b44e81f58a0db385b1ad8c Mon Sep 17 00:00:00 2001 From: KMY Date: Thu, 30 May 2024 23:33:59 +0900 Subject: [PATCH 05/16] Normalize language code of incoming posts (#30403) --- app/lib/activitypub/parser/status_parser.rb | 11 ++-- .../activitypub/parser/status_parser_spec.rb | 50 +++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 spec/lib/activitypub/parser/status_parser_spec.rb diff --git a/app/lib/activitypub/parser/status_parser.rb b/app/lib/activitypub/parser/status_parser.rb index a77686dcb3..e982a12e63 100644 --- a/app/lib/activitypub/parser/status_parser.rb +++ b/app/lib/activitypub/parser/status_parser.rb @@ -3,6 +3,8 @@ class ActivityPub::Parser::StatusParser include JsonLdHelper + NORMALIZED_LOCALE_NAMES = LanguagesHelper::SUPPORTED_LOCALES.keys.index_by(&:downcase).freeze + # @param [Hash] json # @param [Hash] options # @option options [String] :followers_collection @@ -118,10 +120,13 @@ class ActivityPub::Parser::StatusParser end def language - @language ||= original_language || (misskey_software? ? 'ja' : nil) + lang = raw_language_code || (misskey_software? ? 'ja' : nil) + lang.presence && NORMALIZED_LOCALE_NAMES.fetch(lang.downcase.to_sym, lang) end - def original_language + private + + def raw_language_code if content_language_map? @object['contentMap'].keys.first elsif name_language_map? @@ -131,8 +136,6 @@ class ActivityPub::Parser::StatusParser end end - private - def audience_to as_array(@object['to'] || @json['to']).map { |x| value_or_id(x) } end diff --git a/spec/lib/activitypub/parser/status_parser_spec.rb b/spec/lib/activitypub/parser/status_parser_spec.rb new file mode 100644 index 0000000000..5d9f008db1 --- /dev/null +++ b/spec/lib/activitypub/parser/status_parser_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe ActivityPub::Parser::StatusParser do + subject { described_class.new(json) } + + let(:sender) { Fabricate(:account, followers_url: 'http://example.com/followers', domain: 'example.com', uri: 'https://example.com/actor') } + let(:follower) { Fabricate(:account, username: 'bob') } + + let(:json) do + { + '@context': 'https://www.w3.org/ns/activitystreams', + id: [ActivityPub::TagManager.instance.uri_for(sender), '#foo'].join, + type: 'Create', + actor: ActivityPub::TagManager.instance.uri_for(sender), + object: object_json, + }.with_indifferent_access + end + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), 'post1'].join('/'), + type: 'Note', + to: [ + 'https://www.w3.org/ns/activitystreams#Public', + ActivityPub::TagManager.instance.uri_for(follower), + ], + content: '@bob lorem ipsum', + contentMap: { + EN: '@bob lorem ipsum', + }, + published: 1.hour.ago.utc.iso8601, + updated: 1.hour.ago.utc.iso8601, + tag: { + type: 'Mention', + href: ActivityPub::TagManager.instance.uri_for(follower), + }, + } + end + + it 'correctly parses status' do + expect(subject).to have_attributes( + text: '@bob lorem ipsum', + uri: [ActivityPub::TagManager.instance.uri_for(sender), 'post1'].join('/'), + reply: false, + language: :en + ) + end +end From 8e788e260ed74ed8afaff85c233d63b28df6860e Mon Sep 17 00:00:00 2001 From: Emelia Smith Date: Wed, 29 May 2024 16:00:05 +0200 Subject: [PATCH 06/16] Fix: remove broken OAuth Application vacuuming & throttle OAuth Application registrations (#30316) Co-authored-by: Claire --- app/lib/vacuum/applications_vacuum.rb | 10 ---- app/workers/scheduler/vacuum_scheduler.rb | 5 -- config/initializers/rack_attack.rb | 4 ++ spec/config/initializers/rack/attack_spec.rb | 18 ++++++++ spec/lib/vacuum/applications_vacuum_spec.rb | 48 -------------------- 5 files changed, 22 insertions(+), 63 deletions(-) delete mode 100644 app/lib/vacuum/applications_vacuum.rb delete mode 100644 spec/lib/vacuum/applications_vacuum_spec.rb diff --git a/app/lib/vacuum/applications_vacuum.rb b/app/lib/vacuum/applications_vacuum.rb deleted file mode 100644 index ba88655f16..0000000000 --- a/app/lib/vacuum/applications_vacuum.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -class Vacuum::ApplicationsVacuum - def perform - Doorkeeper::Application.where(owner_id: nil) - .where.missing(:created_users, :access_tokens, :access_grants) - .where(created_at: ...1.day.ago) - .in_batches.delete_all - end -end diff --git a/app/workers/scheduler/vacuum_scheduler.rb b/app/workers/scheduler/vacuum_scheduler.rb index 5c6e74e3d4..417d0ab592 100644 --- a/app/workers/scheduler/vacuum_scheduler.rb +++ b/app/workers/scheduler/vacuum_scheduler.rb @@ -22,7 +22,6 @@ class Scheduler::VacuumScheduler preview_cards_vacuum, backups_vacuum, access_tokens_vacuum, - applications_vacuum, feeds_vacuum, imports_vacuum, list_statuses_vacuum, @@ -62,10 +61,6 @@ class Scheduler::VacuumScheduler Vacuum::ImportsVacuum.new end - def applications_vacuum - Vacuum::ApplicationsVacuum.new - end - def ng_histories_vacuum Vacuum::NgHistoriesVacuum.new end diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index fa1bdca544..1757ce5df1 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -105,6 +105,10 @@ class Rack::Attack req.authenticated_user_id if (req.post? && req.path.match?(API_DELETE_REBLOG_REGEX)) || (req.delete? && req.path.match?(API_DELETE_STATUS_REGEX)) end + throttle('throttle_oauth_application_registrations/ip', limit: 5, period: 10.minutes) do |req| + req.throttleable_remote_ip if req.post? && req.path == '/api/v1/apps' + end + throttle('throttle_sign_up_attempts/ip', limit: 25, period: 5.minutes) do |req| req.throttleable_remote_ip if req.post? && req.path_matches?('/auth') end diff --git a/spec/config/initializers/rack/attack_spec.rb b/spec/config/initializers/rack/attack_spec.rb index e25b7dfde9..0a388c2f41 100644 --- a/spec/config/initializers/rack/attack_spec.rb +++ b/spec/config/initializers/rack/attack_spec.rb @@ -131,4 +131,22 @@ describe Rack::Attack, type: :request do it_behaves_like 'throttled endpoint' end end + + describe 'throttle excessive oauth application registration requests by IP address' do + let(:throttle) { 'throttle_oauth_application_registrations/ip' } + let(:limit) { 5 } + let(:period) { 10.minutes } + let(:path) { '/api/v1/apps' } + let(:params) do + { + client_name: 'Throttle Test', + redirect_uris: 'urn:ietf:wg:oauth:2.0:oob', + scopes: 'read', + } + end + + let(:request) { -> { post path, params: params, headers: { 'REMOTE_ADDR' => remote_ip } } } + + it_behaves_like 'throttled endpoint' + end end diff --git a/spec/lib/vacuum/applications_vacuum_spec.rb b/spec/lib/vacuum/applications_vacuum_spec.rb deleted file mode 100644 index df5c860602..0000000000 --- a/spec/lib/vacuum/applications_vacuum_spec.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe Vacuum::ApplicationsVacuum do - subject { described_class.new } - - describe '#perform' do - let!(:app_with_token) { Fabricate(:application, created_at: 1.month.ago) } - let!(:app_with_grant) { Fabricate(:application, created_at: 1.month.ago) } - let!(:app_with_signup) { Fabricate(:application, created_at: 1.month.ago) } - let!(:app_with_owner) { Fabricate(:application, created_at: 1.month.ago, owner: Fabricate(:user)) } - let!(:unused_app) { Fabricate(:application, created_at: 1.month.ago) } - let!(:recent_app) { Fabricate(:application, created_at: 1.hour.ago) } - - before do - Fabricate(:access_token, application: app_with_token) - Fabricate(:access_grant, application: app_with_grant) - Fabricate(:user, created_by_application: app_with_signup) - - subject.perform - end - - it 'does not delete applications with valid access tokens' do - expect { app_with_token.reload }.to_not raise_error - end - - it 'does not delete applications with valid access grants' do - expect { app_with_grant.reload }.to_not raise_error - end - - it 'does not delete applications that were used to create users' do - expect { app_with_signup.reload }.to_not raise_error - end - - it 'does not delete owned applications' do - expect { app_with_owner.reload }.to_not raise_error - end - - it 'does not delete applications registered less than a day ago' do - expect { recent_app.reload }.to_not raise_error - end - - it 'deletes unused applications' do - expect { unused_app.reload }.to raise_error ActiveRecord::RecordNotFound - end - end -end From 4bfcf0d3f026e7e6626e5e6ea18619d1a504c134 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 30 May 2024 14:03:13 +0200 Subject: [PATCH 07/16] Merge pull request from GHSA-5fq7-3p3j-9vrf --- app/services/notify_service.rb | 20 ++++++++++++-------- spec/services/notify_service_spec.rb | 13 +++++++++++++ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb index 3716d38f92..b56fd4dd07 100644 --- a/app/services/notify_service.rb +++ b/app/services/notify_service.rb @@ -150,6 +150,9 @@ class NotifyService < BaseService end def statuses_that_mention_sender + # This queries private mentions from the recipient to the sender up in the thread. + # This allows up to 100 messages that do not match in the thread, allowing conversations + # involving multiple people. Status.count_by_sql([<<-SQL.squish, id: @notification.target_status.in_reply_to_id, recipient_id: @recipient.id, sender_id: @sender.id, depth_limit: 100]) WITH RECURSIVE ancestors(id, in_reply_to_id, mention_id, path, depth) AS ( SELECT s.id, s.in_reply_to_id, m.id, ARRAY[s.id], 0 @@ -157,16 +160,17 @@ class NotifyService < BaseService LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id WHERE s.id = :id UNION ALL - SELECT s.id, s.in_reply_to_id, m.id, st.path || s.id, st.depth + 1 - FROM ancestors st - JOIN statuses s ON s.id = st.in_reply_to_id - LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id - WHERE st.mention_id IS NULL AND NOT s.id = ANY(path) AND st.depth < :depth_limit + SELECT s.id, s.in_reply_to_id, m.id, ancestors.path || s.id, ancestors.depth + 1 + FROM ancestors + JOIN statuses s ON s.id = ancestors.in_reply_to_id + /* early exit if we already have a mention matching our requirements */ + LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id AND s.account_id = :recipient_id + WHERE ancestors.mention_id IS NULL AND NOT s.id = ANY(path) AND ancestors.depth < :depth_limit ) SELECT COUNT(*) - FROM ancestors st - JOIN statuses s ON s.id = st.id - WHERE st.mention_id IS NOT NULL AND s.visibility = 3 + FROM ancestors + JOIN statuses s ON s.id = ancestors.id + WHERE ancestors.mention_id IS NOT NULL AND s.account_id = :recipient_id AND s.visibility = 3 SQL end end diff --git a/spec/services/notify_service_spec.rb b/spec/services/notify_service_spec.rb index 514f634d7f..6064d2b050 100644 --- a/spec/services/notify_service_spec.rb +++ b/spec/services/notify_service_spec.rb @@ -309,6 +309,19 @@ RSpec.describe NotifyService do expect(subject.filter?).to be false end end + + context 'when the sender is mentioned in an unrelated message chain' do + before do + original_status = Fabricate(:status, visibility: :direct) + intermediary_status = Fabricate(:status, visibility: :direct, thread: original_status) + notification.target_status.update(thread: intermediary_status) + Fabricate(:mention, status: original_status, account: notification.from_account) + end + + it 'returns true' do + expect(subject.filter?).to be true + end + end end end end From 993bae2850db2a2c1c54da52610657abf50b6e86 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 30 May 2024 14:14:04 +0200 Subject: [PATCH 08/16] Merge pull request from GHSA-q3rg-xx5v-4mxh --- config/initializers/rack_attack.rb | 10 ++++++- spec/config/initializers/rack/attack_spec.rb | 31 ++++++++++++++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index 1757ce5df1..034fb7444d 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -30,13 +30,17 @@ class Rack::Attack end def authenticated_user_id - authenticated_token&.resource_owner_id + authenticated_token&.resource_owner_id || warden_user_id end def authenticated_token_id authenticated_token&.id end + def warden_user_id + @env['warden']&.user&.id + end + def unauthenticated? !authenticated_user_id end @@ -141,6 +145,10 @@ class Rack::Attack req.session[:attempt_user_id] || req.params.dig('user', 'email').presence if req.post? && req.path_matches?('/auth/sign_in') end + throttle('throttle_password_change/account', limit: 10, period: 10.minutes) do |req| + req.authenticated_user_id if req.put? || (req.patch? && req.path_matches?('/auth')) + end + self.throttled_responder = lambda do |request| now = Time.now.utc match_data = request.env['rack.attack.match_data'] diff --git a/spec/config/initializers/rack/attack_spec.rb b/spec/config/initializers/rack/attack_spec.rb index 0a388c2f41..19de480898 100644 --- a/spec/config/initializers/rack/attack_spec.rb +++ b/spec/config/initializers/rack/attack_spec.rb @@ -56,7 +56,7 @@ describe Rack::Attack, type: :request do end def throttle_count - described_class.cache.read("#{counter_prefix}:#{throttle}:#{remote_ip}") || 0 + described_class.cache.read("#{counter_prefix}:#{throttle}:#{discriminator}") || 0 end def counter_prefix @@ -64,11 +64,12 @@ describe Rack::Attack, type: :request do end def increment_counter - described_class.cache.count("#{throttle}:#{remote_ip}", period) + described_class.cache.count("#{throttle}:#{discriminator}", period) end end let(:remote_ip) { '1.2.3.5' } + let(:discriminator) { remote_ip } describe 'throttle excessive sign-up requests by IP address' do context 'when accessed through the website' do @@ -149,4 +150,30 @@ describe Rack::Attack, type: :request do it_behaves_like 'throttled endpoint' end + + describe 'throttle excessive password change requests by account' do + let(:user) { Fabricate(:user, email: 'user@host.example') } + let(:throttle) { 'throttle_password_change/account' } + let(:limit) { 10 } + let(:period) { 10.minutes } + let(:request) { -> { put path, headers: { 'REMOTE_ADDR' => remote_ip } } } + let(:path) { '/auth' } + let(:discriminator) { user.id } + + before do + sign_in user, scope: :user + + # Unfortunately, devise's `sign_in` helper causes the `session` to be + # loaded in the next request regardless of whether it's actually accessed + # by the client code. + # + # So, we make an extra query to clear issue a session cookie instead. + # + # A less resource-intensive way to deal with that would be to generate the + # session cookie manually, but this seems pretty involved. + get '/' + end + + it_behaves_like 'throttled endpoint' + end end From b75b26bab47d9e545eb577bb82925a8423cdef05 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 30 May 2024 14:24:29 +0200 Subject: [PATCH 09/16] Merge pull request from GHSA-c2r5-cfqr-c553 * Add hardening monkey-patch to prevent IP spoofing on misconfigured installations * Remove rack-attack safelist --- config/application.rb | 1 + config/initializers/rack_attack.rb | 4 -- lib/action_dispatch/remote_ip_extensions.rb | 72 +++++++++++++++++++++ 3 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 lib/action_dispatch/remote_ip_extensions.rb diff --git a/config/application.rb b/config/application.rb index 1b38789927..a74100e1c4 100644 --- a/config/application.rb +++ b/config/application.rb @@ -47,6 +47,7 @@ require_relative '../lib/chewy/strategy/bypass_with_warning' require_relative '../lib/webpacker/manifest_extensions' require_relative '../lib/webpacker/helper_extensions' require_relative '../lib/rails/engine_extensions' +require_relative '../lib/action_dispatch/remote_ip_extensions' require_relative '../lib/active_record/database_tasks_extensions' require_relative '../lib/active_record/batches' require_relative '../lib/simple_navigation/item_extensions' diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index 034fb7444d..b3739429e8 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -62,10 +62,6 @@ class Rack::Attack end end - Rack::Attack.safelist('allow from localhost') do |req| - req.remote_ip == '127.0.0.1' || req.remote_ip == '::1' - end - Rack::Attack.blocklist('deny from blocklist') do |req| IpBlock.blocked?(req.remote_ip) end diff --git a/lib/action_dispatch/remote_ip_extensions.rb b/lib/action_dispatch/remote_ip_extensions.rb new file mode 100644 index 0000000000..e5c48bf3c5 --- /dev/null +++ b/lib/action_dispatch/remote_ip_extensions.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +# Mastodon is not made to be directly accessed without a reverse proxy. +# This monkey-patch prevents remote IP address spoofing when being accessed +# directly. +# +# See PR: https://github.com/rails/rails/pull/51610 + +# In addition to the PR above, it also raises an error if a request with +# `X-Forwarded-For` or `Client-Ip` comes directly from a client without +# going through a trusted proxy. + +# rubocop:disable all -- This is a mostly vendored file + +module ActionDispatch + class RemoteIp + module GetIpExtensions + def calculate_ip + # Set by the Rack web server, this is a single value. + remote_addr = ips_from(@req.remote_addr).last + + # Could be a CSV list and/or repeated headers that were concatenated. + client_ips = ips_from(@req.client_ip).reverse! + forwarded_ips = ips_from(@req.x_forwarded_for).reverse! + + # `Client-Ip` and `X-Forwarded-For` should not, generally, both be set. If they + # are both set, it means that either: + # + # 1) This request passed through two proxies with incompatible IP header + # conventions. + # + # 2) The client passed one of `Client-Ip` or `X-Forwarded-For` + # (whichever the proxy servers weren't using) themselves. + # + # Either way, there is no way for us to determine which header is the right one + # after the fact. Since we have no idea, if we are concerned about IP spoofing + # we need to give up and explode. (If you're not concerned about IP spoofing you + # can turn the `ip_spoofing_check` option off.) + should_check_ip = @check_ip && client_ips.last && forwarded_ips.last + if should_check_ip && !forwarded_ips.include?(client_ips.last) + # We don't know which came from the proxy, and which from the user + raise IpSpoofAttackError, "IP spoofing attack?! " \ + "HTTP_CLIENT_IP=#{@req.client_ip.inspect} " \ + "HTTP_X_FORWARDED_FOR=#{@req.x_forwarded_for.inspect}" + end + + # NOTE: Mastodon addition to make sure we don't get requests from a non-trusted client + if @check_ip && (forwarded_ips.last || client_ips.last) && !@proxies.any? { |proxy| proxy === remote_addr } + raise IpSpoofAttackError, "IP spoofing attack?! client #{remote_addr} is not a trusted proxy " \ + "HTTP_CLIENT_IP=#{@req.client_ip.inspect} " \ + "HTTP_X_FORWARDED_FOR=#{@req.x_forwarded_for.inspect}" + end + + # We assume these things about the IP headers: + # + # - X-Forwarded-For will be a list of IPs, one per proxy, or blank + # - Client-Ip is propagated from the outermost proxy, or is blank + # - REMOTE_ADDR will be the IP that made the request to Rack + ips = forwarded_ips + client_ips + ips.compact! + + # If every single IP option is in the trusted list, return the IP that's + # furthest away + filter_proxies([remote_addr] + ips).first || ips.last || remote_addr + end + end + end +end + +ActionDispatch::RemoteIp::GetIp.prepend(ActionDispatch::RemoteIp::GetIpExtensions) + +# rubocop:enable all From 2a05566c5c4d756ad8f3df6e8bc2b00a15d30970 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 30 May 2024 14:56:18 +0200 Subject: [PATCH 10/16] Fix rate-limiting incorrectly triggering a session cookie on most endpoints (#30483) --- config/initializers/rack_attack.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index b3739429e8..14fab7ecda 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -30,7 +30,7 @@ class Rack::Attack end def authenticated_user_id - authenticated_token&.resource_owner_id || warden_user_id + authenticated_token&.resource_owner_id end def authenticated_token_id @@ -142,7 +142,7 @@ class Rack::Attack end throttle('throttle_password_change/account', limit: 10, period: 10.minutes) do |req| - req.authenticated_user_id if req.put? || (req.patch? && req.path_matches?('/auth')) + req.warden_user_id if req.put? || (req.patch? && req.path_matches?('/auth')) end self.throttled_responder = lambda do |request| From d65f8a119688a9cad86286cd7de537bcf5fc1ed3 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 30 May 2024 15:34:46 +0200 Subject: [PATCH 11/16] Bump version to v4.3.0-alpha.4 (#30482) --- CHANGELOG.md | 55 +++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 10 +++----- lib/mastodon/version.rb | 2 +- 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a53790afaf..c9b24d6f15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,61 @@ All notable changes to this project will be documented in this file. +## [4.2.9] - 2024-05-30 + +### Security + +- Update dependencies +- Fix private mention filtering ([GHSA-5fq7-3p3j-9vrf](https://github.com/mastodon/mastodon/security/advisories/GHSA-5fq7-3p3j-9vrf)) +- Fix password change endpoint not being rate-limited ([GHSA-q3rg-xx5v-4mxh](https://github.com/mastodon/mastodon/security/advisories/GHSA-q3rg-xx5v-4mxh)) +- Add hardening around rate-limit bypass ([GHSA-c2r5-cfqr-c553](https://github.com/mastodon/mastodon/security/advisories/GHSA-c2r5-cfqr-c553)) + +### Added + +- Add rate-limit on OAuth application registration ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30316)) +- Add fallback redirection when getting a webfinger query `WEB_DOMAIN@WEB_DOMAIN` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28592)) +- Add `digest` attribute to `Admin::DomainBlock` entity in REST API ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/29092)) + +### Removed + +- Remove superfluous application-level caching in some controllers ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29862)) +- Remove aggressive OAuth application vacuuming ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30316)) + +### Fixed + +- Fix leaking Elasticsearch connections in Sidekiq processes ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30450)) +- Fix language of remote posts not being recognized when using unusual casing ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30403)) +- Fix off-by-one in `tootctl media` commands ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30306)) +- Fix removal of allowed domains (in `LIMITED_FEDERATION_MODE`) not being recorded in the audit log ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30125)) +- Fix not being able to block a subdomain of an already-blocked domain through the API ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30119)) +- Fix `Idempotency-Key` being ignored when scheduling a post ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30084)) +- Fix crash when supplying the `FFMPEG_BINARY` environment variable ([timothyjrogers](https://github.com/mastodon/mastodon/pull/30022)) +- Fix improper email address validation ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29838)) +- Fix results/query in `api/v1/featured_tags/suggestions` ([mjankowski](https://github.com/mastodon/mastodon/pull/29597)) +- Fix unblocking internationalized domain names under certain conditions ([tribela](https://github.com/mastodon/mastodon/pull/29530)) +- Fix admin account created by `mastodon:setup` not being auto-approved ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29379)) +- Fix reference to non-existent var in CLI maintenance command ([mjankowski](https://github.com/mastodon/mastodon/pull/28363)) + +## [4.2.8] - 2024-02-23 + +### Added + +- Add hourly task to automatically require approval for new registrations in the absence of moderators ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29318), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/29355)) + In order to prevent future abandoned Mastodon servers from being used for spam, harassment and other malicious activity, Mastodon will now automatically switch new user registrations to require moderator approval whenever they are left open and no activity (including non-moderation actions from apps) from any logged-in user with permission to access moderation reports has been detected in a full week. + When this happens, users with the permission to change server settings will receive an email notification. + This feature is disabled when `EMAIL_DOMAIN_ALLOWLIST` is used, and can also be disabled with `DISABLE_AUTOMATIC_SWITCHING_TO_APPROVED_REGISTRATIONS=true`. + +### Changed + +- Change registrations to be closed by default on new installations ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29280)) + If you are running a server and never changed your registrations mode from the default, updating will automatically close your registrations. + Simply re-enable them through the administration interface or using `tootctl settings registrations open` if you want to enable them again. + +### Fixed + +- Fix processing of remote ActivityPub actors making use of `Link` objects as `Image` `url` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29335)) +- Fix link verifications when page size exceeds 1MB ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29358)) + ## [4.2.7] - 2024-02-16 ### Fixed diff --git a/docker-compose.yml b/docker-compose.yml index 526ed896c7..d50c2353f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,7 +56,7 @@ services: web: build: . - image: ghcr.io/mastodon/mastodon:v4.2.7 + image: ghcr.io/mastodon/mastodon:v4.2.9 restart: always env_file: .env.production command: bundle exec puma -C config/puma.rb @@ -76,10 +76,8 @@ services: - ./public/system:/mastodon/public/system streaming: - build: - context: . - dockerfile: streaming/Dockerfile - image: ghcr.io/mastodon/mastodon:v4.2.7 + build: . + image: ghcr.io/mastodon/mastodon:v4.2.9 restart: always env_file: .env.production command: node ./streaming @@ -97,7 +95,7 @@ services: sidekiq: build: . - image: ghcr.io/mastodon/mastodon:v4.2.7 + image: ghcr.io/mastodon/mastodon:v4.2.9 restart: always env_file: .env.production command: bundle exec sidekiq diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 53a520b75a..4c740d2255 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -31,7 +31,7 @@ module Mastodon end def default_prerelease - 'alpha.3' + 'alpha.4' end def prerelease From 28b74eabace197233fbdad1feb77d499163fb8d7 Mon Sep 17 00:00:00 2001 From: KMY Date: Thu, 30 May 2024 23:36:44 +0900 Subject: [PATCH 12/16] Bump version to 12.1 --- lib/mastodon/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 4c740d2255..a1050219f8 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -9,7 +9,7 @@ module Mastodon end def kmyblue_minor - 0 + 1 end def kmyblue_flag From 35bd3ea5b70a3afe3425ff5d39a992edded6d0e4 Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 31 May 2024 08:08:12 +0900 Subject: [PATCH 13/16] Fix dicker-compose --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d50c2353f5..e7ae95ea7a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3' services: db: restart: always From 7615e12e8890e1a0bba375620993e7c297446bb4 Mon Sep 17 00:00:00 2001 From: KMY Date: Fri, 31 May 2024 08:11:57 +0900 Subject: [PATCH 14/16] Fix test --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index a10613b30b..475adea0e3 100644 --- a/Gemfile +++ b/Gemfile @@ -64,7 +64,7 @@ gem 'kaminari', '~> 1.2' gem 'link_header', '~> 0.0' gem 'mario-redis-lock', '~> 1.2', require: 'redis_lock' gem 'mime-types', '~> 3.5.0', require: 'mime/types/columnar' -gem 'nokogiri', '~> 1.15' +gem 'nokogiri', '~> 1.16.5' gem 'nsa' gem 'oj', '~> 3.14' gem 'ox', '~> 2.14' diff --git a/Gemfile.lock b/Gemfile.lock index b4ac5e4f22..6a871fac0f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -454,7 +454,7 @@ GEM net-smtp (0.5.0) net-protocol nio4r (2.7.1) - nokogiri (1.16.4) + nokogiri (1.16.5) mini_portile2 (~> 2.8.2) racc (~> 1.4) nsa (0.3.0) @@ -887,7 +887,7 @@ DEPENDENCIES mime-types (~> 3.5.0) net-http (~> 0.4.0) net-ldap (~> 0.18) - nokogiri (~> 1.15) + nokogiri (~> 1.16.5) nsa oj (~> 3.14) omniauth (~> 2.0) From 0bd26af2dd7a22c34b92169cc1391abcb44564de Mon Sep 17 00:00:00 2001 From: KMY Date: Sun, 2 Jun 2024 11:03:56 +0900 Subject: [PATCH 15/16] =?UTF-8?q?Fix:=20=E7=B5=B5=E6=96=87=E5=AD=97?= =?UTF-8?q?=E3=83=AA=E3=82=A2=E3=82=AF=E3=82=B7=E3=83=A7=E3=83=B3=E3=81=AB?= =?UTF-8?q?=E5=8E=B3=E3=81=97=E3=81=84=E3=83=AC=E3=83=BC=E3=83=88=E3=83=AA?= =?UTF-8?q?=E3=83=9F=E3=83=83=E3=83=88=E3=81=8C=E9=81=A9=E7=94=A8=E3=81=95?= =?UTF-8?q?=E3=82=8C=E3=82=8B=E5=95=8F=E9=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/initializers/rack_attack.rb | 4 +- spec/config/initializers/rack/attack_spec.rb | 83 ++++++++++++++++---- 2 files changed, 70 insertions(+), 17 deletions(-) diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index 14fab7ecda..a3c2b821cf 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -141,8 +141,10 @@ class Rack::Attack req.session[:attempt_user_id] || req.params.dig('user', 'email').presence if req.post? && req.path_matches?('/auth/sign_in') end + API_CREATE_EMOJI_REACTION_REGEX = %r{\A/api/v1/statuses/\d+/emoji_reactions} + throttle('throttle_password_change/account', limit: 10, period: 10.minutes) do |req| - req.warden_user_id if req.put? || (req.patch? && req.path_matches?('/auth')) + req.warden_user_id if (req.put? && !req.path.match?(API_CREATE_EMOJI_REACTION_REGEX)) || (req.patch? && req.path_matches?('/auth')) end self.throttled_responder = lambda do |request| diff --git a/spec/config/initializers/rack/attack_spec.rb b/spec/config/initializers/rack/attack_spec.rb index 19de480898..59d0462695 100644 --- a/spec/config/initializers/rack/attack_spec.rb +++ b/spec/config/initializers/rack/attack_spec.rb @@ -7,7 +7,7 @@ describe Rack::Attack, type: :request do Rails.application end - shared_examples 'throttled endpoint' do + shared_context 'with throttled endpoint base' do before do # Rack::Attack periods are not rolling, so avoid flaky tests by setting the time in a way # to avoid crossing period boundaries. @@ -19,6 +19,30 @@ describe Rack::Attack, type: :request do travel_to Time.zone.at(counter_prefix * period.seconds) end + def below_limit + limit - 1 + end + + def above_limit + limit * 2 + end + + def throttle_count + described_class.cache.read("#{counter_prefix}:#{throttle}:#{discriminator}") || 0 + end + + def counter_prefix + (Time.now.to_i / period.seconds).to_i + end + + def increment_counter + described_class.cache.count("#{throttle}:#{discriminator}", period) + end + end + + shared_examples 'throttled endpoint' do + include_examples 'with throttled endpoint base' + context 'when the number of requests is lower than the limit' do before do below_limit.times { increment_counter } @@ -46,25 +70,32 @@ describe Rack::Attack, type: :request do expect(response).to_not have_http_status(429) end end + end - def below_limit - limit - 1 + shared_examples 'does not throttle endpoint' do + include_examples 'with throttled endpoint base' + + context 'when the number of requests is lower than the limit' do + before do + below_limit.times { increment_counter } + end + + it 'does not change the request status' do + expect { request.call }.to change { throttle_count }.by(0) + + expect(response).to_not have_http_status(429) + end end - def above_limit - limit * 2 - end + context 'when the number of requests is higher than the limit' do + before do + above_limit.times { increment_counter } + end - def throttle_count - described_class.cache.read("#{counter_prefix}:#{throttle}:#{discriminator}") || 0 - end - - def counter_prefix - (Time.now.to_i / period.seconds).to_i - end - - def increment_counter - described_class.cache.count("#{throttle}:#{discriminator}", period) + it 'returns http too many requests after limit and returns to normal status after period' do + expect { request.call }.to change { throttle_count }.by(0) + expect(response).to_not have_http_status(429) + end end end @@ -176,4 +207,24 @@ describe Rack::Attack, type: :request do it_behaves_like 'throttled endpoint' end + + describe 'throttle excessive emoji reaction requests by account' do + let(:user) { Fabricate(:user, email: 'user@host.example') } + let(:throttle) { 'throttle_password_change/account' } + let(:limit) { 10 } + let(:period) { 10.minutes } + let(:request) { -> { put path, headers: { 'REMOTE_ADDR' => remote_ip } } } + let(:status) { Fabricate(:status) } + let(:emoji) { Fabricate(:custom_emoji) } + let(:path) { "/api/v1/statuses/#{status.id}/emoji_reactions/#{emoji.shortcode}" } + let(:discriminator) { user.id } + + before do + sign_in user, scope: :user + + get '/' + end + + it_behaves_like 'does not throttle endpoint' + end end From 58040c5aa7c4aba5f83bed033c2ed5fa31eb8c1c Mon Sep 17 00:00:00 2001 From: KMY Date: Sun, 2 Jun 2024 11:21:08 +0900 Subject: [PATCH 16/16] Bump version to 12.2 --- lib/mastodon/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index a1050219f8..f77f02d514 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -9,7 +9,7 @@ module Mastodon end def kmyblue_minor - 1 + 2 end def kmyblue_flag