Merge remote-tracking branch 'parent/main' into kbtopic-remove-quote

app/models/account.rb

@@ -89,6 +89,7 @@ class Account < ApplicationRecord
   include Account::Associations
   include Account::Avatar
   include Account::Counters
+  include Account::FaspConcern
   include Account::FinderConcern
   include Account::Header
   include Account::Interactions

app/models/concerns/account/fasp_concern.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Account::FaspConcern
+  extend ActiveSupport::Concern
+
+  included do
+    after_commit :announce_new_account_to_subscribed_fasp, on: :create
+    after_commit :announce_updated_account_to_subscribed_fasp, on: :update
+    after_commit :announce_deleted_account_to_subscribed_fasp, on: :destroy
+  end
+
+  private
+
+  def announce_new_account_to_subscribed_fasp
+    return unless Mastodon::Feature.fasp_enabled?
+    return unless discoverable?
+
+    uri = ActivityPub::TagManager.instance.uri_for(self)
+    Fasp::AnnounceAccountLifecycleEventWorker.perform_async(uri, 'new')
+  end
+
+  def announce_updated_account_to_subscribed_fasp
+    return unless Mastodon::Feature.fasp_enabled?
+    return unless discoverable? || saved_change_to_discoverable?
+
+    uri = ActivityPub::TagManager.instance.uri_for(self)
+    Fasp::AnnounceAccountLifecycleEventWorker.perform_async(uri, 'update')
+  end
+
+  def announce_deleted_account_to_subscribed_fasp
+    return unless Mastodon::Feature.fasp_enabled?
+    return unless discoverable?
+
+    uri = ActivityPub::TagManager.instance.uri_for(self)
+    Fasp::AnnounceAccountLifecycleEventWorker.perform_async(uri, 'delete')
+  end
+end

app/models/concerns/favourite/fasp_concern.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Favourite::FaspConcern
+  extend ActiveSupport::Concern
+
+  included do
+    after_commit :announce_trends_to_subscribed_fasp, on: :create
+  end
+
+  private
+
+  def announce_trends_to_subscribed_fasp
+    return unless Mastodon::Feature.fasp_enabled?
+
+    Fasp::AnnounceTrendWorker.perform_async(status_id, 'favourite')
+  end
+end

app/models/concerns/legacy_otp_secret.rb

@@ -1,77 +0,0 @@
-# frozen_string_literal: true
-
-# TODO: This file is here for legacy support during devise-two-factor upgrade.
-# It should be removed after all records have been migrated.
-
-module LegacyOtpSecret
-  extend ActiveSupport::Concern
-
-  private
-
-  # Decrypt and return the `encrypted_otp_secret` attribute which was used in
-  # prior versions of devise-two-factor
-  # @return [String] The decrypted OTP secret
-  def legacy_otp_secret
-    return nil unless self[:encrypted_otp_secret]
-    return nil unless self.class.otp_secret_encryption_key
-
-    hmac_iterations = 2000 # a default set by the Encryptor gem
-    key = self.class.otp_secret_encryption_key
-    salt = Base64.decode64(encrypted_otp_secret_salt)
-    iv = Base64.decode64(encrypted_otp_secret_iv)
-
-    raw_cipher_text = Base64.decode64(encrypted_otp_secret)
-    # The last 16 bytes of the ciphertext are the authentication tag - we use
-    # Galois Counter Mode which is an authenticated encryption mode
-    cipher_text = raw_cipher_text[0..-17]
-    auth_tag =  raw_cipher_text[-16..-1] # rubocop:disable Style/SlicingWithRange
-
-    # this alrorithm lifted from
-    # https://github.com/attr-encrypted/encryptor/blob/master/lib/encryptor.rb#L54
-
-    # create an OpenSSL object which will decrypt the AES cipher with 256 bit
-    # keys in Galois Counter Mode (GCM). See
-    # https://ruby.github.io/openssl/OpenSSL/Cipher.html
-    cipher = OpenSSL::Cipher.new('aes-256-gcm')
-
-    # tell the cipher we want to decrypt. Symmetric algorithms use a very
-    # similar process for encryption and decryption, hence the same object can
-    # do both.
-    cipher.decrypt
-
-    # Use a Password-Based Key Derivation Function to generate the key actually
-    # used for encryptoin from the key we got as input.
-    cipher.key = OpenSSL::PKCS5.pbkdf2_hmac_sha1(key, salt, hmac_iterations, cipher.key_len)
-
-    # set the Initialization Vector (IV)
-    cipher.iv = iv
-
-    # The tag must be set after calling Cipher#decrypt, Cipher#key= and
-    # Cipher#iv=, but before calling Cipher#final. After all decryption is
-    # performed, the tag is verified automatically in the call to Cipher#final.
-    #
-    # If the auth_tag does not verify, then #final will raise OpenSSL::Cipher::CipherError
-    cipher.auth_tag = auth_tag
-
-    # auth_data must be set after auth_tag has been set when decrypting See
-    # http://ruby-doc.org/stdlib-2.0.0/libdoc/openssl/rdoc/OpenSSL/Cipher.html#method-i-auth_data-3D
-    # we are not adding any authenticated data but OpenSSL docs say this should
-    # still be called.
-    cipher.auth_data = ''
-
-    # #update is (somewhat confusingly named) the method which actually
-    # performs the decryption on the given chunk of data. Our OTP secret is
-    # short so we only need to call it once.
-    #
-    # It is very important that we call #final because:
-    #
-    # 1. The authentication tag is checked during the call to #final
-    # 2. Block based cipher modes (e.g. CBC) work on fixed size chunks. We need
-    #    to call #final to get it to process the last chunk properly. The output
-    #    of #final should be appended to the decrypted value. This isn't
-    #    required for streaming cipher modes but including it is a best practice
-    #    so that your code will continue to function correctly even if you later
-    #    change to a block cipher mode.
-    cipher.update(cipher_text) + cipher.final
-  end
-end

app/models/concerns/status/fasp_concern.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Status::FaspConcern
+  extend ActiveSupport::Concern
+
+  included do
+    after_commit :announce_new_content_to_subscribed_fasp, on: :create
+    after_commit :announce_updated_content_to_subscribed_fasp, on: :update
+    after_commit :announce_deleted_content_to_subscribed_fasp, on: :destroy
+    after_commit :announce_trends_to_subscribed_fasp, on: :create
+  end
+
+  private
+
+  def announce_new_content_to_subscribed_fasp
+    return unless Mastodon::Feature.fasp_enabled?
+    return unless account_indexable? && public_visibility?
+
+    # We need the uri here, but it is set in another `after_commit`
+    # callback. Hooks included from modules are run before the ones
+    # in the class itself and can neither be reordered nor is there
+    # a way to declare dependencies.
+    store_uri if uri.nil?
+    Fasp::AnnounceContentLifecycleEventWorker.perform_async(uri, 'new')
+  end
+
+  def announce_updated_content_to_subscribed_fasp
+    return unless Mastodon::Feature.fasp_enabled?
+    return unless account_indexable? && public_visibility?
+
+    Fasp::AnnounceContentLifecycleEventWorker.perform_async(uri, 'update')
+  end
+
+  def announce_deleted_content_to_subscribed_fasp
+    return unless Mastodon::Feature.fasp_enabled?
+    return unless account_indexable? && public_visibility?
+
+    Fasp::AnnounceContentLifecycleEventWorker.perform_async(uri, 'delete')
+  end
+
+  def announce_trends_to_subscribed_fasp
+    return unless Mastodon::Feature.fasp_enabled?
+    return unless account_indexable?
+
+    candidate_id, trend_source =
+      if reblog_of_id
+        [reblog_of_id, 'reblog']
+      elsif in_reply_to_id
+        [in_reply_to_id, 'reply']
+      end
+    Fasp::AnnounceTrendWorker.perform_async(candidate_id, trend_source) if candidate_id
+  end
+end

app/models/fasp.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module Fasp
+  DATA_CATEGORIES = %w(account content).freeze
+
   def self.table_name_prefix
     'fasp_'
   end

app/models/fasp/backfill_request.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: fasp_backfill_requests
+#
+#  id               :bigint(8)        not null, primary key
+#  category         :string           not null
+#  cursor           :string
+#  fulfilled        :boolean          default(FALSE), not null
+#  max_count        :integer          default(100), not null
+#  created_at       :datetime         not null
+#  updated_at       :datetime         not null
+#  fasp_provider_id :bigint(8)        not null
+#
+class Fasp::BackfillRequest < ApplicationRecord
+  belongs_to :fasp_provider, class_name: 'Fasp::Provider'
+
+  validates :category, presence: true, inclusion: Fasp::DATA_CATEGORIES
+  validates :max_count, presence: true,
+                        numericality: { only_integer: true }
+
+  after_commit :queue_fulfillment_job, on: :create
+
+  def next_objects
+    @next_objects ||= base_scope.to_a
+  end
+
+  def next_uris
+    next_objects.map { |o| ActivityPub::TagManager.instance.uri_for(o) }
+  end
+
+  def more_objects_available?
+    return false if next_objects.empty?
+
+    base_scope.where(id: ...(next_objects.last.id)).any?
+  end
+
+  def advance!
+    if more_objects_available?
+      update!(cursor: next_objects.last.id)
+    else
+      update!(fulfilled: true)
+    end
+  end
+
+  private
+
+  def base_scope
+    result = category_scope.limit(max_count).order(id: :desc)
+    result = result.where(id: ...cursor) if cursor.present?
+    result
+  end
+
+  def category_scope
+    case category
+    when 'account'
+      Account.discoverable.without_instance_actor
+    when 'content'
+      Status.indexable
+    end
+  end
+
+  def queue_fulfillment_job
+    Fasp::BackfillWorker.perform_async(id)
+  end
+end

app/models/fasp/provider.rb

@@ -22,7 +22,9 @@
 class Fasp::Provider < ApplicationRecord
   include DebugConcern
 
+  has_many :fasp_backfill_requests, inverse_of: :fasp_provider, class_name: 'Fasp::BackfillRequest', dependent: :delete_all
   has_many :fasp_debug_callbacks, inverse_of: :fasp_provider, class_name: 'Fasp::DebugCallback', dependent: :delete_all
+  has_many :fasp_subscriptions, inverse_of: :fasp_provider, class_name: 'Fasp::Subscription', dependent: :delete_all
 
   validates :name, presence: true
   validates :base_url, presence: true, url: true

app/models/fasp/subscription.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: fasp_subscriptions
+#
+#  id                  :bigint(8)        not null, primary key
+#  category            :string           not null
+#  max_batch_size      :integer          not null
+#  subscription_type   :string           not null
+#  threshold_likes     :integer
+#  threshold_replies   :integer
+#  threshold_shares    :integer
+#  threshold_timeframe :integer
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#  fasp_provider_id    :bigint(8)        not null
+#
+class Fasp::Subscription < ApplicationRecord
+  TYPES = %w(lifecycle trends).freeze
+
+  belongs_to :fasp_provider, class_name: 'Fasp::Provider'
+
+  validates :category, presence: true, inclusion: Fasp::DATA_CATEGORIES
+  validates :subscription_type, presence: true,
+                                inclusion: TYPES
+
+  scope :category_content, -> { where(category: 'content') }
+  scope :category_account, -> { where(category: 'account') }
+  scope :lifecycle, -> { where(subscription_type: 'lifecycle') }
+  scope :trends, -> { where(subscription_type: 'trends') }
+
+  def threshold=(threshold)
+    self.threshold_timeframe = threshold['timeframe'] || 15
+    self.threshold_shares    = threshold['shares'] || 3
+    self.threshold_likes     = threshold['likes'] || 3
+    self.threshold_replies   = threshold['replies'] || 3
+  end
+
+  def timeframe_start
+    threshold_timeframe.minutes.ago
+  end
+end

app/models/favourite.rb

@@ -14,6 +14,7 @@
 
 class Favourite < ApplicationRecord
   include Paginable
+  include Favourite::FaspConcern
 
   update_index('statuses', :status)
 

app/models/form/admin_settings.rb

@@ -36,6 +36,7 @@ class Form::AdminSettings
     trendable_by_default
     show_domain_blocks
     show_domain_blocks_rationale
+    allow_referrer_origin
     noindex
     require_invite_text
     media_cache_retention_period
@@ -85,6 +86,7 @@ class Form::AdminSettings
   ).freeze
 
   BOOLEAN_KEYS = %i(
+    allow_referrer_origin
     timeline_preview
     activity_api_enabled
     peers_api_enabled

app/models/media_attachment.rb

@@ -123,6 +123,7 @@ class MediaAttachment < ApplicationRecord
         output: {
           'loglevel' => 'fatal',
           'map_metadata' => '-1',
+          'movflags' => 'faststart', # Move metadata to start of file so playback can begin before download finishes
           'c:v' => 'copy',
           'c:a' => 'copy',
         }.freeze,

app/models/rule.rb

@@ -19,7 +19,29 @@ class Rule < ApplicationRecord
 
   self.discard_column = :deleted_at
 
+  has_many :translations, inverse_of: :rule, class_name: 'RuleTranslation', dependent: :destroy
+  accepts_nested_attributes_for :translations, reject_if: :all_blank, allow_destroy: true
+
   validates :text, presence: true, length: { maximum: TEXT_SIZE_LIMIT }
 
   scope :ordered, -> { kept.order(priority: :asc, id: :asc) }
+
+  def move!(offset)
+    rules = Rule.ordered.to_a
+    position = rules.index(self)
+
+    rules.delete_at(position)
+    rules.insert(position + offset, self)
+
+    transaction do
+      rules.each.with_index do |rule, index|
+        rule.update!(priority: index)
+      end
+    end
+  end
+
+  def translation_for(locale)
+    @cached_translations ||= {}
+    @cached_translations[locale] ||= translations.where(language: [locale, locale.to_s.split('-').first]).order('length(language) desc').first || RuleTranslation.new(language: locale, text: text, hint: hint)
+  end
 end

app/models/rule_translation.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: rule_translations
+#
+#  id         :bigint(8)        not null, primary key
+#  hint       :text             default(""), not null
+#  language   :string           not null
+#  text       :text             default(""), not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#  rule_id    :bigint(8)        not null
+#
+class RuleTranslation < ApplicationRecord
+  belongs_to :rule
+
+  validates :language, presence: true, uniqueness: { scope: :rule_id }
+  validates :text, presence: true, length: { maximum: Rule::TEXT_SIZE_LIMIT }
+end

app/models/status.rb

@@ -42,6 +42,7 @@ class Status < ApplicationRecord
   include Paginable
   include RateLimitable
   include Status::DomainBlockConcern
+  include Status::FaspConcern
   include Status::FetchRepliesConcern
   include Status::SafeReblogInsert
   include Status::SearchConcern
@@ -214,7 +215,7 @@ class Status < ApplicationRecord
                    ],
                    thread: :account
 
-  delegate :domain, to: :account, prefix: true
+  delegate :domain, :indexable?, to: :account, prefix: true
 
   REAL_TIME_WINDOW = 6.hours
 

app/models/terms_of_service.rb

@@ -23,6 +23,8 @@ class TermsOfService < ApplicationRecord
 
   validate :effective_date_cannot_be_in_the_past
 
+  NOTIFICATION_ACTIVITY_CUTOFF = 1.year.freeze
+
   def published?
     published_at.present?
   end
@@ -39,8 +41,20 @@ class TermsOfService < ApplicationRecord
     notification_sent_at.present?
   end
 
+  def base_user_scope
+    User.confirmed.where(created_at: ..published_at).joins(:account)
+  end
+
+  def email_notification_cutoff
+    published_at - NOTIFICATION_ACTIVITY_CUTOFF
+  end
+
+  def scope_for_interstitial
+    base_user_scope.merge(Account.suspended).or(base_user_scope.where(current_sign_in_at: [nil, ...email_notification_cutoff]))
+  end
+
   def scope_for_notification
-    User.confirmed.joins(:account).merge(Account.without_suspended).where(created_at: (..published_at))
+    base_user_scope.merge(Account.without_suspended).where(current_sign_in_at: email_notification_cutoff...)
   end
 
   private

app/models/user.rb

@@ -15,9 +15,6 @@
 #  current_sign_in_at        :datetime
 #  disabled                  :boolean          default(FALSE), not null
 #  email                     :string           default(""), not null
-#  encrypted_otp_secret      :string
-#  encrypted_otp_secret_iv   :string
-#  encrypted_otp_secret_salt :string
 #  encrypted_password        :string           default(""), not null
 #  last_emailed_at           :datetime
 #  last_sign_in_at           :datetime
@@ -25,6 +22,7 @@
 #  otp_backup_codes          :string           is an Array
 #  otp_required_for_login    :boolean          default(FALSE), not null
 #  otp_secret                :string
+#  require_tos_interstitial  :boolean          default(FALSE), not null
 #  reset_password_sent_at    :datetime
 #  reset_password_token      :string
 #  settings                  :text
@@ -45,14 +43,17 @@
 
 class User < ApplicationRecord
   self.ignored_columns += %w(
+    admin
+    current_sign_in_ip
+    encrypted_otp_secret
+    encrypted_otp_secret_iv
+    encrypted_otp_secret_salt
+    filtered_languages
+    last_sign_in_ip
+    moderator
     remember_created_at
     remember_token
-    current_sign_in_ip
-    last_sign_in_ip
     skip_sign_in_token
-    filtered_languages
-    admin
-    moderator
   )
 
   include LanguagesHelper
@@ -75,11 +76,8 @@ class User < ApplicationRecord
   REACTION_DECK_MAX = 256
 
   devise :two_factor_authenticatable,
-         otp_secret_encryption_key: Rails.configuration.x.otp_secret,
          otp_secret_length: 32
 
-  include LegacyOtpSecret # Must be after the above `devise` line in order to override the legacy method
-
   devise :two_factor_backupable,
          otp_number_of_backup_codes: 10
 

app/models/user_settings.rb

@@ -21,6 +21,7 @@ class UserSettings
   setting :default_language, default: nil
   setting :default_sensitive, default: false
   setting :default_privacy, default: nil, in: %w(public public_unlisted login unlisted private)
+  setting :default_quote_policy, default: 'public', in: %w(public followers nobody)
   setting :stay_privacy, default: false
   setting :default_reblog_privacy, default: nil
   setting :disabled_visibilities, default: %w()