Merge commit 'fd284311e7' into kb_migration

app/lib/activitypub/activity/flag.rb

@@ -9,7 +9,7 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
 
     target_accounts.each do |target_account|
       target_statuses     = target_statuses_by_account[target_account.id]
-      replied_to_accounts = Account.local.where(id: target_statuses.filter_map(&:in_reply_to_account_id))
+      replied_to_accounts = target_statuses.nil? ? [] : Account.local.where(id: target_statuses.filter_map(&:in_reply_to_account_id))
 
       next if target_account.suspended? || (!target_account.local? && replied_to_accounts.none?)
 

app/lib/attachment_batch.rb

@@ -76,6 +76,9 @@ class AttachmentBatch
           when :fog
             logger.debug { "Deleting #{attachment.path(style)}" }
             attachment.directory.files.new(key: attachment.path(style)).destroy
+          when :azure
+            logger.debug { "Deleting #{attachment.path(style)}" }
+            attachment.destroy
           end
         end
       end

app/lib/importer/base_importer.rb

@@ -45,8 +45,11 @@ class Importer::BaseImporter
   # Remove documents from the index that no longer exist in the database
   def clean_up!
     index.scroll_batches do |documents|
+      primary_key = index.adapter.target.primary_key
+      raise ActiveRecord::UnknownPrimaryKey, index.adapter.target if primary_key.nil?
+
       ids           = documents.pluck('_id')
-      existence_map = index.adapter.target.where(id: ids).pluck(:id).each_with_object({}) { |id, map| map[id.to_s] = true }
+      existence_map = index.adapter.target.where(primary_key => ids).pluck(primary_key).each_with_object({}) { |id, map| map[id.to_s] = true }
       tmp           = ids.reject { |id| existence_map[id] }
 
       next if tmp.empty?

app/lib/importer/instances_index_importer.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class Importer::InstancesIndexImporter < Importer::BaseImporter
+  def import!
+    index.adapter.default_scope.find_in_batches(batch_size: @batch_size) do |tmp|
+      in_work_unit(tmp) do |instances|
+        bulk = Chewy::Index::Import::BulkBuilder.new(index, to_index: instances).bulk_body
+
+        indexed = bulk.count { |entry| entry[:index] }
+        deleted = bulk.count { |entry| entry[:delete] }
+
+        Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
+
+        [indexed, deleted]
+      end
+    end
+
+    wait!
+  end
+
+  private
+
+  def index
+    InstancesIndex
+  end
+end

app/lib/link_details_extractor.rb

@@ -124,7 +124,7 @@ class LinkDetailsExtractor
       author_url: author_url || '',
       embed_url: embed_url || '',
       language: language,
-      created_at: published_at.presence || Time.now.utc,
+      published_at: published_at.presence,
     }
   end
 

app/lib/request.rb

@@ -68,13 +68,26 @@ class Request
   # about 15s in total
   TIMEOUT = { connect_timeout: 5, read_timeout: 10, write_timeout: 10, read_deadline: 30 }.freeze
 
+  # Workaround for overly-eager decoding of percent-encoded characters in Addressable::URI#normalized_path
+  # https://github.com/sporkmonger/addressable/issues/366
+  URI_NORMALIZER = lambda do |uri|
+    uri = HTTP::URI.parse(uri)
+
+    HTTP::URI.new(
+      scheme: uri.normalized_scheme,
+      authority: uri.normalized_authority,
+      path: Addressable::URI.normalize_path(uri.path),
+      query: uri.query
+    )
+  end
+
   include RoutingHelper
 
   def initialize(verb, url, **options)
     raise ArgumentError if url.blank?
 
     @verb        = verb
-    @url         = Addressable::URI.parse(url).normalize
+    @url         = URI_NORMALIZER.call(url)
     @http_client = options.delete(:http_client)
     @allow_local = options.delete(:allow_local)
     @options     = options.merge(socket_class: use_proxy? || @allow_local ? ProxySocket : Socket)
@@ -139,7 +152,7 @@ class Request
     end
 
     def http_client
-      HTTP.use(:auto_inflate).follow(max_hops: 3)
+      HTTP.use(:auto_inflate).use(normalize_uri: { normalizer: URI_NORMALIZER }).follow(max_hops: 3)
     end
   end
 

app/lib/rss/builder.rb

@@ -14,13 +14,14 @@ class RSS::Builder
   end
 
   def to_xml
-    ('<?xml version="1.0" encoding="UTF-8"?>'.dup << Ox.dump(wrap_in_document, effort: :tolerant)).force_encoding('UTF-8')
+    Ox.dump(wrap_in_document, effort: :tolerant).force_encoding('UTF-8')
   end
 
   private
 
   def wrap_in_document
     Ox::Document.new(version: '1.0').tap do |document|
+      document << xml_instruct
       document << Ox::Element.new('rss').tap do |rss|
         rss['version']        = '2.0'
         rss['xmlns:webfeeds'] = 'http://webfeeds.org/rss/1.0'
@@ -30,4 +31,11 @@ class RSS::Builder
       end
     end
   end
+
+  def xml_instruct
+    Ox::Instruct.new(:xml).tap do |instruct|
+      instruct[:version] = '1.0'
+      instruct[:encoding] = 'UTF-8'
+    end
+  end
 end

app/lib/status_cache_hydrator.rb

@@ -11,7 +11,7 @@ class StatusCacheHydrator
 
     # If we're delivering to the author who disabled the display of the application used to create the
     # status, we need to hydrate the application, since it was not rendered for the basic payload
-    payload[:application] = @status.application.present? ? ActiveModelSerializers::SerializableResource.new(@status.application, serializer: REST::StatusSerializer::ApplicationSerializer).as_json : nil if payload[:application].nil? && @status.account_id == account_id
+    payload[:application] = payload_application if payload[:application].nil? && @status.account_id == account_id
 
     # We take advantage of the fact that some relationships can only occur with an original status, not
     # the reblog that wraps it, so we can assume that some values are always false
@@ -19,11 +19,13 @@ class StatusCacheHydrator
       payload[:muted]      = false
       payload[:bookmarked] = false
       payload[:pinned]     = false if @status.account_id == account_id
-      payload[:filtered]   = CustomFilter.apply_cached_filters(CustomFilter.cached_filters_for(account_id), @status.reblog).map { |filter| ActiveModelSerializers::SerializableResource.new(filter, serializer: REST::FilterResultSerializer).as_json }
+      payload[:filtered]   = CustomFilter
+                             .apply_cached_filters(CustomFilter.cached_filters_for(account_id), @status.reblog)
+                             .map { |filter| serialized_filter(filter) }
 
       # If the reblogged status is being delivered to the author who disabled the display of the application
       # used to create the status, we need to hydrate it here too
-      payload[:reblog][:application] = @status.reblog.application.present? ? ActiveModelSerializers::SerializableResource.new(@status.reblog.application, serializer: REST::StatusSerializer::ApplicationSerializer).as_json : nil if payload[:reblog][:application].nil? && @status.reblog.account_id == account_id
+      payload[:reblog][:application] = payload_reblog_application if payload[:reblog][:application].nil? && @status.reblog.account_id == account_id
 
       payload[:reblog][:favourited] = Favourite.where(account_id: account_id, status_id: @status.reblog_of_id).exists?
       payload[:reblog][:reblogged]  = Status.where(account_id: account_id, reblog_of_id: @status.reblog_of_id).exists?
@@ -51,7 +53,9 @@ class StatusCacheHydrator
       payload[:muted]      = ConversationMute.where(account_id: account_id, conversation_id: @status.conversation_id).exists?
       payload[:bookmarked] = Bookmark.where(account_id: account_id, status_id: @status.id).exists?
       payload[:pinned]     = StatusPin.where(account_id: account_id, status_id: @status.id).exists? if @status.account_id == account_id
-      payload[:filtered]   = CustomFilter.apply_cached_filters(CustomFilter.cached_filters_for(account_id), @status).map { |filter| ActiveModelSerializers::SerializableResource.new(filter, serializer: REST::FilterResultSerializer).as_json }
+      payload[:filtered]   = CustomFilter
+                             .apply_cached_filters(CustomFilter.cached_filters_for(account_id), @status)
+                             .map { |filter| serialized_filter(filter) }
 
       if payload[:poll]
         payload[:poll][:voted] = @status.account_id == account_id
@@ -61,4 +65,35 @@ class StatusCacheHydrator
 
     payload
   end
+
+  private
+
+  def serialized_filter(filter)
+    ActiveModelSerializers::SerializableResource.new(
+      filter,
+      serializer: REST::FilterResultSerializer
+    ).as_json
+  end
+
+  def payload_application
+    @status.application.present? ? serialized_status_application_json : nil
+  end
+
+  def serialized_status_application_json
+    ActiveModelSerializers::SerializableResource.new(
+      @status.application,
+      serializer: REST::StatusSerializer::ApplicationSerializer
+    ).as_json
+  end
+
+  def payload_reblog_application
+    @status.reblog.application.present? ? serialized_status_reblog_application_json : nil
+  end
+
+  def serialized_status_reblog_application_json
+    ActiveModelSerializers::SerializableResource.new(
+      @status.reblog.application,
+      serializer: REST::StatusSerializer::ApplicationSerializer
+    ).as_json
+  end
 end

app/lib/text_formatter.rb

@@ -80,7 +80,7 @@ class TextFormatter
       entity[:indices].first
     end
 
-    result = ''.dup
+    result = +''
 
     last_index = entities.reduce(0) do |index, entity|
       indices = entity[:indices]

app/lib/webfinger_resource.rb

@@ -11,6 +11,8 @@ class WebfingerResource
 
   def username
     case resource
+    when %r{\A(https?://)?#{instance_actor_regexp}/?\Z}
+      Rails.configuration.x.local_domain
     when /\Ahttps?/i
       username_from_url
     when /@/
@@ -22,6 +24,13 @@ class WebfingerResource
 
   private
 
+  def instance_actor_regexp
+    hosts = [Rails.configuration.x.local_domain, Rails.configuration.x.web_domain]
+    hosts.concat(Rails.configuration.x.alternate_domains) if Rails.configuration.x.alternate_domains.present?
+
+    Regexp.union(hosts)
+  end
+
   def username_from_url
     if account_show_page?
       path_params[:username]