Merge remote-tracking branch 'parent/main' into kb_migration

Gemfile.lock

@@ -532,7 +532,7 @@ GEM
       premailer (~> 1.7, >= 1.7.9)
     private_address_check (0.5.0)
     public_suffix (5.0.3)
-    puma (6.3.0)
+    puma (6.3.1)
       nio4r (~> 2.0)
     pundit (2.3.0)
       activesupport (>= 3.0.0)

app/javascript/packs/public.jsx

@@ -65,7 +65,6 @@ function loaded() {
     };
   };
 
-  ready(() => {
   const locale = document.documentElement.lang;
 
   const dateTimeFormat = new Intl.DateTimeFormat(locale, {
@@ -219,9 +218,9 @@ function loaded() {
     const message = (statusEl.dataset.spoiler === 'expanded') ? (localeData['status.show_less'] || 'Show less') : (localeData['status.show_more'] || 'Show more');
     spoilerLink.textContent = (new IntlMessageFormat(message, locale)).format();
   });
-  });
+}
 
-  delegate(document, '#account_display_name', 'input', ({ target }) => {
+delegate(document, '#account_display_name', 'input', ({ target }) => {
   const name = document.querySelector('.card .display-name strong');
   if (name) {
     if (target.value) {
@@ -230,17 +229,17 @@ function loaded() {
       name.textContent = target.dataset.default;
     }
   }
-  });
+});
 
-  delegate(document, '#account_avatar', 'change', ({ target }) => {
+delegate(document, '#edit_profile input[type=file]', 'change', ({ target }) => {
-    const avatar = document.querySelector('.card .avatar img');
+  const avatar = document.getElementById(target.id + '-preview');
   const [file] = target.files || [];
   const url = file ? URL.createObjectURL(file) : avatar.dataset.originalSrc;
 
   avatar.src = url;
-  });
+});
 
-  const getProfileAvatarAnimationHandler = (swapTo) => {
+const getProfileAvatarAnimationHandler = (swapTo) => {
   //animate avatar gifs on the profile page when moused over
   return ({ target }) => {
     const swapSrc = target.getAttribute(swapTo);
@@ -249,21 +248,13 @@ function loaded() {
       target.src = swapSrc;
     }
   };
-  };
+};
 
-  delegate(document, 'img#profile_page_avatar', 'mouseover', getProfileAvatarAnimationHandler('data-original'));
+delegate(document, 'img#profile_page_avatar', 'mouseover', getProfileAvatarAnimationHandler('data-original'));
 
-  delegate(document, 'img#profile_page_avatar', 'mouseout', getProfileAvatarAnimationHandler('data-static'));
+delegate(document, 'img#profile_page_avatar', 'mouseout', getProfileAvatarAnimationHandler('data-static'));
 
-  delegate(document, '#account_header', 'change', ({ target }) => {
+delegate(document, '#account_locked', 'change', ({ target }) => {
-    const header = document.querySelector('.card .card__img img');
-    const [file] = target.files || [];
-    const url = file ? URL.createObjectURL(file) : header.dataset.originalSrc;
-
-    header.src = url;
-  });
-
-  delegate(document, '#account_locked', 'change', ({ target }) => {
   const lock = document.querySelector('.card .display-name i');
 
   if (lock) {
@@ -273,15 +264,15 @@ function loaded() {
       lock.dataset.hidden = 'true';
     }
   }
-  });
+});
 
-  delegate(document, '.input-copy input', 'click', ({ target }) => {
+delegate(document, '.input-copy input', 'click', ({ target }) => {
   target.focus();
   target.select();
   target.setSelectionRange(0, target.value.length);
-  });
+});
 
-  delegate(document, '.input-copy button', 'click', ({ target }) => {
+delegate(document, '.input-copy button', 'click', ({ target }) => {
   const input = target.parentNode.querySelector('.input-copy__wrapper input');
 
   const oldReadOnly = input.readonly;
@@ -305,9 +296,9 @@ function loaded() {
   }
 
   input.readonly = oldReadOnly;
-  });
+});
 
-  const toggleSidebar = () => {
+const toggleSidebar = () => {
   const sidebar = document.querySelector('.sidebar ul');
   const toggleButton = document.querySelector('.sidebar__toggle__icon');
 
@@ -321,31 +312,29 @@ function loaded() {
 
   toggleButton.classList.toggle('active');
   sidebar.classList.toggle('visible');
-  };
+};
 
-  delegate(document, '.sidebar__toggle__icon', 'click', () => {
+delegate(document, '.sidebar__toggle__icon', 'click', () => {
   toggleSidebar();
-  });
+});
 
-  delegate(document, '.sidebar__toggle__icon', 'keydown', e => {
+delegate(document, '.sidebar__toggle__icon', 'keydown', e => {
   if (e.key === ' ' || e.key === 'Enter') {
     e.preventDefault();
     toggleSidebar();
   }
-  });
+});
 
-  // Empty the honeypot fields in JS in case something like an extension
+// Empty the honeypot fields in JS in case something like an extension
-  // automatically filled them.
+// automatically filled them.
-  delegate(document, '#registration_new_user,#new_user', 'submit', () => {
+delegate(document, '#registration_new_user,#new_user', 'submit', () => {
   ['user_website', 'user_confirm_password', 'registration_user_website', 'registration_user_confirm_password'].forEach(id => {
     const field = document.getElementById(id);
     if (field) {
       field.value = '';
     }
   });
-  });
+});
-}
-
 
 function main() {
   ready(loaded);

app/javascript/styles/mastodon/forms.scss

@@ -310,9 +310,19 @@ code {
       border-radius: 4px;
       background: url('images/void.png');
 
+      &[src$='missing.png'] {
+        visibility: hidden;
+      }
+
       &:last-child {
         margin-bottom: 0;
       }
+
+      &#account_avatar-preview {
+        width: 90px;
+        height: 90px;
+        object-fit: cover;
+      }
     }
   }
 

app/lib/cache_buster.rb

@@ -2,8 +2,14 @@
 
 class CacheBuster
   def initialize(options = {})
-    @secret_header = options[:secret_header] || 'Secret-Header'
+    ActiveSupport::Deprecation.warn('Default values for the cache buster secret header name and values will be removed in Mastodon 4.3. Please set them explicitely if you rely on those.') unless options[:http_method] || (options[:secret] && options[:secret_header])
-    @secret        = options[:secret] || 'True'
+
+    @secret_header = options[:secret_header] ||
+                     (options[:http_method] ? nil : 'Secret-Header')
+    @secret = options[:secret] ||
+              (options[:http_method] ? nil : 'True')
+
+    @http_method = options[:http_method] || 'GET'
   end
 
   def bust(url)
@@ -21,8 +27,9 @@ class CacheBuster
   end
 
   def build_request(url, http_client)
-    Request.new(:get, url, http_client: http_client).tap do |request|
+    request = Request.new(@http_method.downcase.to_sym, url, http_client: http_client)
-      request.add_headers(@secret_header => @secret)
+    request.add_headers(@secret_header => @secret) if @secret_header.present? && @secret && !@secret.empty?
-    end
+
+    request
   end
 end

app/lib/request.rb

@@ -117,7 +117,7 @@ class Request
 
   def perform
     begin
-      response = http_client.public_send(@verb, @url.to_s, @options.merge(headers: headers))
+      response = http_client.request(@verb, @url.to_s, @options.merge(headers: headers))
     rescue => e
       raise e.class, "#{e.message} on #{@url}", e.backtrace[0]
     end

app/lib/video_metadata_extractor.rb

@@ -43,6 +43,9 @@ class VideoMetadataExtractor
         @height      = video_stream[:height]
         @frame_rate  = video_stream[:avg_frame_rate] == '0/0' ? nil : Rational(video_stream[:avg_frame_rate])
         @r_frame_rate = video_stream[:r_frame_rate] == '0/0' ? nil : Rational(video_stream[:r_frame_rate])
+        # For some video streams the frame_rate reported by `ffprobe` will be 0/0, but for these streams we
+        # should use `r_frame_rate` instead. Video screencast generated by Gnome Screencast have this issue.
+        @frame_rate ||= @r_frame_rate
       end
 
       if (audio_stream = audio_streams.first)

app/models/follow_recommendation.rb

@@ -2,7 +2,7 @@
 
 # == Schema Information
 #
-# Table name: follow_recommendations
+# Table name: global_follow_recommendations
 #
 #  account_id :bigint(8)        primary key
 #  rank       :decimal(, )
@@ -11,6 +11,7 @@
 
 class FollowRecommendation < ApplicationRecord
   self.primary_key = :account_id
+  self.table_name = :global_follow_recommendations
 
   belongs_to :account_summary, foreign_key: :account_id, inverse_of: false
   belongs_to :account

app/models/status.rb

@@ -466,13 +466,25 @@ class Status < ApplicationRecord
 
       account_ids.uniq!
 
+      status_ids = cached_items.map { |item| item.reblog? ? item.reblog_of_id : item.id }.uniq
+
       return if account_ids.empty?
 
       accounts = Account.where(id: account_ids).includes(:account_stat, :user).index_by(&:id)
 
+      status_stats = StatusStat.where(status_id: status_ids).index_by(&:status_id)
+
       cached_items.each do |item|
         item.account = accounts[item.account_id]
         item.reblog.account = accounts[item.reblog.account_id] if item.reblog?
+
+        if item.reblog?
+          status_stat = status_stats[item.reblog.id]
+          item.reblog.status_stat = status_stat if status_stat.present?
+        else
+          status_stat = status_stats[item.id]
+          item.status_stat = status_stat if status_stat.present?
+        end
       end
     end
 

app/views/settings/profiles/show.html.haml

@@ -38,10 +38,10 @@
       .fields-group
         = f.input :avatar, wrapper: :with_block_label, input_html: { accept: AccountAvatar::IMAGE_MIME_TYPES.join(',') }, hint: t('simple_form.hints.defaults.avatar', dimensions: '400x400', size: number_to_human_size(AccountAvatar::LIMIT))
 
-    - if @account.avatar.present?
     .fields-row__column.fields-row__column-6
       .fields-group
-          = image_tag @account.avatar.url, class: 'fields-group__thumbnail', width: 90, height: 90
+        = image_tag @account.avatar.url, class: 'fields-group__thumbnail', id: 'account_avatar-preview'
+        - if @account.avatar.present?
           = link_to settings_profile_picture_path('avatar'), data: { method: :delete }, class: 'link-button link-button--destructive' do
             = fa_icon 'trash fw'
             = t('generic.delete')
@@ -51,10 +51,10 @@
       .fields-group
         = f.input :header, wrapper: :with_block_label, input_html: { accept: AccountHeader::IMAGE_MIME_TYPES.join(',') }, hint: t('simple_form.hints.defaults.header', dimensions: '1500x500', size: number_to_human_size(AccountHeader::LIMIT))
 
-    - if @account.header.present?
     .fields-row__column.fields-row__column-6
       .fields-group
-          = image_tag @account.header.url, class: 'fields-group__thumbnail'
+        = image_tag @account.header.url, class: 'fields-group__thumbnail', id: 'account_header-preview'
+        - if @account.header.present?
           = link_to settings_profile_picture_path('header'), data: { method: :delete }, class: 'link-button link-button--destructive' do
             = fa_icon 'trash fw'
             = t('generic.delete')

config/application.rb

@@ -51,6 +51,7 @@ require_relative '../lib/rails/engine_extensions'
 require_relative '../lib/active_record/database_tasks_extensions'
 require_relative '../lib/active_record/batches'
 require_relative '../lib/simple_navigation/item_extensions'
+require_relative '../lib/http_extensions'
 
 Dotenv::Railtie.load
 

config/initializers/cache_buster.rb

@@ -6,5 +6,6 @@ Rails.application.configure do
   config.x.cache_buster = {
     secret_header: ENV['CACHE_BUSTER_SECRET_HEADER'],
     secret: ENV['CACHE_BUSTER_SECRET'],
+    http_method: ENV['CACHE_BUSTER_HTTP_METHOD'] || 'GET',
   }
 end

db/migrate/20230818141056_create_global_follow_recommendations.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class CreateGlobalFollowRecommendations < ActiveRecord::Migration[7.0]
+  def change
+    create_view :global_follow_recommendations, materialized: { no_data: true }
+    safety_assured { add_index :global_follow_recommendations, :account_id, unique: true }
+  end
+end

db/post_migrate/20230818142253_drop_follow_recommendations.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class DropFollowRecommendations < ActiveRecord::Migration[7.0]
+  def up
+    drop_view :follow_recommendations, materialized: true
+  end
+
+  def down
+    create_view :follow_recommendations, version: 2, materialized: { no_data: true }
+    safety_assured { add_index :follow_recommendations, :account_id, unique: true }
+  end
+end

db/schema.rb

@@ -1498,34 +1498,36 @@ ActiveRecord::Schema[7.0].define(version: 2023_08_19_084858) do
   SQL
   add_index "account_summaries", ["account_id"], name: "index_account_summaries_on_account_id", unique: true
 
-  create_view "follow_recommendations", materialized: true, sql_definition: <<-SQL
+  create_view "global_follow_recommendations", materialized: true, sql_definition: <<-SQL
       SELECT t0.account_id,
       sum(t0.rank) AS rank,
       array_agg(t0.reason) AS reason
      FROM ( SELECT account_summaries.account_id,
               ((count(follows.id))::numeric / (1.0 + (count(follows.id))::numeric)) AS rank,
               'most_followed'::text AS reason
-             FROM (((follows
+             FROM ((follows
                JOIN account_summaries ON ((account_summaries.account_id = follows.target_account_id)))
                JOIN users ON ((users.account_id = follows.account_id)))
-               LEFT JOIN follow_recommendation_suppressions ON ((follow_recommendation_suppressions.account_id = follows.target_account_id)))
+            WHERE ((users.current_sign_in_at >= (now() - 'P30D'::interval)) AND (account_summaries.sensitive = false) AND (NOT (EXISTS ( SELECT 1
-            WHERE ((users.current_sign_in_at >= (now() - 'P30D'::interval)) AND (account_summaries.sensitive = false) AND (follow_recommendation_suppressions.id IS NULL))
+                     FROM follow_recommendation_suppressions
+                    WHERE (follow_recommendation_suppressions.account_id = follows.target_account_id)))))
             GROUP BY account_summaries.account_id
            HAVING (count(follows.id) >= 5)
           UNION ALL
            SELECT account_summaries.account_id,
               (sum((status_stats.reblogs_count + status_stats.favourites_count)) / (1.0 + sum((status_stats.reblogs_count + status_stats.favourites_count)))) AS rank,
               'most_interactions'::text AS reason
-             FROM (((status_stats
+             FROM ((status_stats
                JOIN statuses ON ((statuses.id = status_stats.status_id)))
                JOIN account_summaries ON ((account_summaries.account_id = statuses.account_id)))
-               LEFT JOIN follow_recommendation_suppressions ON ((follow_recommendation_suppressions.account_id = statuses.account_id)))
+            WHERE ((statuses.id >= (((date_part('epoch'::text, (now() - 'P30D'::interval)) * (1000)::double precision))::bigint << 16)) AND (account_summaries.sensitive = false) AND (NOT (EXISTS ( SELECT 1
-            WHERE ((statuses.id >= (((date_part('epoch'::text, (now() - 'P30D'::interval)) * (1000)::double precision))::bigint << 16)) AND (account_summaries.sensitive = false) AND (follow_recommendation_suppressions.id IS NULL))
+                     FROM follow_recommendation_suppressions
+                    WHERE (follow_recommendation_suppressions.account_id = statuses.account_id)))))
             GROUP BY account_summaries.account_id
            HAVING (sum((status_stats.reblogs_count + status_stats.favourites_count)) >= (5)::numeric)) t0
     GROUP BY t0.account_id
     ORDER BY (sum(t0.rank)) DESC;
   SQL
-  add_index "follow_recommendations", ["account_id"], name: "index_follow_recommendations_on_account_id", unique: true
+  add_index "global_follow_recommendations", ["account_id"], name: "index_global_follow_recommendations_on_account_id", unique: true
 
 end

db/views/global_follow_recommendations_v01.sql

@@ -0,0 +1,32 @@
+SELECT
+  account_id,
+  sum(rank) AS rank,
+  array_agg(reason) AS reason
+FROM (
+  SELECT
+    account_summaries.account_id AS account_id,
+    count(follows.id) / (1.0 + count(follows.id)) AS rank,
+    'most_followed' AS reason
+  FROM follows
+  INNER JOIN account_summaries ON account_summaries.account_id = follows.target_account_id
+  INNER JOIN users ON users.account_id = follows.account_id
+  WHERE users.current_sign_in_at >= (now() - interval '30 days')
+    AND account_summaries.sensitive = 'f'
+    AND NOT EXISTS (SELECT 1 FROM follow_recommendation_suppressions WHERE follow_recommendation_suppressions.account_id = follows.target_account_id)
+  GROUP BY account_summaries.account_id
+  HAVING count(follows.id) >= 5
+  UNION ALL
+  SELECT account_summaries.account_id AS account_id,
+         sum(status_stats.reblogs_count + status_stats.favourites_count) / (1.0 + sum(status_stats.reblogs_count + status_stats.favourites_count)) AS rank,
+         'most_interactions' AS reason
+  FROM status_stats
+  INNER JOIN statuses ON statuses.id = status_stats.status_id
+  INNER JOIN account_summaries ON account_summaries.account_id = statuses.account_id
+  WHERE statuses.id >= ((date_part('epoch', now() - interval '30 days') * 1000)::bigint << 16)
+    AND account_summaries.sensitive = 'f'
+    AND NOT EXISTS (SELECT 1 FROM follow_recommendation_suppressions WHERE follow_recommendation_suppressions.account_id = statuses.account_id)
+  GROUP BY account_summaries.account_id
+  HAVING sum(status_stats.reblogs_count + status_stats.favourites_count) >= 5
+) t0
+GROUP BY account_id
+ORDER BY rank DESC

lib/http_extensions.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# Monkey patching until https://github.com/httprb/http/pull/757 is merged
+unless HTTP::Request::METHODS.include?(:purge)
+  methods = HTTP::Request::METHODS.dup
+  HTTP::Request.send(:remove_const, :METHODS)
+  HTTP::Request.const_set(:METHODS, methods.push(:purge).freeze)
+end

spec/controllers/concerns/cache_concern_spec.rb

@@ -13,12 +13,17 @@ RSpec.describe CacheConcern do
     def empty_relation
       render plain: cache_collection(Status.none, Status).size
     end
+
+    def account_statuses_favourites
+      render plain: cache_collection(Status.where(account_id: params[:id]), Status).map(&:favourites_count)
+    end
   end
 
   before do
     routes.draw do
       get 'empty_array' => 'anonymous#empty_array'
-      post 'empty_relation' => 'anonymous#empty_relation'
+      get 'empty_relation' => 'anonymous#empty_relation'
+      get 'account_statuses_favourites' => 'anonymous#account_statuses_favourites'
     end
   end
 
@@ -36,5 +41,20 @@ RSpec.describe CacheConcern do
         expect(response.body).to eq '0'
       end
     end
+
+    context 'when given a collection of statuses' do
+      let!(:account) { Fabricate(:account) }
+      let!(:status)  { Fabricate(:status, account: account) }
+
+      it 'correctly updates with new interactions' do
+        get :account_statuses_favourites, params: { id: account.id }
+        expect(response.body).to eq '[0]'
+
+        FavouriteService.new.call(account, status)
+
+        get :account_statuses_favourites, params: { id: account.id }
+        expect(response.body).to eq '[1]'
+      end
+    end
   end
 end

spec/lib/cache_buster_spec.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe CacheBuster do
+  subject { described_class.new(secret_header: secret_header, secret: secret, http_method: http_method) }
+
+  let(:secret_header) { nil }
+  let(:secret) { nil }
+  let(:http_method) { nil }
+
+  let(:purge_url) { 'https://example.com/test_purge' }
+
+  describe '#bust' do
+    shared_examples 'makes_request' do
+      it 'makes an HTTP purging request' do
+        method = http_method&.to_sym || :get
+        stub_request(method, purge_url).to_return(status: 200)
+
+        subject.bust(purge_url)
+
+        test_request = a_request(method, purge_url)
+
+        test_request = test_request.with(headers: { secret_header => secret }) if secret && secret_header
+
+        expect(test_request).to have_been_made.once
+      end
+    end
+
+    context 'when using default options' do
+      include_examples 'makes_request'
+    end
+
+    context 'when specifying a secret header' do
+      let(:secret_header) { 'X-Purge-Secret' }
+      let(:secret) { SecureRandom.hex(20) }
+
+      include_examples 'makes_request'
+    end
+
+    context 'when specifying a PURGE method' do
+      let(:http_method) { 'purge' }
+
+      context 'when not using headers' do
+        include_examples 'makes_request'
+      end
+
+      context 'when specifying a secret header' do
+        let(:secret_header) { 'X-Purge-Secret' }
+        let(:secret) { SecureRandom.hex(20) }
+
+        include_examples 'makes_request'
+      end
+    end
+  end
+end

streaming/index.js

@@ -110,6 +110,11 @@ const pgConfigFromEnv = (env) => {
 
   if (env.DATABASE_URL) {
     baseConfig = dbUrlToConfig(env.DATABASE_URL);
+
+    // Support overriding the database password in the connection URL
+    if (!baseConfig.password && env.DB_PASS) {
+      baseConfig.password = env.DB_PASS;
+    }
   } else {
     baseConfig = pgConfigs[environment];
 

yarn.lock

@@ -1677,9 +1677,9 @@
     "@jridgewell/sourcemap-codec" "^1.4.14"
 
 "@material-design-icons/svg@^0.14.10":
-  version "0.14.10"
+  version "0.14.11"
-  resolved "https://registry.yarnpkg.com/@material-design-icons/svg/-/svg-0.14.10.tgz#25804b66d0740b0bf8d6841fa343dfdd60f22e82"
+  resolved "https://registry.yarnpkg.com/@material-design-icons/svg/-/svg-0.14.11.tgz#f90a2c8de801523c3b17e606c89313121c8bb3b4"
-  integrity sha512-rXxfqj5Su8i51aG8s8QRIe7mX1gB+C/ZCroLu3JvIsO3+Vx6PcWP97HLwIl7AQH/jYIHQlKq0E6OMqU91u5fCg==
+  integrity sha512-jpAksWZIVLB5/qTAeqANns7pH/faIQR3jgV2yROUNKZkzpJ428h7e1/byJB+rFZNI0hgZpY9nOVMLhc1J41HtA==
 
 "@nodelib/fs.scandir@2.1.5":
   version "2.1.5"
@@ -9206,9 +9206,9 @@ pg-types@^4.0.1:
     postgres-range "^1.1.1"
 
 pg@^8.5.0:
-  version "8.11.2"
+  version "8.11.3"
-  resolved "https://registry.yarnpkg.com/pg/-/pg-8.11.2.tgz#1a23f6de7bfb65ba56e4dd15df96668d319900c4"
+  resolved "https://registry.yarnpkg.com/pg/-/pg-8.11.3.tgz#d7db6e3fe268fcedd65b8e4599cda0b8b4bf76cb"
-  integrity sha512-l4rmVeV8qTIrrPrIR3kZQqBgSN93331s9i6wiUiLOSk0Q7PmUxZD/m1rQI622l3NfqBby9Ar5PABfS/SulfieQ==
+  integrity sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==
   dependencies:
     buffer-writer "2.0.0"
     packet-reader "1.0.0"