gitea/nas
1
0
Fork 2
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'parent/main' into upstream-20241206

Browse source
This commit is contained in:
KMY 2024-12-06 12:17:45 +09:00
commit 3c3ee557d0
187 changed files with 1105 additions and 537 deletions
Download patch file Download diff file Expand all files Collapse all files

9
app/models/preview_card.rb
View file

@ -134,7 +134,7 @@ class PreviewCard < ApplicationRecord
end
def authors
@authors ||= [PreviewCard::Author.new(self)]
@authors ||= Array(serialized_authors)
end
class Author < ActiveModelSerializers::Model
@ -169,6 +169,13 @@ class PreviewCard < ApplicationRecord
private
def serialized_authors
if author_name? || author_url?
PreviewCard::Author
.new(self)
end
end
def extract_dimensions
file = image.queued_for_write[:original]

10
app/models/setting.rb
View file

@ -7,10 +7,8 @@
# id :bigint(8) not null, primary key
# var :string not null
# value :text
# thing_type :string
# created_at :datetime
# updated_at :datetime
# thing_id :bigint(8)
#
# This file is derived from a fork of the `rails-settings-cached` gem available at
@ -46,10 +44,10 @@ class Setting < ApplicationRecord
after_commit :rewrite_cache, on: %i(create update)
after_commit :expire_cache, on: %i(destroy)
# Settings are server-wide settings only, but they were previously
# used for users too. This can be dropped later with a database
# migration dropping any scoped setting.
default_scope { where(thing_type: nil, thing_id: nil) }
self.ignored_columns += %w(
thing_id
thing_type
)
class << self
# get or set a variable with the variable as the called method

1
app/models/tag.rb
View file

@ -33,6 +33,7 @@ class Tag < ApplicationRecord
has_many :followers, through: :passive_relationships, source: :account
has_one :antenna_tag, dependent: :destroy, inverse_of: :tag
has_one :trend, class_name: 'TagTrend', inverse_of: :tag, dependent: :destroy
HASHTAG_SEPARATORS = "_\u00B7\u30FB\u200c"
HASHTAG_FIRST_SEQUENCE_CHUNK_ONE = "[[:word:]_][[:word:]#{HASHTAG_SEPARATORS}]*[[:alpha:]#{HASHTAG_SEPARATORS}]"

21
app/models/tag_trend.rb Normal file
View file

@ -0,0 +1,21 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: tag_trends
#
# id :bigint(8) not null, primary key
# allowed :boolean default(FALSE), not null
# language :string default(""), not null
# rank :integer default(0), not null
# score :float default(0.0), not null
# tag_id :bigint(8) not null
#
class TagTrend < ApplicationRecord
include RankedTrend
belongs_to :tag
scope :allowed, -> { where(allowed: true) }
scope :not_allowed, -> { where(allowed: false) }
end

47
app/models/trends/base.rb
View file

@ -34,19 +34,7 @@ class Trends::Base
end
def query
Trends::Query.new(key_prefix, klass)
end
def score(id, locale: nil)
redis.zscore([key_prefix, 'all', locale].compact.join(':'), id) || 0
end
def rank(id, locale: nil)
redis.zrevrank([key_prefix, 'allowed', locale].compact.join(':'), id)
end
def currently_trending_ids(allowed, limit)
redis.zrevrange(allowed ? "#{key_prefix}:allowed" : "#{key_prefix}:all", 0, limit.positive? ? limit - 1 : limit).map(&:to_i)
Trends::Query.new(klass)
end
protected
@ -64,42 +52,9 @@ class Trends::Base
redis.expire(used_key(at_time), 1.day.seconds)
end
def score_at_rank(rank)
redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0
end
def replace_items(suffix, items)
tmp_prefix = "#{key_prefix}:tmp:#{SecureRandom.alphanumeric(6)}#{suffix}"
allowed_items = filter_for_allowed_items(items)
redis.pipelined do |pipeline|
items.each { |item| pipeline.zadd("#{tmp_prefix}:all", item[:score], item[:item].id) }
allowed_items.each { |item| pipeline.zadd("#{tmp_prefix}:allowed", item[:score], item[:item].id) }
rename_set(pipeline, "#{tmp_prefix}:all", "#{key_prefix}:all#{suffix}", items)
rename_set(pipeline, "#{tmp_prefix}:allowed", "#{key_prefix}:allowed#{suffix}", allowed_items)
end
end
def filter_for_allowed_items(items)
raise NotImplementedError
end
private
def used_key(at_time)
"#{key_prefix}:used:#{at_time.beginning_of_day.to_i}"
end
def rename_set(pipeline, from_key, to_key, set_items)
if set_items.empty?
pipeline.del(to_key)
else
pipeline.rename(from_key, to_key)
end
end
def skip_review?
Setting.trendable_by_default
end
end

17
app/models/trends/links.rb
View file

@ -14,15 +14,6 @@ class Trends::Links < Trends::Base
}
class Query < Trends::Query
def filtered_for!(account)
@account = account
self
end
def filtered_for(account)
clone.filtered_for!(account)
end
def to_arel
scope = PreviewCard.joins(:trend).reorder(score: :desc)
scope = scope.reorder(language_order_clause.desc, score: :desc) if preferred_languages.present?
@ -37,14 +28,6 @@ class Trends::Links < Trends::Base
def language_order_clause
Arel::Nodes::Case.new.when(PreviewCardTrend.arel_table[:language].in(preferred_languages)).then(1).else(0)
end
def preferred_languages
if @account&.chosen_languages.present?
@account.chosen_languages
else
@locale
end
end
end
def register(status, at_time = Time.now.utc)

57
app/models/trends/query.rb
View file

@ -1,19 +1,18 @@
# frozen_string_literal: true
class Trends::Query
include Redisable
include Enumerable
attr_reader :prefix, :klass, :loaded
attr_reader :klass, :loaded
alias loaded? loaded
def initialize(prefix, klass)
@prefix = prefix
def initialize(_prefix, klass)
@klass = klass
@records = []
@loaded = false
@allowed = false
@account = nil
@limit = nil
@offset = nil
end
@ -27,6 +26,15 @@ class Trends::Query
clone.allowed!
end
def filtered_for!(account)
@account = account
self
end
def filtered_for(account)
clone.filtered_for!(account)
end
def in_locale!(value)
@locale = value
self
@ -68,22 +76,11 @@ class Trends::Query
alias to_a to_ary
def to_arel
if ids_for_key.empty?
klass.none
else
scope = klass.joins(sanitized_join_sql).reorder('x.ordering')
scope = scope.offset(@offset) if @offset.present?
scope = scope.limit(@limit) if @limit.present?
scope
end
raise NotImplementedError
end
private
def key
[@prefix, @allowed ? 'allowed' : 'all', @locale].compact.join(':')
end
def load
unless loaded?
@records = perform_queries
@ -93,29 +90,15 @@ class Trends::Query
self
end
def ids_for_key
@ids_for_key ||= redis.zrevrange(key, 0, -1).map(&:to_i)
end
def sanitized_join_sql
ActiveRecord::Base.sanitize_sql_array(join_sql_array)
end
def join_sql_array
[join_sql_query, ids_for_key]
end
def join_sql_query
<<~SQL.squish
JOIN unnest(array[?]) WITH ordinality AS x (id, ordering) ON #{klass.table_name}.id = x.id
SQL
end
def perform_queries
apply_scopes(to_arel).to_a
to_arel.to_a
end
def apply_scopes(scope)
scope
def preferred_languages
if @account&.chosen_languages.present?
@account.chosen_languages
else
@locale
end
end
end

17
app/models/trends/statuses.rb
View file

@ -13,15 +13,6 @@ class Trends::Statuses < Trends::Base
}
class Query < Trends::Query
def filtered_for!(account)
@account = account
self
end
def filtered_for(account)
clone.filtered_for!(account)
end
def to_arel
scope = Status.joins(:trend).reorder(score: :desc)
scope = scope.reorder(language_order_clause.desc, score: :desc) if preferred_languages.present?
@ -37,14 +28,6 @@ class Trends::Statuses < Trends::Base
def language_order_clause
Arel::Nodes::Case.new.when(StatusTrend.arel_table[:language].in(preferred_languages)).then(1).else(0)
end
def preferred_languages
if @account&.chosen_languages.present?
@account.chosen_languages
else
@locale
end
end
end
def register(status, at_time = Time.now.utc)

36
app/models/trends/tag_filter.rb
View file

@ -6,6 +6,8 @@ class Trends::TagFilter
status
).freeze
IGNORED_PARAMS = %w(page).freeze
attr_reader :params
def initialize(params)
@ -13,14 +15,10 @@ class Trends::TagFilter
end
def results
scope = if params[:status] == 'pending_review'
Tag.unscoped.order(id: :desc)
else
trending_scope
end
scope = initial_scope
params.each do |key, value|
next if key.to_s == 'page'
next if IGNORED_PARAMS.include?(key.to_s)
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
@ -30,19 +28,24 @@ class Trends::TagFilter
private
def initial_scope
Tag.select(Tag.arel_table[Arel.star])
.joins(:trend)
.eager_load(:trend)
.reorder(score: :desc)
end
def scope_for(key, value)
case key.to_s
when 'status'
status_scope(value)
when 'trending'
trending_scope(value)
else
raise "Unknown filter: #{key}"
raise Mastodon::InvalidParameterError, "Unknown filter: #{key}"
end
end
def trending_scope
Trends.tags.query.to_arel
end
def status_scope(value)
case value.to_s
when 'approved'
@ -52,7 +55,16 @@ class Trends::TagFilter
when 'pending_review'
Tag.pending_review
else
raise "Unknown status: #{value}"
raise Mastodon::InvalidParameterError, "Unknown status: #{value}"
end
end
def trending_scope(value)
case value
when 'allowed'
TagTrend.allowed
else
TagTrend.all
end
end
end

72
app/models/trends/tags.rb
View file

@ -3,6 +3,8 @@
class Trends::Tags < Trends::Base
PREFIX = 'trending_tags'
BATCH_SIZE = 100
self.default_options = {
threshold: 5,
review_threshold: 3,
@ -11,6 +13,22 @@ class Trends::Tags < Trends::Base
decay_threshold: 1,
}
class Query < Trends::Query
def to_arel
scope = Tag.joins(:trend).reorder(language_order_clause.desc, score: :desc)
scope = scope.merge(TagTrend.allowed) if @allowed
scope = scope.offset(@offset) if @offset.present?
scope = scope.limit(@limit) if @limit.present?
scope
end
private
def language_order_clause
Arel::Nodes::Case.new.when(TagTrend.arel_table[:language].in(preferred_languages)).then(1).else(0)
end
end
def register(status, at_time = Time.now.utc)
return unless !status.reblog? && %i(public public_unlisted login).include?(status.visibility.to_sym) && !status.account.silenced?
return if !status.account.local? && DomainBlock.block_trends?(status.account.domain)
@ -25,19 +43,39 @@ class Trends::Tags < Trends::Base
record_used_id(tag.id, at_time)
end
def query
Query.new(key_prefix, klass)
end
def refresh(at_time = Time.now.utc)
tags = Tag.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
calculate_scores(tags, at_time)
# First, recalculate scores for tags that were trending previously. We split the queries
# to avoid having to load all of the IDs into Ruby just to send them back into Postgres
Tag.where(id: TagTrend.select(:tag_id)).find_in_batches(batch_size: BATCH_SIZE) do |tags|
calculate_scores(tags, at_time)
end
# Then, calculate scores for tags that were used today. There are potentially some
# duplicate items here that we might process one more time, but that should be fine
Tag.where(id: recently_used_ids(at_time)).find_in_batches(batch_size: BATCH_SIZE) do |tags|
calculate_scores(tags, at_time)
end
# Now that all trends have up-to-date scores, and all the ones below the threshold have
# been removed, we can recalculate their positions
TagTrend.recalculate_ordered_rank
end
def request_review
tags = Tag.where(id: currently_trending_ids(false, -1))
score_at_threshold = TagTrend.allowed.by_rank.ranked_below(options[:review_threshold]).first&.score || 0
tag_trends = TagTrend.not_allowed.includes(:tag)
tags.filter_map do |tag|
next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification?
tag_trends.filter_map do |trend|
tag = trend.tag
tag.touch(:requested_review_at)
tag
if trend.score > score_at_threshold && !tag.trendable? && tag.requires_review_notification?
tag.touch(:requested_review_at)
tag
end
end
end
@ -54,9 +92,7 @@ class Trends::Tags < Trends::Base
private
def calculate_scores(tags, at_time)
items = []
tags.each do |tag|
items = tags.map do |tag|
expected = tag.history.get(at_time - 1.day).accounts.to_f
expected = 1.0 if expected.zero?
observed = tag.history.get(at_time).accounts.to_f
@ -80,19 +116,13 @@ class Trends::Tags < Trends::Base
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
next unless decaying_score >= options[:decay_threshold]
items << { score: decaying_score, item: tag }
[decaying_score, tag]
end
replace_items('', items)
end
to_insert = items.filter { |(score, _)| score >= options[:decay_threshold] }
to_delete = items.filter { |(score, _)| score < options[:decay_threshold] }
def filter_for_allowed_items(items)
items.select { |item| item[:item].trendable? }
end
def would_be_trending?(id)
score(id) > score_at_rank(options[:review_threshold] - 1)
TagTrend.upsert_all(to_insert.map { |(score, tag)| { tag_id: tag.id, score: score, language: '', allowed: tag.trendable? || false } }, unique_by: %w(tag_id language)) if to_insert.any?
TagTrend.where(tag_id: to_delete.map { |(_, tag)| tag.id }).delete_all if to_delete.any?
end
end