gitea/nas
1
0
Fork 2
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

Fix trend calculation working on too many items at a time (#25835)

Browse source
This commit is contained in:
Eugen Rochko 2023-07-08 20:16:48 +02:00 • committed by GitHub
parent 338a0e70cc
commit 610cf6c371
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 38 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

26
app/models/trends/statuses.rb
View file

@ -3,6 +3,8 @@
class Trends::Statuses < Trends::Base
PREFIX = 'trending_statuses'
BATCH_SIZE = 100
self.default_options = {
threshold: 5,
review_threshold: 3,
@ -58,8 +60,21 @@ class Trends::Statuses < Trends::Base
end
def refresh(at_time = Time.now.utc)
statuses = Status.where(id: (recently_used_ids(at_time) + StatusTrend.pluck(:status_id)).uniq).includes(:status_stat, :account)
calculate_scores(statuses, at_time)
# First, recalculate scores for statuses that were trending previously. We split the queries
# to avoid having to load all of the IDs into Ruby just to send them back into Postgres
Status.where(id: StatusTrend.select(:status_id)).includes(:status_stat, :account).find_in_batches(batch_size: BATCH_SIZE) do |statuses|
calculate_scores(statuses, at_time)
end
# Then, calculate scores for statuses that were used today. There are potentially some
# duplicate items here that we might process one more time, but that should be fine
Status.where(id: recently_used_ids(at_time)).includes(:status_stat, :account).find_in_batches(batch_size: BATCH_SIZE) do |statuses|
calculate_scores(statuses, at_time)
end
# Now that all trends have up-to-date scores, and all the ones below the threshold have
# been removed, we can recalculate their positions
StatusTrend.connection.exec_update('UPDATE status_trends SET rank = t0.calculated_rank FROM (SELECT id, row_number() OVER w AS calculated_rank FROM status_trends WINDOW w AS (PARTITION BY language ORDER BY score DESC)) t0 WHERE status_trends.id = t0.id')
end
def request_review
@ -117,10 +132,7 @@ class Trends::Statuses < Trends::Base
to_insert = items.filter { |(score, _)| score >= options[:decay_threshold] }
to_delete = items.filter { |(score, _)| score < options[:decay_threshold] }
StatusTrend.transaction do
StatusTrend.upsert_all(to_insert.map { |(score, status)| { status_id: status.id, account_id: status.account_id, score: score, language: status.language, allowed: status.trendable? || false } }, unique_by: :status_id) if to_insert.any?
StatusTrend.where(status_id: to_delete.map { |(_, status)| status.id }).delete_all if to_delete.any?
StatusTrend.connection.exec_update('UPDATE status_trends SET rank = t0.calculated_rank FROM (SELECT id, row_number() OVER w AS calculated_rank FROM status_trends WINDOW w AS (PARTITION BY language ORDER BY score DESC)) t0 WHERE status_trends.id = t0.id')
end
StatusTrend.upsert_all(to_insert.map { |(score, status)| { status_id: status.id, account_id: status.account_id, score: score, language: status.language, allowed: status.trendable? || false } }, unique_by: :status_id) if to_insert.any?
StatusTrend.where(status_id: to_delete.map { |(_, status)| status.id }).delete_all if to_delete.any?
end
end