gitea/nas
1
0
Fork 2
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

Fix trend calculation working on too many items at a time (#25835)

Browse source
This commit is contained in:
Eugen Rochko 2023-07-08 20:16:48 +02:00 • committed by GitHub
parent 338a0e70cc
commit 610cf6c371
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 38 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

26
app/models/trends/links.rb
View file

@ -3,6 +3,8 @@
class Trends::Links < Trends::Base
PREFIX = 'trending_links'
BATCH_SIZE = 100
self.default_options = {
threshold: 5,
review_threshold: 3,
@ -67,8 +69,21 @@ class Trends::Links < Trends::Base
end
def refresh(at_time = Time.now.utc)
preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + PreviewCardTrend.pluck(:preview_card_id)).uniq)
calculate_scores(preview_cards, at_time)
# First, recalculate scores for links that were trending previously. We split the queries
# to avoid having to load all of the IDs into Ruby just to send them back into Postgres
PreviewCard.where(id: PreviewCardTrend.select(:preview_card_id)).find_in_batches(batch_size: BATCH_SIZE) do |preview_cards|
calculate_scores(preview_cards, at_time)
end
# Then, calculate scores for links that were used today. There are potentially some
# duplicate items here that we might process one more time, but that should be fine
PreviewCard.where(id: recently_used_ids(at_time)).find_in_batches(batch_size: BATCH_SIZE) do |preview_cards|
calculate_scores(preview_cards, at_time)
end
# Now that all trends have up-to-date scores, and all the ones below the threshold have
# been removed, we can recalculate their positions
PreviewCardTrend.connection.exec_update('UPDATE preview_card_trends SET rank = t0.calculated_rank FROM (SELECT id, row_number() OVER w AS calculated_rank FROM preview_card_trends WINDOW w AS (PARTITION BY language ORDER BY score DESC)) t0 WHERE preview_card_trends.id = t0.id')
end
def request_review
@ -139,10 +154,7 @@ class Trends::Links < Trends::Base
to_insert = items.filter { |(score, _)| score >= options[:decay_threshold] }
to_delete = items.filter { |(score, _)| score < options[:decay_threshold] }
PreviewCardTrend.transaction do
PreviewCardTrend.upsert_all(to_insert.map { |(score, preview_card)| { preview_card_id: preview_card.id, score: score, language: preview_card.language, allowed: preview_card.trendable? || false } }, unique_by: :preview_card_id) if to_insert.any?
PreviewCardTrend.where(preview_card_id: to_delete.map { |(_, preview_card)| preview_card.id }).delete_all if to_delete.any?
PreviewCardTrend.connection.exec_update('UPDATE preview_card_trends SET rank = t0.calculated_rank FROM (SELECT id, row_number() OVER w AS calculated_rank FROM preview_card_trends WINDOW w AS (PARTITION BY language ORDER BY score DESC)) t0 WHERE preview_card_trends.id = t0.id')
end
PreviewCardTrend.upsert_all(to_insert.map { |(score, preview_card)| { preview_card_id: preview_card.id, score: score, language: preview_card.language, allowed: preview_card.trendable? || false } }, unique_by: :preview_card_id) if to_insert.any?
PreviewCardTrend.where(preview_card_id: to_delete.map { |(_, preview_card)| preview_card.id }).delete_all if to_delete.any?
end
end