Add trending links (#16917)

* Add trending links

* Add overriding specific links trendability

* Add link type to preview cards and only trend articles

Change trends review notifications from being sent every 5 minutes to being sent every 2 hours

Change threshold from 5 unique accounts to 15 unique accounts

* Fix tests
This commit is contained in:
Eugen Rochko 2021-11-25 13:07:38 +01:00 committed by GitHub
parent 46e62fc4b3
commit 6e50134a42
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
97 changed files with 2071 additions and 722 deletions

View file

@ -67,7 +67,7 @@
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "app/models/account.rb",
"line": 479,
"line": 484,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "find_by_sql([\" WITH first_degree AS (\\n SELECT target_account_id\\n FROM follows\\n WHERE account_id = ?\\n UNION ALL\\n SELECT ?\\n )\\n SELECT\\n accounts.*,\\n (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n FROM accounts\\n LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?)\\n WHERE accounts.id IN (SELECT * FROM first_degree)\\n AND #{query} @@ #{textsearch}\\n AND accounts.suspended_at IS NULL\\n AND accounts.moved_to_account_id IS NULL\\n GROUP BY accounts.id\\n ORDER BY rank DESC\\n LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, account.id, limit, offset])",
"render_path": null,
@ -100,6 +100,26 @@
"confidence": "Weak",
"note": ""
},
{
"warning_type": "SQL Injection",
"warning_code": 0,
"fingerprint": "75fcd147b7611763ab6915faf8c5b0709e612b460f27c05c72d8b9bd0a6a77f8",
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "lib/mastodon/snowflake.rb",
"line": 87,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "connection.execute(\"CREATE OR REPLACE FUNCTION timestamp_id(table_name text)\\nRETURNS bigint AS\\n$$\\n DECLARE\\n time_part bigint;\\n sequence_base bigint;\\n tail bigint;\\n BEGIN\\n time_part := (\\n -- Get the time in milliseconds\\n ((date_part('epoch', now()) * 1000))::bigint\\n -- And shift it over two bytes\\n << 16);\\n\\n sequence_base := (\\n 'x' ||\\n -- Take the first two bytes (four hex characters)\\n substr(\\n -- Of the MD5 hash of the data we documented\\n md5(table_name || '#{SecureRandom.hex(16)}' || time_part::text),\\n 1, 4\\n )\\n -- And turn it into a bigint\\n )::bit(16)::bigint;\\n\\n -- Finally, add our sequence number to our base, and chop\\n -- it to the last two bytes\\n tail := (\\n (sequence_base + nextval(table_name || '_id_seq'))\\n & 65535);\\n\\n -- Return the time part and the sequence part. OR appears\\n -- faster here than addition, but they're equivalent:\\n -- time_part has no trailing two bytes, and tail is only\\n -- the last two bytes.\\n RETURN time_part | tail;\\n END\\n$$ LANGUAGE plpgsql VOLATILE;\\n\")",
"render_path": null,
"location": {
"type": "method",
"class": "Mastodon::Snowflake",
"method": "define_timestamp_id"
},
"user_input": "SecureRandom.hex(16)",
"confidence": "Medium",
"note": ""
},
{
"warning_type": "Mass Assignment",
"warning_code": 105,
@ -140,6 +160,26 @@
"confidence": "High",
"note": ""
},
{
"warning_type": "SQL Injection",
"warning_code": 0,
"fingerprint": "8c1d8c4b76c1cd3960e90dff999f854a6ff742fcfd8de6c7184ac5a1b1a4d7dd",
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "app/models/preview_card_filter.rb",
"line": 50,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "PreviewCard.joins(\"join unnest(array[#{(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id\")",
"render_path": null,
"location": {
"type": "method",
"class": "PreviewCardFilter",
"method": "trending_scope"
},
"user_input": "(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")",
"confidence": "Medium",
"note": ""
},
{
"warning_type": "SQL Injection",
"warning_code": 0,
@ -147,7 +187,7 @@
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "app/models/account.rb",
"line": 448,
"line": 453,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "find_by_sql([\" SELECT\\n accounts.*,\\n ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n FROM accounts\\n WHERE #{query} @@ #{textsearch}\\n AND accounts.suspended_at IS NULL\\n AND accounts.moved_to_account_id IS NULL\\n ORDER BY rank DESC\\n LIMIT ? OFFSET ?\\n\".squish, limit, offset])",
"render_path": null,
@ -160,26 +200,6 @@
"confidence": "Medium",
"note": ""
},
{
"warning_type": "SQL Injection",
"warning_code": 0,
"fingerprint": "9ccb9ba6a6947400e187d515e0bf719d22993d37cfc123c824d7fafa6caa9ac3",
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "lib/mastodon/snowflake.rb",
"line": 87,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "connection.execute(\" CREATE OR REPLACE FUNCTION timestamp_id(table_name text)\\n RETURNS bigint AS\\n $$\\n DECLARE\\n time_part bigint;\\n sequence_base bigint;\\n tail bigint;\\n BEGIN\\n time_part := (\\n -- Get the time in milliseconds\\n ((date_part('epoch', now()) * 1000))::bigint\\n -- And shift it over two bytes\\n << 16);\\n\\n sequence_base := (\\n 'x' ||\\n -- Take the first two bytes (four hex characters)\\n substr(\\n -- Of the MD5 hash of the data we documented\\n md5(table_name ||\\n '#{SecureRandom.hex(16)}' ||\\n time_part::text\\n ),\\n 1, 4\\n )\\n -- And turn it into a bigint\\n )::bit(16)::bigint;\\n\\n -- Finally, add our sequence number to our base, and chop\\n -- it to the last two bytes\\n tail := (\\n (sequence_base + nextval(table_name || '_id_seq'))\\n & 65535);\\n\\n -- Return the time part and the sequence part. OR appears\\n -- faster here than addition, but they're equivalent:\\n -- time_part has no trailing two bytes, and tail is only\\n -- the last two bytes.\\n RETURN time_part | tail;\\n END\\n $$ LANGUAGE plpgsql VOLATILE;\\n\")",
"render_path": null,
"location": {
"type": "method",
"class": "Mastodon::Snowflake",
"method": "define_timestamp_id"
},
"user_input": "SecureRandom.hex(16)",
"confidence": "Medium",
"note": ""
},
{
"warning_type": "Redirect",
"warning_code": 18,
@ -201,23 +221,53 @@
"note": ""
},
{
"warning_type": "Redirect",
"warning_code": 18,
"fingerprint": "ba699ddcc6552c422c4ecd50d2cd217f616a2446659e185a50b05a0f2dad8d33",
"check_name": "Redirect",
"message": "Possible unprotected redirect",
"file": "app/controllers/media_controller.rb",
"line": 20,
"link": "https://brakemanscanner.org/docs/warning_types/redirect/",
"code": "redirect_to(MediaAttachment.attached.find_by!(:shortcode => ((params[:id] or params[:medium_id]))).file.url(:original))",
"warning_type": "SQL Injection",
"warning_code": 0,
"fingerprint": "c32a484ccd9da46abd3bc93d08b72029d7dbc0576ccf4e878a9627e9a83cad2e",
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "app/models/tag_filter.rb",
"line": 50,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "Tag.joins(\"join unnest(array[#{Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id\")",
"render_path": null,
"location": {
"type": "method",
"class": "MediaController",
"method": "show"
"class": "TagFilter",
"method": "trending_scope"
},
"user_input": "MediaAttachment.attached.find_by!(:shortcode => ((params[:id] or params[:medium_id]))).file.url(:original)",
"confidence": "High",
"user_input": "Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")",
"confidence": "Medium",
"note": ""
},
{
"warning_type": "Cross-Site Scripting",
"warning_code": 4,
"fingerprint": "cd5cfd7f40037fbfa753e494d7129df16e358bfc43ef0da3febafbf4ee1ed3ac",
"check_name": "LinkToHref",
"message": "Potentially unsafe model attribute in `link_to` href",
"file": "app/views/admin/trends/links/_preview_card.html.haml",
"line": 7,
"link": "https://brakemanscanner.org/docs/warning_types/link_to_href",
"code": "link_to((Unresolved Model).new.title, (Unresolved Model).new.url)",
"render_path": [
{
"type": "template",
"name": "admin/trends/links/index",
"line": 37,
"file": "app/views/admin/trends/links/index.html.haml",
"rendered": {
"name": "admin/trends/links/_preview_card",
"file": "app/views/admin/trends/links/_preview_card.html.haml"
}
}
],
"location": {
"type": "template",
"template": "admin/trends/links/_preview_card"
},
"user_input": "(Unresolved Model).new.url",
"confidence": "Weak",
"note": ""
},
{
@ -227,7 +277,7 @@
"check_name": "SQL",
"message": "Possible SQL injection",
"file": "app/models/account.rb",
"line": 495,
"line": 500,
"link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
"code": "find_by_sql([\" SELECT\\n accounts.*,\\n (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n FROM accounts\\n LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?) OR (accounts.id = f.target_account_id AND f.account_id = ?)\\n WHERE #{query} @@ #{textsearch}\\n AND accounts.suspended_at IS NULL\\n AND accounts.moved_to_account_id IS NULL\\n GROUP BY accounts.id\\n ORDER BY rank DESC\\n LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, limit, offset])",
"render_path": null,
@ -261,6 +311,6 @@
"note": ""
}
],
"updated": "2021-05-11 20:22:27 +0900",
"brakeman_version": "5.0.1"
"updated": "2021-11-14 05:26:09 +0100",
"brakeman_version": "5.1.2"
}

View file

@ -674,8 +674,8 @@ en:
desc_html: Affects hashtags that have not been previously disallowed
title: Allow hashtags to trend without prior review
trends:
desc_html: Publicly display previously reviewed hashtags that are currently trending
title: Trending hashtags
desc_html: Publicly display previously reviewed content that is currently trending
title: Trends
site_uploads:
delete: Delete uploaded file
destroyed_msg: Site upload successfully deleted!
@ -702,21 +702,51 @@ en:
sidekiq_process_check:
message_html: No Sidekiq process running for the %{value} queue(s). Please review your Sidekiq configuration
tags:
accounts_today: Unique uses today
accounts_week: Unique uses this week
breakdown: Breakdown of today's usage by source
last_active: Recently used
most_popular: Most popular
most_recent: Recently created
name: Hashtag
review: Review status
reviewed: Reviewed
title: Hashtags
trending_right_now: Trending right now
unique_uses_today: "%{count} posting today"
unreviewed: Not reviewed
updated_msg: Hashtag settings updated successfully
title: Administration
trends:
allow: Allow
approved: Approved
disallow: Disallow
links:
allow: Allow link
allow_provider: Allow publisher
disallow: Disallow link
disallow_provider: Disallow publisher
shared_by_over_week:
one: Shared by one person over the last week
other: Shared by %{count} people over the last week
title: Trending links
usage_comparison: Shared %{today} times today, compared to %{yesterday} yesterday
pending_review: Pending review
preview_card_providers:
allowed: Links from this publisher can trend
rejected: Links from this publisher won't trend
title: Publishers
rejected: Rejected
tags:
current_score: Current score %{score}
dashboard:
tag_accounts_measure: unique uses
tag_languages_dimension: Top languages
tag_servers_dimension: Top servers
tag_servers_measure: different servers
tag_uses_measure: total uses
listable: Can be suggested
not_listable: Won't be suggested
not_trendable: Won't appear under trends
not_usable: Cannot be used
peaked_on_and_decaying: Peaked on %{date}, now decaying
title: Trending hashtags
trendable: Can appear under trends
trending_rank: 'Trending #%{rank}'
usable: Can be used
usage_comparison: Used %{today} times today, compared to %{yesterday} yesterday
used_by_over_week:
one: Used by one person over the last week
other: Used by %{count} people over the last week
title: Trends
warning_presets:
add_new: Add new
delete: Delete
@ -731,9 +761,16 @@ en:
body: "%{reporter} has reported %{target}"
body_remote: Someone from %{domain} has reported %{target}
subject: New report for %{instance} (#%{id})
new_trending_tag:
body: 'The hashtag #%{name} is trending today, but has not been previously reviewed. It will not be displayed publicly unless you allow it to, or just save the form as it is to never hear about it again.'
subject: New hashtag up for review on %{instance} (#%{name})
new_trending_links:
body: The following links are trending today, but their publishers have not been previously reviewed. They will not be displayed publicly unless you approve them. Further notifications from the same publishers will not be generated.
no_approved_links: There are currently no approved trending links.
requirements: The lowest approved trending link is currently "%{lowest_link_title}" with a score of %{lowest_link_score}.
subject: New trending links up for review on %{instance}
new_trending_tags:
body: 'The following hashtags are trending today, but they have not been previously reviewed. They will not be displayed publicly unless you approve them:'
no_approved_tags: There are currently no approved trending hashtags.
requirements: 'The lowest approved trending hashtag is currently #%{lowest_tag_name} with a score of %{lowest_tag_score}.'
subject: New trending hashtags up for review on %{instance}
aliases:
add_new: Create alias
created_msg: Successfully created a new alias. You can now initiate the move from the old account.
@ -940,7 +977,7 @@ en:
changes_saved_msg: Changes successfully saved!
copy: Copy
delete: Delete
no_batch_actions_available: No batch actions available on this page
none: None
order_by: Order by
save_changes: Save changes
validation_errors:

View file

@ -204,8 +204,8 @@ en:
mention: Someone mentioned you
pending_account: New account needs review
reblog: Someone boosted your post
report: New report is submitted
trending_tag: An unreviewed hashtag is trending
report: A new report is submitted
trending_tag: A new trend requires approval
rule:
text: Rule
tag:

View file

@ -34,12 +34,16 @@ SimpleNavigation::Configuration.run do |navigation|
n.item :invites, safe_join([fa_icon('user-plus fw'), t('invites.title')]), invites_path, if: proc { Setting.min_invite_role == 'user' && current_user.functional? }
n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_url, if: -> { current_user.functional? }
n.item :trends, safe_join([fa_icon('fire fw'), t('admin.trends.title')]), admin_trends_tags_path, if: proc { current_user.staff? } do |s|
s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.trends.tags.title')]), admin_trends_tags_path, highlights_on: %r{/admin/tags|/admin/trends/tags}
s.item :links, safe_join([fa_icon('newspaper-o fw'), t('admin.trends.links.title')]), admin_trends_links_path, highlights_on: %r{/admin/trends/links}
end
n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), admin_reports_url, if: proc { current_user.staff? } do |s|
s.item :action_logs, safe_join([fa_icon('bars fw'), t('admin.action_logs.title')]), admin_action_logs_url
s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_url, highlights_on: %r{/admin/reports}
s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts}
s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path
s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.tags.title')]), admin_tags_path, highlights_on: %r{/admin/tags}
s.item :follow_recommendations, safe_join([fa_icon('user-plus fw'), t('admin.follow_recommendations.title')]), admin_follow_recommendations_path, highlights_on: %r{/admin/follow_recommendations}
s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? }
s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? }

View file

@ -301,12 +301,27 @@ Rails.application.routes.draw do
resources :account_moderation_notes, only: [:create, :destroy]
resource :follow_recommendations, only: [:show, :update]
resources :tags, only: [:show, :update]
resources :tags, only: [:index, :show, :update] do
collection do
post :approve_all
post :reject_all
post :batch
namespace :trends do
resources :links, only: [:index] do
collection do
post :batch
end
end
resources :tags, only: [:index] do
collection do
post :batch
end
end
namespace :links do
resources :preview_card_providers, only: [:index], path: :publishers do
collection do
post :batch
end
end
end
end
end
@ -399,7 +414,7 @@ Rails.application.routes.draw do
resources :favourites, only: [:index]
resources :bookmarks, only: [:index]
resources :reports, only: [:create]
resources :trends, only: [:index]
resources :trends, only: [:index], controller: 'trends/tags'
resources :filters, only: [:index, :create, :show, :update, :destroy]
resources :endorsements, only: [:index]
resources :markers, only: [:index, :create]
@ -410,6 +425,11 @@ Rails.application.routes.draw do
resources :apps, only: [:create]
namespace :trends do
resources :links, only: [:index]
resources :tags, only: [:index]
end
namespace :emails do
resources :confirmations, only: [:create]
end
@ -512,7 +532,9 @@ Rails.application.routes.draw do
end
end
resources :trends, only: [:index]
namespace :trends do
resources :tags, only: [:index]
end
post :measures, to: 'measures#create'
post :dimensions, to: 'dimensions#create'

View file

@ -13,9 +13,13 @@
every: '5m'
class: Scheduler::ScheduledStatusesScheduler
queue: scheduler
trending_tags_scheduler:
trends_refresh_scheduler:
every: '5m'
class: Scheduler::TrendingTagsScheduler
class: Scheduler::Trends::RefreshScheduler
queue: scheduler
trends_review_notifications_scheduler:
every: '2h'
class: Scheduler::Trends::ReviewNotificationsScheduler
queue: scheduler
media_cleanup_scheduler:
cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'