diff --git a/.bundler-audit.yml b/.bundler-audit.yml deleted file mode 100644 index f84ec80872..0000000000 --- a/.bundler-audit.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -ignore: - - CVE-2015-9284 # Mitigation following https://github.com/omniauth/omniauth/wiki/Resolving-CVE-2015-9284#mitigating-in-rails-applications diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fe1ac3803..cfcc188362 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,240 @@ All notable changes to this project will be documented in this file. +## [4.2.0] - UNRELEASED + +The following changelog entries focus on changes visible to users, administrators, client developers or federated software developers, but there has also been a lot of code modernization, refactoring, and tooling work, in particular by [@danielmbrasil](https://github.com/danielmbrasil), [@mjankowski](https://github.com/mjankowski), [@nschonni](https://github.com/nschonni), [@renchap](https://github.com/renchap), and [@takayamaki](https://github.com/takayamaki). + +### Added + +- **Add role badges to the web interface** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25649), [Gargron](https://github.com/mastodon/mastodon/pull/26281)) +- **Add ability to pick domains to forward reports to using the `forward_to_domains` parameter in `POST /api/v1/reports`** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25866)) + The `forward_to_domains` REST API parameter is a list of strings. If it is empty or omitted, the previous behavior is maintained. + The `forward` parameter still needs to be set for `forward_to_domains` to be taken into account. + The forwarded-to domains can only include that of the original author and people being replied to. +- **Add forwarding of reported replies to servers being replied to** ([Gargron](https://github.com/mastodon/mastodon/pull/25341), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26189)) +- Add direct link to the Single-Sign On provider if there is only one sign up method available ([CSDUMMI](https://github.com/mastodon/mastodon/pull/26083), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26368)) +- **Add webhook templating** ([Gargron](https://github.com/mastodon/mastodon/pull/23289)) +- **Add webhooks for local `status.created`, `status.updated`, `account.updated` and `report.updated`** ([VyrCossont](https://github.com/mastodon/mastodon/pull/24133), [VyrCossont](https://github.com/mastodon/mastodon/pull/24243), [VyrCossont](https://github.com/mastodon/mastodon/pull/24211)) +- **Add exclusive lists** ([dariusk](https://github.com/mastodon/mastodon/pull/22048), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25324)) +- **Add a confirmation screen when suspending a domain** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25144), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25603)) +- **Add support for importing lists** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25203), [mgmn](https://github.com/mastodon/mastodon/pull/26120), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26372)) +- **Add optional hCaptcha support** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25019), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25057), [Gargron](https://github.com/mastodon/mastodon/pull/25395), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26388)) +- **Add lines to threads in web UI** ([Gargron](https://github.com/mastodon/mastodon/pull/24549), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24677), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24696), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24711), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24714), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24713), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24715), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24800), [teeerevor](https://github.com/mastodon/mastodon/pull/25706), [renchap](https://github.com/mastodon/mastodon/pull/25807)) +- **Add new onboarding flow to web UI** ([Gargron](https://github.com/mastodon/mastodon/pull/24619), [Gargron](https://github.com/mastodon/mastodon/pull/24646), [Gargron](https://github.com/mastodon/mastodon/pull/24705), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24872), [ThisIsMissEm](https://github.com/mastodon/mastodon/pull/24883), [Gargron](https://github.com/mastodon/mastodon/pull/24954), [stevenjlm](https://github.com/mastodon/mastodon/pull/24959), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25010), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25275), [Gargron](https://github.com/mastodon/mastodon/pull/25559), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25561)) +- Add `GET /api/v1/instance/languages` to REST API ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24443)) +- Add primary key to `preview_cards_statuses` join table ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25243), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26384)) +- Add client-side timeout on resend confirmation button ([Gargron](https://github.com/mastodon/mastodon/pull/26300)) +- Add published date and author to news on the explore screen in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/26155)) +- Add `lang` attribute to various UI components ([c960657](https://github.com/mastodon/mastodon/pull/23869), [c960657](https://github.com/mastodon/mastodon/pull/23891), [c960657](https://github.com/mastodon/mastodon/pull/26111), [c960657](https://github.com/mastodon/mastodon/pull/26149)) +- Add stricter protocol fields validation for accounts ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25937)) +- Add support for Azure blob storage ([mistydemeo](https://github.com/mastodon/mastodon/pull/23607), [mistydemeo](https://github.com/mastodon/mastodon/pull/26080)) +- Add toast with option to open post after publishing in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25564), [Signez](https://github.com/mastodon/mastodon/pull/25919)) +- Add canonical link tags in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25715)) +- Add button to see results for polls in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25726)) +- Add at-symbol prepended to mention span title ([forsamori](https://github.com/mastodon/mastodon/pull/25684)) +- Add users index on `unconfirmed_email` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25672), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25702)) +- Add superapp index on `oauth_applications` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25670)) +- Add index to backups on `user_id` column ([mjankowski](https://github.com/mastodon/mastodon/pull/25647)) +- Add onboarding prompt when home feed too slow in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25267), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25556), [Gargron](https://github.com/mastodon/mastodon/pull/25579), [renchap](https://github.com/mastodon/mastodon/pull/25580), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25581), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25617), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25917)) +- Add `POST /api/v1/conversations/:id/unread` API endpoint to mark a conversation as unread ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25509)) +- Add `translate="no"` to outgoing mentions and links ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25524)) +- Add unsubscribe link and headers to e-mails ([Gargron](https://github.com/mastodon/mastodon/pull/25378), [c960657](https://github.com/mastodon/mastodon/pull/26085)) +- Add logging of websocket send errors ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25280)) +- Add time zone preference ([Gargron](https://github.com/mastodon/mastodon/pull/25342), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26025)) +- Add `legal` as report category ([Gargron](https://github.com/mastodon/mastodon/pull/23941), [renchap](https://github.com/mastodon/mastodon/pull/25400)) +- Add `data-nosnippet` so Google doesn't use trending posts in snippets for `/` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25279)) +- Add card with who invited you to join when displaying rules on sign-up ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23475)) +- Add missing primary keys to `accounts_tags` and `statuses_tags` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25210)) +- Add support for custom sign-up URLs ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25014), [renchap](https://github.com/mastodon/mastodon/pull/25108), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25190), [mgmn](https://github.com/mastodon/mastodon/pull/25531)) + This is set using `SSO_ACCOUNT_SIGN_UP` and reflected in the REST API by adding `registrations.sign_up_url` to the `/api/v2/instance` endpoint. +- Add polling and automatic redirection to `/start` on email confirmation ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25013)) +- Add ability to block sign-ups from IP using the CLI ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24870)) +- Add ALT badges to media that has alternative text in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/24782), [c960657](https://github.com/mastodon/mastodon/pull/26166) +- Add ability to include accounts with pending follow requests in lists ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/19727), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24810)) +- Add trend management to admin API ([rrgeorge](https://github.com/mastodon/mastodon/pull/24257)) + - `POST /api/v1/admin/trends/statuses/:id/approve` + - `POST /api/v1/admin/trends/statuses/:id/reject` + - `POST /api/v1/admin/trends/links/:id/approve` + - `POST /api/v1/admin/trends/links/:id/reject` + - `POST /api/v1/admin/trends/tags/:id/approve` + - `POST /api/v1/admin/trends/tags/:id/reject` + - `GET /api/v1/admin/trends/links/publishers` + - `POST /api/v1/admin/trends/links/publishers/:id/approve` + - `POST /api/v1/admin/trends/links/publishers/:id/reject` +- Add user handle to notification mail recipient address ([HeitorMC](https://github.com/mastodon/mastodon/pull/24240)) +- Add progress indicator to sign-up flow ([Gargron](https://github.com/mastodon/mastodon/pull/24545)) +- Add client-side validation for taken username in sign-up form ([Gargron](https://github.com/mastodon/mastodon/pull/24546)) +- Add `--approve` option to `tootctl accounts create` ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24533)) +- Add “In Memoriam” banner back to profiles ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23591), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/23614)) + This adds the `memorial` attribute to the `Account` REST API entity. +- Add colour to follow button when hashtag is being followed ([c960657](https://github.com/mastodon/mastodon/pull/24361)) +- Add further explanations to the profile link verification instructions ([drzax](https://github.com/mastodon/mastodon/pull/19723)) +- Add a link to Identity provider's account settings from the account settings ([CSDUMMI](https://github.com/mastodon/mastodon/pull/24100), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24628)) +- Add support for streaming server to connect to postgres with self-signed certs through the `sslmode` URL parameter ([ramuuns](https://github.com/mastodon/mastodon/pull/21431)) +- Add support for specifying S3 storage classes through the `S3_STORAGE_CLASS` environment variable ([hyl](https://github.com/mastodon/mastodon/pull/22480)) +- Add support for incoming rich text ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23913)) +- Add support for Ruby 3.2 ([tenderlove](https://github.com/mastodon/mastodon/pull/22928), [casperisfine](https://github.com/mastodon/mastodon/pull/24142), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24202)) +- Add API parameter to safeguard unexpected mentions in new posts ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/18350)) + +### Changed + +- **Change reblogs to be excluded from "Posts and replies" tab in web UI** ([Gargron](https://github.com/mastodon/mastodon/pull/26302)) +- **Change interaction modal in web interface** ([Gargron, ClearlyClaire](https://github.com/mastodon/mastodon/pull/26075), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26269), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26268), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26267)) +- **Change design of link previews in web UI** ([Gargron](https://github.com/mastodon/mastodon/pull/26136), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26151), [Gargron](https://github.com/mastodon/mastodon/pull/26153), [Gargron](https://github.com/mastodon/mastodon/pull/26250), [Gargron](https://github.com/mastodon/mastodon/pull/26287), [Gargron](https://github.com/mastodon/mastodon/pull/26286), [c960657](https://github.com/mastodon/mastodon/pull/26184)) +- **Change "direct message" nomenclature to "private mention" in web UI** ([Gargron](https://github.com/mastodon/mastodon/pull/24248)) +- **Change translation feature to cover Content Warnings, poll options and media descriptions** ([c960657](https://github.com/mastodon/mastodon/pull/24175), [S-H-GAMELINKS](https://github.com/mastodon/mastodon/pull/25251), [c960657](https://github.com/mastodon/mastodon/pull/26168)) +- **Change account search to match by text when opted-in** ([jsgoldstein](https://github.com/mastodon/mastodon/pull/25599), [Gargron](https://github.com/mastodon/mastodon/pull/26378)) +- **Change import feature to be clearer, less error-prone and more reliable** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/21054), [mgmn](https://github.com/mastodon/mastodon/pull/24874)) +- **Change local and federated timelines to be in a single “Live feeds” column** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25641), [Gargron](https://github.com/mastodon/mastodon/pull/25683), [mgmn](https://github.com/mastodon/mastodon/pull/25694), [Plastikmensch](https://github.com/mastodon/mastodon/pull/26247)) +- **Change user archive export to be faster and more reliable, and export `.zip` archives instead of `.tar.gz` ones** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23360), [TheEssem](https://github.com/mastodon/mastodon/pull/25034)) +- **Change `mastodon-streaming` systemd unit files to be templated** ([e-nomem](https://github.com/mastodon/mastodon/pull/24751)) +- **Change `statsd` integration to disable sidekiq metrics by default** ([mjankowski](https://github.com/mastodon/mastodon/pull/25265), [mjankowski](https://github.com/mastodon/mastodon/pull/25336), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26310)) + This deprecates `statsd` support and disables the sidekiq integration unless `STATSD_SIDEKIQ` is set to `true`. + This is because the `nsa` gem is unmaintained, and its sidekiq integration is known to add very significant overhead. + Later versions of Mastodon will have other ways to get the same metrics. +- **Change replica support to native Rails adapter** ([krainboltgreene](https://github.com/mastodon/mastodon/pull/25693), [Gargron](https://github.com/mastodon/mastodon/pull/25849), [Gargron](https://github.com/mastodon/mastodon/pull/25874), [Gargron](https://github.com/mastodon/mastodon/pull/25851), [Gargron](https://github.com/mastodon/mastodon/pull/25977), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26074), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26326), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26386)) + This is a breaking change, dropping `makara` support, and requiring you to update your database configuration if you are using replicas. + To tell Mastodon to use a read replica, you can either set the `REPLICA_DB_NAME` environment variable (along with `REPLICA_DB_USER`, `REPLICA_DB_PASS`, `REPLICA_DB_HOST`, and `REPLICA_DB_PORT`, if they differ from the primary database), or the `REPLICA_DATABASE_URL` environment variable if your configuration is based on `DATABASE_URL`. +- Change header of hashtag timelines in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/26362)) +- Change streaming `/metrics` to include additional metrics ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/26299)) +- Change indexing frequency from 5 minutes to 1 minute, add locks to schedulers ([Gargron](https://github.com/mastodon/mastodon/pull/26304)) +- Change column link to add a better keyboard focus indicator ([teeerevor](https://github.com/mastodon/mastodon/pull/26278)) +- Change poll form element colors to fit with the rest of the ui ([teeerevor](https://github.com/mastodon/mastodon/pull/26139), [teeerevor](https://github.com/mastodon/mastodon/pull/26162), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26164)) +- Change 'favourite' to 'favorite' for American English ([marekr](https://github.com/mastodon/mastodon/pull/24667), [gunchleoc](https://github.com/mastodon/mastodon/pull/26009), [nabijaczleweli](https://github.com/mastodon/mastodon/pull/26109)) +- Change ActivityStreams representation of suspended accounts to not use a blank `name` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25276)) +- Change focus UI for keyboard only input ([teeerevor](https://github.com/mastodon/mastodon/pull/25935), [Gargron](https://github.com/mastodon/mastodon/pull/26125)) +- Change thread view to scroll to the selected post rather than the post being replied to ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24685)) +- Change links in multi-column mode so tabs are open in single-column mode ([Signez](https://github.com/mastodon/mastodon/pull/25893), [Signez](https://github.com/mastodon/mastodon/pull/26070), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25973)) +- Change searching with `#` to include account index ([jsgoldstein](https://github.com/mastodon/mastodon/pull/25638)) +- Change label and design of sensitive and unavailable media in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25712), [Gargron](https://github.com/mastodon/mastodon/pull/26135), [Gargron](https://github.com/mastodon/mastodon/pull/26330)) +- Change button colors to increase hover/focus contrast and consistency ([teeerevor](https://github.com/mastodon/mastodon/pull/25677), [Gargron](https://github.com/mastodon/mastodon/pull/25679)) +- Change dropdown icon above compose form from ellipsis to bars in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25661)) +- Change header backgrounds to use fewer different colors in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25577)) +- Change files to be deleted in batches instead of one-by-one ([Gargron](https://github.com/mastodon/mastodon/pull/23302), [S-H-GAMELINKS](https://github.com/mastodon/mastodon/pull/25586), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25587)) +- Change emoji picker icon ([iparr](https://github.com/mastodon/mastodon/pull/25479)) +- Change edit profile page ([Gargron](https://github.com/mastodon/mastodon/pull/25413)) +- Change "bot" label to "automated" ([Gargron](https://github.com/mastodon/mastodon/pull/25356)) +- Change design of dropdowns in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25107)) +- Change wording of “Content cache retention period” setting to highlight destructive implications ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23261)) +- Change autolinking to allow carets in URL search params ([renchap](https://github.com/mastodon/mastodon/pull/25216)) +- Change share action from being in action bar to being in dropdown in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25105)) +- Change remote report processing to accept reports with long comments, but truncate them ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25028)) +- Change sessions to be ordered from most-recent to least-recently updated ([frankieroberto](https://github.com/mastodon/mastodon/pull/25005)) +- Change vacuum scheduler to also delete expired tokens and unused application records ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24868), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24871)) +- Change "Sign in" to "Login" ([Gargron](https://github.com/mastodon/mastodon/pull/24942)) +- Change domain suspensions to also be checked before trying to fetch unknown remote resources ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24535)) +- Change media components to use aspect-ratio rather than compute height themselves ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24686), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24943)) +- Change logo version in header based on screen size in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/24707)) +- Change label from "For you" to "People" on explore screen in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/24706)) +- Change logged-out WebUI HTML pages to be cached for a few seconds ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24708)) +- Change unauthenticated responses to be cached in REST API ([Gargron](https://github.com/mastodon/mastodon/pull/24348), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24662), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24665)) +- Change HTTP caching logic ([Gargron](https://github.com/mastodon/mastodon/pull/24347), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24604)) +- Change hashtags and mentions in bios to open in-app in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/24643)) +- Change styling of the recommended accounts to allow bio to be more visible ([chike00](https://github.com/mastodon/mastodon/pull/24480)) +- Change account search in moderation interface to allow searching by username including the leading `@` ([HeitorMC](https://github.com/mastodon/mastodon/pull/24242)) +- Change all components to use the same error page in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/24512)) +- Change search pop-out in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/24305)) +- Change user settings to be stored in a more optimal way ([Gargron](https://github.com/mastodon/mastodon/pull/23630), [c960657](https://github.com/mastodon/mastodon/pull/24321), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24453), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24460), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24558), [Gargron](https://github.com/mastodon/mastodon/pull/24761), [Gargron](https://github.com/mastodon/mastodon/pull/24783), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25508), [jsgoldstein](https://github.com/mastodon/mastodon/pull/25340)) +- Change media upload limits and remove client-side resizing ([Gargron](https://github.com/mastodon/mastodon/pull/23726)) +- Change design of account rows in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/24247), [Gargron](https://github.com/mastodon/mastodon/pull/24343), [Gargron](https://github.com/mastodon/mastodon/pull/24956), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25131)) +- Change log-out to use Single Logout when using external log-in through OIDC ([CSDUMMI](https://github.com/mastodon/mastodon/pull/24020)) +- Change sidekiq-bulk's batch size from 10,000 to 1,000 jobs in one Redis call ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24034)) +- Change translation to only be offered for supported languages ([c960657](https://github.com/mastodon/mastodon/pull/23879), [c960657](https://github.com/mastodon/mastodon/pull/24037)) + This adds the `/api/v1/instance/translation_languages` REST API endpoint that returns an object with the supported translation language pairs in the form: + ```json + { + "fr": ["en", "de"] + } + ``` + (where `fr` is a supported source language and `en` and `de` or supported output language when translating a `fr` string) +- Change compose form checkbox to native input with `appearance: none` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/22949)) +- Change posts' clickable area to be larger ([c960657](https://github.com/mastodon/mastodon/pull/23621)) +- Change `followed_by` link to `location=all` if account is local on /admin/accounts/:id page ([tribela](https://github.com/mastodon/mastodon/pull/23467)) + +### Removed + +- **Remove support for Node.js 14** ([renchap](https://github.com/mastodon/mastodon/pull/25198)) +- **Remove support for Ruby 2.7** ([nschonni](https://github.com/mastodon/mastodon/pull/24237)) +- **Remove clustering from streaming API** ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/24655)) +- **Remove anonymous access to the streaming API** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23989)) +- Remove 16:9 cropping from web UI ([Gargron](https://github.com/mastodon/mastodon/pull/26132)) +- Remove back button from bookmarks, favourites and lists screens in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/26126)) +- Remove display name input from sign-up form ([Gargron](https://github.com/mastodon/mastodon/pull/24704)) +- Remove `tai` locale ([c960657](https://github.com/mastodon/mastodon/pull/23880)) +- Remove empty Kushubian (csb) local files ([nschonni](https://github.com/mastodon/mastodon/pull/24151)) +- Remove `Permissions-Policy` header from all responses ([Gargron](https://github.com/mastodon/mastodon/pull/24124)) + +### Fixed + +- **Fix filters not being applying in the explore page** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25887)) +- **Fix being unable to load past a full page of filtered posts in Home timeline** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24930)) +- **Fix log-in flow when involving both OAuth and external authentication** ([CSDUMMI](https://github.com/mastodon/mastodon/pull/24073)) +- **Fix broken links in account gallery** ([c960657](https://github.com/mastodon/mastodon/pull/24218)) +- Fix adding column with default value taking longer on Postgres >= 11 ([Gargron](https://github.com/mastodon/mastodon/pull/26375)) +- Fix light theme select option for hashtags ([teeerevor](https://github.com/mastodon/mastodon/pull/26311)) +- Fix AVIF attachments ([c960657](https://github.com/mastodon/mastodon/pull/26264)) +- Fix incorrect URL normalization when fetching remote resources ([c960657](https://github.com/mastodon/mastodon/pull/26219), [c960657](https://github.com/mastodon/mastodon/pull/26285)) +- Fix being unable to filter posts for individual Chinese languages ([gunchleoc](https://github.com/mastodon/mastodon/pull/26066)) +- Fix preview card sometimes linking to 4xx error pages ([c960657](https://github.com/mastodon/mastodon/pull/26200)) +- Fix emoji picker button scrolling with textarea content in single-column view ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25304)) +- Fix missing border on error screen in light theme in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/26152)) +- Fix UI overlap with the loupe icon in the Explore Tab ([gol-cha](https://github.com/mastodon/mastodon/pull/26113)) +- Fix unexpected redirection to `/explore` after sign-in ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26143)) +- Fix `/api/v1/statuses/:id/unfavourite` and `/api/v1/statuses/:id/unreblog` returning non-updated counts ([c960657](https://github.com/mastodon/mastodon/pull/24365)) +- Fix clicking the “Back” button sometimes leading out of Mastodon ([c960657](https://github.com/mastodon/mastodon/pull/23953), [CSFlorin](https://github.com/mastodon/mastodon/pull/24835), [S-H-GAMELINKS](https://github.com/mastodon/mastodon/pull/24867), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25281)) +- Fix processing of `null` ActivityPub activities ([tribela](https://github.com/mastodon/mastodon/pull/26021)) +- Fix hashtag posts not being removed from home feed on hashtag unfollow ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26028)) +- Fix for "follows you" indicator in light web UI not readable ([vmstan](https://github.com/mastodon/mastodon/pull/25993)) +- Fix incorrect line break between icon and number of reposts & favourites ([edent](https://github.com/mastodon/mastodon/pull/26004)) +- Fix sounds not being loaded from assets host ([Signez](https://github.com/mastodon/mastodon/pull/25931)) +- Fix buttons showing inconsistent styles ([teeerevor](https://github.com/mastodon/mastodon/pull/25903), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25965), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26341)) +- Fix trend calculation working on too many items at a time ([Gargron](https://github.com/mastodon/mastodon/pull/25835)) +- Fix dropdowns being disabled for logged out users in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25714), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25964)) +- Fix explore page being inaccessible when opted-out of trends in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25716)) +- Fix re-activated accounts possibly getting deleted by `AccountDeletionWorker` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25711)) +- Fix `/api/v2/search` not working with following query param ([danielmbrasil](https://github.com/mastodon/mastodon/pull/25681)) +- Fix inefficient query when requesting a new confirmation email from a logged-in account ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25669)) +- Fix unnecessary concurrent calls to `/api/*/instance` in web UI ([mgmn](https://github.com/mastodon/mastodon/pull/25663)) +- Fix resolving local URL for remote content ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25637)) +- Fix search not being easily findable on smaller screens in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25576), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25631)) +- Fix j/k keyboard shortcuts on some status lists ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25554)) +- Fix missing validation on `default_privacy` setting ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25513)) +- Fix incorrect pagination headers in `/api/v2/admin/accounts` ([danielmbrasil](https://github.com/mastodon/mastodon/pull/25477)) +- Fix non-interactive upload container being given a `button` role and tabIndex ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25462)) +- Fix always redirecting to onboarding in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/25396)) +- Fix inconsistent use of middle dot (·) instead of bullet (•) to separate items ([j-f1](https://github.com/mastodon/mastodon/pull/25248)) +- Fix spacing of middle dots in the detailed status meta section ([j-f1](https://github.com/mastodon/mastodon/pull/25247)) +- Fix prev/next buttons color in media viewer ([renchap](https://github.com/mastodon/mastodon/pull/25231)) +- Fix email addresses not being properly updated in `tootctl maintenance fix-duplicates` ([mjankowski](https://github.com/mastodon/mastodon/pull/25118)) +- Fix unicode surrogate pairs sometimes being broken in page title ([eai04191](https://github.com/mastodon/mastodon/pull/25148)) +- Fix various inefficient queries against account domains ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25126)) +- Fix video player offering to expand in a lightbox when it's in an `iframe` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25067)) +- Fix post embed previews ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25071)) +- Fix inadequate error handling in several API controllers when given invalid parameters ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24947), [danielmbrasil](https://github.com/mastodon/mastodon/pull/24958), [danielmbrasil](https://github.com/mastodon/mastodon/pull/25063), [danielmbrasil](https://github.com/mastodon/mastodon/pull/25072), [danielmbrasil](https://github.com/mastodon/mastodon/pull/25386), [danielmbrasil](https://github.com/mastodon/mastodon/pull/25595)) +- Fix uncaught `ActiveRecord::StatementInvalid` in Mastodon::IpBlocksCLI ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24861)) +- Fix various edge cases with local moves ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24812)) +- Fix `tootctl accounts cull` crashing when encountering a domain resolving to a private address ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23378)) +- Fix `tootctl accounts approve --number N` not aproving the N earliest registrations ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24605)) +- Fix being unable to clear media description when editing posts ([c960657](https://github.com/mastodon/mastodon/pull/24720)) +- Fix unavailable translations not falling back to English ([mgmn](https://github.com/mastodon/mastodon/pull/24727)) +- Fix anonymous visitors getting a session cookie on first visit ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24584), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24650), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24664)) +- Fix cutting off first letter of hashtag links sometimes in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/24623)) +- Fix crash in `tootctl accounts create --reattach --force` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24557), [danielmbrasil](https://github.com/mastodon/mastodon/pull/24680)) +- Fix characters being emojified even when using Variation Selector 15 (text) ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/20949), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24615)) +- Fix uncaught ActiveRecord::StatementInvalid exception in `Mastodon::AccountsCLI#approve` ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24590)) +- Fix email confirmation skip option in `tootctl accounts modify USERNAME --email EMAIL --confirm` ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24578)) +- Fix tooltip for dates without time ([c960657](https://github.com/mastodon/mastodon/pull/24244)) +- Fix missing loading spinner and loading more on scroll in Private Mentions column ([c960657](https://github.com/mastodon/mastodon/pull/24446)) +- Fix account header image missing from `/settings/profile` on narrow screens ([c960657](https://github.com/mastodon/mastodon/pull/24433)) +- Fix height of announcements not being updated when using reduced animations ([c960657](https://github.com/mastodon/mastodon/pull/24354)) +- Fix inconsistent radius in advanced interface drawer ([thislight](https://github.com/mastodon/mastodon/pull/24407)) +- Fix loading more trending posts on scroll in the advanced interface ([OmmyZhang](https://github.com/mastodon/mastodon/pull/24314)) +- Fix poll ending notification for edited polls ([c960657](https://github.com/mastodon/mastodon/pull/24311)) +- Fix max width of media in `/about` and `/privacy-policy` ([mgmn](https://github.com/mastodon/mastodon/pull/24180)) +- Fix streaming API not being usable without `DATABASE_URL` ([Gargron](https://github.com/mastodon/mastodon/pull/23960)) +- Fix external authentication not running onboarding code for new users ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23458)) + ## [4.1.6] - 2023-07-31 ### Fixed diff --git a/Gemfile b/Gemfile index a1eba4efe0..27adec7a09 100644 --- a/Gemfile +++ b/Gemfile @@ -35,11 +35,14 @@ group :pam_authentication, optional: true do end gem 'net-ldap', '~> 0.18' -gem 'omniauth-cas', '~> 2.0' -gem 'omniauth-saml', '~> 1.10' + +# TODO: Point back at released omniauth-cas gem when PR merged +# https://github.com/dlindahl/omniauth-cas/pull/68 +gem 'omniauth-cas', github: 'stanhu/omniauth-cas', ref: '4211e6d05941b4a981f9a36b49ec166cecd0e271' +gem 'omniauth-saml', '~> 2.0' gem 'omniauth_openid_connect', '~> 0.6.1' -gem 'omniauth', '~> 1.9' -gem 'omniauth-rails_csrf_protection', '~> 0.1' +gem 'omniauth', '~> 2.0' +gem 'omniauth-rails_csrf_protection', '~> 1.0' gem 'color_diff', '~> 0.1' gem 'discard', '~> 1.2' diff --git a/Gemfile.lock b/Gemfile.lock index a44538d16f..46a10e5acc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -26,6 +26,16 @@ GIT rails-settings-cached (0.6.6) rails (>= 4.2.0) +GIT + remote: https://github.com/stanhu/omniauth-cas.git + revision: 4211e6d05941b4a981f9a36b49ec166cecd0e271 + ref: 4211e6d05941b4a981f9a36b49ec166cecd0e271 + specs: + omniauth-cas (2.0.0) + addressable (~> 2.3) + nokogiri (~> 1.5) + omniauth (>= 1.2, < 3) + GEM remote: https://rubygems.org/ specs: @@ -472,19 +482,16 @@ GEM mini_portile2 (~> 2.8.2) racc (~> 1.4) oj (3.15.0) - omniauth (1.9.2) + omniauth (2.1.1) hashie (>= 3.4.6) - rack (>= 1.6.2, < 3) - omniauth-cas (2.0.0) - addressable (~> 2.3) - nokogiri (~> 1.5) - omniauth (~> 1.2) - omniauth-rails_csrf_protection (0.1.2) + rack (>= 2.2.3) + rack-protection + omniauth-rails_csrf_protection (1.0.1) actionpack (>= 4.2) - omniauth (>= 1.3.1) - omniauth-saml (1.10.3) - omniauth (~> 1.3, >= 1.3.2) - ruby-saml (~> 1.9) + omniauth (~> 2.0) + omniauth-saml (2.1.0) + omniauth (~> 2.0) + ruby-saml (~> 1.12) omniauth_openid_connect (0.6.1) omniauth (>= 1.9, < 3) openid_connect (~> 1.1) @@ -542,6 +549,8 @@ GEM httpclient json-jwt (>= 1.11.0) rack (>= 2.1.0) + rack-protection (3.0.5) + rack rack-proxy (0.7.6) rack rack-test (2.1.0) @@ -871,10 +880,10 @@ DEPENDENCIES nokogiri (~> 1.15) nsa! oj (~> 3.14) - omniauth (~> 1.9) - omniauth-cas (~> 2.0) - omniauth-rails_csrf_protection (~> 0.1) - omniauth-saml (~> 1.10) + omniauth (~> 2.0) + omniauth-cas! + omniauth-rails_csrf_protection (~> 1.0) + omniauth-saml (~> 2.0) omniauth_openid_connect (~> 0.6.1) ox (~> 2.14) parslet diff --git a/app/chewy/accounts_index.rb b/app/chewy/accounts_index.rb index b80fdfed9f..9615421ae7 100644 --- a/app/chewy/accounts_index.rb +++ b/app/chewy/accounts_index.rb @@ -33,7 +33,7 @@ class AccountsIndex < Chewy::Index }, verbatim: { - tokenizer: 'whitespace', + tokenizer: 'standard', filter: %w(lowercase asciifolding cjk_width), }, diff --git a/app/controllers/concerns/captcha_concern.rb b/app/controllers/concerns/captcha_concern.rb index 576304d1ca..170c8f5e03 100644 --- a/app/controllers/concerns/captcha_concern.rb +++ b/app/controllers/concerns/captcha_concern.rb @@ -42,7 +42,7 @@ module CaptchaConcern end def extend_csp_for_captcha! - policy = request.content_security_policy + policy = request.content_security_policy&.clone return unless captcha_required? && policy.present? @@ -54,6 +54,8 @@ module CaptchaConcern policy.send(directive, *values) end + + request.content_security_policy = policy end def render_captcha diff --git a/app/javascript/mastodon/features/interaction_modal/index.jsx b/app/javascript/mastodon/features/interaction_modal/index.jsx index 85ef0511db..d5599a78a7 100644 --- a/app/javascript/mastodon/features/interaction_modal/index.jsx +++ b/app/javascript/mastodon/features/interaction_modal/index.jsx @@ -336,36 +336,24 @@ class InteractionModal extends React.PureComponent { } let signupButton; - let signUpOrSignInButton; if (sso_redirect) { - signUpOrSignInButton = ( - - + signupButton = ( + + - ) + ); + } else if (registrationsOpen) { + signupButton = ( + + + + ); } else { - if(registrationsOpen) { - signupButton = ( - - - - ); - } else { - signupButton = ( - - ); - } - - signUpOrSignInButton = ( - <> - - - - {signupButton} - + signupButton = ( + ); } @@ -376,13 +364,6 @@ class InteractionModal extends React.PureComponent {

{actionDescription}

-
-
-

- {signUpOrSignInButton} -
-
-

diff --git a/app/javascript/mastodon/features/ui/components/sign_in_banner.jsx b/app/javascript/mastodon/features/ui/components/sign_in_banner.jsx index db85c6cbfb..d5f0c00dca 100644 --- a/app/javascript/mastodon/features/ui/components/sign_in_banner.jsx +++ b/app/javascript/mastodon/features/ui/components/sign_in_banner.jsx @@ -17,9 +17,9 @@ const SignInBanner = () => { let signupButton; - const signupUrl = useAppSelector((state) => state.getIn(['server', 'server', 'registrations', 'url'], null) || '/auth/sign_up'); + const signupUrl = useAppSelector((state) => state.getIn(['server', 'server', 'registrations', 'url'], null) || '/auth/sign_up'); - if (sso_redirect) { + if (sso_redirect) { return (

diff --git a/app/models/application_record.rb b/app/models/application_record.rb index c2de8fb2af..567542f91d 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -5,7 +5,7 @@ class ApplicationRecord < ActiveRecord::Base include Remotable - connects_to database: { writing: :primary, reading: ENV['DB_REPLICA_NAME'] || ENV['REPLICA_DATABASE_URL'] ? :replica : :primary } + connects_to database: { writing: :primary, reading: ENV['REPLICA_DB_NAME'] || ENV['REPLICA_DATABASE_URL'] ? :replica : :primary } class << self def update_index(_type_name, *_args, &_block) diff --git a/app/services/account_search_service.rb b/app/services/account_search_service.rb index b732fbcec3..a15b691211 100644 --- a/app/services/account_search_service.rb +++ b/app/services/account_search_service.rb @@ -8,6 +8,143 @@ class AccountSearchService < BaseService # Min. number of characters to look for non-exact matches MIN_QUERY_LENGTH = 5 + class QueryBuilder + def initialize(query, account, options = {}) + @query = query + @account = account + @options = options + end + + def build + AccountsIndex.query( + bool: { + must: { + function_score: { + query: { + bool: { + must: must_clauses, + }, + }, + + functions: [ + reputation_score_function, + followers_score_function, + time_distance_function, + ], + }, + }, + + should: should_clauses, + } + ) + end + + private + + def must_clauses + if @account && @options[:following] + [core_query, only_following_query] + else + [core_query] + end + end + + def should_clauses + if @account && !@options[:following] + [boost_following_query] + else + [] + end + end + + # This function limits results to only the accounts the user is following + def only_following_query + { + terms: { + id: following_ids, + }, + } + end + + # This function promotes accounts the user is following + def boost_following_query + { + terms: { + id: following_ids, + boost: 100, + }, + } + end + + # This function deranks accounts that follow more people than follow them + def reputation_score_function + { + script_score: { + script: { + source: "(Math.max(doc['followers_count'].value, 0) + 0.0) / (Math.max(doc['followers_count'].value, 0) + Math.max(doc['following_count'].value, 0) + 1)", + }, + }, + } + end + + # This function promotes accounts that have more followers + def followers_score_function + { + script_score: { + script: { + source: "(Math.max(doc['followers_count'].value, 0) / (Math.max(doc['followers_count'].value, 0) + 1))", + }, + }, + } + end + + # This function deranks accounts that haven't posted in a long time + def time_distance_function + { + gauss: { + last_status_at: { + scale: '30d', + offset: '30d', + decay: 0.3, + }, + }, + } + end + + def following_ids + @following_ids ||= @account.active_relationships.pluck(:target_account_id) + [@account.id] + end + end + + class AutocompleteQueryBuilder < QueryBuilder + private + + def core_query + { + multi_match: { + query: @query, + type: 'bool_prefix', + fields: %w(username username.* display_name display_name.*), + }, + } + end + end + + class FullQueryBuilder < QueryBuilder + private + + def core_query + { + multi_match: { + query: @query, + type: 'most_fields', + fields: %w(username^2 display_name^2 text text.*), + operator: 'and', + }, + } + end + end + def call(query, account = nil, options = {}) @query = query&.strip&.gsub(/\A@/, '') @limit = options[:limit].to_i @@ -71,27 +208,15 @@ class AccountSearchService < BaseService end def from_elasticsearch - must_clauses = must_clause - should_clauses = should_clause - - if account - return [] if options[:following] && following_ids.empty? - - if options[:following] - must_clauses << { terms: { id: following_ids } } - elsif following_ids.any? - should_clauses << { terms: { id: following_ids, boost: 100 } } + query_builder = begin + if options[:use_searchable_text] + FullQueryBuilder.new(terms_for_query, account, options.slice(:following)) + else + AutocompleteQueryBuilder.new(terms_for_query, account, options.slice(:following)) end end - query = { bool: { must: must_clauses, should: should_clauses } } - functions = [reputation_score_function, followers_score_function, time_distance_function] - - records = AccountsIndex.query(function_score: { query: query, functions: functions }) - .limit(limit_for_non_exact_results) - .offset(offset) - .objects - .compact + records = query_builder.build.limit(limit_for_non_exact_results).offset(offset).objects.compact ActiveRecord::Associations::Preloader.new(records: records, associations: :account_stat) @@ -100,76 +225,6 @@ class AccountSearchService < BaseService nil end - def reputation_score_function - { - script_score: { - script: { - source: "(Math.max(doc['followers_count'].value, 0) + 0.0) / (Math.max(doc['followers_count'].value, 0) + Math.max(doc['following_count'].value, 0) + 1)", - }, - }, - } - end - - def followers_score_function - { - script_score: { - script: { - source: "Math.log10(Math.max(doc['followers_count'].value, 0) + 2)", - }, - }, - } - end - - def time_distance_function - { - gauss: { - last_status_at: { - scale: '30d', - offset: '30d', - decay: 0.3, - }, - }, - } - end - - def must_clause - if options[:start_with_hashtag] - fields = %w(text text.*) - else - fields = %w(username username.* display_name display_name.*) - fields << 'text' << 'text.*' if options[:use_searchable_text] - end - - [ - { - multi_match: { - query: terms_for_query, - fields: fields, - type: 'best_fields', - operator: 'or', - }, - }, - ] - end - - def should_clause - [ - { - multi_match: { - query: terms_for_query, - fields: %w(username username.* display_name display_name.*), - type: 'best_fields', - operator: 'and', - boost: 10, - }, - }, - ] - end - - def following_ids - @following_ids ||= account.active_relationships.pluck(:target_account_id) + [account.id] - end - def limit_for_non_exact_results return 0 if @account.nil? && query.size < MIN_QUERY_LENGTH diff --git a/app/services/bulk_import_service.rb b/app/services/bulk_import_service.rb index 5c14adc499..591b112127 100644 --- a/app/services/bulk_import_service.rb +++ b/app/services/bulk_import_service.rb @@ -162,10 +162,9 @@ class BulkImportService < BaseService def import_lists! rows = @import.rows.to_a + included_lists = rows.map { |row| row.data['list_name'] }.uniq if @import.overwrite? - included_lists = rows.map { |row| row.data['list_name'] }.uniq - @account.owned_lists.where.not(title: included_lists).destroy_all # As list membership changes do not retroactively change timeline @@ -175,6 +174,10 @@ class BulkImportService < BaseService end end + included_lists.each do |title| + @account.owned_lists.find_or_create_by!(title: title) + end + Import::RowWorker.push_bulk(rows) do |row| [row.id] end diff --git a/config/database.yml b/config/database.yml index 0907193feb..b995f23178 100644 --- a/config/database.yml +++ b/config/database.yml @@ -55,10 +55,10 @@ production: prepared_statements: <%= ENV['PREPARED_STATEMENTS'] || 'true' %> replica: <<: *default - database: <%= ENV['DB_REPLICA_NAME'] ||ENV['DB_NAME'] || 'mastodon_production' %> - username: <%= ENV['DB_REPLICA_USER'] ||ENV['DB_USER'] || 'mastodon' %> - password: <%= (ENV['DB_REPLICA_PASS'] || ENV['DB_PASS'] || '').to_json %> - host: <%= ENV['DB_REPLICA_HOST'] ||ENV['DB_HOST'] || 'localhost' %> - port: <%= ENV['DB_REPLICA_PORT'] ||ENV['DB_PORT'] || 5432 %> + database: <%= ENV['REPLICA_DB_NAME'] ||ENV['DB_NAME'] || 'mastodon_production' %> + username: <%= ENV['REPLICA_DB_USER'] ||ENV['DB_USER'] || 'mastodon' %> + password: <%= (ENV['REPLICA_DB_PASS'] || ENV['DB_PASS'] || '').to_json %> + host: <%= ENV['REPLICA_DB_HOST'] ||ENV['DB_HOST'] || 'localhost' %> + port: <%= ENV['REPLICA_DB_PORT'] ||ENV['DB_PORT'] || 5432 %> prepared_statements: <%= ENV['PREPARED_STATEMENTS'] || 'true' %> replica: true diff --git a/db/post_migrate/20230803082451_add_unique_index_on_preview_cards_statuses.rb b/db/post_migrate/20230803082451_add_unique_index_on_preview_cards_statuses.rb index 936d7840eb..c35ad80028 100644 --- a/db/post_migrate/20230803082451_add_unique_index_on_preview_cards_statuses.rb +++ b/db/post_migrate/20230803082451_add_unique_index_on_preview_cards_statuses.rb @@ -18,7 +18,7 @@ class AddUniqueIndexOnPreviewCardsStatuses < ActiveRecord::Migration[6.1] def deduplicate_and_reindex! deduplicate_preview_cards! - safety_assured { execute 'REINDEX INDEX preview_cards_statuses_pkey' } + safety_assured { execute 'REINDEX INDEX CONCURRENTLY preview_cards_statuses_pkey' } rescue ActiveRecord::RecordNotUnique retry end diff --git a/dist/mastodon-streaming.service b/dist/mastodon-streaming.service index 533feb5341..66cc3474ef 100644 --- a/dist/mastodon-streaming.service +++ b/dist/mastodon-streaming.service @@ -1,51 +1,12 @@ [Unit] Description=mastodon-streaming After=network.target +Wants=mastodon-streaming@4000.service [Service] -Type=simple -User=mastodon -WorkingDirectory=/home/mastodon/live -Environment="NODE_ENV=production" -Environment="PORT=4000" -ExecStart=/usr/bin/node ./streaming -TimeoutSec=15 -Restart=always -LimitNOFILE=65536 -# Proc filesystem -ProcSubset=pid -ProtectProc=invisible -# Capabilities -CapabilityBoundingSet= -# Security -NoNewPrivileges=true -# Sandboxing -ProtectSystem=strict -PrivateTmp=true -PrivateDevices=true -PrivateUsers=true -ProtectHostname=true -ProtectKernelLogs=true -ProtectKernelModules=true -ProtectKernelTunables=true -ProtectControlGroups=true -RestrictAddressFamilies=AF_INET -RestrictAddressFamilies=AF_INET6 -RestrictAddressFamilies=AF_NETLINK -RestrictAddressFamilies=AF_UNIX -RestrictNamespaces=true -LockPersonality=true -RestrictRealtime=true -RestrictSUIDSGID=true -RemoveIPC=true -PrivateMounts=true -ProtectClock=true -# System Call Filtering -SystemCallArchitectures=native -SystemCallFilter=~@cpu-emulation @debug @keyring @ipc @memlock @mount @obsolete @privileged @resources @setuid -SystemCallFilter=pipe -SystemCallFilter=pipe2 -ReadWritePaths=/home/mastodon/live +Type=oneshot +ExecStart=/bin/echo "mastodon-streaming exists only to collectively start and stop mastodon-streaming@ instances, shimming over the migration to templated mastodon-streaming systemd unit" +RemainAfterExit=yes [Install] WantedBy=multi-user.target diff --git a/dist/mastodon-streaming@.service b/dist/mastodon-streaming@.service new file mode 100644 index 0000000000..f43c4ea4a6 --- /dev/null +++ b/dist/mastodon-streaming@.service @@ -0,0 +1,54 @@ +[Unit] +Description=mastodon-streaming on port %I +After=network.target +# handles using `systemctl restart mastodon-streaming` +PartOf=mastodon-streaming.service +ReloadPropagatedFrom=mastodon-streaming.service + +[Service] +Type=simple +User=mastodon +WorkingDirectory=/home/mastodon/live +Environment="NODE_ENV=production" +Environment="PORT=%i" +ExecStart=/usr/bin/node ./streaming +TimeoutSec=15 +Restart=always +LimitNOFILE=65536 +# Proc filesystem +ProcSubset=pid +ProtectProc=invisible +# Capabilities +CapabilityBoundingSet= +# Security +NoNewPrivileges=true +# Sandboxing +ProtectSystem=strict +PrivateTmp=true +PrivateDevices=true +PrivateUsers=true +ProtectHostname=true +ProtectKernelLogs=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_INET +RestrictAddressFamilies=AF_INET6 +RestrictAddressFamilies=AF_NETLINK +RestrictAddressFamilies=AF_UNIX +RestrictNamespaces=true +LockPersonality=true +RestrictRealtime=true +RestrictSUIDSGID=true +RemoveIPC=true +PrivateMounts=true +ProtectClock=true +# System Call Filtering +SystemCallArchitectures=native +SystemCallFilter=~@cpu-emulation @debug @keyring @ipc @memlock @mount @obsolete @privileged @resources @setuid +SystemCallFilter=pipe +SystemCallFilter=pipe2 +ReadWritePaths=/home/mastodon/live + +[Install] +WantedBy=multi-user.target mastodon-streaming.service diff --git a/dist/nginx.conf b/dist/nginx.conf index fc68e9a6d1..39fa58e50d 100644 --- a/dist/nginx.conf +++ b/dist/nginx.conf @@ -8,7 +8,15 @@ upstream backend { } upstream streaming { + # Instruct nginx to send connections to the server with the least number of connections + # to ensure load is distributed evenly. + least_conn; + server 127.0.0.1:4000 fail_timeout=0; + # Uncomment these lines for load-balancing multiple instances of streaming for scaling, + # this assumes your running the streaming server on ports 4000, 4001, and 4002: + # server 127.0.0.1:4001 fail_timeout=0; + # server 127.0.0.1:4002 fail_timeout=0; } proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHE:10m inactive=7d max_size=1g; diff --git a/lib/mastodon/migration_helpers.rb b/lib/mastodon/migration_helpers.rb index 5a252b3512..4a43f67c27 100644 --- a/lib/mastodon/migration_helpers.rb +++ b/lib/mastodon/migration_helpers.rb @@ -195,7 +195,14 @@ module Mastodon def supports_drop_index_concurrently? version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i - version >= 90200 + version >= 90_200 + end + + # Only available on Postgresql >= 11 + def supports_add_column_with_default? + version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i + + version >= 11_000 end # Adds a foreign key with only minimal locking on the tables involved. @@ -414,6 +421,11 @@ module Mastodon # This method can also take a block which is passed directly to the # `update_column_in_batches` method. def add_column_with_default(table, column, type, default:, limit: nil, allow_null: false, &block) + if supports_add_column_with_default? + add_column(table, column, type, default: default, limit: limit, null: allow_null) + return + end + if transaction_open? raise 'add_column_with_default can not be run inside a transaction, ' \ 'you can disable transactions by calling disable_ddl_transaction! ' \ diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index f95c105eee..236204de7b 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -9,15 +9,15 @@ module Mastodon end def minor - 1 + 2 end def patch - 6 + 0 end def flags - ENV.fetch('MASTODON_VERSION_FLAGS', '') + ENV.fetch('MASTODON_VERSION_FLAGS', '-beta1') end def suffix diff --git a/spec/fixtures/files/attachment.gif b/spec/fixtures/files/attachment.gif index 2937f5abef..89dd73ad3e 100644 Binary files a/spec/fixtures/files/attachment.gif and b/spec/fixtures/files/attachment.gif differ diff --git a/spec/fixtures/files/attachment.webm b/spec/fixtures/files/attachment.webm index 3babe5cf59..edde443960 100644 Binary files a/spec/fixtures/files/attachment.webm and b/spec/fixtures/files/attachment.webm differ diff --git a/spec/models/media_attachment_spec.rb b/spec/models/media_attachment_spec.rb index d142875aa9..d57d43eda7 100644 --- a/spec/models/media_attachment_spec.rb +++ b/spec/models/media_attachment_spec.rb @@ -6,7 +6,7 @@ RSpec.describe MediaAttachment, paperclip_processing: true do describe 'local?' do subject { media_attachment.local? } - let(:media_attachment) { Fabricate(:media_attachment, remote_url: remote_url) } + let(:media_attachment) { described_class.new(remote_url: remote_url) } context 'when remote_url is blank' do let(:remote_url) { '' } @@ -28,7 +28,7 @@ RSpec.describe MediaAttachment, paperclip_processing: true do describe 'needs_redownload?' do subject { media_attachment.needs_redownload? } - let(:media_attachment) { Fabricate(:media_attachment, remote_url: remote_url, file: file) } + let(:media_attachment) { described_class.new(remote_url: remote_url, file: file) } context 'when file is blank' do let(:file) { nil } @@ -64,11 +64,11 @@ RSpec.describe MediaAttachment, paperclip_processing: true do end describe '#to_param' do - let(:media_attachment) { Fabricate(:media_attachment, shortcode: shortcode) } - let(:shortcode) { nil } + let(:media_attachment) { Fabricate.build(:media_attachment, shortcode: shortcode, id: id) } context 'when media attachment has a shortcode' do let(:shortcode) { 'foo' } + let(:id) { 123 } it 'returns shortcode' do expect(media_attachment.to_param).to eq shortcode @@ -77,9 +77,10 @@ RSpec.describe MediaAttachment, paperclip_processing: true do context 'when media attachment does not have a shortcode' do let(:shortcode) { nil } + let(:id) { 123 } it 'returns string representation of id' do - expect(media_attachment.to_param).to eq media_attachment.id.to_s + expect(media_attachment.to_param).to eq id.to_s end end end @@ -89,38 +90,33 @@ RSpec.describe MediaAttachment, paperclip_processing: true do media.destroy end - it 'saves media attachment' do + it 'saves media attachment with correct file metadata' do expect(media.persisted?).to be true expect(media.file).to_not be_nil - end - it 'completes processing' do + # completes processing expect(media.processing_complete?).to be true - end - it 'sets type' do + # sets type expect(media.type).to eq 'image' - end - it 'sets content type' do + # sets content type expect(media.file_content_type).to eq content_type - end - it 'sets file extension' do + # sets file extension expect(media.file_file_name).to end_with extension end - it 'strips original file name' do + it 'saves media attachment with correct size metadata' do + # strips original file name expect(media.file_file_name).to_not start_with '600x400' - end - it 'sets meta for original' do + # sets meta for original expect(media.file.meta['original']['width']).to eq 600 expect(media.file.meta['original']['height']).to eq 400 expect(media.file.meta['original']['aspect']).to eq 1.5 - end - it 'sets meta for thumbnail' do + # sets meta for thumbnail expect(media.file.meta['small']['width']).to eq 588 expect(media.file.meta['small']['height']).to eq 392 expect(media.file.meta['small']['aspect']).to eq 1.5 @@ -128,54 +124,48 @@ RSpec.describe MediaAttachment, paperclip_processing: true do end describe 'jpeg' do - let(:media) { described_class.create(account: Fabricate(:account), file: attachment_fixture('600x400.jpeg')) } + let(:media) { Fabricate(:media_attachment, file: attachment_fixture('600x400.jpeg')) } it_behaves_like 'static 600x400 image', 'image/jpeg', '.jpeg' end describe 'png' do - let(:media) { described_class.create(account: Fabricate(:account), file: attachment_fixture('600x400.png')) } + let(:media) { Fabricate(:media_attachment, file: attachment_fixture('600x400.png')) } it_behaves_like 'static 600x400 image', 'image/png', '.png' end describe 'webp' do - let(:media) { described_class.create(account: Fabricate(:account), file: attachment_fixture('600x400.webp')) } + let(:media) { Fabricate(:media_attachment, file: attachment_fixture('600x400.webp')) } it_behaves_like 'static 600x400 image', 'image/webp', '.webp' end describe 'avif' do - let(:media) { described_class.create(account: Fabricate(:account), file: attachment_fixture('600x400.avif')) } + let(:media) { Fabricate(:media_attachment, file: attachment_fixture('600x400.avif')) } it_behaves_like 'static 600x400 image', 'image/jpeg', '.jpeg' end describe 'heic' do - let(:media) { described_class.create(account: Fabricate(:account), file: attachment_fixture('600x400.heic')) } + let(:media) { Fabricate(:media_attachment, file: attachment_fixture('600x400.heic')) } it_behaves_like 'static 600x400 image', 'image/jpeg', '.jpeg' end describe 'base64-encoded image' do let(:base64_attachment) { "data:image/jpeg;base64,#{Base64.encode64(attachment_fixture('600x400.jpeg').read)}" } - let(:media) { described_class.create(account: Fabricate(:account), file: base64_attachment) } + let(:media) { Fabricate(:media_attachment, file: base64_attachment) } it_behaves_like 'static 600x400 image', 'image/jpeg', '.jpeg' end describe 'animated gif' do - let(:media) { described_class.create(account: Fabricate(:account), file: attachment_fixture('avatar.gif')) } + let(:media) { Fabricate(:media_attachment, file: attachment_fixture('avatar.gif')) } - it 'sets type to gifv' do + it 'sets correct file metadata' do expect(media.type).to eq 'gifv' - end - - it 'converts original file to mp4' do expect(media.file_content_type).to eq 'video/mp4' - end - - it 'sets meta' do expect(media.file.meta['original']['width']).to eq 128 expect(media.file.meta['original']['height']).to eq 128 end @@ -189,17 +179,11 @@ RSpec.describe MediaAttachment, paperclip_processing: true do fixtures.each do |fixture| context fixture[:filename] do - let(:media) { described_class.create(account: Fabricate(:account), file: attachment_fixture(fixture[:filename])) } + let(:media) { Fabricate(:media_attachment, file: attachment_fixture(fixture[:filename])) } - it 'sets type to image' do + it 'sets correct file metadata' do expect(media.type).to eq 'image' - end - - it 'leaves original file as-is' do expect(media.file_content_type).to eq 'image/gif' - end - - it 'sets meta' do expect(media.file.meta['original']['width']).to eq fixture[:width] expect(media.file.meta['original']['height']).to eq fixture[:height] expect(media.file.meta['original']['aspect']).to eq fixture[:aspect] @@ -209,31 +193,19 @@ RSpec.describe MediaAttachment, paperclip_processing: true do end describe 'ogg with cover art' do - let(:media) { described_class.create(account: Fabricate(:account), file: attachment_fixture('boop.ogg')) } + let(:media) { Fabricate(:media_attachment, file: attachment_fixture('boop.ogg')) } - it 'detects it as an audio file' do + it 'sets correct file metadata' do expect(media.type).to eq 'audio' - end - - it 'sets meta for the duration' do expect(media.file.meta['original']['duration']).to be_within(0.05).of(0.235102) - end - - it 'extracts thumbnail' do expect(media.thumbnail.present?).to be true - end - - it 'extracts colors from thumbnail' do expect(media.file.meta['colors']['background']).to eq '#3088d4' - end - - it 'gives the file a random name' do expect(media.file_file_name).to_not eq 'boop.ogg' end end describe 'mp3 with large cover art' do - let(:media) { described_class.create(account: Fabricate(:account), file: attachment_fixture('boop.mp3')) } + let(:media) { Fabricate(:media_attachment, file: attachment_fixture('boop.mp3')) } it 'detects it as an audio file' do expect(media.type).to eq 'audio' @@ -253,34 +225,36 @@ RSpec.describe MediaAttachment, paperclip_processing: true do end it 'is invalid without file' do - media = described_class.new(account: Fabricate(:account)) + media = described_class.new + expect(media.valid?).to be false + expect(media).to model_have_error_on_field(:file) end describe 'size limit validation' do it 'rejects video files that are too large' do stub_const 'MediaAttachment::IMAGE_LIMIT', 100.megabytes stub_const 'MediaAttachment::VIDEO_LIMIT', 1.kilobyte - expect { described_class.create!(account: Fabricate(:account), file: attachment_fixture('attachment.webm')) }.to raise_error(ActiveRecord::RecordInvalid) + expect { Fabricate(:media_attachment, file: attachment_fixture('attachment.webm')) }.to raise_error(ActiveRecord::RecordInvalid) end it 'accepts video files that are small enough' do stub_const 'MediaAttachment::IMAGE_LIMIT', 1.kilobyte stub_const 'MediaAttachment::VIDEO_LIMIT', 100.megabytes - media = described_class.create!(account: Fabricate(:account), file: attachment_fixture('attachment.webm')) + media = Fabricate(:media_attachment, file: attachment_fixture('attachment.webm')) expect(media.valid?).to be true end it 'rejects image files that are too large' do stub_const 'MediaAttachment::IMAGE_LIMIT', 1.kilobyte stub_const 'MediaAttachment::VIDEO_LIMIT', 100.megabytes - expect { described_class.create!(account: Fabricate(:account), file: attachment_fixture('attachment.jpg')) }.to raise_error(ActiveRecord::RecordInvalid) + expect { Fabricate(:media_attachment, file: attachment_fixture('attachment.jpg')) }.to raise_error(ActiveRecord::RecordInvalid) end it 'accepts image files that are small enough' do stub_const 'MediaAttachment::IMAGE_LIMIT', 100.megabytes stub_const 'MediaAttachment::VIDEO_LIMIT', 1.kilobyte - media = described_class.create!(account: Fabricate(:account), file: attachment_fixture('attachment.jpg')) + media = Fabricate(:media_attachment, file: attachment_fixture('attachment.jpg')) expect(media.valid?).to be true end end diff --git a/spec/services/bulk_import_row_service_spec.rb b/spec/services/bulk_import_row_service_spec.rb index 5e09845b53..a77acc0732 100644 --- a/spec/services/bulk_import_row_service_spec.rb +++ b/spec/services/bulk_import_row_service_spec.rb @@ -161,6 +161,12 @@ RSpec.describe BulkImportRowService do end include_examples 'common behavior' + + it 'does not create a new list' do + account.follow!(target_account) + + expect { subject.call(import_row) }.to_not(change { List.where(title: 'my list').count }) + end end end end