gitea/nas
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 17

Remove deprecated Import model (#34371)

Browse source
This commit is contained in:
Matt Jankowski 2025-04-10 08:49:24 -04:00 • committed by GitHub
parent e74d682b21
commit 4c2f64907b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 13 additions and 565 deletions
Download patch file Download diff file Expand all files Collapse all files

1
app/lib/admin/metrics/dimension/space_usage_dimension.rb
View file

@ -45,7 +45,6 @@ class Admin::Metrics::Dimension::SpaceUsageDimension < Admin::Metrics::Dimension
PreviewCard.sum(:image_file_size),
Account.sum(Arel.sql('COALESCE(avatar_file_size, 0) + COALESCE(header_file_size, 0)')),
Backup.sum(:dump_file_size),
Import.sum(:data_file_size),
SiteUpload.sum(:file_file_size),
].sum

46
app/models/import.rb
View file

@ -1,46 +0,0 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: imports
#
# id :bigint(8) not null, primary key
# type :integer not null
# approved :boolean default(FALSE), not null
# created_at :datetime not null
# updated_at :datetime not null
# data_file_name :string
# data_content_type :string
# data_file_size :integer
# data_updated_at :datetime
# account_id :bigint(8) not null
# overwrite :boolean default(FALSE), not null
#
# NOTE: This is a deprecated model, only kept to not break ongoing imports
# on upgrade. See `BulkImport` and `Form::Import` for its replacements.
class Import < ApplicationRecord
FILE_TYPES = %w(text/plain text/csv application/csv).freeze
MODES = %i(merge overwrite).freeze
self.inheritance_column = false
belongs_to :account
enum :type, { following: 0, blocking: 1, muting: 2, domain_blocking: 3, bookmarks: 4 }
validates :type, presence: true
has_attached_file :data
validates_attachment_content_type :data, content_type: FILE_TYPES
validates_attachment_presence :data
def mode
overwrite? ? :overwrite : :merge
end
def mode=(str)
self.overwrite = str.to_sym == :overwrite
end
end

144
app/services/import_service.rb
View file

@ -1,144 +0,0 @@
# frozen_string_literal: true
require 'csv'
# NOTE: This is a deprecated service, only kept to not break ongoing imports
# on upgrade. See `BulkImportService` for its replacement.
class ImportService < BaseService
ROWS_PROCESSING_LIMIT = 20_000
def call(import)
@import = import
@account = @import.account
case @import.type
when 'following'
import_follows!
when 'blocking'
import_blocks!
when 'muting'
import_mutes!
when 'domain_blocking'
import_domain_blocks!
when 'bookmarks'
import_bookmarks!
end
end
private
def import_follows!
parse_import_data!(['Account address'])
import_relationships!('follow', 'unfollow', @account.following, ROWS_PROCESSING_LIMIT, reblogs: { header: 'Show boosts', default: true }, notify: { header: 'Notify on new posts', default: false }, languages: { header: 'Languages', default: nil })
end
def import_blocks!
parse_import_data!(['Account address'])
import_relationships!('block', 'unblock', @account.blocking, ROWS_PROCESSING_LIMIT)
end
def import_mutes!
parse_import_data!(['Account address'])
import_relationships!('mute', 'unmute', @account.muting, ROWS_PROCESSING_LIMIT, notifications: { header: 'Hide notifications', default: true })
end
def import_domain_blocks!
parse_import_data!(['#domain'])
items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row['#domain'].strip }
if @import.overwrite?
presence_hash = items.index_with(true)
@account.domain_blocks.find_each do |domain_block|
if presence_hash[domain_block.domain]
items.delete(domain_block.domain)
else
@account.unblock_domain!(domain_block.domain)
end
end
end
items.each do |domain|
@account.block_domain!(domain)
end
AfterAccountDomainBlockWorker.push_bulk(items) do |domain|
[@account.id, domain]
end
end
def import_relationships!(action, undo_action, overwrite_scope, limit, extra_fields = {})
local_domain_suffix = "@#{Rails.configuration.x.local_domain}"
items = @data.take(limit).map { |row| [row['Account address']&.strip&.delete_suffix(local_domain_suffix), extra_fields.to_h { |key, field_settings| [key, row[field_settings[:header]]&.strip || field_settings[:default]] }] }.reject { |(id, _)| id.blank? }
if @import.overwrite?
presence_hash = items.each_with_object({}) { |(id, extra), mapping| mapping[id] = [true, extra] }
overwrite_scope.reorder(nil).find_each do |target_account|
if presence_hash[target_account.acct]
items.delete(target_account.acct)
extra = presence_hash[target_account.acct][1]
Import::RelationshipWorker.perform_async(@account.id, target_account.acct, action, extra.stringify_keys)
else
Import::RelationshipWorker.perform_async(@account.id, target_account.acct, undo_action)
end
end
end
head_items = items.uniq { |acct, _| acct.split('@')[1] }
tail_items = items - head_items
Import::RelationshipWorker.push_bulk(head_items + tail_items) do |acct, extra|
[@account.id, acct, action, extra.stringify_keys]
end
end
def import_bookmarks!
parse_import_data!(['#uri'])
items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row['#uri'].strip }
if @import.overwrite?
presence_hash = items.index_with(true)
@account.bookmarks.find_each do |bookmark|
if presence_hash[bookmark.status.uri]
items.delete(bookmark.status.uri)
else
bookmark.destroy!
end
end
end
statuses = items.filter_map do |uri|
status = ActivityPub::TagManager.instance.uri_to_resource(uri, Status)
next if status.nil? && ActivityPub::TagManager.instance.local_uri?(uri)
status || ActivityPub::FetchRemoteStatusService.new.call(uri)
rescue *Mastodon::HTTP_CONNECTION_ERRORS, Mastodon::UnexpectedResponseError
nil
rescue => e
Rails.logger.warn "Unexpected error when importing bookmark: #{e}"
nil
end
account_ids = statuses.map(&:account_id)
preloaded_relations = @account.relations_map(account_ids, skip_blocking_and_muting: true)
statuses.keep_if { |status| StatusPolicy.new(@account, status, preloaded_relations).show? }
statuses.each do |status|
@account.bookmarks.find_or_create_by!(account: @account, status: status)
end
end
def parse_import_data!(default_headers)
data = CSV.parse(import_data, headers: true)
data = CSV.parse(import_data, headers: default_headers) unless data.headers&.first&.strip&.include?(' ')
@data = data.compact_blank
end
def import_data
Paperclip.io_adapters.for(@import.data).read.force_encoding(Encoding::UTF_8)
end
end

57
app/workers/import/relationship_worker.rb
View file

@ -1,57 +0,0 @@
# frozen_string_literal: true
# NOTE: This is a deprecated worker, only kept to not break ongoing imports
# on upgrade. See `Import::RowWorker` for its replacement.
class Import::RelationshipWorker
include Sidekiq::Worker
sidekiq_options queue: 'pull', retry: 8, dead: false
def perform(account_id, target_account_uri, relationship, options)
from_account = Account.find(account_id)
target_domain = domain(target_account_uri)
target_account = stoplight_wrapper(target_domain).run { ResolveAccountService.new.call(target_account_uri, { check_delivery_availability: true }) }
options.symbolize_keys!
return if target_account.nil?
case relationship
when 'follow'
begin
FollowService.new.call(from_account, target_account, **options)
rescue ActiveRecord::RecordInvalid
raise if FollowLimitValidator.limit_for_account(from_account) < from_account.following_count
end
when 'unfollow'
UnfollowService.new.call(from_account, target_account)
when 'block'
BlockService.new.call(from_account, target_account)
when 'unblock'
UnblockService.new.call(from_account, target_account)
when 'mute'
MuteService.new.call(from_account, target_account, **options)
when 'unmute'
UnmuteService.new.call(from_account, target_account)
end
rescue ActiveRecord::RecordNotFound
true
end
def domain(uri)
domain = uri.is_a?(Account) ? uri.domain : uri.split('@')[1]
TagManager.instance.local_domain?(domain) ? nil : TagManager.instance.normalize_domain(domain)
end
def stoplight_wrapper(domain)
if domain.present?
Stoplight("source:#{domain}")
.with_fallback { nil }
.with_threshold(1)
.with_cool_off_time(5.minutes.seconds)
.with_error_handler { |error, handle| error.is_a?(HTTP::Error) || error.is_a?(OpenSSL::SSL::SSLError) ? handle.call(error) : raise(error) }
else
Stoplight('domain-blank')
end
end
end

17
app/workers/import_worker.rb
View file

@ -1,17 +0,0 @@
# frozen_string_literal: true
# NOTE: This is a deprecated worker, only kept to not break ongoing imports
# on upgrade. See `ImportWorker` for its replacement.
class ImportWorker
include Sidekiq::Worker
sidekiq_options queue: 'pull', retry: false
def perform(import_id)
import = Import.find(import_id)
ImportService.new.call(import)
ensure
import&.destroy
end
end