github-mirror/discourse
github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-04-13 14:25:14 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

FIX: copy uploads quickly in discourse_merger.rb, and fix user avatar upload id for copied users

Browse Source
This commit is contained in:
Neil Lalonde 2018-07-18 16:42:59 -04:00
parent afc94ac9e4
commit 24da2940a7
1 changed files with 4 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
script/bulk_import/discourse_merger.rb
View File

@ -281,10 +281,7 @@ class BulkImport::DiscourseMerger < BulkImport::Base
)
end
# This method of copying uploads directly copies the files and then upload records.
# It's fast, but something is wrong because all the images are white squares.
# Also, some remapping needs to be done once this is finished.
def copy_uploads_experimental
def copy_uploads
puts ''
print "copying uploads..."
@ -300,14 +297,11 @@ class BulkImport::DiscourseMerger < BulkImport::Base
@raw_connection.copy_data(sql, @encoder) do
source_raw_connection.exec("SELECT #{columns.map { |c| "\"#{c}\"" }.join(', ')} FROM uploads").each do |row|
next if Upload.where(sha1: row['sha1']).exists? # built-in images
next if Upload.where(sha1: row['sha1']).exists?
rel_filename = row['url'].gsub(/^\/uploads\/[^\/]+\//, '')
rel_filename = rel_filename.gsub(/^\/\/[^\/]+\.amazonaws\.com\//, '')
absolute_filename = File.join(@uploads_path, rel_filename)
print '.'
next unless File.exists?(absolute_filename)
old_id = row['id']
if old_id && last_id
@ -318,19 +312,10 @@ class BulkImport::DiscourseMerger < BulkImport::Base
old_user_id = row['user_id'].to_i
if old_user_id >= 1
row['user_id'] = user_id_from_imported_id(old_user_id)
next if row['user_id'].nil? # associated record for a deleted user
next if row['user_id'].nil?
end
old_url = row['url']
row['url'] = "/uploads/default/#{rel_filename}"
# Too slow to do remapping three times for each upload!!!
# if @source_cdn
# DbHelper.remap(UrlHelper.absolute(old_url, @source_cdn), row['url'])
# end
# DbHelper.remap(UrlHelper.absolute(old_url, @source_base_url), row['url'])
# DbHelper.remap(old_url, row['url'])
row['url'] = "/uploads/default/#{rel_filename}" if File.exists?(absolute_filename)
@raw_connection.put_copy_data(row.values)
end
@ -339,39 +324,6 @@ class BulkImport::DiscourseMerger < BulkImport::Base
copy_model(PostUpload)
copy_model(UserAvatar)
end
# The super slow method of copying uploads:
def copy_uploads
puts ''
print "copying uploads..."
source_raw_connection.exec("SELECT * FROM uploads").each do |row|
user_id = row['user_id'].to_i
user_id = user_id_from_imported_id(user_id) if user_id > 0
rel_filename = row['url'].gsub(/^\/uploads\/[^\/]+\//, '')
rel_filename = rel_filename.gsub(/^\/\/[^\/]+\.amazonaws\.com\//, '')
absolute_filename = File.join(@uploads_path, rel_filename)
print '.'
next unless File.exists?(absolute_filename)
upload = create_upload(user_id, absolute_filename, File.basename(absolute_filename))
if upload&.persisted?
@uploads[row['id']] = upload.id
if @source_cdn
DbHelper.remap(UrlHelper.absolute(row['url'], @source_cdn), upload.url)
end
DbHelper.remap(UrlHelper.absolute(row['url'], @source_base_url), upload.url)
DbHelper.remap(row['url'], upload.url)
else
puts "Error: Upload did not persist for #{absolute_filename}! #{upload&.errors&.full_messages}"
end
end
puts ''
copy_model(PostUpload)
copy_model(UserAvatar)
# Users have a column "uploaded_avatar_id" which needs to be mapped now.
User.where("id >= ?", @first_new_user_id).find_each do |u|