github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2024-11-22 11:23:25 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

FIX: Add compatibility for bucket folder paths in migrate_to_s3 task (#6855)

Browse Source 
* FIX: Add compatibility for bucket folder paths in migrate_to_s3 task
* Refactor bucket_name split logic into S3Helper
This commit is contained in:
Rishabh 2019-01-08 20:04:48 +05:30 committed by GitHub
parent 733a60e888
commit f181e9cc08
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 24 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
lib/s3_helper.rb
View File

@ -12,7 +12,7 @@ class S3Helper
@s3_bucket_name, @s3_bucket_folder_path = begin
raise Discourse::InvalidParameters.new("s3_bucket_name") if s3_bucket_name.blank?
s3_bucket_name.downcase.split("/".freeze, 2)
self.class.get_bucket_and_folder_path(s3_bucket_name)
end
@tombstone_prefix =
@ -23,6 +23,10 @@ class S3Helper
end
end
def self.get_bucket_and_folder_path(s3_bucket_name)
s3_bucket_name.downcase.split("/".freeze, 2)
end
def upload(file, path, options = {})
path = get_path_for_s3_upload(path)
obj = s3_bucket.object(path)
@ -62,10 +66,10 @@ class S3Helper
options[:copy_source] = File.join(@s3_bucket_name, source)
else
if @s3_bucket_folder_path
bucket_folder, filename = begin
folder, filename = begin
source.split("/".freeze, 2)
end
options[:copy_source] = File.join(@s3_bucket_name, bucket_folder, multisite_upload_path, filename)
options[:copy_source] = File.join(@s3_bucket_name, folder, multisite_upload_path, filename)
else
options[:copy_source] = File.join(@s3_bucket_name, multisite_upload_path, source)
end

24
lib/tasks/uploads.rake
View File

@ -214,6 +214,7 @@ def migrate_to_s3
db = RailsMultisite::ConnectionManagement.current_db
dry_run = !!ENV["DRY_RUN"]
bucket_has_folder_path = true if ENV["DISCOURSE_S3_BUCKET"].include? "/"
puts "*" * 30 + " DRY RUN " + "*" * 30 if dry_run
puts "Migrating uploads to S3 for '#{db}'..."
@ -246,11 +247,18 @@ def migrate_to_s3
s3 = Aws::S3::Client.new(S3Helper.s3_options(GlobalSetting))
if bucket_has_folder_path
bucket, folder = S3Helper.get_bucket_and_folder_path(ENV["DISCOURSE_S3_BUCKET"])
folder = File.join(folder, "/")
else
bucket, folder = GlobalSetting.s3_bucket, ""
end
begin
s3.head_bucket(bucket: GlobalSetting.s3_bucket)
s3.head_bucket(bucket: bucket)
rescue Aws::S3::Errors::NotFound
puts "Bucket '#{GlobalSetting.s3_bucket}' not found. Creating it..."
s3.create_bucket(bucket: GlobalSetting.s3_bucket) unless dry_run
puts "Bucket '#{bucket}' not found. Creating it..."
s3.create_bucket(bucket: bucket) unless dry_run
end
puts "Uploading files to S3..."
@ -267,7 +275,7 @@ def migrate_to_s3
s3_objects = []
prefix = Rails.configuration.multisite ? "#{db}/original/" : "original/"
options = { bucket: GlobalSetting.s3_bucket, prefix: prefix }
options = { bucket: bucket, prefix: prefix }
loop do
response = s3.list_objects_v2(options)
@ -287,6 +295,8 @@ def migrate_to_s3
path = File.join("public", file)
name = File.basename(path)
etag = Digest::MD5.file(path).hexdigest
key = file[file.index(prefix)..-1]
key.prepend(folder) if bucket_has_folder_path
if s3_object = s3_objects.find { |obj| file.ends_with?(obj.key) }
next if File.size(path) == s3_object.size && s3_object.etag[etag]
@ -295,9 +305,9 @@ def migrate_to_s3
options = {
acl: "public-read",
body: File.open(path, "rb"),
bucket: GlobalSetting.s3_bucket,
bucket: bucket,
content_type: MiniMime.lookup_by_filename(name)&.content_type,
key: file[file.index(prefix)..-1],
key: key,
}
if !FileHelper.is_supported_image?(name)
@ -337,7 +347,7 @@ def migrate_to_s3
}
from = "/uploads/#{db}/original/(\\dX/(?:[a-f0-9]/)*[a-f0-9]{40}[a-z0-9\\.]*)"
to = "#{SiteSetting.Upload.s3_base_url}/#{prefix}\\1"
to = "#{SiteSetting.Upload.s3_base_url}/#{folder}#{prefix}\\1"
if dry_run
puts "REPLACING '#{from}' WITH '#{to}'"