2017-10-03 15:00:42 +08:00
|
|
|
require "aws-sdk-s3"
|
2014-09-25 04:52:09 +08:00
|
|
|
|
|
|
|
class S3Helper
|
|
|
|
|
2016-08-17 16:16:00 +08:00
|
|
|
class SettingMissing < StandardError; end
|
|
|
|
|
2018-05-23 05:21:52 +08:00
|
|
|
attr_reader :s3_bucket_name, :s3_bucket_folder_path
|
2016-08-19 14:08:04 +08:00
|
|
|
|
2017-10-06 13:20:01 +08:00
|
|
|
def initialize(s3_bucket_name, tombstone_prefix = '', options = {})
|
2016-08-17 16:16:00 +08:00
|
|
|
@s3_options = default_s3_options.merge(options)
|
2016-08-16 11:13:59 +08:00
|
|
|
|
2016-08-19 14:08:04 +08:00
|
|
|
@s3_bucket_name, @s3_bucket_folder_path = begin
|
2017-10-06 13:20:01 +08:00
|
|
|
raise Discourse::InvalidParameters.new("s3_bucket_name") if s3_bucket_name.blank?
|
|
|
|
s3_bucket_name.downcase.split("/".freeze, 2)
|
2016-08-15 16:06:29 +08:00
|
|
|
end
|
2014-09-25 04:52:09 +08:00
|
|
|
|
2016-08-15 16:06:29 +08:00
|
|
|
@tombstone_prefix =
|
|
|
|
if @s3_bucket_folder_path
|
|
|
|
File.join(@s3_bucket_folder_path, tombstone_prefix)
|
|
|
|
else
|
|
|
|
tombstone_prefix
|
|
|
|
end
|
2014-09-25 04:52:09 +08:00
|
|
|
end
|
|
|
|
|
2017-07-28 09:20:09 +08:00
|
|
|
def upload(file, path, options = {})
|
2016-08-15 16:06:29 +08:00
|
|
|
path = get_path_for_s3_upload(path)
|
2016-08-15 11:21:24 +08:00
|
|
|
obj = s3_bucket.object(path)
|
2015-05-25 15:57:06 +08:00
|
|
|
obj.upload_file(file, options)
|
2016-08-15 16:06:29 +08:00
|
|
|
path
|
2014-09-25 04:52:09 +08:00
|
|
|
end
|
|
|
|
|
2017-07-28 09:20:09 +08:00
|
|
|
def remove(s3_filename, copy_to_tombstone = false)
|
2014-09-25 04:52:09 +08:00
|
|
|
# copy the file in tombstone
|
2016-08-15 16:06:29 +08:00
|
|
|
if copy_to_tombstone && @tombstone_prefix.present?
|
2018-08-08 11:26:05 +08:00
|
|
|
self.copy(
|
|
|
|
File.join(@tombstone_prefix, s3_filename),
|
|
|
|
get_path_for_s3_upload(s3_filename)
|
|
|
|
)
|
2014-09-25 04:52:09 +08:00
|
|
|
end
|
2016-08-15 11:21:24 +08:00
|
|
|
|
2014-09-25 04:52:09 +08:00
|
|
|
# delete the file
|
2018-08-08 11:26:05 +08:00
|
|
|
s3_bucket.object(get_path_for_s3_upload(s3_filename)).delete
|
2015-05-25 15:57:06 +08:00
|
|
|
rescue Aws::S3::Errors::NoSuchKey
|
2014-09-25 04:52:09 +08:00
|
|
|
end
|
|
|
|
|
2018-08-08 11:26:05 +08:00
|
|
|
def copy(source, destination)
|
|
|
|
s3_bucket
|
|
|
|
.object(source)
|
|
|
|
.copy_from(copy_source: File.join(@s3_bucket_name, destination))
|
|
|
|
end
|
|
|
|
|
2017-10-09 07:26:58 +08:00
|
|
|
# make sure we have a cors config for assets
|
|
|
|
# otherwise we will have no fonts
|
|
|
|
def ensure_cors!
|
|
|
|
rule = nil
|
|
|
|
|
|
|
|
begin
|
|
|
|
rule = s3_resource.client.get_bucket_cors(
|
|
|
|
bucket: @s3_bucket_name
|
|
|
|
).cors_rules&.first
|
|
|
|
rescue Aws::S3::Errors::NoSuchCORSConfiguration
|
|
|
|
# no rule
|
|
|
|
end
|
|
|
|
|
|
|
|
unless rule
|
|
|
|
puts "installing CORS rule"
|
|
|
|
|
|
|
|
s3_resource.client.put_bucket_cors(
|
|
|
|
bucket: @s3_bucket_name,
|
|
|
|
cors_configuration: {
|
|
|
|
cors_rules: [{
|
|
|
|
allowed_headers: ["Authorization"],
|
|
|
|
allowed_methods: ["GET", "HEAD"],
|
|
|
|
allowed_origins: ["*"],
|
|
|
|
max_age_seconds: 3000
|
|
|
|
}]
|
|
|
|
}
|
|
|
|
)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def update_lifecycle(id, days, prefix: nil, tag: nil)
|
|
|
|
|
|
|
|
filter = {}
|
|
|
|
|
|
|
|
if prefix
|
|
|
|
filter[:prefix] = prefix
|
|
|
|
elsif tag
|
|
|
|
filter[:tag] = tag
|
|
|
|
end
|
2015-05-25 23:59:00 +08:00
|
|
|
|
2014-09-25 04:52:09 +08:00
|
|
|
# cf. http://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html
|
2017-10-03 15:00:42 +08:00
|
|
|
rule = {
|
|
|
|
id: id,
|
|
|
|
status: "Enabled",
|
2017-10-09 07:26:58 +08:00
|
|
|
expiration: { days: days },
|
|
|
|
filter: filter
|
2017-10-03 15:00:42 +08:00
|
|
|
}
|
|
|
|
|
2017-10-09 07:26:58 +08:00
|
|
|
rules = []
|
2017-10-03 15:00:42 +08:00
|
|
|
|
2017-10-09 07:26:58 +08:00
|
|
|
begin
|
|
|
|
rules = s3_resource.client.get_bucket_lifecycle_configuration(bucket: @s3_bucket_name).rules
|
|
|
|
rescue Aws::S3::Errors::NoSuchLifecycleConfiguration
|
|
|
|
# skip trying to merge
|
|
|
|
end
|
2017-10-03 15:00:42 +08:00
|
|
|
|
2017-11-13 12:36:45 +08:00
|
|
|
# in the past we has a rule that was called purge-tombstone vs purge_tombstone
|
|
|
|
# just go ahead and normalize for our bucket
|
2017-10-03 15:00:42 +08:00
|
|
|
rules.delete_if do |r|
|
2017-11-13 12:36:45 +08:00
|
|
|
r.id.gsub('_', '-') == id.gsub('_', '-')
|
2017-10-03 15:00:42 +08:00
|
|
|
end
|
|
|
|
|
|
|
|
rules << rule
|
|
|
|
|
2017-11-13 12:36:45 +08:00
|
|
|
# normalize filter in rules, due to AWS library bug
|
|
|
|
rules = rules.map do |r|
|
|
|
|
r = r.to_h
|
|
|
|
prefix = r.delete(:prefix)
|
|
|
|
if prefix
|
|
|
|
r[:filter] = { prefix: prefix }
|
|
|
|
end
|
|
|
|
r
|
|
|
|
end
|
|
|
|
|
2017-10-09 07:26:58 +08:00
|
|
|
s3_resource.client.put_bucket_lifecycle_configuration(
|
|
|
|
bucket: @s3_bucket_name,
|
|
|
|
lifecycle_configuration: {
|
2017-10-03 15:00:42 +08:00
|
|
|
rules: rules
|
|
|
|
})
|
|
|
|
end
|
|
|
|
|
|
|
|
def update_tombstone_lifecycle(grace_period)
|
|
|
|
return if @tombstone_prefix.blank?
|
|
|
|
update_lifecycle("purge_tombstone", grace_period, prefix: @tombstone_prefix)
|
|
|
|
end
|
|
|
|
|
2017-10-06 13:20:01 +08:00
|
|
|
def list(prefix = "")
|
2018-05-23 05:30:32 +08:00
|
|
|
s3_bucket.objects(prefix: "#{@s3_bucket_folder_path}/#{prefix}")
|
2017-10-03 15:00:42 +08:00
|
|
|
end
|
|
|
|
|
|
|
|
def tag_file(key, tags)
|
|
|
|
tag_array = []
|
|
|
|
tags.each do |k, v|
|
|
|
|
tag_array << { key: k.to_s, value: v.to_s }
|
|
|
|
end
|
|
|
|
|
|
|
|
s3_resource.client.put_object_tagging(
|
|
|
|
bucket: @s3_bucket_name,
|
|
|
|
key: key,
|
|
|
|
tagging: {
|
|
|
|
tag_set: tag_array
|
|
|
|
}
|
|
|
|
)
|
2014-09-25 04:52:09 +08:00
|
|
|
end
|
|
|
|
|
2017-10-06 13:20:01 +08:00
|
|
|
def self.s3_options(obj)
|
2018-07-16 12:44:55 +08:00
|
|
|
opts = { region: obj.s3_region,
|
|
|
|
endpoint: SiteSetting.s3_endpoint,
|
|
|
|
force_path_style: SiteSetting.s3_force_path_style }
|
2014-09-25 04:52:09 +08:00
|
|
|
|
2017-10-06 13:20:01 +08:00
|
|
|
unless obj.s3_use_iam_profile
|
|
|
|
opts[:access_key_id] = obj.s3_access_key_id
|
|
|
|
opts[:secret_access_key] = obj.s3_secret_access_key
|
|
|
|
end
|
|
|
|
|
|
|
|
opts
|
2016-08-15 16:06:29 +08:00
|
|
|
end
|
2014-09-25 04:52:09 +08:00
|
|
|
|
2017-10-06 13:20:01 +08:00
|
|
|
private
|
2014-09-25 04:52:09 +08:00
|
|
|
|
2017-10-06 13:20:01 +08:00
|
|
|
def default_s3_options
|
|
|
|
if SiteSetting.enable_s3_uploads?
|
|
|
|
options = self.class.s3_options(SiteSetting)
|
|
|
|
check_missing_site_options
|
|
|
|
options
|
|
|
|
elsif GlobalSetting.use_s3?
|
|
|
|
self.class.s3_options(GlobalSetting)
|
|
|
|
else
|
|
|
|
{}
|
2014-09-25 04:52:09 +08:00
|
|
|
end
|
2017-10-06 13:20:01 +08:00
|
|
|
end
|
2014-09-25 04:52:09 +08:00
|
|
|
|
2017-10-06 13:20:01 +08:00
|
|
|
def get_path_for_s3_upload(path)
|
|
|
|
path = File.join(@s3_bucket_folder_path, path) if @s3_bucket_folder_path
|
|
|
|
path
|
2016-08-15 22:04:24 +08:00
|
|
|
end
|
|
|
|
|
|
|
|
def s3_resource
|
2016-08-16 11:13:59 +08:00
|
|
|
Aws::S3::Resource.new(@s3_options)
|
2016-08-15 16:06:29 +08:00
|
|
|
end
|
2014-09-25 04:52:09 +08:00
|
|
|
|
2016-08-15 16:06:29 +08:00
|
|
|
def s3_bucket
|
2018-08-08 11:26:05 +08:00
|
|
|
@s3_bucket ||= begin
|
|
|
|
bucket = s3_resource.bucket(@s3_bucket_name)
|
|
|
|
bucket.create unless bucket.exists?
|
|
|
|
bucket
|
|
|
|
end
|
2016-08-15 16:06:29 +08:00
|
|
|
end
|
2016-08-17 16:16:00 +08:00
|
|
|
|
2017-10-06 13:20:01 +08:00
|
|
|
def check_missing_site_options
|
2016-08-17 16:16:00 +08:00
|
|
|
unless SiteSetting.s3_use_iam_profile
|
2017-10-06 13:20:01 +08:00
|
|
|
raise SettingMissing.new("access_key_id") if SiteSetting.s3_access_key_id.blank?
|
|
|
|
raise SettingMissing.new("secret_access_key") if SiteSetting.s3_secret_access_key.blank?
|
2016-08-17 16:16:00 +08:00
|
|
|
end
|
|
|
|
end
|
2014-09-25 04:52:09 +08:00
|
|
|
end
|