2019-05-03 06:17:27 +08:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2017-10-03 15:00:42 +08:00
|
|
|
require_dependency "s3_helper"
|
|
|
|
|
|
|
|
def brotli_s3_path(path)
|
|
|
|
ext = File.extname(path)
|
|
|
|
"#{path[0..-ext.length]}br#{ext}"
|
|
|
|
end
|
|
|
|
|
|
|
|
def gzip_s3_path(path)
|
|
|
|
ext = File.extname(path)
|
|
|
|
"#{path[0..-ext.length]}gz#{ext}"
|
|
|
|
end
|
|
|
|
|
|
|
|
def should_skip?(path)
|
2017-10-06 13:20:01 +08:00
|
|
|
return false if ENV['FORCE_S3_UPLOADS']
|
|
|
|
@existing_assets ||= Set.new(helper.list("assets/").map(&:key))
|
|
|
|
@existing_assets.include?(path)
|
2017-10-03 15:00:42 +08:00
|
|
|
end
|
|
|
|
|
2017-10-06 13:20:01 +08:00
|
|
|
def upload(path, remote_path, content_type, content_encoding = nil)
|
2017-10-03 15:00:42 +08:00
|
|
|
|
|
|
|
options = {
|
|
|
|
cache_control: 'max-age=31556952, public, immutable',
|
|
|
|
content_type: content_type,
|
|
|
|
acl: 'public-read',
|
|
|
|
tagging: ''
|
|
|
|
}
|
|
|
|
|
|
|
|
if content_encoding
|
|
|
|
options[:content_encoding] = content_encoding
|
|
|
|
end
|
|
|
|
|
2017-10-06 13:20:01 +08:00
|
|
|
if should_skip?(remote_path)
|
|
|
|
puts "Skipping: #{remote_path}"
|
2017-10-03 15:00:42 +08:00
|
|
|
else
|
2017-10-06 13:20:01 +08:00
|
|
|
puts "Uploading: #{remote_path}"
|
2019-01-04 18:00:45 +08:00
|
|
|
|
|
|
|
File.open(path) do |file|
|
|
|
|
helper.upload(file, remote_path, options)
|
|
|
|
end
|
2017-10-03 15:00:42 +08:00
|
|
|
end
|
2017-10-06 13:20:01 +08:00
|
|
|
end
|
2017-10-03 15:00:42 +08:00
|
|
|
|
2018-11-15 15:10:39 +08:00
|
|
|
def use_db_s3_config
|
|
|
|
ENV["USE_DB_S3_CONFIG"]
|
|
|
|
end
|
|
|
|
|
2017-10-06 13:20:01 +08:00
|
|
|
def helper
|
2018-11-15 15:10:39 +08:00
|
|
|
@helper ||= begin
|
2018-11-15 15:48:40 +08:00
|
|
|
bucket, options =
|
|
|
|
if use_db_s3_config
|
|
|
|
[
|
|
|
|
SiteSetting.s3_upload_bucket.downcase,
|
|
|
|
S3Helper.s3_options(SiteSetting)
|
|
|
|
]
|
|
|
|
else
|
|
|
|
[
|
|
|
|
GlobalSetting.s3_bucket.downcase,
|
|
|
|
S3Helper.s3_options(GlobalSetting)
|
|
|
|
]
|
|
|
|
end
|
|
|
|
|
|
|
|
S3Helper.new(bucket, '', options)
|
2018-11-15 15:10:39 +08:00
|
|
|
end
|
2017-10-03 15:00:42 +08:00
|
|
|
end
|
|
|
|
|
|
|
|
def assets
|
2017-10-03 15:27:09 +08:00
|
|
|
cached = Rails.application.assets&.cached
|
2017-10-03 15:00:42 +08:00
|
|
|
manifest = Sprockets::Manifest.new(cached, Rails.root + 'public/assets', Rails.application.config.assets.manifest)
|
|
|
|
|
2017-10-06 13:20:01 +08:00
|
|
|
results = []
|
2017-10-03 15:00:42 +08:00
|
|
|
|
2017-10-06 13:20:01 +08:00
|
|
|
manifest.assets.each do |_, path|
|
|
|
|
fullpath = (Rails.root + "public/assets/#{path}").to_s
|
|
|
|
|
|
|
|
content_type = MiniMime.lookup_by_filename(fullpath).content_type
|
|
|
|
|
|
|
|
asset_path = "assets/#{path}"
|
|
|
|
results << [fullpath, asset_path, content_type]
|
|
|
|
|
|
|
|
if File.exist?(fullpath + '.br')
|
|
|
|
results << [fullpath + '.br', brotli_s3_path(asset_path), content_type, 'br']
|
|
|
|
end
|
|
|
|
|
|
|
|
if File.exist?(fullpath + '.gz')
|
|
|
|
results << [fullpath + '.gz', gzip_s3_path(asset_path), content_type, 'gzip']
|
|
|
|
end
|
|
|
|
|
|
|
|
if File.exist?(fullpath + '.map')
|
|
|
|
results << [fullpath + '.map', asset_path + '.map', 'application/json']
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
results
|
2017-10-03 15:00:42 +08:00
|
|
|
end
|
|
|
|
|
2017-10-09 07:26:58 +08:00
|
|
|
def asset_paths
|
|
|
|
Set.new(assets.map { |_, asset_path| asset_path })
|
2017-10-03 15:00:42 +08:00
|
|
|
end
|
|
|
|
|
2017-10-06 13:20:01 +08:00
|
|
|
def ensure_s3_configured!
|
2018-11-15 15:10:39 +08:00
|
|
|
unless GlobalSetting.use_s3? || use_db_s3_config
|
2019-05-28 16:32:24 +08:00
|
|
|
STDERR.puts "ERROR: Ensure S3 is configured in config/discourse.conf or environment vars"
|
2017-10-06 13:20:01 +08:00
|
|
|
exit 1
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2019-01-04 05:13:06 +08:00
|
|
|
task 's3:correct_acl' => :environment do
|
|
|
|
ensure_s3_configured!
|
|
|
|
|
|
|
|
puts "ensuring public-read is set on every upload and optimized image"
|
|
|
|
|
|
|
|
i = 0
|
|
|
|
|
|
|
|
base_url = Discourse.store.absolute_base_url
|
|
|
|
|
|
|
|
objects = Upload.pluck(:id, :url).map { |array| array << :upload }
|
|
|
|
objects.concat(OptimizedImage.pluck(:id, :url).map { |array| array << :optimized_image })
|
|
|
|
|
|
|
|
puts "#{objects.length} objects found"
|
|
|
|
|
|
|
|
objects.each do |id, url, type|
|
|
|
|
i += 1
|
|
|
|
if !url.start_with?(base_url)
|
|
|
|
puts "Skipping #{type} #{id} since it is not stored on s3, url is #{url}"
|
|
|
|
else
|
2019-01-04 05:32:09 +08:00
|
|
|
begin
|
|
|
|
key = url[(base_url.length + 1)..-1]
|
|
|
|
object = Discourse.store.s3_helper.object(key)
|
|
|
|
object.acl.put(acl: "public-read")
|
|
|
|
rescue => e
|
|
|
|
puts "Skipping #{type} #{id} url is #{url} #{e}"
|
|
|
|
end
|
2019-01-04 05:13:06 +08:00
|
|
|
end
|
|
|
|
if i % 100 == 0
|
|
|
|
puts "#{i} done"
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
end
|
|
|
|
|
2017-10-03 15:00:42 +08:00
|
|
|
task 's3:upload_assets' => :environment do
|
2017-10-06 13:20:01 +08:00
|
|
|
ensure_s3_configured!
|
2018-10-15 09:43:31 +08:00
|
|
|
|
|
|
|
puts "installing CORS rule"
|
2017-10-09 07:26:58 +08:00
|
|
|
helper.ensure_cors!
|
2017-10-06 13:20:01 +08:00
|
|
|
|
|
|
|
assets.each do |asset|
|
|
|
|
upload(*asset)
|
2017-10-03 15:00:42 +08:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
task 's3:expire_missing_assets' => :environment do
|
2017-10-06 13:20:01 +08:00
|
|
|
ensure_s3_configured!
|
2017-10-03 15:00:42 +08:00
|
|
|
|
|
|
|
count = 0
|
2017-10-09 07:26:58 +08:00
|
|
|
keep = 0
|
|
|
|
|
|
|
|
in_manifest = asset_paths
|
|
|
|
|
2017-10-03 15:00:42 +08:00
|
|
|
puts "Ensuring AWS assets are tagged correctly for removal"
|
2017-10-09 07:26:58 +08:00
|
|
|
helper.list('assets/').each do |f|
|
|
|
|
if !in_manifest.include?(f.key)
|
2017-10-03 15:00:42 +08:00
|
|
|
helper.tag_file(f.key, old: true)
|
|
|
|
count += 1
|
|
|
|
else
|
|
|
|
# ensure we do not delete this by mistake
|
|
|
|
helper.tag_file(f.key, {})
|
2017-10-09 07:26:58 +08:00
|
|
|
keep += 1
|
2017-10-03 15:00:42 +08:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-10-09 07:26:58 +08:00
|
|
|
puts "#{count} assets were flagged for removal in 10 days (#{keep} assets will be retained)"
|
2017-10-03 15:00:42 +08:00
|
|
|
|
|
|
|
puts "Ensuring AWS rule exists for purging old assets"
|
2017-10-09 07:26:58 +08:00
|
|
|
helper.update_lifecycle("delete_old_assets", 10, tag: { key: 'old', value: 'true' })
|
2017-10-03 15:00:42 +08:00
|
|
|
|
|
|
|
end
|