discourse/lib/tasks/s3.rake

require_dependency "s3_helper"

def brotli_s3_path(path)
  ext = File.extname(path)
  "#{path[0..-ext.length]}br#{ext}"
end

def gzip_s3_path(path)
  ext = File.extname(path)
  "#{path[0..-ext.length]}gz#{ext}"
end

def should_skip?(path)
  return true if ENV['FORCE_S3_UPLOADS']
  @existing_assets ||= Set.new(helper.list.map(&:key))
  @existing_assets.include?('assets/' + path)
end

def upload_asset(helper, path, recurse: true, content_type: nil, fullpath: nil, content_encoding: nil)
  fullpath ||= (Rails.root + "public/assets/#{path}").to_s

  content_type ||= MiniMime.lookup_by_filename(path).content_type

  options = {
    cache_control: 'max-age=31556952, public, immutable',
    content_type: content_type,
    acl: 'public-read',
    tagging: ''
  }

  if content_encoding
    options[:content_encoding] = content_encoding
  end

  if should_skip?(path)
    puts "Skipping: #{path}"
  else
    puts "Uploading: #{path}"
    helper.upload(fullpath, path, options)
  end

  if recurse
    if File.exist?(fullpath + ".br")
      brotli_path = brotli_s3_path(path)
      upload_asset(helper, brotli_path,
        fullpath: fullpath + ".br",
        recurse: false,
        content_type: content_type,
        content_encoding: 'br'
      )
    end

    if File.exist?(fullpath + ".gz")
      gzip_path = gzip_s3_path(path)
      upload_asset(helper, gzip_path,
        fullpath: fullpath + ".gz",
        recurse: false,
        content_type: content_type,
        content_encoding: 'gzip'
      )
    end

    if File.exist?(fullpath + ".map")
      upload_asset(helper, path + ".map", recurse: false, content_type: 'application/json')
    end
  end
end

def assets
  cached = Rails.application.assets.cached
  manifest = Sprockets::Manifest.new(cached, Rails.root + 'public/assets', Rails.application.config.assets.manifest)

  raise Discourse::SiteSettingMissing.new("s3_upload_bucket") if SiteSetting.s3_upload_bucket.blank?
  manifest.assets
end

def helper
  @helper ||= S3Helper.new(SiteSetting.s3_upload_bucket.downcase + '/assets')
end

def in_manifest
  found = []
  assets.each do |_, path|
    fullpath = (Rails.root + "public/assets/#{path}").to_s

    asset_path = "assets/#{path}"
    found << asset_path

    if File.exist?(fullpath + '.br')
      found << brotli_s3_path(asset_path)
    end

    if File.exist?(fullpath + '.gz')
      found << gzip_s3_path(asset_path)
    end

    if File.exist?(fullpath + '.map')
      found << asset_path + '.map'
    end

  end
  Set.new(found)
end

task 's3:upload_assets' => :environment do
  assets.each do |name, fingerprint|
    upload_asset(helper, fingerprint)
  end
end

task 's3:expire_missing_assets' => :environment do
  keep = in_manifest

  count = 0
  puts "Ensuring AWS assets are tagged correctly for removal"
  helper.list.each do |f|
    if keep.include?(f.key)
      helper.tag_file(f.key, old: true)
      count += 1
    else
      # ensure we do not delete this by mistake
      helper.tag_file(f.key, {})
    end
  end

  puts "#{count} assets were flagged for removal in 10 days"

  puts "Ensuring AWS rule exists for purging old assets"
  #helper.update_lifecycle("delete_old_assets", 10, prefix: 'old=true')

  puts "Waiting on https://github.com/aws/aws-sdk-ruby/issues/1623"

end