From f079dd63ed58930e06f823505e8c34bda9024671 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 25 May 2015 17:57:06 +1000 Subject: [PATCH] PERF: remove "fog" dependency --- Gemfile | 3 +- Gemfile.lock | 89 +++++------------------------ lib/file_store/s3_store.rb | 2 +- lib/s3_helper.rb | 111 +++++++++++++++---------------------- 4 files changed, 61 insertions(+), 144 deletions(-) diff --git a/Gemfile b/Gemfile index b3e58c654fb..bab99a6024c 100644 --- a/Gemfile +++ b/Gemfile @@ -57,7 +57,8 @@ gem 'fast_xor' # while we sort out https://github.com/sdsykes/fastimage/pull/46 gem 'fastimage_discourse', require: 'fastimage' -gem 'fog', '1.26.0', require: false +gem 'aws-sdk', require: false +gem 'excon', require: false gem 'unf', require: false gem 'email_reply_parser' diff --git a/Gemfile.lock b/Gemfile.lock index 2b207db1d01..fcd40076fa2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,7 +6,6 @@ PATH GEM remote: https://rubygems.org/ specs: - CFPropertyList (2.2.8) actionmailer (4.1.10) actionpack (= 4.1.10) actionview (= 4.1.10) @@ -41,6 +40,14 @@ GEM activerecord (>= 2.3.0) rake (~> 10.4.2, >= 10.4.2) arel (5.0.1.20140414130214) + aws-sdk (2.0.45) + aws-sdk-resources (= 2.0.45) + aws-sdk-core (2.0.45) + builder (~> 3.0) + jmespath (~> 1.0) + multi_json (~> 1.0) + aws-sdk-resources (2.0.45) + aws-sdk-core (= 2.0.45) babel-source (4.6.6) babel-transpiler (0.6.0) babel-source (>= 4.0, < 5) @@ -84,7 +91,7 @@ GEM ember-source (1.11.3.1) erubis (2.7.0) eventmachine (1.0.7) - excon (0.44.4) + excon (0.45.3) execjs (2.5.2) exifr (1.1.3) fabrication (2.9.8) @@ -101,79 +108,11 @@ GEM fast_xs (0.8.0) fastimage_discourse (1.6.6) ffi (1.9.6) - fission (0.5.0) - CFPropertyList (~> 2.2) flamegraph (0.1.0) fast_stack - fog (1.26.0) - fog-atmos - fog-brightbox (~> 0.4) - fog-core (~> 1.27, >= 1.27.1) - fog-ecloud - fog-json - fog-profitbricks - fog-radosgw (>= 0.0.2) - fog-sakuracloud (>= 0.0.4) - fog-softlayer - fog-storm_on_demand - fog-terremark - fog-vmfusion - fog-voxel - fog-xml (~> 0.1.1) - ipaddress (~> 0.5) - nokogiri (~> 1.5, >= 1.5.11) - fog-atmos (0.1.0) - fog-core - fog-xml - fog-brightbox (0.7.1) - fog-core (~> 1.22) - fog-json - inflecto (~> 0.0.2) - fog-core (1.27.2) - builder - excon (~> 0.38) - formatador (~> 0.2) - mime-types - net-scp (~> 1.1) - net-ssh (>= 2.1.3) - fog-ecloud (0.0.2) - fog-core - fog-xml - fog-json (1.0.0) - multi_json (~> 1.0) - fog-profitbricks (0.0.1) - fog-core - fog-xml - nokogiri - fog-radosgw (0.0.3) - fog-core (>= 1.21.0) - fog-json - fog-xml (>= 0.0.1) - fog-sakuracloud (0.1.1) - fog-core - fog-json - fog-softlayer (0.3.26) - fog-core - fog-json - fog-storm_on_demand (0.1.0) - fog-core - fog-json - fog-terremark (0.0.3) - fog-core - fog-xml - fog-vmfusion (0.0.1) - fission - fog-core - fog-voxel (0.0.2) - fog-core - fog-xml - fog-xml (0.1.1) - fog-core - nokogiri (~> 1.5, >= 1.5.11) foreman (0.77.0) dotenv (~> 1.0.2) thor (~> 0.19.1) - formatador (0.2.5) fspath (2.0.6) gctools (0.2.3) given_core (3.5.4) @@ -195,8 +134,8 @@ GEM progress (~> 3.0.0) image_size (1.1.5) in_threads (1.2.2) - inflecto (0.0.2) - ipaddress (0.8.0) + jmespath (1.0.2) + multi_json (~> 1.0) jquery-rails (3.1.2) railties (>= 3.0, < 5.0) thor (>= 0.14, < 2.0) @@ -231,9 +170,6 @@ GEM multi_xml (0.5.5) multipart-post (2.0.0) mustache (0.99.8) - net-scp (1.2.1) - net-ssh (>= 2.6.5) - net-ssh (2.9.2) netrc (0.10.3) nokogiri (1.6.6.2) mini_portile (~> 0.6.0) @@ -459,6 +395,7 @@ DEPENDENCIES actionpack-action_caching active_model_serializers (~> 0.8.3) annotate + aws-sdk babel-transpiler barber better_errors @@ -468,6 +405,7 @@ DEPENDENCIES email_reply_parser ember-rails ember-source (= 1.11.3.1) + excon fabrication (= 2.9.8) fakeweb (~> 1.3.0) fast_blank @@ -475,7 +413,6 @@ DEPENDENCIES fast_xs fastimage_discourse flamegraph - fog (= 1.26.0) foreman gctools handlebars-source (= 2.0.0) diff --git a/lib/file_store/s3_store.rb b/lib/file_store/s3_store.rb index 5d24f0054ec..ea8a8257575 100644 --- a/lib/file_store/s3_store.rb +++ b/lib/file_store/s3_store.rb @@ -80,7 +80,7 @@ module FileStore def store_file(file, path, filename=nil, content_type=nil) # stored uploaded are public by default - options = { public: true } + options = { acl: 'public-read' } # add a "content disposition" header for "attachments" options[:content_disposition] = "attachment; filename=\"#{filename}\"" if filename && !FileHelper.is_image?(filename) # add a "content type" header when provided (ie. for "attachments") diff --git a/lib/s3_helper.rb b/lib/s3_helper.rb index d512b655b54..29b1078c97f 100644 --- a/lib/s3_helper.rb +++ b/lib/s3_helper.rb @@ -1,50 +1,76 @@ -require "fog" +require "aws-sdk" class S3Helper - def initialize(s3_bucket, tombstone_prefix=nil, fog=nil) + def initialize(s3_bucket, tombstone_prefix=nil) raise Discourse::InvalidParameters.new("s3_bucket") if s3_bucket.blank? @s3_bucket = s3_bucket @tombstone_prefix = tombstone_prefix check_missing_site_settings - - @fog = fog || Fog::Storage.new(s3_options) end def upload(file, unique_filename, options={}) - args = { - body: file, - key: unique_filename, - public: false, - } - - args.merge!(options) - - directory = get_or_create_directory(@s3_bucket) - directory.files.create(args) + obj = s3_bucket.object(unique_filename) + obj.upload_file(file, options) end def remove(unique_filename, copy_to_tombstone=false) + bucket = s3_bucket + # copy the file in tombstone if copy_to_tombstone && @tombstone_prefix.present? - @fog.copy_object(unique_filename, @s3_bucket, @tombstone_prefix + unique_filename, @s3_bucket) + bucket.object(@tombstone_prefix + unique_filename).copy_from(copy_source: "#{@s3_bucket}/#{unique_filename}") end # delete the file - @fog.delete_object(@s3_bucket, unique_filename) - rescue Excon::Errors::NotFound - # if the file cannot be found, don't raise an error + bucket.object(unique_filename).delete + rescue Aws::S3::Errors::NoSuchKey end def update_tombstone_lifecycle(grace_period) + return if @tombstone_prefix.blank? # cf. http://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html - @fog.put_bucket_lifecycle(@s3_bucket, lifecycle(grace_period)) + s3_resource.client.put_bucket_lifecycle({ + bucket: @s3_bucket, + lifecycle_configuration: { + rules: [ + { + id: 'purge-tombstone', + status: 'Enabled', + expiration: { + days: grace_period + }, + prefix: @tombstone_prefix + } + ] + } + }) end private + def s3_resource + opts = {} + + opts = { + access_key_id: SiteSetting.s3_access_key_id, + secret_access_key: SiteSetting.s3_secret_access_key + } unless SiteSetting.s3_use_iam_profile + + opts[:region] = SiteSetting.s3_region unless SiteSetting.s3_region.blank? + + Aws::S3::Resource.new(opts) + end + + def s3_bucket + bucket = s3_resource.bucket(@s3_bucket) + bucket.create unless bucket.exists? + bucket + end + + def check_missing_site_settings unless SiteSetting.s3_use_iam_profile raise Discourse::SiteSettingMissing.new("s3_access_key_id") if SiteSetting.s3_access_key_id.blank? @@ -52,51 +78,4 @@ class S3Helper end end - def s3_options - options = { provider: 'AWS', scheme: SiteSetting.scheme } - - # cf. https://github.com/fog/fog/issues/2381 - options[:path_style] = dns_compatible?(@s3_bucket, SiteSetting.use_https?) - - options[:region] = SiteSetting.s3_region unless SiteSetting.s3_region.blank? - - if SiteSetting.s3_use_iam_profile - options.merge!(use_iam_profile: true) - else - options.merge!(aws_access_key_id: SiteSetting.s3_access_key_id, - aws_secret_access_key: SiteSetting.s3_secret_access_key) - end - - options - end - - def get_or_create_directory(bucket) - directory = @fog.directories.get(bucket) - directory = @fog.directories.create(key: bucket) unless directory - directory - end - - def lifecycle(grace_period) - { - "Rules" => [{ - "Prefix" => @tombstone_prefix, - "Enabled" => true, - "Expiration" => { "Days" => grace_period } - }] - } - end - - # cf. https://github.com/aws/aws-sdk-core-ruby/blob/master/aws-sdk-core/lib/aws-sdk-core/plugins/s3_bucket_dns.rb#L65-L80 - def dns_compatible?(bucket_name, ssl) - return false unless valid_subdomain?(bucket_name) - bucket_name.match(/\./) && ssl ? false : true - end - - def valid_subdomain?(bucket_name) - bucket_name.size < 64 && - bucket_name =~ /^[a-z0-9][a-z0-9.-]+[a-z0-9]$/ && - bucket_name !~ /(\d+\.){3}\d+/ && - bucket_name !~ /[.-]{2}/ - end - end