From 1051b8de84612b768f6384df8e442a685cf39e2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Hanol?= Date: Mon, 17 Jun 2013 22:14:36 +0200 Subject: [PATCH 1/3] removed useless ignored directories --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index 4cf8d7c8ca0..046f5d10cfd 100644 --- a/.gitignore +++ b/.gitignore @@ -64,8 +64,6 @@ config/fog_credentials.yml /public/uploads /public/stylesheet-cache/* -/public/downloads -/public/images # Scripts used for downloading/refreshing db script/download_db From ae3543872c3415e66fd38dcf519f603d84d1d5c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Hanol?= Date: Mon, 17 Jun 2013 22:16:14 +0200 Subject: [PATCH 2/3] renamed the `sha` column to the proper `sha1` --- app/models/upload.rb | 18 +++++++++--------- db/migrate/20130617180009_rename_sha_column.rb | 15 +++++++++++++++ lib/local_store.rb | 2 +- lib/s3.rb | 4 ++-- spec/models/upload_spec.rb | 2 +- 5 files changed, 28 insertions(+), 13 deletions(-) create mode 100644 db/migrate/20130617180009_rename_sha_column.rb diff --git a/app/models/upload.rb b/app/models/upload.rb index 277b5af3e96..869e69fb582 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -38,9 +38,9 @@ class Upload < ActiveRecord::Base def self.create_for(user_id, file) # compute the sha - sha = Digest::SHA1.file(file.tempfile).hexdigest + sha1 = Digest::SHA1.file(file.tempfile).hexdigest # check if the file has already been uploaded - upload = Upload.where(sha: sha).first + upload = Upload.where(sha1: sha1).first # otherwise, create it if upload.blank? @@ -53,7 +53,7 @@ class Upload < ActiveRecord::Base user_id: user_id, original_filename: file.original_filename, filesize: File.size(file.tempfile), - sha: sha, + sha1: sha1, width: width, height: height, url: "" @@ -61,7 +61,7 @@ class Upload < ActiveRecord::Base # make sure we're at the beginning of the file (FastImage is moving the pointer) file.rewind # store the file and update its url - upload.url = Upload.store_file(file, sha, image_info, upload.id) + upload.url = Upload.store_file(file, sha1, image_info, upload.id) # save the url upload.save end @@ -69,9 +69,9 @@ class Upload < ActiveRecord::Base upload end - def self.store_file(file, sha, image_info, upload_id) - return S3.store_file(file, sha, image_info, upload_id) if SiteSetting.enable_s3_uploads? - return LocalStore.store_file(file, sha, image_info, upload_id) + def self.store_file(file, sha1, image_info, upload_id) + return S3.store_file(file, sha1, image_info, upload_id) if SiteSetting.enable_s3_uploads? + return LocalStore.store_file(file, sha1, image_info, upload_id) end def self.uploaded_regex @@ -105,11 +105,11 @@ end # url :string(255) not null # created_at :datetime not null # updated_at :datetime not null -# sha :string(255) +# sha1 :string(40) # # Indexes # -# index_uploads_on_sha (sha) UNIQUE +# index_uploads_on_sha1 (sha1) UNIQUE # index_uploads_on_user_id (user_id) # diff --git a/db/migrate/20130617180009_rename_sha_column.rb b/db/migrate/20130617180009_rename_sha_column.rb new file mode 100644 index 00000000000..c621a485deb --- /dev/null +++ b/db/migrate/20130617180009_rename_sha_column.rb @@ -0,0 +1,15 @@ +class RenameShaColumn < ActiveRecord::Migration + def up + remove_index :uploads, :sha + rename_column :uploads, :sha, :sha1 + change_column :uploads, :sha1, :string, limit: 40 + add_index :uploads, :sha1, unique: true + end + + def down + remove_index :uploads, :sha1 + change_column :uploads, :sha1, :string, limit: 255 + rename_column :uploads, :sha1, :sha + add_index :uploads, :sha, unique: true + end +end diff --git a/lib/local_store.rb b/lib/local_store.rb index 928d38dc02f..11ae1a6ea8d 100644 --- a/lib/local_store.rb +++ b/lib/local_store.rb @@ -1,6 +1,6 @@ module LocalStore - def self.store_file(file, sha, image_info, upload_id) + def self.store_file(file, sha1, image_info, upload_id) clean_name = Digest::SHA1.hexdigest("#{Time.now.to_s}#{file.original_filename}")[0,16] + ".#{image_info.type}" url_root = "/uploads/#{RailsMultisite::ConnectionManagement.current_db}/#{upload_id}" path = "#{Rails.root}/public#{url_root}" diff --git a/lib/s3.rb b/lib/s3.rb index 5df38f0de86..7ef07e0e63e 100644 --- a/lib/s3.rb +++ b/lib/s3.rb @@ -1,13 +1,13 @@ module S3 - def self.store_file(file, sha, image_info, upload_id) + def self.store_file(file, sha1, image_info, upload_id) raise Discourse::SiteSettingMissing.new("s3_upload_bucket") if SiteSetting.s3_upload_bucket.blank? raise Discourse::SiteSettingMissing.new("s3_access_key_id") if SiteSetting.s3_access_key_id.blank? raise Discourse::SiteSettingMissing.new("s3_secret_access_key") if SiteSetting.s3_secret_access_key.blank? @fog_loaded = require 'fog' unless @fog_loaded - remote_filename = "#{upload_id}#{sha}.#{image_info.type}" + remote_filename = "#{upload_id}#{sha1}.#{image_info.type}" options = S3.generate_options directory = S3.get_or_create_directory(SiteSetting.s3_upload_bucket, options) diff --git a/spec/models/upload_spec.rb b/spec/models/upload_spec.rb index d20464a539d..68d47f72089 100644 --- a/spec/models/upload_spec.rb +++ b/spec/models/upload_spec.rb @@ -34,7 +34,7 @@ describe Upload do upload.user_id.should == user_id upload.original_filename.should == logo.original_filename upload.filesize.should == File.size(logo.tempfile) - upload.sha.should == Digest::SHA1.file(logo.tempfile).hexdigest + upload.sha1.should == Digest::SHA1.file(logo.tempfile).hexdigest upload.width.should == 244 upload.height.should == 66 upload.url.should == url From 7bdc616040c934225e512e0e3eed27ab2b3404e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Hanol?= Date: Mon, 17 Jun 2013 22:46:48 +0200 Subject: [PATCH 3/3] keep uploads reverse index up to date --- lib/cooked_post_processor.rb | 9 +++++++++ spec/components/cooked_post_processor_spec.rb | 17 +++++++++++++++++ spec/fabricators/post_fabricator.rb | 7 +++++++ spec/fabricators/upload_fabricator.rb | 2 +- 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/lib/cooked_post_processor.rb b/lib/cooked_post_processor.rb index 192502d4ae0..7ee2ec65ebf 100644 --- a/lib/cooked_post_processor.rb +++ b/lib/cooked_post_processor.rb @@ -36,6 +36,8 @@ class CookedPostProcessor # retrieve the associated upload, if any upload = get_upload_from_url(img['src']) if upload.present? + # update reverse index + associate_to_post upload # create a thumbnail upload.create_thumbnail! # optimize image @@ -87,6 +89,13 @@ class CookedPostProcessor end end + def associate_to_post(upload) + return if PostUpload.where(post_id: @post.id, upload_id: upload.id).count > 0 + PostUpload.create({ post_id: @post.id, upload_id: upload.id }) + rescue ActiveRecord::RecordNotUnique + # do not care if it's already associated + end + def optimize_image(img) return img["src"] # 1) optimize using image_optim diff --git a/spec/components/cooked_post_processor_spec.rb b/spec/components/cooked_post_processor_spec.rb index 8371999a685..6ace798c320 100644 --- a/spec/components/cooked_post_processor_spec.rb +++ b/spec/components/cooked_post_processor_spec.rb @@ -57,6 +57,23 @@ describe CookedPostProcessor do end + context 'with uploaded images in the post' do + before do + @topic = Fabricate(:topic) + @post = Fabricate(:post_with_uploads, topic: @topic, user: @topic.user) + @cpp = CookedPostProcessor.new(@post) + @cpp.expects(:get_upload_from_url).returns(Fabricate(:upload)) + @cpp.expects(:get_size).returns([100,200]) + end + + it "keeps reverse index up to date" do + @cpp.post_process_images + @post.uploads.reload + @post.uploads.count.should == 1 + end + + end + context 'with unsized images in the post' do let(:user) { Fabricate(:user) } let(:topic) { Fabricate(:topic, user: user) } diff --git a/spec/fabricators/post_fabricator.rb b/spec/fabricators/post_fabricator.rb index 80919d57889..132a942c559 100644 --- a/spec/fabricators/post_fabricator.rb +++ b/spec/fabricators/post_fabricator.rb @@ -40,6 +40,13 @@ Fabricator(:post_with_s3_image_url, from: :post) do " end +Fabricator(:post_with_uploads, from: :post) do + cooked " + + " +end + + Fabricator(:basic_reply, from: :post) do user(:coding_horror) reply_to_post_number 1 diff --git a/spec/fabricators/upload_fabricator.rb b/spec/fabricators/upload_fabricator.rb index 1435dc11c9e..16fed88372a 100644 --- a/spec/fabricators/upload_fabricator.rb +++ b/spec/fabricators/upload_fabricator.rb @@ -4,5 +4,5 @@ Fabricator(:upload) do filesize 1234 width 100 height 200 - url "/uploads/default/123456789.jpg" + url "/uploads/default/1/1234567890123456.jpg" end