github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2024-11-22 11:44:49 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

PERF: use url instead of file key in temporary inventory table.

Browse Source
This commit is contained in:
Vinoth Kannan 2019-06-13 22:03:58 +05:30
parent ed21128ee6
commit 35d6fff69e
2 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
lib/s3_inventory.rb
View File

@ -37,13 +37,14 @@ class S3Inventory
multisite_prefix = "uploads/#{RailsMultisite::ConnectionManagement.current_db}/"
ActiveRecord::Base.transaction do
begin
connection.exec("CREATE TEMP TABLE #{table_name}(key text UNIQUE, etag text, PRIMARY KEY(etag, key))")
connection.exec("CREATE TEMP TABLE #{table_name}(url text UNIQUE, etag text, PRIMARY KEY(etag, url))")
connection.copy_data("COPY #{table_name} FROM STDIN CSV") do
files.each do |file|
CSV.foreach(file[:filename][0...-3], headers: false) do |row|
key = row[CSV_KEY_INDEX]
next if Rails.configuration.multisite && key.exclude?(multisite_prefix)
connection.put_copy_data("#{key},#{row[CSV_ETAG_INDEX]}\n")
url = File.join(Discourse.store.absolute_base_url, key)
connection.put_copy_data("#{url},#{row[CSV_ETAG_INDEX]}\n")
end
end
end
@ -53,7 +54,7 @@ class S3Inventory
SET etag = #{table_name}.etag
FROM #{table_name}
WHERE #{model.table_name}.etag IS NULL
AND url ILIKE '%' || #{table_name}.key")
AND #{model.table_name}.url = #{table_name}.url")
list_missing_post_uploads if type == "original"
@ -85,11 +86,12 @@ class S3Inventory
next if sha1.blank?
upload_id = nil
result = connection.exec("SELECT * FROM #{table_name} WHERE key LIKE '%original/%/#{sha1}%'")
result = connection.exec("SELECT * FROM #{table_name} WHERE url LIKE '%original/%/#{sha1}%'")
if result.count >= 1
begin
key = result[0]["key"]
url = result[0]["url"]
key = url.sub(/^#{Discourse.store.absolute_base_url}\//, "")
data = @s3_helper.object(key).data
filename = (data.content_disposition&.match(/filename=\"(.*)\"/) || [])[1]
@ -97,7 +99,7 @@ class S3Inventory
user_id: Discourse.system_user.id,
original_filename: filename || File.basename(key),
filesize: data.content_length,
url: File.join(Discourse.store.absolute_base_url, key),
url: url,
sha1: sha1,
etag: result[0]["etag"]
)

4
spec/components/s3_inventory_spec.rb
View File

@ -82,8 +82,8 @@ describe "S3Inventory" do
it "should backfill etags to uploads table correctly" do
files = [
["//bucket.amazonaws.com/original/0184537a4f419224404d013414e913a4f56018f2.jpg", "defcaac0b4aca535c284e95f30d608d0"],
["//bucket.amazonaws.com/original/0789fbf5490babc68326b9cec90eeb0d6590db05.png", "25c02eaceef4cb779fc17030d33f7f06"]
["#{Discourse.store.absolute_base_url}/original/0184537a4f419224404d013414e913a4f56018f2.jpg", "defcaac0b4aca535c284e95f30d608d0"],
["#{Discourse.store.absolute_base_url}/original/0789fbf5490babc68326b9cec90eeb0d6590db05.png", "25c02eaceef4cb779fc17030d33f7f06"]
]
files.each { |file| Fabricate(:upload, url: file[0]) }