github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2024-11-22 09:12:45 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

FIX: Create and consume temp table inside a transaction (#7030)

Browse Source 
To prevent access issue in pgbouncer which runs in transaction pooling
This commit is contained in:
Vinoth Kannan 2019-02-20 08:22:40 +05:30 committed by Sam
parent 1f4ace4f56
commit da1ff2da2c
1 changed files with 26 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
lib/s3_inventory.rb
View File

@ -34,37 +34,39 @@ class S3Inventory
download_inventory_files_to_tmp_directory
decompress_inventory_files
begin
table_name = "#{inventory_id}_inventory"
connection = ActiveRecord::Base.connection.raw_connection
connection.exec("CREATE TEMP TABLE #{table_name}(key text UNIQUE, etag text, PRIMARY KEY(etag, key))")
connection.copy_data("COPY #{table_name} FROM STDIN CSV") do
files.each do |file|
CSV.foreach(file[:filename][0...-3], headers: false) do |row|
connection.put_copy_data("#{row[CSV_KEY_INDEX]},#{row[CSV_ETAG_INDEX]}\n")
ActiveRecord::Base.transaction do
begin
table_name = "#{inventory_id}_inventory"
connection = ActiveRecord::Base.connection.raw_connection
connection.exec("CREATE TEMP TABLE #{table_name}(key text UNIQUE, etag text, PRIMARY KEY(etag, key))")
connection.copy_data("COPY #{table_name} FROM STDIN CSV") do
files.each do |file|
CSV.foreach(file[:filename][0...-3], headers: false) do |row|
connection.put_copy_data("#{row[CSV_KEY_INDEX]},#{row[CSV_ETAG_INDEX]}\n")
end
end
end
end
# backfilling etags
connection.async_exec("UPDATE #{model.table_name}
SET etag = #{table_name}.etag
FROM #{table_name}
WHERE #{model.table_name}.etag IS NULL
AND url ILIKE '%' || #{table_name}.key")
# backfilling etags
connection.async_exec("UPDATE #{model.table_name}
SET etag = #{table_name}.etag
FROM #{table_name}
WHERE #{model.table_name}.etag IS NULL
AND url ILIKE '%' || #{table_name}.key")
uploads = (model == Upload) ? model.where("created_at < ?", inventory_date) : model
missing_uploads = uploads.joins("LEFT JOIN #{table_name} ON #{table_name}.etag = #{model.table_name}.etag").where("#{table_name}.etag is NULL")
uploads = (model == Upload) ? model.where("created_at < ?", inventory_date) : model
missing_uploads = uploads.joins("LEFT JOIN #{table_name} ON #{table_name}.etag = #{model.table_name}.etag").where("#{table_name}.etag is NULL")
if (missing_count = missing_uploads.count) > 0
missing_uploads.select(:id, :url).find_each do |upload|
log upload.url
if (missing_count = missing_uploads.count) > 0
missing_uploads.select(:id, :url).find_each do |upload|
log upload.url
end
log "#{missing_count} of #{uploads.count} #{model.name.underscore.pluralize} are missing"
end
log "#{missing_count} of #{uploads.count} #{model.name.underscore.pluralize} are missing"
ensure
connection.exec("DROP TABLE #{table_name}") unless connection.nil?
end
ensure
connection.exec("DROP TABLE #{table_name}") unless connection.nil?
end
end
end