github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2024-11-23 16:46:12 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

DEV: Do one query per month when exporting chat messages (#22746)

Browse Source 
We did some testing and saw that making one query per month is 
cheaper than querying all chat messages at ones. Note that even 
though the export job will be performing one query per month, 
the exported messages will be streamed into a single CSV file, so 
nothing changes from the user's point of view.
This commit is contained in:
Andrei Prigorshnev 2023-07-27 21:56:32 +04:00 committed by GitHub
parent 80f5018924
commit ad05924bdf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 50 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
plugins/chat/lib/chat/messages_exporter.rb
View File

@ -4,32 +4,13 @@ module Chat
module MessagesExporter
LIMIT = 10_000
def chat_message_export
Chat::Message
.unscoped
.where(created_at: 6.months.ago..Time.current)
.includes(:chat_channel)
.includes(:user)
.includes(:last_editor)
.limit(LIMIT)
.find_each do |chat_message|
yield(
[
chat_message.id,
chat_message.chat_channel.id,
chat_message.chat_channel.name,
chat_message.user.id,
chat_message.user.username,
chat_message.message,
chat_message.cooked,
chat_message.created_at,
chat_message.updated_at,
chat_message.deleted_at,
chat_message.in_reply_to&.id,
chat_message.last_editor&.id,
chat_message.last_editor&.username,
]
)
def chat_message_export(&block)
# perform 1 db query per month:
now = Time.now
from = 6.months.ago
while from <= now
export(from, from + 1.month, &block)
from = from + 1.month
end
end
@ -54,5 +35,36 @@ module Chat
super
end
end
private
def export(from, to)
Chat::Message
.unscoped
.where(created_at: from..to)
.includes(:chat_channel)
.includes(:user)
.includes(:last_editor)
.limit(LIMIT)
.find_each do |chat_message|
yield(
[
chat_message.id,
chat_message.chat_channel.id,
chat_message.chat_channel.name,
chat_message.user.id,
chat_message.user.username,
chat_message.message,
chat_message.cooked,
chat_message.created_at,
chat_message.updated_at,
chat_message.deleted_at,
chat_message.in_reply_to&.id,
chat_message.last_editor&.id,
chat_message.last_editor&.username,
]
)
end
end
end
end

18
plugins/chat/spec/system/admin/csv_exports_spec.rb
View File

@ -11,7 +11,10 @@ RSpec.describe "Chat CSV exports", type: :system do
end
xit "exports chat messages" do
message = Fabricate(:chat_message)
message_1 = Fabricate(:chat_message, created_at: 12.months.ago)
message_2 = Fabricate(:chat_message, created_at: 6.months.ago)
message_3 = Fabricate(:chat_message, created_at: 1.months.ago)
message_4 = Fabricate(:chat_message, created_at: Time.now)
visit "/admin/plugins/chat"
click_button "Create export"
@ -39,8 +42,17 @@ RSpec.describe "Chat CSV exports", type: :system do
],
)
assert_message(exported_data.second, message_1)
assert_message(exported_data.third, message_2)
assert_message(exported_data.fourth, message_3)
assert_message(exported_data.fifth, message_4)
ensure
csv_export_pm_page.clear_downloads
end
def assert_message(exported_message, message)
time_format = "%Y-%m-%d %H:%M:%S UTC"
expect(exported_data.second).to eq(
expect(exported_message).to eq(
[
message.id.to_s,
message.chat_channel.id.to_s,
@ -57,7 +69,5 @@ RSpec.describe "Chat CSV exports", type: :system do
message.last_editor.username,
],
)
ensure
csv_export_pm_page.clear_downloads
end
end