github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2024-11-23 01:33:24 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #4194 from gschlager/phpbb3-importer-no-offset

Browse Source 
Stop using OFFSET in DB queries of phpBB3 importer
This commit is contained in:
Régis Hanol 2016-04-25 10:25:02 +02:00
commit c7a6fb005b
4 changed files with 65 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
script/import_scripts/phpbb3/database/database_3_0.rb
View File

@ -12,8 +12,8 @@ module ImportScripts::PhpBB3
SQL
end
def fetch_users(offset)
query(<<-SQL)
def fetch_users(last_user_id)
query(<<-SQL, :user_id)
SELECT u.user_id, u.user_email, u.username, u.user_password, u.user_regdate, u.user_lastvisit, u.user_ip,
u.user_type, u.user_inactive_reason, g.group_name, b.ban_start, b.ban_end, b.ban_reason,
u.user_posts, u.user_website, u.user_from, u.user_birthday, u.user_avatar_type, u.user_avatar
@ -23,10 +23,9 @@ module ImportScripts::PhpBB3
u.user_id = b.ban_userid AND b.ban_exclude = 0 AND
(b.ban_end = 0 OR b.ban_end >= UNIX_TIMESTAMP())
)
WHERE u.user_type != #{Constants::USER_TYPE_IGNORE}
ORDER BY u.user_id ASC
WHERE u.user_id > #{last_user_id} AND u.user_type != #{Constants::USER_TYPE_IGNORE}
ORDER BY u.user_id
LIMIT #{@batch_size}
OFFSET #{offset}
SQL
end
@ -38,15 +37,16 @@ module ImportScripts::PhpBB3
SQL
end
def fetch_anonymous_users(offset)
query(<<-SQL)
def fetch_anonymous_users(last_username)
last_username = escape(last_username)
query(<<-SQL, :post_username)
SELECT post_username, MIN(post_time) AS first_post_time
FROM #{@table_prefix}_posts
WHERE post_username <> ''
WHERE post_username > '#{last_username}'
GROUP BY post_username
ORDER BY post_username ASC
ORDER BY post_username
LIMIT #{@batch_size}
OFFSET #{offset}
SQL
end
@ -60,7 +60,7 @@ module ImportScripts::PhpBB3
GROUP BY forum_id
) x ON (f.forum_id = x.forum_id)
WHERE f.forum_type != #{Constants::FORUM_TYPE_LINK}
ORDER BY f.parent_id ASC, f.left_id ASC
ORDER BY f.parent_id, f.left_id
SQL
end
@ -71,17 +71,17 @@ module ImportScripts::PhpBB3
SQL
end
def fetch_posts(offset)
query(<<-SQL)
def fetch_posts(last_post_id)
query(<<-SQL, :post_id)
SELECT p.post_id, p.topic_id, t.forum_id, t.topic_title, t.topic_first_post_id, p.poster_id,
p.post_text, p.post_time, p.post_username, t.topic_status, t.topic_type, t.poll_title,
CASE WHEN t.poll_length > 0 THEN t.poll_start + t.poll_length ELSE NULL END AS poll_end,
t.poll_max_options, p.post_attachment
FROM #{@table_prefix}_posts p
JOIN #{@table_prefix}_topics t ON (p.topic_id = t.topic_id)
ORDER BY p.post_id ASC
WHERE p.post_id > #{last_post_id}
ORDER BY p.post_id
LIMIT #{@batch_size}
OFFSET #{offset}
SQL
end
@ -140,7 +140,7 @@ module ImportScripts::PhpBB3
SELECT physical_filename, real_filename
FROM #{@table_prefix}_attachments
WHERE topic_id = #{topic_id} AND post_msg_id = #{post_id}
ORDER BY filetime DESC, post_msg_id ASC
ORDER BY filetime DESC, post_msg_id
SQL
end
@ -158,9 +158,9 @@ module ImportScripts::PhpBB3
end
end
def fetch_messages(use_fixed_messages, offset)
def fetch_messages(use_fixed_messages, last_msg_id)
if use_fixed_messages
query(<<-SQL)
query(<<-SQL, :msg_id)
SELECT m.msg_id, i.root_msg_id, m.author_id, m.message_time, m.message_subject, m.message_text,
IFNULL(a.attachment_count, 0) AS attachment_count
FROM #{@table_prefix}_privmsgs m
@ -171,12 +171,12 @@ module ImportScripts::PhpBB3
WHERE topic_id = 0
GROUP BY post_msg_id
) a ON (m.msg_id = a.post_msg_id)
ORDER BY i.root_msg_id ASC, m.msg_id ASC
WHERE m.msg_id > #{last_msg_id}
ORDER BY i.root_msg_id, m.msg_id
LIMIT #{@batch_size}
OFFSET #{offset}
SQL
else
query(<<-SQL)
query(<<-SQL, :msg_id)
SELECT m.msg_id, m.root_level AS root_msg_id, m.author_id, m.message_time, m.message_subject,
m.message_text, IFNULL(a.attachment_count, 0) AS attachment_count
FROM #{@table_prefix}_privmsgs m
@ -186,9 +186,9 @@ module ImportScripts::PhpBB3
WHERE topic_id = 0
GROUP BY post_msg_id
) a ON (m.msg_id = a.post_msg_id)
ORDER BY m.root_level ASC, m.msg_id ASC
WHERE m.msg_id > #{last_msg_id}
ORDER BY m.root_level, m.msg_id
LIMIT #{@batch_size}
OFFSET #{offset}
SQL
end
end
@ -229,14 +229,14 @@ module ImportScripts::PhpBB3
SQL
end
def fetch_bookmarks(offset)
query(<<-SQL)
def fetch_bookmarks(last_user_id, last_topic_id)
query(<<-SQL, :user_id, :topic_first_post_id)
SELECT b.user_id, t.topic_first_post_id
FROM #{@table_prefix}_bookmarks b
JOIN #{@table_prefix}_topics t ON (b.topic_id = t.topic_id)
ORDER BY b.user_id ASC, b.topic_id ASC
WHERE b.user_id > #{last_user_id} AND b.topic_id > #{last_topic_id}
ORDER BY b.user_id, b.topic_id
LIMIT #{@batch_size}
OFFSET #{offset}
SQL
end
@ -322,7 +322,7 @@ module ImportScripts::PhpBB3
AND b.normalized_subject = i.normalized_subject
AND a.msg_id <> m.msg_id
AND a.message_time < m.message_time
ORDER BY a.message_time ASC
ORDER BY a.message_time
LIMIT 1
), 0) ELSE i.root_msg_id END AS root_msg_id
FROM #{@table_prefix}_privmsgs m

5
script/import_scripts/phpbb3/database/database_3_1.rb
View File

@ -3,7 +3,7 @@ require_relative '../support/constants'
module ImportScripts::PhpBB3
class Database_3_1 < Database_3_0
def fetch_users(offset)
def fetch_users(last_user_id)
query(<<-SQL)
SELECT u.user_id, u.user_email, u.username,
CASE WHEN u.user_password LIKE '$2y$%'
@ -20,10 +20,9 @@ module ImportScripts::PhpBB3
u.user_id = b.ban_userid AND b.ban_exclude = 0 AND
(b.ban_end = 0 OR b.ban_end >= UNIX_TIMESTAMP())
)
WHERE u.user_type != #{Constants::USER_TYPE_IGNORE}
WHERE u.user_id > #{last_user_id} AND u.user_type != #{Constants::USER_TYPE_IGNORE}
ORDER BY u.user_id ASC
LIMIT #{@batch_size}
OFFSET #{offset}
SQL
end
end

27
script/import_scripts/phpbb3/database/database_base.rb
View File

@ -12,13 +12,36 @@ module ImportScripts::PhpBB3
protected
# Executes a database query.
def query(sql)
@database_client.query(sql, cache_rows: true, symbolize_keys: true)
def query(sql, *last_columns)
rows = @database_client.query(sql, cache_rows: true, symbolize_keys: true)
return rows if last_columns.length == 0
result = [rows]
last_row = find_last_row(rows)
last_columns.each { |column| result.push(last_row ? last_row[column] : nil) }
result
end
# Executes a database query and returns the value of the 'count' column.
def count(sql)
query(sql).first[:count]
end
def escape(value)
@database_client.escape(value)
end
private
def find_last_row(rows)
last_index = rows.size - 1
rows.each_with_index do |row, index|
return row if index == last_index
end
nil
end
end
end

15
script/import_scripts/phpbb3/importer.rb
View File

@ -57,9 +57,10 @@ module ImportScripts::PhpBB3
puts '', 'creating users'
total_count = @database.count_users
importer = @importers.user_importer
last_user_id = 0
batches do |offset|
rows = @database.fetch_users(offset)
rows, last_user_id = @database.fetch_users(last_user_id)
break if rows.size < 1
next if all_records_exist?(:users, importer.map_users_to_import_ids(rows))
@ -74,9 +75,10 @@ module ImportScripts::PhpBB3
puts '', 'creating anonymous users'
total_count = @database.count_anonymous_users
importer = @importers.user_importer
last_username = ''
batches do |offset|
rows = @database.fetch_anonymous_users(offset)
rows, last_username = @database.fetch_anonymous_users(last_username)
break if rows.size < 1
next if all_records_exist?(:users, importer.map_anonymous_users_to_import_ids(rows))
@ -101,9 +103,10 @@ module ImportScripts::PhpBB3
puts '', 'creating topics and posts'
total_count = @database.count_posts
importer = @importers.post_importer
last_post_id = 0
batches do |offset|
rows = @database.fetch_posts(offset)
rows, last_post_id = @database.fetch_posts(last_post_id)
break if rows.size < 1
next if all_records_exist?(:posts, importer.map_to_import_ids(rows))
@ -123,9 +126,10 @@ module ImportScripts::PhpBB3
puts '', 'creating private messages'
total_count = @database.count_messages(@settings.fix_private_messages)
importer = @importers.message_importer
last_msg_id = 0
batches do |offset|
rows = @database.fetch_messages(@settings.fix_private_messages, offset)
rows, last_msg_id = @database.fetch_messages(@settings.fix_private_messages, last_msg_id)
break if rows.size < 1
next if all_records_exist?(:posts, importer.map_to_import_ids(rows))
@ -140,9 +144,10 @@ module ImportScripts::PhpBB3
puts '', 'creating bookmarks'
total_count = @database.count_bookmarks
importer = @importers.bookmark_importer
last_user_id = last_topic_id = 0
batches do |offset|
rows = @database.fetch_bookmarks(offset)
rows, last_user_id, last_topic_id = @database.fetch_bookmarks(last_user_id, last_topic_id)
break if rows.size < 1
create_bookmarks(rows, total: total_count, offset: offset) do |row|