github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2024-11-26 14:03:39 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Improve vBulletin bulk import script to support table prefix.

Browse Source 
Improve base bulk import script to convert list tags to ul/li.
This commit is contained in:
Arpit Jalan 2018-12-10 10:10:44 +05:30
parent e17d0c0526
commit 0365d50797
2 changed files with 50 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
script/bulk_import/base.rb
View File

@ -552,8 +552,12 @@ class BulkImport::Base
# [URL]...[/URL] # [URL]...[/URL]
# [MP3]...[/MP3] # [MP3]...[/MP3]
# [EMAIL]...[/EMAIL]
# [LEFT]...[/LEFT]
raw.gsub!(/\[\/?URL\]/i, "") raw.gsub!(/\[\/?URL\]/i, "")
raw.gsub!(/\[\/?MP3\]/i, "") raw.gsub!(/\[\/?MP3\]/i, "")
raw.gsub!(/\[\/?EMAIL\]/i, "")
raw.gsub!(/\[\/?LEFT\]/i, "")
# [FONT=blah] and [COLOR=blah] # [FONT=blah] and [COLOR=blah]
raw.gsub!(/\[FONT=.*?\](.*?)\[\/FONT\]/im, "\\1") raw.gsub!(/\[FONT=.*?\](.*?)\[\/FONT\]/im, "\\1")
@ -610,6 +614,19 @@ class BulkImport::Base
# [SPOILER=Some hidden stuff]SPOILER HERE!![/SPOILER] # [SPOILER=Some hidden stuff]SPOILER HERE!![/SPOILER]
raw.gsub!(/\[SPOILER="?(.+?)"?\](.+?)\[\/SPOILER\]/im) { "\n#{$1}\n[spoiler]#{$2}[/spoiler]\n" } raw.gsub!(/\[SPOILER="?(.+?)"?\](.+?)\[\/SPOILER\]/im) { "\n#{$1}\n[spoiler]#{$2}[/spoiler]\n" }
# convert list tags to ul and list=1 tags to ol
# (basically, we're only missing list=a here...)
# (https://meta.discourse.org/t/phpbb-3-importer-old/17397)
raw.gsub!(/\[list\](.*?)\[\/list\]/im, '[ul]\1[/ul]')
raw.gsub!(/\[list=1\](.*?)\[\/list\]/im, '[ol]\1[/ol]')
raw.gsub!(/\[list\](.*?)\[\/list:u\]/im, '[ul]\1[/ul]')
raw.gsub!(/\[list=1\](.*?)\[\/list:o\]/im, '[ol]\1[/ol]')
# convert *-tags to li-tags so bbcode-to-md can do its magic on phpBB's lists:
raw.gsub!(/\[\*\]\n/, '')
raw.gsub!(/\[\*\](.*?)\[\/\*:m\]/, '[li]\1[/li]')
raw.gsub!(/\[\*\](.*?)\n/, '[li]\1[/li]')
raw.gsub!(/\[\*=1\]/, '')
raw raw
end end

65
script/bulk_import/vbulletin.rb
View File

@ -5,12 +5,13 @@ require "htmlentities"
class BulkImport::VBulletin < BulkImport::Base class BulkImport::VBulletin < BulkImport::Base
TABLE_PREFIX = "vb_"
SUSPENDED_TILL ||= Date.new(3000, 1, 1) SUSPENDED_TILL ||= Date.new(3000, 1, 1)
def initialize def initialize
super super
host = ENV["DB_HOST"] host = ENV["DB_HOST"] || "localhost"
username = ENV["DB_USERNAME"] || "root" username = ENV["DB_USERNAME"] || "root"
password = ENV["DB_PASSWORD"] password = ENV["DB_PASSWORD"]
database = ENV["DB_NAME"] || "vbulletin" database = ENV["DB_NAME"] || "vbulletin"
@ -67,7 +68,7 @@ class BulkImport::VBulletin < BulkImport::Base
groups = mysql_stream <<-SQL groups = mysql_stream <<-SQL
SELECT usergroupid, title, description, usertitle SELECT usergroupid, title, description, usertitle
FROM usergroup FROM #{TABLE_PREFIX}usergroup
WHERE usergroupid > #{@last_imported_group_id} WHERE usergroupid > #{@last_imported_group_id}
ORDER BY usergroupid ORDER BY usergroupid
SQL SQL
@ -86,11 +87,11 @@ class BulkImport::VBulletin < BulkImport::Base
puts "Importing users..." puts "Importing users..."
users = mysql_stream <<-SQL users = mysql_stream <<-SQL
SELECT user.userid, username, email, joindate, birthday, ipaddress, user.usergroupid, bandate, liftdate SELECT u.userid, username, email, joindate, birthday, ipaddress, u.usergroupid, bandate, liftdate
FROM user FROM #{TABLE_PREFIX}user u
LEFT JOIN userban ON userban.userid = user.userid LEFT JOIN #{TABLE_PREFIX}userban ub ON ub.userid = u.userid
WHERE user.userid > #{@last_imported_user_id} WHERE u.userid > #{@last_imported_user_id}
ORDER BY user.userid ORDER BY u.userid
SQL SQL
create_users(users) do |row| create_users(users) do |row|
@ -115,10 +116,10 @@ class BulkImport::VBulletin < BulkImport::Base
puts "Importing user emails..." puts "Importing user emails..."
users = mysql_stream <<-SQL users = mysql_stream <<-SQL
SELECT user.userid, email, joindate SELECT u.userid, email, joindate
FROM user FROM #{TABLE_PREFIX}user u
WHERE user.userid > #{@last_imported_user_id} WHERE u.userid > #{@last_imported_user_id}
ORDER BY user.userid ORDER BY u.userid
SQL SQL
create_user_emails(users) do |row| create_user_emails(users) do |row|
@ -135,14 +136,14 @@ class BulkImport::VBulletin < BulkImport::Base
puts "Importing user stats..." puts "Importing user stats..."
users = mysql_stream <<-SQL users = mysql_stream <<-SQL
SELECT user.userid, joindate, posts, COUNT(thread.threadid) AS threads, post.dateline SELECT u.userid, joindate, posts, COUNT(t.threadid) AS threads, p.dateline
#{", post_thanks_user_amount, post_thanks_thanked_times" if @has_post_thanks} #{", post_thanks_user_amount, post_thanks_thanked_times" if @has_post_thanks}
FROM user FROM #{TABLE_PREFIX}user u
LEFT OUTER JOIN post ON post.postid = user.lastpostid LEFT OUTER JOIN #{TABLE_PREFIX}post p ON p.postid = u.lastpostid
LEFT OUTER JOIN thread ON user.userid = thread.postuserid LEFT OUTER JOIN #{TABLE_PREFIX}thread t ON u.userid = t.postuserid
WHERE user.userid > #{@last_imported_user_id} WHERE u.userid > #{@last_imported_user_id}
GROUP BY user.userid GROUP BY u.userid
ORDER BY user.userid ORDER BY u.userid
SQL SQL
create_user_stats(users) do |row| create_user_stats(users) do |row|
@ -169,7 +170,7 @@ class BulkImport::VBulletin < BulkImport::Base
group_users = mysql_stream <<-SQL group_users = mysql_stream <<-SQL
SELECT usergroupid, userid SELECT usergroupid, userid
FROM user FROM #{TABLE_PREFIX}user
WHERE userid > #{@last_imported_user_id} WHERE userid > #{@last_imported_user_id}
SQL SQL
@ -186,7 +187,7 @@ class BulkImport::VBulletin < BulkImport::Base
user_passwords = mysql_stream <<-SQL user_passwords = mysql_stream <<-SQL
SELECT userid, password SELECT userid, password
FROM user FROM #{TABLE_PREFIX}user
WHERE userid > #{@last_imported_user_id} WHERE userid > #{@last_imported_user_id}
ORDER BY userid ORDER BY userid
SQL SQL
@ -204,7 +205,7 @@ class BulkImport::VBulletin < BulkImport::Base
user_salts = mysql_stream <<-SQL user_salts = mysql_stream <<-SQL
SELECT userid, salt SELECT userid, salt
FROM user FROM #{TABLE_PREFIX}user
WHERE userid > #{@last_imported_user_id} WHERE userid > #{@last_imported_user_id}
AND LENGTH(COALESCE(salt, '')) > 0 AND LENGTH(COALESCE(salt, '')) > 0
ORDER BY userid ORDER BY userid
@ -223,7 +224,7 @@ class BulkImport::VBulletin < BulkImport::Base
user_profiles = mysql_stream <<-SQL user_profiles = mysql_stream <<-SQL
SELECT userid, homepage, profilevisits SELECT userid, homepage, profilevisits
FROM user FROM #{TABLE_PREFIX}user
WHERE userid > #{@last_imported_user_id} WHERE userid > #{@last_imported_user_id}
ORDER BY userid ORDER BY userid
SQL SQL
@ -242,7 +243,7 @@ class BulkImport::VBulletin < BulkImport::Base
categories = mysql_query(<<-SQL categories = mysql_query(<<-SQL
SELECT forumid, parentid, title, description, displayorder SELECT forumid, parentid, title, description, displayorder
FROM forum FROM #{TABLE_PREFIX}forum
WHERE forumid > #{@last_imported_category_id} WHERE forumid > #{@last_imported_category_id}
ORDER BY forumid ORDER BY forumid
SQL SQL
@ -289,9 +290,9 @@ class BulkImport::VBulletin < BulkImport::Base
topics = mysql_stream <<-SQL topics = mysql_stream <<-SQL
SELECT threadid, title, forumid, postuserid, open, dateline, views, visible, sticky SELECT threadid, title, forumid, postuserid, open, dateline, views, visible, sticky
FROM thread FROM #{TABLE_PREFIX}thread t
WHERE threadid > #{@last_imported_topic_id} WHERE threadid > #{@last_imported_topic_id}
AND EXISTS (SELECT 1 FROM post WHERE post.threadid = thread.threadid) AND EXISTS (SELECT 1 FROM #{TABLE_PREFIX}post p WHERE p.threadid = t.threadid)
ORDER BY threadid ORDER BY threadid
SQL SQL
@ -319,11 +320,11 @@ class BulkImport::VBulletin < BulkImport::Base
puts "Importing posts..." puts "Importing posts..."
posts = mysql_stream <<-SQL posts = mysql_stream <<-SQL
SELECT postid, post.threadid, parentid, userid, post.dateline, post.visible, pagetext SELECT postid, p.threadid, parentid, userid, p.dateline, p.visible, pagetext
#{", post_thanks_amount" if @has_post_thanks} #{", post_thanks_amount" if @has_post_thanks}
FROM post FROM #{TABLE_PREFIX}post p
JOIN thread ON thread.threadid = post.threadid JOIN #{TABLE_PREFIX}thread t ON t.threadid = p.threadid
WHERE postid > #{@last_imported_post_id} WHERE postid > #{@last_imported_post_id}
ORDER BY postid ORDER BY postid
SQL SQL
@ -357,7 +358,7 @@ class BulkImport::VBulletin < BulkImport::Base
post_thanks = mysql_stream <<-SQL post_thanks = mysql_stream <<-SQL
SELECT postid, userid, date SELECT postid, userid, date
FROM post_thanks FROM #{TABLE_PREFIX}post_thanks
WHERE postid > #{@last_imported_post_id} WHERE postid > #{@last_imported_post_id}
ORDER BY postid ORDER BY postid
SQL SQL
@ -385,7 +386,7 @@ class BulkImport::VBulletin < BulkImport::Base
topics = mysql_stream <<-SQL topics = mysql_stream <<-SQL
SELECT pmtextid, title, fromuserid, touserarray, dateline SELECT pmtextid, title, fromuserid, touserarray, dateline
FROM pmtext FROM #{TABLE_PREFIX}pmtext
WHERE pmtextid > (#{@last_imported_private_topic_id - PRIVATE_OFFSET}) WHERE pmtextid > (#{@last_imported_private_topic_id - PRIVATE_OFFSET})
ORDER BY pmtextid ORDER BY pmtextid
SQL SQL
@ -414,7 +415,7 @@ class BulkImport::VBulletin < BulkImport::Base
mysql_stream(<<-SQL mysql_stream(<<-SQL
SELECT pmtextid, touserarray SELECT pmtextid, touserarray
FROM pmtext FROM #{TABLE_PREFIX}pmtext
WHERE pmtextid > (#{@last_imported_private_topic_id - PRIVATE_OFFSET}) WHERE pmtextid > (#{@last_imported_private_topic_id - PRIVATE_OFFSET})
ORDER BY pmtextid ORDER BY pmtextid
SQL SQL
@ -439,7 +440,7 @@ class BulkImport::VBulletin < BulkImport::Base
posts = mysql_stream <<-SQL posts = mysql_stream <<-SQL
SELECT pmtextid, title, fromuserid, touserarray, dateline, message SELECT pmtextid, title, fromuserid, touserarray, dateline, message
FROM pmtext FROM #{TABLE_PREFIX}pmtext
WHERE pmtextid > #{@last_imported_private_post_id - PRIVATE_OFFSET} WHERE pmtextid > #{@last_imported_private_post_id - PRIVATE_OFFSET}
ORDER BY pmtextid ORDER BY pmtextid
SQL SQL