From 0365d5079710e457cc6d51898efe1dcf70c47c64 Mon Sep 17 00:00:00 2001 From: Arpit Jalan Date: Mon, 10 Dec 2018 10:10:44 +0530 Subject: [PATCH] Improve vBulletin bulk import script to support table prefix. Improve base bulk import script to convert list tags to ul/li. --- script/bulk_import/base.rb | 17 +++++++++ script/bulk_import/vbulletin.rb | 65 +++++++++++++++++---------------- 2 files changed, 50 insertions(+), 32 deletions(-) diff --git a/script/bulk_import/base.rb b/script/bulk_import/base.rb index 5fa8e964ca5..ce6ef27d3d8 100644 --- a/script/bulk_import/base.rb +++ b/script/bulk_import/base.rb @@ -552,8 +552,12 @@ class BulkImport::Base # [URL]...[/URL] # [MP3]...[/MP3] + # [EMAIL]...[/EMAIL] + # [LEFT]...[/LEFT] raw.gsub!(/\[\/?URL\]/i, "") raw.gsub!(/\[\/?MP3\]/i, "") + raw.gsub!(/\[\/?EMAIL\]/i, "") + raw.gsub!(/\[\/?LEFT\]/i, "") # [FONT=blah] and [COLOR=blah] raw.gsub!(/\[FONT=.*?\](.*?)\[\/FONT\]/im, "\\1") @@ -610,6 +614,19 @@ class BulkImport::Base # [SPOILER=Some hidden stuff]SPOILER HERE!![/SPOILER] raw.gsub!(/\[SPOILER="?(.+?)"?\](.+?)\[\/SPOILER\]/im) { "\n#{$1}\n[spoiler]#{$2}[/spoiler]\n" } + # convert list tags to ul and list=1 tags to ol + # (basically, we're only missing list=a here...) + # (https://meta.discourse.org/t/phpbb-3-importer-old/17397) + raw.gsub!(/\[list\](.*?)\[\/list\]/im, '[ul]\1[/ul]') + raw.gsub!(/\[list=1\](.*?)\[\/list\]/im, '[ol]\1[/ol]') + raw.gsub!(/\[list\](.*?)\[\/list:u\]/im, '[ul]\1[/ul]') + raw.gsub!(/\[list=1\](.*?)\[\/list:o\]/im, '[ol]\1[/ol]') + # convert *-tags to li-tags so bbcode-to-md can do its magic on phpBB's lists: + raw.gsub!(/\[\*\]\n/, '') + raw.gsub!(/\[\*\](.*?)\[\/\*:m\]/, '[li]\1[/li]') + raw.gsub!(/\[\*\](.*?)\n/, '[li]\1[/li]') + raw.gsub!(/\[\*=1\]/, '') + raw end diff --git a/script/bulk_import/vbulletin.rb b/script/bulk_import/vbulletin.rb index bde627df94c..7361edb63b0 100644 --- a/script/bulk_import/vbulletin.rb +++ b/script/bulk_import/vbulletin.rb @@ -5,12 +5,13 @@ require "htmlentities" class BulkImport::VBulletin < BulkImport::Base + TABLE_PREFIX = "vb_" SUSPENDED_TILL ||= Date.new(3000, 1, 1) def initialize super - host = ENV["DB_HOST"] + host = ENV["DB_HOST"] || "localhost" username = ENV["DB_USERNAME"] || "root" password = ENV["DB_PASSWORD"] database = ENV["DB_NAME"] || "vbulletin" @@ -67,7 +68,7 @@ class BulkImport::VBulletin < BulkImport::Base groups = mysql_stream <<-SQL SELECT usergroupid, title, description, usertitle - FROM usergroup + FROM #{TABLE_PREFIX}usergroup WHERE usergroupid > #{@last_imported_group_id} ORDER BY usergroupid SQL @@ -86,11 +87,11 @@ class BulkImport::VBulletin < BulkImport::Base puts "Importing users..." users = mysql_stream <<-SQL - SELECT user.userid, username, email, joindate, birthday, ipaddress, user.usergroupid, bandate, liftdate - FROM user - LEFT JOIN userban ON userban.userid = user.userid - WHERE user.userid > #{@last_imported_user_id} - ORDER BY user.userid + SELECT u.userid, username, email, joindate, birthday, ipaddress, u.usergroupid, bandate, liftdate + FROM #{TABLE_PREFIX}user u + LEFT JOIN #{TABLE_PREFIX}userban ub ON ub.userid = u.userid + WHERE u.userid > #{@last_imported_user_id} + ORDER BY u.userid SQL create_users(users) do |row| @@ -115,10 +116,10 @@ class BulkImport::VBulletin < BulkImport::Base puts "Importing user emails..." users = mysql_stream <<-SQL - SELECT user.userid, email, joindate - FROM user - WHERE user.userid > #{@last_imported_user_id} - ORDER BY user.userid + SELECT u.userid, email, joindate + FROM #{TABLE_PREFIX}user u + WHERE u.userid > #{@last_imported_user_id} + ORDER BY u.userid SQL create_user_emails(users) do |row| @@ -135,14 +136,14 @@ class BulkImport::VBulletin < BulkImport::Base puts "Importing user stats..." users = mysql_stream <<-SQL - SELECT user.userid, joindate, posts, COUNT(thread.threadid) AS threads, post.dateline + SELECT u.userid, joindate, posts, COUNT(t.threadid) AS threads, p.dateline #{", post_thanks_user_amount, post_thanks_thanked_times" if @has_post_thanks} - FROM user - LEFT OUTER JOIN post ON post.postid = user.lastpostid - LEFT OUTER JOIN thread ON user.userid = thread.postuserid - WHERE user.userid > #{@last_imported_user_id} - GROUP BY user.userid - ORDER BY user.userid + FROM #{TABLE_PREFIX}user u + LEFT OUTER JOIN #{TABLE_PREFIX}post p ON p.postid = u.lastpostid + LEFT OUTER JOIN #{TABLE_PREFIX}thread t ON u.userid = t.postuserid + WHERE u.userid > #{@last_imported_user_id} + GROUP BY u.userid + ORDER BY u.userid SQL create_user_stats(users) do |row| @@ -169,7 +170,7 @@ class BulkImport::VBulletin < BulkImport::Base group_users = mysql_stream <<-SQL SELECT usergroupid, userid - FROM user + FROM #{TABLE_PREFIX}user WHERE userid > #{@last_imported_user_id} SQL @@ -186,7 +187,7 @@ class BulkImport::VBulletin < BulkImport::Base user_passwords = mysql_stream <<-SQL SELECT userid, password - FROM user + FROM #{TABLE_PREFIX}user WHERE userid > #{@last_imported_user_id} ORDER BY userid SQL @@ -204,7 +205,7 @@ class BulkImport::VBulletin < BulkImport::Base user_salts = mysql_stream <<-SQL SELECT userid, salt - FROM user + FROM #{TABLE_PREFIX}user WHERE userid > #{@last_imported_user_id} AND LENGTH(COALESCE(salt, '')) > 0 ORDER BY userid @@ -223,7 +224,7 @@ class BulkImport::VBulletin < BulkImport::Base user_profiles = mysql_stream <<-SQL SELECT userid, homepage, profilevisits - FROM user + FROM #{TABLE_PREFIX}user WHERE userid > #{@last_imported_user_id} ORDER BY userid SQL @@ -242,7 +243,7 @@ class BulkImport::VBulletin < BulkImport::Base categories = mysql_query(<<-SQL SELECT forumid, parentid, title, description, displayorder - FROM forum + FROM #{TABLE_PREFIX}forum WHERE forumid > #{@last_imported_category_id} ORDER BY forumid SQL @@ -289,9 +290,9 @@ class BulkImport::VBulletin < BulkImport::Base topics = mysql_stream <<-SQL SELECT threadid, title, forumid, postuserid, open, dateline, views, visible, sticky - FROM thread + FROM #{TABLE_PREFIX}thread t WHERE threadid > #{@last_imported_topic_id} - AND EXISTS (SELECT 1 FROM post WHERE post.threadid = thread.threadid) + AND EXISTS (SELECT 1 FROM #{TABLE_PREFIX}post p WHERE p.threadid = t.threadid) ORDER BY threadid SQL @@ -319,11 +320,11 @@ class BulkImport::VBulletin < BulkImport::Base puts "Importing posts..." posts = mysql_stream <<-SQL - SELECT postid, post.threadid, parentid, userid, post.dateline, post.visible, pagetext + SELECT postid, p.threadid, parentid, userid, p.dateline, p.visible, pagetext #{", post_thanks_amount" if @has_post_thanks} - FROM post - JOIN thread ON thread.threadid = post.threadid + FROM #{TABLE_PREFIX}post p + JOIN #{TABLE_PREFIX}thread t ON t.threadid = p.threadid WHERE postid > #{@last_imported_post_id} ORDER BY postid SQL @@ -357,7 +358,7 @@ class BulkImport::VBulletin < BulkImport::Base post_thanks = mysql_stream <<-SQL SELECT postid, userid, date - FROM post_thanks + FROM #{TABLE_PREFIX}post_thanks WHERE postid > #{@last_imported_post_id} ORDER BY postid SQL @@ -385,7 +386,7 @@ class BulkImport::VBulletin < BulkImport::Base topics = mysql_stream <<-SQL SELECT pmtextid, title, fromuserid, touserarray, dateline - FROM pmtext + FROM #{TABLE_PREFIX}pmtext WHERE pmtextid > (#{@last_imported_private_topic_id - PRIVATE_OFFSET}) ORDER BY pmtextid SQL @@ -414,7 +415,7 @@ class BulkImport::VBulletin < BulkImport::Base mysql_stream(<<-SQL SELECT pmtextid, touserarray - FROM pmtext + FROM #{TABLE_PREFIX}pmtext WHERE pmtextid > (#{@last_imported_private_topic_id - PRIVATE_OFFSET}) ORDER BY pmtextid SQL @@ -439,7 +440,7 @@ class BulkImport::VBulletin < BulkImport::Base posts = mysql_stream <<-SQL SELECT pmtextid, title, fromuserid, touserarray, dateline, message - FROM pmtext + FROM #{TABLE_PREFIX}pmtext WHERE pmtextid > #{@last_imported_private_post_id - PRIVATE_OFFSET} ORDER BY pmtextid SQL