github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2024-12-20 14:53:50 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity
bbb31b05ca
discourse/script/import_scripts/vbulletin3.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

1699 lines
54 KiB
Ruby
Raw Normal View History

DEV: vBulletin 3 import script (#29478)
2024-11-13 08:07:18 +08:00
# frozen_string_literal: true
require "mysql2"
require File.expand_path(File.dirname(__FILE__) + "/base.rb")
require "htmlentities"
begin
require "php_serialize" # https://github.com/jqr/php-serialize
rescue LoadError
puts
puts "php_serialize not found."
puts "Add to Gemfile, like this: "
puts
puts "echo gem \\'php-serialize\\' >> Gemfile"
puts "bundle install"
exit
end
# For vBulletin 3, based on vbulletin.rb which is for vBulletin 4.
class ImportScripts::VBulletin < ImportScripts::Base
BATCH_SIZE = 1000
# CHANGE THESE BEFORE RUNNING THE IMPORTER
DEV: Fix ruby linting (#29731)
2024-11-13 08:47:10 +08:00
DB_HOST = ENV["DB_HOST"] || "localhost"
DB_NAME = ENV["DB_NAME"] || "vbulletin"
DB_PW = ENV["DB_PW"] || ""
DB_USER = ENV["DB_USER"] || "root"
TIMEZONE = ENV["TIMEZONE"] || "America/Los_Angeles"
TABLE_PREFIX = ENV["TABLE_PREFIX"] || "vb_"
ATTACHMENT_DIR = ENV["ATTACHMENT_DIR"] || "/path/to/your/attachment/folder"
IMAGES_DIR = ENV["IMAGES_DIR"] || "/path/to/your/images/folder"
DEV: vBulletin 3 import script (#29478)
2024-11-13 08:07:18 +08:00
# Hostname + path of the forum. Used to transform deeplinks to posts and attachments to internal links
DEV: Fix ruby linting (#29731)
2024-11-13 08:47:10 +08:00
FORUM_URL = ENV["FORUM_URL"] || "localhost/"
DEV: vBulletin 3 import script (#29478)
2024-11-13 08:07:18 +08:00
# vBulletin forum ID to make to pre-seeded categories
DEV: Fix ruby linting (#29731)
2024-11-13 08:47:10 +08:00
FORUM_GENERAL_ID = ENV["FORUM_GENERAL_ID"].to_i || -1
FORUM_FEEDBACK_ID = ENV["FORUM_FEEDBACK_ID"].to_i || -1
FORUM_STAFF_ID = ENV["FORUM_STAFF_ID"].to_i || -1
DEV: vBulletin 3 import script (#29478)
2024-11-13 08:07:18 +08:00
# If non zero, create a user field containing the user title
DEV: Fix ruby linting (#29731)
2024-11-13 08:47:10 +08:00
CREATE_USERTITLE_FIELD = ENV["CREATE_USERTITLE_FIELD"].to_i != 0 || false
DEV: vBulletin 3 import script (#29478)
2024-11-13 08:07:18 +08:00
# You might also want to change the title and message for the imported private message archive
# search for "PM ARCHIVE MESSAGE" in this script
puts "#{DB_USER}:#{DB_PW}@#{DB_HOST} wants #{DB_NAME}"
def initialize
@bbcode_to_md = true
super
@usernames = {}
@tz = TZInfo::Timezone.get(TIMEZONE)
@htmlentities = HTMLEntities.new
@client =
Mysql2::Client.new(host: DB_HOST, username: DB_USER, password: DB_PW, database: DB_NAME)
rescue Exception => e
puts "=" * 50
puts e.message
puts <<EOM
Cannot connect in to database.
Hostname: #{DB_HOST}
Username: #{DB_USER}
Password: #{DB_PW}
database: #{DB_NAME}
Edit the script or set these environment variables:
export DB_HOST="localhost"
export DB_NAME="vbulletin"
export DB_PW=""
export DB_USER="root"
export TABLE_PREFIX="vb_"
export ATTACHMENT_DIR="/path/to/your/attachment/folder"
export IMAGES_DIR="/path/to/your/images/folder"
export FORUM_URL="hostname/path"
export FORUM_GENERAL_ID=-1
export FORUM_FEEDBACK_ID=-1
export FORUM_STAFF_ID=-1
export CREATE_USERTITLE_FIELD=0
Exiting.
EOM
exit
end
def get_site_settings_for_import
settings = super
settings[:automatically_download_gravatars] = false
settings[:max_post_length] = 150_000
settings
end
def execute
SiteSetting.enable_category_group_moderation = true
SiteSetting.max_category_nesting = 3
SiteSetting.fixed_category_positions = true
# showthread.php?.*t=19326.* -> showthread.php?t=19326
# showthread.php?.*p=19326.* -> showpost.php?p=19326
# showpost.php?.*p=19326.* -> showpost.php?p=19326
SiteSetting.permalink_normalizations = <<~EOL.split("\n").join("|")
/showthread\\.php.*[?&]t=(\\d+).*/showthread.php?t=\\1
/showthread\\.php.*[?&]p=(\\d+).*/showpost.php?p=\\1
/showpost\\.php.*[?&]p=(\\d+).*/showpost.php?p=\\1
EOL
begin
mysql_query("CREATE INDEX firstpostid_index ON #{TABLE_PREFIX}thread (firstpostid)")
rescue StandardError
nil
end
import_settings
import_groups
# Do not enable while creating users, affects performance
SiteSetting.migratepassword_enabled = false if SiteSetting.has_setting?(
"migratepassword_enabled",
)
import_users
SiteSetting.migratepassword_enabled = true if SiteSetting.has_setting?(
"migratepassword_enabled",
)
create_groups_membership
setup_group_membership_requests
setup_default_categories
import_categories
setup_category_moderator_groups
import_topics
import_posts
import_pm_archive
import_attachments
close_topics
post_process_posts
suspend_users
end
def import_settings
puts "", "importing important forum settings..."
settings =
mysql_query(
"SELECT varname, value FROM setting WHERE varname IN ('bbtitle', 'hometitle', 'companyname', 'webmasteremail')",
).map { |s| [s["varname"], s["value"]] }.to_h
SiteSetting.title = settings["bbtitle"] if settings["bbtitle"] &&
(SiteSetting.title == "Discourse" || !SiteSetting.title)
SiteSetting.notification_email = settings["webmasteremail"] if settings["webmasteremail"] &&
SiteSetting.notification_email == "noreply@unconfigured.discourse.org"
if SiteSetting.company_name.nil? || SiteSetting.company_name.empty?
if !settings["companyname"].empty?
SiteSetting.company_name = settings["companyname"]
elsif !settings["hometitle"].empty?
SiteSetting.company_name = settings["hometitle"]
end
end
end
def import_groups
puts "", "importing groups..."
groups = mysql_query <<-SQL
SELECT usergroupid, title, description, genericoptions, (SELECT count(*) > 0 FROM usergroupleader l WHERE l.usergroupid = g.usergroupid) as hasleaders
FROM #{TABLE_PREFIX}usergroup g
WHERE ispublicgroup = 1
ORDER BY usergroupid
SQL
create_groups(groups) do |group|
{
id: group["usergroupid"],
name: @htmlentities.decode(group["title"]).strip.downcase,
full_name: group["title"],
bio_raw: group["description"],
public_admission: group["hasleaders"].to_i == 0,
public_exit: true,
visibility_level: 1,
members_visibility_level: group["genericoptions"].to_i & 4 == 0 ? 4 : 0,
}
end
end
def setup_group_membership_requests
puts "", "setting group membership requests..."
groups = mysql_query <<-SQL
SELECT distinct usergroupid
FROM usergroupleader
SQL
groups.each do |gid|
group_id = group_id_from_imported_group_id(gid["usergroupid"])
next if !group_id
group = Group.find(group_id)
next if !group
puts "\t#{group.name}"
group.allow_membership_requests = true
group.save()
end
end
def get_username_for_old_username(old_username)
@usernames.has_key?(old_username) ? @usernames[old_username] : old_username
end
def import_users
puts "", "importing users..."
leaders = mysql_query <<-SQL
SELECT usergroupid, userid
FROM usergroupleader
SQL
usergroupLeaders =
leaders
.map { |gl| [gl["usergroupid"], gl["userid"]] }
.group_by { |gl| gl.shift }
.transform_values { |values| values.flatten }
# Exclude new users without confirmed email signed up more than 90 days ago
user_count =
mysql_query(
"SELECT COUNT(userid) count FROM #{TABLE_PREFIX}user WHERE (usergroupid != 3 OR posts > 0 OR joindate > (unix_timestamp() - 90*259200))",
).first[
"count"
]
last_user_id = -1
if CREATE_USERTITLE_FIELD
userTitleField = UserField.find_by(name: "User title")
if userTitleField.nil?
userTitleField =
UserField.new(
name: "User title",
description: "One line description about you.",
editable: true,
show_on_profile: true,
requirement: "optional",
field_type_enum: "text",
)
userTitleField.save!
puts "created 'user title' user field"
end
end
batches(BATCH_SIZE) do |offset|
users = mysql_query(<<-SQL).to_a
SELECT userid
, username
, homepage
, usertitle
, usergroupid
, joindate
, email
, password
, salt
, membergroupids
, ipaddress
, birthday
FROM #{TABLE_PREFIX}user
WHERE userid > #{last_user_id}
AND (usergroupid != 3 OR posts > 0 OR joindate > (unix_timestamp() - 90*259200))
ORDER BY userid
LIMIT #{BATCH_SIZE}
SQL
break if users.empty?
last_user_id = users[-1]["userid"]
users.reject! { |u| @lookup.user_already_imported?(u["userid"]) }
create_users(users, total: user_count, offset: offset) do |user|
email = user["email"].presence || fake_email
email = fake_email if !EmailAddressValidator.valid_value?(email)
password = [user["password"].presence, user["salt"].presence].compact
.join(":")
.delete("\000")
username = @htmlentities.decode(user["username"]).strip
group_ids = user["membergroupids"].split(",").map(&:to_i)
ip_addr =
begin
IPAddr.new(user["ipaddress"])
rescue StandardError
nil
end
cfields = {}
cfields["import_pass"] = password
{
id: user["userid"],
username: username,
password: password,
email: email,
merge: true,
website: user["homepage"].strip,
primary_group_id: group_id_from_imported_group_id(user["usergroupid"].to_i),
created_at: parse_timestamp(user["joindate"]),
last_seen_at: parse_timestamp(user["lastvisit"]),
registration_ip_address: ip_addr,
date_of_birth: parse_birthday(user["birthday"]),
custom_fields: cfields,
post_create_action:
proc do |u|
import_profile_picture(user, u)
import_profile_background(user, u)
if CREATE_USERTITLE_FIELD
u.set_user_field(userTitleField.id, @htmlentities.decode(user["usertitle"]).strip)
end
u.grant_admin! if user["usergroupid"] == 6
u.grant_moderation! if user["usergroupid"] == 5
GroupUser.transaction do
group_ids.each do |gid|
(group_id = group_id_from_imported_group_id(gid)) &&
GroupUser.find_or_create_by(user: u, group_id: group_id) do |groupUser|
groupUser.owner =
usergroupLeaders.include?(gid) &&
usergroupLeaders[gid].include?(user["userid"].to_i)
end
end
end
end,
}
end
end
@usernames =
UserCustomField
.joins(:user)
.where(name: "import_username")
.pluck("user_custom_fields.value", "users.username")
.to_h
end
def parse_birthday(birthday)
return if birthday.blank?
date_of_birth =
begin
Date.strptime(birthday.gsub(/[^\d-]+/, ""), "%m-%d-%Y")
rescue StandardError
nil
end
return if date_of_birth.nil?
if date_of_birth.year < 1904
Date.new(1904, date_of_birth.month, date_of_birth.day)
else
date_of_birth
end
end
def create_groups_membership
puts "", "creating groups membership..."
Group.find_each do |group|
begin
next if group.automatic
puts "\t#{group.name}"
next if GroupUser.where(group_id: group.id).count > 0
user_ids_in_group = User.where(primary_group_id: group.id).pluck(:id).to_a
next if user_ids_in_group.size == 0
values =
user_ids_in_group
.map { |user_id| "(#{group.id}, #{user_id}, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" }
.join(",")
DB.exec <<~SQL
INSERT INTO group_users (group_id, user_id, created_at, updated_at) VALUES #{values}
SQL
Group.reset_counters(group.id, :group_users)
rescue Exception => e
puts e.message
puts e.backtrace.join("\n")
end
end
end
def import_profile_picture(old_user, imported_user)
query = mysql_query <<-SQL
SELECT c.filedata, c.filename, a.avatarpath
FROM #{TABLE_PREFIX}user u
LEFT OUTER JOIN #{TABLE_PREFIX}customavatar c ON c.userid = u.userid AND c.visible = 1
LEFT OUTER JOIN #{TABLE_PREFIX}avatar a ON a.avatarid = u.avatarid
WHERE u.userid = #{old_user["userid"]}
ORDER BY dateline DESC
LIMIT 1
SQL
picture = query.first
return if picture.nil?
return if picture["filedata"].nil? && picture["avatarpath"].nil?
customavatar = false
file = nil
filename = nil
if !picture["filedata"].nil?
file = Tempfile.new("profile-picture")
file.write(picture["filedata"].encode("ASCII-8BIT").force_encoding("UTF-8"))
file.rewind
customavatar = true
filename = picture["filename"]
else
filename = File.join(IMAGES_DIR, picture["avatarpath"])
return unless File.exist?(filename)
file = File.open(filename, "rb")
filename = File.basename(filename)
end
upload = UploadCreator.new(file, filename).create_for(imported_user.id)
return if !upload.persisted?
imported_user.create_user_avatar
imported_user.user_avatar.update(custom_upload_id: upload.id)
imported_user.update(uploaded_avatar_id: upload.id)
ensure
begin
file.close
rescue StandardError
nil
end
if customavatar
begin
file.unlind
rescue StandardError
nil
end
end
end
def import_profile_background(old_user, imported_user)
query = mysql_query <<-SQL
SELECT filedata, filename
FROM #{TABLE_PREFIX}customprofilepic
WHERE userid = #{old_user["userid"]}
ORDER BY dateline DESC
LIMIT 1
SQL
background = query.first
return if background.nil?
return if background["filedata"].nil?
file = Tempfile.new("profile-background")
file.write(background["filedata"].encode("ASCII-8BIT").force_encoding("UTF-8"))
file.rewind
upload = UploadCreator.new(file, background["filename"]).create_for(imported_user.id)
return if !upload.persisted?
imported_user.user_profile.upload_profile_background(upload)
ensure
begin
file.close
rescue StandardError
nil
end
begin
file.unlink
rescue StandardError
nil
end
end
def setup_default_categories
set_category_importid(SiteSetting.general_category_id, FORUM_GENERAL_ID)
set_category_importid(SiteSetting.meta_category_id, FORUM_FEEDBACK_ID)
set_category_importid(SiteSetting.staff_category_id, FORUM_STAFF_ID)
end
def set_category_importid(category_id, import_id)
return if category_id == -1 || !import_id || import_id <= 0
cat = Category.find_by_id(category_id)
cat.custom_fields["import_id"] = import_id
cat.save!
add_category(import_id, cat)
end
def import_categories
puts "", "importing top level categories..."
categories = mysql_query(<<-SQL).to_a
SELECT forumid, title, description, displayorder, parentid, parentId AS origParentId,
(options & 2 > 0) AS is_allow_posting,
(options & 4 = 0) AS is_category_only,
(SELECT forumpermissions & 524288 > 0 FROM #{TABLE_PREFIX}forumpermission fp WHERE fp.forumid = f.forumid AND usergroupid = 1) AS public_access,
(SELECT forumpermissions & 524288 > 0 FROM #{TABLE_PREFIX}forumpermission fp WHERE fp.forumid = f.forumid AND usergroupid = 2) AS registered_access,
(SELECT forumpermissions & 16 > 0 FROM #{TABLE_PREFIX}forumpermission fp WHERE fp.forumid = f.forumid AND usergroupid = 2) AS registered_create,
(SELECT forumpermissions & 96 > 0 FROM #{TABLE_PREFIX}forumpermission fp WHERE fp.forumid = f.forumid AND usergroupid = 2) AS registered_reply,
(SELECT max(forumpermissions & 524288 > 0) FROM #{TABLE_PREFIX}forumpermission fp WHERE fp.forumid = f.forumid AND usergroupid IN (5,6)) AS staff_access,
(SELECT count(DISTINCT coalesce(fp.forumpermissions & 524288 > 0, 2)) > 1 FROM #{TABLE_PREFIX}usergroup ug LEFT OUTER JOIN #{TABLE_PREFIX}forumpermission fp ON fp.forumid = f.forumid AND fp.usergroupid = ug.usergroupid WHERE ug.ispublicgroup = 1) AS special_access
FROM #{TABLE_PREFIX}forum f
ORDER BY forumid
SQL
categories.each { |c| c["parent"] = categories.detect { |p| p["forumid"] == c["parentid"] } }
top_level_categories = categories.select { |c| c["parentid"] == -1 }
create_categories(top_level_categories) do |category|
{
id: category["forumid"],
name: @htmlentities.decode(category["title"]).strip,
position: category["displayorder"],
description: @htmlentities.decode(category["description"]).strip,
}
end
top_level_categories.each { |c| import_subcategories(c, categories, 1) }
categories.each do |forum|
cat_id = category_id_from_imported_category_id(forum["forumid"])
if cat_id
Permalink.find_or_create_by(
url: "forumdisplay.php?f=#{forum["forumid"]}",
category_id: cat_id,
)
end
end
puts "", "applying category permissions..."
top_level_categories.each { |c| process_category_permissions(c, categories) }
end
def process_category_permissions(cat, categories)
access = flatten_forum_access(cat, categories)
apply_category_permissions(
Category.find(category_id_from_imported_category_id(cat["forumid"])),
cat,
access,
)
children_categories = categories.select { |c| c["parentid"] == cat["forumid"] }
children_categories.each { |c| process_category_permissions(c, categories) }
end
def flatten_forum_access(category, categories)
access = {
public_access: nil,
registered_access: nil,
registered_create: nil,
registered_reply: nil,
staff_access: nil,
special_access: 0,
}
while category
access.keys.each { |p| access[p] = category[p.to_s] if access[p].nil? }
access[:special_access] = category["special_access"] if access[:special_access] == 0
category = categories.detect { |c| c["forumid"] == category["origParentId"] }
end
# Assume standard access
access[:public_access] = 1 if access[:public_access].nil?
access[:registered_access] = 1 if access[:registered_access].nil?
access[:registered_create] = 1 if access[:registered_create].nil?
access[:registered_reply] = 1 if access[:registered_reply].nil?
access
end
def apply_category_permissions(category, forum, access)
if !category.subcategories.empty?
category.show_subcategory_list = true
category.subcategory_list_style = "rows_with_featured_topics"
end
if forum["is_category_only"] == 1
# Mimmic vbulletin behavior to not show any posts
category.default_list_filter = "none"
end
if !category.subcategories.empty? && forum["is_category_only"] == 0
# Also a forum, don't show content of subforum
category.default_list_filter = "none"
end
if (forum["is_category_only"] == 1 && access[:public_access] == 1)
puts "\t#{category.name} is a public category only"
category.permissions = { everyone: :readonly }
category.save()
return
end
permissions = {}
base_level = "trust_level_0"
base_level = :everyone if access[:public_access] == 1
if access[:registered_access] == 1
if forum["is_allow_posting"] == 0 || forum["is_category_only"] == 1
permissions[base_level] = :readonly
elsif access[:registered_create] == 1
permissions[base_level] = :full
elsif access[:registered_reply] == 1
permissions[base_level] = :create_post
else
permissions[base_level] = :readonly
end
end
permissions["staff"] = :full if access[:staff_access] == 1
apply_special_category_permissions(category, forum, permissions) if access[:special_access] == 1
puts "\t#{category.name} permissions: #{permissions}"
category.permissions = permissions
category.save()
end
def apply_special_category_permissions(category, forum, permissions)
parent_public = true
parent_parent_public = true
if !category.parent_category.nil?
parent_public =
category.parent_category.category_groups.any? { |g| g.group.id == 0 } ||
category.parent_category.category_groups.empty?
if !category.parent_category.parent_category.nil?
parent_parent_public =
category.parent_category.parent_category.category_groups.any? { |g| g.group.id == 0 } ||
category.parent_category.parent_category.category_groups.empty?
end
end
apply_defaults = permissions.empty?
specials = {}
while !forum.nil?
forumid = forum["forumid"]
result = mysql_query(<<-SQL)
SELECT ug.usergroupid, ug.title,
fp.forumpermissions IS NOT NULL as non_default,
coalesce(fp.forumpermissions & 524288 > 0, ug.forumpermissions & 524288 > 0) as can_see,
coalesce(fp.forumpermissions & 16 > 0, ug.forumpermissions & 16> 0) as can_create,
coalesce(fp.forumpermissions & 96 > 0, ug.forumpermissions & 96 > 0) as can_reply
FROM #{TABLE_PREFIX}usergroup ug
LEFT JOIN #{TABLE_PREFIX}forumpermission fp ON fp.usergroupid = ug.usergroupid AND fp.forumid = #{forumid}
WHERE ug.ispublicgroup = 1
AND (fp.forumpermissions & 524288 > 0
OR ug.forumpermissions & 524288 > 0)
SQL
forum = forum["parent"]
result.each do |perms|
groupid = perms["usergroupid"]
if specials[groupid].nil?
specials[groupid] = perms
elsif specials[groupid]["non_default"] == 0
specials[groupid] = perms
end
end
end
specials.each_value do |perms|
next if perms["can_see"] == 0
next if !apply_defaults && perms["non_default"] == 0
groupid = group_id_from_imported_group_id(perms["usergroupid"])
if perms["can_create"] == 1
permissions[groupid] = :full
elsif perms["can_reply"] == 1
permissions[groupid] = :create_post
else
permissions[groupid] = :readonly
end
# If parents are not public the group must also have readonly access to parent
if !parent_parent_public
add_minimal_access_to_category(category.parent_category.parent_category, groupid)
end
add_minimal_access_to_category(category.parent_category, groupid) if !parent_public
end
end
def add_minimal_access_to_category(category, groupid)
return if category.parent_category.category_groups.any? { |g| g.group.id == groupid }
category.category_groups.build(group_id: groupid, permission_type: :readonly)
category.save()
end
def import_subcategories(parent, categories, depth)
children_categories = categories.select { |c| c["parentid"] == parent["forumid"] }
return if children_categories.empty?
puts "",
"importing #{children_categories.length} child categories for \"#{parent["title"]}\" (depth #{depth})..."
if depth >= SiteSetting.max_category_nesting
puts "\treducing category depth"
children_categories.each do |cc|
while cc["parentid"] != parent["forumid"]
cc["parentid"] = categories.detect { |c| c["forumid"] == cc["parentid"] }["parentid"]
end
end
end
create_categories(children_categories) do |category|
{
id: category["forumid"],
name: @htmlentities.decode(category["title"]).strip,
position: category["displayorder"],
description: @htmlentities.decode(category["description"]).strip,
parent_category_id: category_id_from_imported_category_id(category["parentid"]),
}
end
children_categories.each { |c| import_subcategories(c, categories, depth + 1) }
end
def setup_category_moderator_groups
puts "", "creating category moderator groups..."
forums = mysql_query("SELECT forumid, parentid, title FROM #{TABLE_PREFIX}forum").to_a
forums.each { |f| f["children"] = forums.select { |c| c["parentid"] == f["forumid"] } }
forum_map = forums.map { |f| [f["forumid"], f] }.to_h
modentries = mysql_query(<<-SQL).to_a
SELECT m.forumid, m.userid, u.usergroupid IN (5,6) is_staff
FROM #{TABLE_PREFIX}moderator m
JOIN #{TABLE_PREFIX}user u ON u.userid = m.userid
WHERE permissions & 1 > 0
AND forumid > -1
SQL
forum_mods = {}
modentries.each do |mod|
forumid = mod["forumid"]
forum_mods[forumid] = [] if forum_mods[forumid].nil?
forum_mods[forumid] << mod
end
forum_mods.each do |forumid, mods|
forum = forum_map[forumid]
puts "\tcreating moderator group for #{forumid}: " + forum["title"]
group = {
id: "forummod-#{forumid}",
name: "mods_" + @htmlentities.decode(forum["title"]).strip.downcase,
full_name: "Moderators: " + forum["title"],
public_admission: false,
public_exit: true,
visibility_level: 2,
members_visibility_level: 2,
}
group_id = group_id_from_imported_group_id(group[:id])
group_id = create_group(group, group[:id]).id if !group_id
mods.each do |m|
GroupUser.find_or_create_by(
user_id: user_id_from_imported_user_id(m["userid"]),
group_id: group_id,
)
end
parent_forum = forum_map[forum["parentid"]]
while parent_forum
parent_id = parent_forum["forumid"]
parent_mods = forum_mods[parent_id]
if parent_mods
parent_mods.each do |m|
GroupUser.find_or_create_by(
user_id: user_id_from_imported_user_id(m["userid"]),
group_id: group_id,
)
end
end
parent_forum = forum_map[parent_forum["parentid"]]
end
apply_category_moderator_group(forum_map, forumid, Group.find(group_id))
end
end
def apply_category_moderator_group(forum_map, forum_id, group)
category = Category.find(category_id_from_imported_category_id(forum_id))
category.reviewable_by_group = group
category.save()
forum_map[forum_id]["children"].each do |c|
apply_category_moderator_group(forum_map, c["forumid"], group)
end
end
def import_topics
puts "", "importing topics..."
topic_count =
mysql_query(
"SELECT COUNT(threadid) count FROM #{TABLE_PREFIX}thread WHERE visible <> 2 AND firstpostid <> 0",
).first[
"count"
]
last_topic_id = -1
batches(BATCH_SIZE) do |offset|
topics = mysql_query(<<-SQL).to_a
SELECT t.threadid threadid, t.title title, forumid, open, postuserid, t.dateline dateline, views, t.visible visible, sticky,
p.postid, p.pagetext raw, t.pollid pollid
FROM #{TABLE_PREFIX}thread t
JOIN #{TABLE_PREFIX}post p ON p.postid = t.firstpostid
WHERE t.threadid > #{last_topic_id} AND t.visible <> 2
ORDER BY t.threadid
LIMIT #{BATCH_SIZE}
SQL
break if topics.empty?
last_topic_id = topics[-1]["threadid"]
topics.reject! { |t| @lookup.post_already_imported?("thread-#{t["threadid"]}") }
create_posts(topics, total: topic_count, offset: offset) do |topic|
raw =
begin
preprocess_post_raw(topic["raw"])
rescue StandardError => e
puts "",
"\tFailed preprocessing raw for thread #{topic["threadid"]}",
e.message,
e.backtrace
nil
end
if raw.blank?
puts "", "\tNo body for thread #{topic["threadid"]}"
next
end
poll_data, poll_raw = retrieve_poll_data(topic["pollid"])
raw = poll_raw << "\n\n" << raw if poll_raw
topic_id = "thread-#{topic["threadid"]}"
t = {
id: topic_id,
user_id: user_id_from_imported_user_id(topic["postuserid"]) || Discourse::SYSTEM_USER_ID,
title: @htmlentities.decode(topic["title"]).strip[0...255],
category: category_id_from_imported_category_id(topic["forumid"]),
raw: raw,
created_at: parse_timestamp(topic["dateline"]),
visible: topic["visible"].to_i == 1,
views: topic["views"],
custom_fields: {
import_post_id: topic["postid"],
},
post_create_action:
proc do |post|
add_post(topic["postid"].to_s, post)
post_process_poll(post, poll_data) if poll_data
Permalink.create(
url: "showthread.php?t=#{topic["threadid"]}",
topic_id: post.topic_id,
)
Permalink.create(url: "showpost.php?p=#{topic["postid"]}", topic_id: post.topic_id)
end,
}
t[:pinned_at] = t[:created_at] if topic["sticky"].to_i == 1
t
end
end
end
def retrieve_poll_data(pollid)
return nil, nil if pollid <= 0
poll_data = mysql_query("SELECT * FROM #{TABLE_PREFIX}poll WHERE pollid = #{pollid}").first.to_h
return nil, nil if !poll_data["pollid"]
options = poll_data["options"].split("|||")
# Ensure unique values
options.each_index do |x|
cnt = 1
val = preprocess_post_raw(options[x])
val.strip!
# escape some markdown which probably shouldn't be there
val.gsub!(/^([*#>_-])/) { "\\#{$1}" }
val = "." if val == ""
idx = options.find_index(val)
while !idx.nil? && idx < x
cnt += 1
val = options[x].strip << " (#{cnt})"
idx = options.find_index(val)
end
options[x] = val
end
arguments = ["results=on_vote"]
arguments << "status=closed" if poll_data["active"] == 0
arguments << "type=multiple" if poll_data["multiple"] == 1
arguments << "public=true" if poll_data["public"] == 1
if poll_data["timeout"] > 0
arguments << "close=" + parse_timestamp(poll_data["timeout"]).iso8601
end
raw = poll_data["question"].dup
raw << "\n\n[poll #{arguments.join(" ")}]"
options.each { |opt| raw << "\n* #{opt}" }
raw << "\n[/poll]"
[poll_data, raw]
end
def post_process_poll(post, poll_data)
poll = post.polls.first
return if !poll
option_map = {}
poll.poll_options.each_with_index { |option, index| option_map[index + 1] = option.id }
poll_votes =
mysql_query(
"SELECT * FROM #{TABLE_PREFIX}pollvote WHERE pollid = #{poll_data["pollid"]} AND votetype = 0",
)
poll_votes.each do |vote|
PollVote.create!(
poll: poll,
poll_option_id: option_map[vote["voteoption"]],
user_id: user_id_from_imported_user_id(vote["userid"]),
)
end
end
def import_posts
puts "", "importing posts..."
post_count = mysql_query(<<-SQL).first["count"]
SELECT COUNT(postid) count
FROM #{TABLE_PREFIX}post p
JOIN #{TABLE_PREFIX}thread t ON t.threadid = p.threadid
WHERE t.firstpostid <> p.postid
AND p.visible <> 2
AND t.visible <> 2
SQL
last_post_id = -1
batches(BATCH_SIZE) do |offset|
posts = mysql_query(<<-SQL).to_a
SELECT p.postid, p.userid, p.threadid, p.pagetext raw, p.dateline, p.visible, p.parentid, p.attach
FROM #{TABLE_PREFIX}post p
JOIN #{TABLE_PREFIX}thread t ON t.threadid = p.threadid
WHERE t.firstpostid <> p.postid AND t.visible <> 2 AND p.visible <> 2
AND p.postid > #{last_post_id}
ORDER BY p.postid
LIMIT #{BATCH_SIZE}
SQL
break if posts.empty?
last_post_id = posts[-1]["postid"]
posts.reject! { |p| @lookup.post_already_imported?(p["postid"].to_i) }
create_posts(posts, total: post_count, offset: offset) do |post|
raw =
begin
preprocess_post_raw(post["raw"])
rescue StandardError => e
puts "", "\tFailed preprocessing raw for post #{post["postid"]}", e.message, e.backtrace
nil
end
if raw.blank?
if post["attach"] > 0
# Post with no text, but does have attachments
raw = "[attach]0[/attach]"
else
puts "", "\tNo body for post #{post["postid"]}"
next
end
end
unless topic = topic_lookup_from_imported_post_id("thread-#{post["threadid"]}")
puts "", "\tMissing thread for post #{post["postid"]}: thread-#{post["threadid"]}"
next
end
p = {
id: post["postid"],
user_id: user_id_from_imported_user_id(post["userid"]) || Discourse::SYSTEM_USER_ID,
topic_id: topic[:topic_id],
raw: raw,
created_at: parse_timestamp(post["dateline"]),
hidden: post["visible"].to_i != 1,
post_create_action:
proc do |realpost|
Permalink.create(url: "showpost.php?p=#{post["postid"]}", post_id: realpost.id)
end,
}
if parent = topic_lookup_from_imported_post_id(post["parentid"])
p[:reply_to_post_number] = parent[:post_number]
end
p
end
end
end
# find the uploaded file information from the db
def find_upload(post, attachment_id)
sql =
"SELECT a.attachmentid attachment_id, a.userid user_id, a.attachmentid file_id, a.filename filename,
LENGTH(a.filedata) AS dbsize, filedata
FROM #{TABLE_PREFIX}attachment a
WHERE a.attachmentid = #{attachment_id}"
results = mysql_query(sql)
unless row = results.first
puts "",
"\tCouldn't find attachment record #{attachment_id} for post.id = #{post.id}, import_id = #{post.custom_fields["import_id"]}"
return nil, nil
end
filename =
File.join(ATTACHMENT_DIR, row["user_id"].to_s.split("").join("/"), "#{row["file_id"]}.attach")
real_filename = row["filename"]
real_filename.prepend SecureRandom.hex if real_filename[0] == "."
unless File.exist?(filename)
if row["dbsize"].to_i == 0
puts "",
"\tAttachment file #{row["attachment_id"]} doesn't exist. Filename: #{real_filename}. Path: #{filename}"
return nil, real_filename
end
tmpfile = "attach_" + row["filedataid"].to_s
filename = File.join("/tmp/", tmpfile)
File.open(filename, "wb") { |f| f.write(row["filedata"]) }
end
upload = create_upload(post.user.id, filename, real_filename)
if upload.nil? || !upload.valid?
puts "", "\tUpload not valid :( Attachment #{attachment_id}"
puts upload.errors.inspect if upload
return nil, real_filename
end
[upload, real_filename]
rescue Mysql2::Error => e
puts "SQL Error"
puts e.message
puts sql
end
def import_pm_archive
puts "", "importing private message archives..."
pm_count = mysql_query("SELECT COUNT(pmid) count FROM #{TABLE_PREFIX}pm").first["count"]
current_count = 0
start = Time.now
users = mysql_query("SELECT distinct userid FROM #{TABLE_PREFIX}pm").to_a
users.each do |row|
userid = row["userid"]
real_userid = user_id_from_imported_user_id(userid)
if @lookup.post_already_imported?("pmarchive-#{userid}") || real_userid.nil?
usrcnt =
mysql_query(
"SELECT COUNT(pmid) count FROM #{TABLE_PREFIX}pm WHERE userid = #{userid}",
).first[
"count"
]
current_count += usrcnt
print_status current_count, pm_count, start
next
end
filename = "pm-archive-#{userid}.txt"
filepath = File.join("/tmp/", "pm-archive-#{userid}.txt")
File.open(filepath, "wb") do |f|
user_pm = mysql_query(<<-SQL)
SELECT p.pmid, p.parentpmid, t.fromuserid, t.fromusername, t.title, t.message, t.dateline, t.touserarray
FROM #{TABLE_PREFIX}pm p
JOIN #{TABLE_PREFIX}pmtext t on t.pmtextid = p.pmtextid
WHERE p.userid = #{userid}
ORDER BY t.dateline
SQL
user_pm.each do |pm|
current_count += 1
print_status current_count, pm_count, start
f << "---\n"
f << "id: #{pm["pmid"]}\n"
f << "in_reply_to: #{pm["parentpmid"]}\n" if pm["parentpmid"] > 0
ts = parse_timestamp(pm["dateline"]).iso8601
f << "timestamp: #{ts}\n"
title =
@htmlentities.decode(
pm["title"].encode("UTF-8", invalid: :replace, undef: :replace, replace: ""),
)
f << "title: #{title}\n"
f << "from: #{pm["fromusername"]}\n"
to_usernames, to_userids = get_pm_recipients(pm)
if to_usernames.length() == 0
f << "to: \n"
elsif to_usernames.length() == 1
f << "to: #{to_usernames[0]}\n"
else
lst = " - " + to_usernames.join("\n -")
f << "to:\n#{lst}\n"
end
f << "message: |+\n "
raw = pm["message"]
raw =
@htmlentities.decode(
raw.encode("UTF-8", invalid: :replace, undef: :replace, replace: ""),
).gsub(/[^[[:print:]]\t\n]/, "")
raw = raw.gsub(/(\r)?\n/, "\n ")
f << raw
f << "\n\n"
end
end
upload = create_upload(real_userid, filepath, filename)
File.delete(filepath)
# PM ARCHIVE MESSAGE
# Private message title
title = "Your private message archive from the previous forum software."
# Private message body explaining the attached PM archive from vBulletin
raw = <<~EOL
Attached is your private message archive from the previous forum software.
The text file contains all the private messages you had saved at the moment of migration.
The file should also be a valid YAML file containing a single document per message.
EOL
raw += html_for_upload(upload, filename)
newpost = {
archetype: Archetype.private_message,
user_id: Discourse::SYSTEM_USER_ID,
target_usernames: User.find_by(id: real_userid).try(:username),
title: title,
raw: raw,
closed: true,
archived: true,
post_create_action:
proc do |post|
UploadReference.ensure_exist!(upload_ids: [upload.id], target: post)
post.topic.closed = true
post.topic.save()
end,
}
create_post(newpost, "pmarchive-#{userid}")
end
end
def get_pm_recipients(pm)
target_usernames = []
target_userids = []
begin
to_user_array = PHP.unserialize(pm["touserarray"])
rescue StandardError
return target_usernames, target_userids
end
begin
to_user_array.each do |to_user|
if to_user[0] == "cc" || to_user[0] == "bcc" # not sure if we should include bcc users
to_user[1].each do |to_user_cc|
user_id = user_id_from_imported_user_id(to_user_cc[0])
username = User.find_by(id: user_id).try(:username)
target_userids << user_id || Discourse::SYSTEM_USER_ID
target_usernames << username if username
end
else
user_id = user_id_from_imported_user_id(to_user[0])
username = User.find_by(id: user_id).try(:username)
target_userids << user_id || Discourse::SYSTEM_USER_ID
target_usernames << username if username
end
end
rescue StandardError
return target_usernames, target_userids
end
[target_usernames, target_userids]
end
def import_attachments
puts "", "importing attachments..."
mapping = {}
attachments = mysql_query(<<-SQL)
SELECT a.attachmentid, a.postid as postid, p.threadid
FROM #{TABLE_PREFIX}attachment a, #{TABLE_PREFIX}post p, #{TABLE_PREFIX}thread t
WHERE a.postid = p.postid
AND t.threadid = p.threadid
AND a.visible = 1
AND p.visible <> 2
AND t.visible <> 2
SQL
attachments.each do |attachment|
post_id = post_id_from_imported_post_id(attachment["postid"])
post_id = post_id_from_imported_post_id("thread-#{attachment["threadid"]}") unless post_id
if post_id.nil?
puts "\tPost for attachment #{attachment["attachmentid"]} not found"
next
end
mapping[post_id] ||= []
mapping[post_id] << attachment["attachmentid"].to_i
end
current_count = 0
total_count = Post.count
success_count = 0
fail_count = 0
start = Time.now
attachment_regex = %r{\[attach[^\]]*\](\d+)\[/attach\]}i
attachment_regex2 =
%r{!\[\]\((https?:)?//#{Regexp.escape(FORUM_URL)}attachment\.php\?attachmentid=(\d+)(&stc=1)?(&d=\d+)?\)}i
Post.find_each do |post|
current_count += 1
print_status current_count, total_count, start
upload_ids = []
new_raw = post.raw.dup
new_raw.gsub!(attachment_regex) do |s|
matches = attachment_regex.match(s)
attachment_id = matches[1]
next "" if attachment_id.to_i == 0
mapping[post.id].delete(attachment_id.to_i) unless mapping[post.id].nil?
upload, filename = find_upload(post, attachment_id)
unless upload
fail_count += 1
next "\n:x: ERROR: missing attachment #{filename}\n"
end
upload_ids << upload.id
html_for_upload(upload, filename)
end
new_raw.gsub!(attachment_regex2) do |s|
matches = attachment_regex2.match(s)
attachment_id = matches[2]
next "" if attachment_id.to_i == 0
mapping[post.id].delete(attachment_id.to_i) unless mapping[post.id].nil?
upload, filename = find_upload(post, attachment_id)
unless upload
fail_count += 1
next "\n:x: ERROR: missing attachment #{filename}\n"
end
upload_ids << upload.id
html_for_upload(upload, filename)
end
# make resumed imports faster
if new_raw == post.raw
unless mapping[post.id].nil? || mapping[post.id].empty?
imported_text = mysql_query(<<-SQL).first["pagetext"]
SELECT p.pagetext
FROM #{TABLE_PREFIX}attachment a, #{TABLE_PREFIX}post p
WHERE a.postid = p.postid
AND a.attachmentid = #{mapping[post.id][0]}
SQL
imported_text.scan(attachment_regex) do |match|
attachment_id = match[0]
mapping[post.id].delete(attachment_id.to_i)
end
imported_text.scan(attachment_regex2) do |match|
attachment_id = match[1]
mapping[post.id].delete(attachment_id.to_i)
end
end
end
unless mapping[post.id].nil? || mapping[post.id].empty?
mapping[post.id].each do |attachment_id|
upload, filename = find_upload(post, attachment_id)
unless upload
fail_count += 1
next
end
upload_ids << upload.id
# internal upload deduplication will make sure that we do not import attachments again
html = html_for_upload(upload, filename)
new_raw += "\n\n#{html}\n\n" if !new_raw[html]
end
end
if new_raw != post.raw
post.raw = new_raw
post.save
end
UploadReference.ensure_exist!(upload_ids: upload_ids, target: post)
success_count += 1
end
end
def close_topics
puts "", "closing topics..."
# keep track of closed topics
closed_topic_ids = []
topics = mysql_query <<-MYSQL
SELECT t.threadid threadid, firstpostid, open
FROM #{TABLE_PREFIX}thread t
JOIN #{TABLE_PREFIX}post p ON p.postid = t.firstpostid
ORDER BY t.threadid
MYSQL
topics.each do |topic|
topic_id = "thread-#{topic["threadid"]}"
closed_topic_ids << topic_id if topic["open"] == 0
end
sql = <<-SQL
WITH closed_topic_ids AS (
SELECT t.id AS topic_id
FROM post_custom_fields pcf
JOIN posts p ON p.id = pcf.post_id
JOIN topics t ON t.id = p.topic_id
WHERE pcf.name = 'import_id'
AND pcf.value IN (?)
)
UPDATE topics
SET closed = true
WHERE id IN (SELECT topic_id FROM closed_topic_ids)
SQL
DB.exec(sql, closed_topic_ids)
end
def post_process_posts
puts "", "postprocessing posts..."
current = 0
max = Post.count
start = Time.now
Post.find_each do |post|
begin
old_raw = post.raw.dup
new_raw = postprocess_post_raw(post, post.raw)
if new_raw != old_raw
post.raw = new_raw
post.save
end
rescue PrettyText::JavaScriptError
nil
ensure
print_status(current += 1, max, start)
end
end
end
def preprocess_post_raw(raw)
return "" if raw.blank?
raw = raw.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")
# decode HTML entities
raw = @htmlentities.decode(raw)
# fix whitespaces
raw.gsub!(/(\r)?\n/, "\n")
# [HTML]...[/HTML]
raw.gsub!(/\[html\]/i, "\n```html\n")
raw.gsub!(%r{\[/html\]}i, "\n```\n")
# [PHP]...[/PHP]
raw.gsub!(/\[php\]/i, "\n```php\n")
raw.gsub!(%r{\[/php\]}i, "\n```\n")
# [CODE]...[/CODE]
# [HIGHLIGHT]...[/HIGHLIGHT]
raw.gsub!(%r{\[/?code\]}i, "\n```\n")
# [SAMP]...[/SAMP]
raw.gsub!(%r{\[/?samp\]}i, "`")
# replace all chevrons with HTML entities
# NOTE: must be done
# - AFTER all the "code" processing
# - BEFORE the "quote" processing
raw.gsub!(/`([^`]+)`/im) { "`" + $1.gsub("<", "\u2603") + "`" }
raw.gsub!("<", "&lt;")
raw.gsub!("\u2603", "<")
raw.gsub!(/`([^`]+)`/im) { "`" + $1.gsub(">", "\u2603") + "`" }
raw.gsub!(">", "&gt;")
raw.gsub!("\u2603", ">")
# Thread/post links via URL
raw.gsub!(
%r{\[url\](https?:)?//#{Regexp.escape(FORUM_URL)}show(thread|post)\.php(.*?)\[/url\]}i,
) do
params = $3
val = ""
/[?&]p(ostid)?=(\d+)/i.match(params) { val = "[post]#{$2}[/post]" }
/[?&]t(hreadid)?=(\d+)/i.match(params) { val = "[thread]#{$2}[/thread]" }
val
end
raw.gsub!(
%r{\[url="?(https?:)?//#{Regexp.escape(FORUM_URL)}show(thread|post)\.php(.*?)"?\](.*?)\[/url\]}im,
) do
params = $3
text = $4
val = $4
/[?&]p(ostid)?=(\d+)/i.match(params) { val = "[post=#{$2}]#{text}[/post]" }
/[?&]t(hreadid)?=(\d+)/i.match(params) { val = "[thread=#{$2}]#{text}[/thread]" }
val
end
# [URL=...]...[/URL]
raw.gsub!(%r{\[url="?([^"]+?)"?\](.*?)\[/url\]}im) { "[#{$2.strip}](#{$1})" }
raw.gsub!(%r{\[url="?(.+?)"?\](.*?)\[/url\]}im) { "[#{$2.strip}](#{$1})" }
# [URL]...[/URL]
# [MP3]...[/MP3]
raw.gsub!(%r{\[/?url\]}i, "")
raw.gsub!(%r{\[/?mp3\]}i, "")
# [MENTION]<username>[/MENTION]
raw.gsub!(%r{\[mention\](.+?)\[/mention\]}i) do
new_username = get_username_for_old_username($1)
"@#{new_username}"
end
# [FONT=blah] and [COLOR=blah]
raw.gsub!(%r{\[/?font(=.*?)?\]}i, "")
raw.gsub!(%r{\[/?color(=.*?)?\]}i, "")
raw.gsub!(%r{\[/?size(=.*?)?\]}i, "")
raw.gsub!(%r{\[/?sup\]}i, "")
raw.gsub!(%r{\[/?big\]}i, "")
raw.gsub!(%r{\[/?small\]}i, "")
raw.gsub!(%r{\[/?h(=.*?)?\]}i, "")
raw.gsub!(%r{\[/?float(=.*?)?\]}i, "")
# [highlight]...[/highlight]
raw.gsub!(%r{\[highlight\](.*?)\[/highlight\]}i, '<mark>\1</mark>')
# [CENTER]...[/CENTER]
raw.gsub!(%r{\[/?center\]}i, "")
raw.gsub!(%r{\[/?left\]}i, "")
raw.gsub!(%r{\[/?right\]}i, "")
# [INDENT]...[/INDENT]
raw.gsub!(%r{\[/?indent\]}i, "")
raw.gsub!(%r{\[/?sigpic\]}i, "")
# [ame]...[/ame]
raw.gsub!(%r{\[ame="?(.*?)"?\](.*?)\[/ame\]}i) { "\n#{$1}\n" }
raw.gsub!(%r{\[ame\](.*?)\[/ame\]}i) { "\n#{$1}\n" }
raw.gsub!(%r{\[/?fp\]}i, "")
# Tables to MD
raw.gsub!(%r{\[TABLE.*?\](.*?)\[/TABLE\]}im) do |t|
rows =
$1.gsub!(%r{\s*\[TR\](.*?)\[/TR\]\s*}im) do |r|
cols = $1.gsub! %r{\s*\[TD.*?\](.*?)\[/TD\]\s*}im, '|\1'
"#{cols}|\n"
end
header, rest = rows.split "\n", 2
c = header.count "|"
sep = "|---" * (c - 1)
"#{header}\n#{sep}|\n#{rest}\n"
end
# Prevent a leading * to make a list
raw.gsub!(/^\*/, '\*')
raw.gsub!(/^-/, '\-')
raw.gsub!(/^\+/, '\+')
# Basic list conversion
#raw.gsub!(%r{\[list(=.*?)?\](.*?)\[/list\]}im) { "\n#{$1}\n" }
#raw.gsub!(/\[\*\]\s*(.*?)\n/) { "* #{$1}\n" }
raw = bbcode_list_to_md(raw)
# [hr]...[/hr]
raw.gsub! %r{\[hr\](.*?)\[/hr\]}im, "\n\n---\n\n"
# [QUOTE(=<username>)]...[/QUOTE]
raw.gsub!(/\n?\[quote(=([^;\]]+))?\]\n?/im) do
if $1
old_username = $2
new_username = get_username_for_old_username(old_username)
"\n[quote=\"#{new_username}\"]\n"
else
"\n[quote]\n"
end
end
raw.gsub! %r{\n?\[/quote\]\n?}im, "\n[/quote]\n"
# [YOUTUBE]<id>[/YOUTUBE]
raw.gsub!(%r{\[youtube\](.+?)\[/youtube\]}i) { "\n//youtu.be/#{$1}\n" }
# [VIDEO=youtube;<id>]...[/VIDEO]
raw.gsub!(%r{\[video=youtube;([^\]]+)\].*?\[/video\]}i) { "\n//youtu.be/#{$1}\n" }
# Fix uppercase B U and I tags
raw.gsub!(%r{(\[/?[BUI]\])}i) { $1.downcase }
# More Additions ....
# [spoiler=Some hidden stuff]SPOILER HERE!![/spoiler]
raw.gsub!(%r{\[spoiler="?(.+?)"?\](.+?)\[/spoiler\]}im) do
"\n#{$1}\n[spoiler]#{$2}[/spoiler]\n"
end
# [IMG][IMG]http://i63.tinypic.com/akga3r.jpg[/IMG][/IMG]
raw.gsub!(%r{\[IMG\]\[IMG\](.+?)\[/IMG\]\[/IMG\]}i) { "![](#{$1})" }
raw.gsub!(%r{\[IMG\](.+?)\[/IMG\]}i) { "![](#{$1})" }
raw
end
def bbcode_list_to_md(input)
head, match, input = input.partition(/\[list(=.*?)?\]/i)
return head unless match
result = head
input.lstrip!
type = []
if /\[list=.*?\]/.match(match)
type << "1. "
else
type << "* "
end
until input.empty?
head, match, input = input.partition(%r{\[(/?list(=.*?)?|\*)\]}i)
result << head
if match == ""
break
elsif match == "[*]"
input.lstrip!
result << "\n" unless result[-1] == "\n" || result.length == 0
if type.length == 0
# List-less list
result << "* "
else
result << (" " * (type.length - 1))
result << type.last
end
elsif match.downcase == "[/list]"
type.pop
if type.length > 0
input.lstrip!
else
result << "\n" unless result[-1] == "\n"
end
else
if type.length == 0
result << "\n" unless result[-1] == "\n" || result.length == 0
end
input.lstrip!
if /\[list=.*?\]/i.match(match)
type << "1. "
else
type << "* "
end
end
end
result
end
def postprocess_post_raw(post, raw)
# [QUOTE=<username>;<post_id>]
raw.gsub!(/\[quote=([^;]+);(\d+)\]/im) do
old_username, post_id = $1, $2
new_username = get_username_for_old_username(old_username)
# There is a bug here when the first post in a topic is quoted.
# The first post in a topic does not have an post_custom_field referring to the post number,
# but it refers to thread-XXX instead, so this lookup fails miserably then.
# Fixing this would imply rewriting that logic completely.
if topic_lookup = topic_lookup_from_imported_post_id(post_id)
post_number = topic_lookup[:post_number]
topic_id = topic_lookup[:topic_id]
"\n[quote=\"#{new_username},post:#{post_number},topic:#{topic_id}\"]\n"
else
"\n[quote=\"#{new_username}\"]\n"
end
end
# remove attachments
raw.gsub!(%r{\[attach[^\]]*\]\d+\[/attach\]}i, "")
# [THREAD]<thread_id>[/THREAD]
# ==> http://my.discourse.org/t/slug/<topic_id>
raw.gsub!(%r{\[thread\](\d+)\[/thread\]}i) do
thread_id = $1
if topic_lookup = topic_lookup_from_imported_post_id("thread-#{thread_id}")
topic_lookup[:url]
else
$&
end
end
# [THREAD=<thread_id>]...[/THREAD]
# ==> [...](http://my.discourse.org/t/slug/<topic_id>)
raw.gsub!(%r{\[thread=(\d+)\](.+?)\[/thread\]}i) do
thread_id, link = $1, $2
if topic_lookup = topic_lookup_from_imported_post_id("thread-#{thread_id}")
url = topic_lookup[:url]
"[#{link}](#{url})"
else
$&
end
end
# [POST]<post_id>[/POST]
# ==> http://my.discourse.org/t/slug/<topic_id>/<post_number>
raw.gsub!(%r{\[post\](\d+)\[/post\]}i) do
post_id = $1
if topic_lookup = topic_lookup_from_imported_post_id(post_id)
topic_lookup[:url]
else
$&
end
end
# [POST=<post_id>]...[/POST]
# ==> [...](http://my.discourse.org/t/<topic_slug>/<topic_id>/<post_number>)
raw.gsub!(%r{\[post=(\d+)\](.+?)\[/post\]}i) do
post_id, link = $1, $2
if topic_lookup = topic_lookup_from_imported_post_id(post_id)
url = topic_lookup[:url]
"[#{link}](#{url})"
else
$&
end
end
raw.gsub!(
%r{\[(.*?)\]\((https?:)?//#{Regexp.escape(FORUM_URL)}attachment\.php\?attachmentid=(\d+).*?\)}i,
) do
upload, filename = find_upload(post, $3)
next "#{$1}\n:x: ERROR: unknown attachment reference #{$3}\n" unless upload
html_for_upload(upload, filename)
end
raw
end
def suspend_users
puts "", "updating banned users"
banned = 0
failed = 0
total = mysql_query("SELECT count(*) count FROM #{TABLE_PREFIX}userban").first["count"]
system_user = Discourse.system_user
mysql_query("SELECT userid, bandate, liftdate, reason FROM #{TABLE_PREFIX}userban").each do |b|
user = User.find_by_id(user_id_from_imported_user_id(b["userid"]))
if user
user.suspended_at = parse_timestamp(b["bandate"])
if b["liftdate"] > 0
user.suspended_till = parse_timestamp(b["liftdate"])
else
user.suspended_till = 200.years.from_now
end
if user.save
StaffActionLogger.new(system_user).log_user_suspend(
user,
"#{b["reason"]} (source: initial import from vBulletin)",
)
banned += 1
else
puts "",
"\tFailed to suspend user #{user.username}. #{user.errors.try(:full_messages).try(:inspect)}"
failed += 1
end
else
puts "", "\tNot found: #{b["userid"]}"
failed += 1
end
print_status banned + failed, total
end
end
def parse_timestamp(timestamp)
return if timestamp.nil?
Time.zone.at(@tz.utc_to_local(TZInfo::Timestamp.new(timestamp)).to_datetime)
end
def mysql_query(sql)
@client.query(sql, cache_rows: true)
end
end
ImportScripts::VBulletin.new.perform
Reference in New Issue Copy Permalink