# frozen_string_literal: true
require_relative "base"
require "pg"
require "htmlentities"
require "ruby-bbcode-to-md"
class BulkImport::PhpBB < BulkImport::Base
SUSPENDED_TILL = Date.new(3000, 1, 1)
TABLE_PREFIX = ENV["TABLE_PREFIX"] || "phpbb_"
def initialize
super
charset = ENV["DB_CHARSET"] || "utf8"
database = ENV["DB_NAME"] || "flightaware"
password = ENV["DB_PASSWORD"] || "discourse"
@html_entities = HTMLEntities.new
@encoding = CHARSET_MAP[charset]
@client = PG.connect(dbname: database, password: password)
@smiley_map = {}
add_default_smilies
end
def execute
import_groups
import_users
import_group_users
import_user_emails
import_user_profiles
import_categories
import_topics
import_posts
import_private_topics
import_topic_allowed_users
import_private_posts
end
def import_groups
puts "Importing groups..."
groups = psql_query <<-SQL
SELECT group_id, group_name, group_desc
FROM #{TABLE_PREFIX}groups
WHERE group_id > #{@last_imported_group_id}
ORDER BY group_id
SQL
create_groups(groups) do |row|
{
imported_id: row["group_id"],
name: normalize_text(row["group_name"]),
bio_raw: normalize_text(row["group_desc"]),
}
end
end
def import_users
puts "Importing users..."
users = psql_query <<-SQL
SELECT u.user_id, u.username, u.user_email, u.user_regdate, u.user_lastvisit, u.user_ip,
u.user_type, u.user_inactive_reason, g.group_id, g.group_name, b.ban_start, b.ban_end, b.ban_reason,
u.user_posts, u.user_website, u.user_from, u.user_birthday, u.user_avatar_type, u.user_avatar
FROM #{TABLE_PREFIX}users u
LEFT OUTER JOIN #{TABLE_PREFIX}groups g ON (g.group_id = u.group_id)
LEFT OUTER JOIN #{TABLE_PREFIX}banlist b ON (
u.user_id = b.ban_userid AND b.ban_exclude = 0 AND
b.ban_end = 0
)
WHERE u.user_id > #{@last_imported_user_id}
ORDER BY u.user_id
SQL
create_users(users) do |row|
u = {
imported_id: row["user_id"],
username: normalize_text(row["username"]),
email: row["user_email"],
created_at: Time.zone.at(row["user_regdate"].to_i),
last_seen_at:
(
if row["user_lastvisit"] == 0
Time.zone.at(row["user_regdate"].to_i)
else
Time.zone.at(row["user_lastvisit"].to_i)
end
),
trust_level: row["user_posts"] == 0 ? TrustLevel[0] : TrustLevel[1],
date_of_birth: parse_birthday(row["user_birthday"]),
primary_group_id: group_id_from_imported_id(row["group_id"]),
}
u[:ip_address] = row["user_ip"][/\b(?:\d{1,3}\.){3}\d{1,3}\b/] if row["user_ip"].present?
if row["ban_start"]
u[:suspended_at] = Time.zone.at(row["ban_start"].to_i)
u[:suspended_till] = (
if row["ban_end"].to_i > 0
Time.zone.at(row["ban_end"].to_i)
else
SUSPENDED_TILL
end
)
end
u
end
end
def import_user_emails
puts "Importing user emails..."
users = psql_query <<-SQL
SELECT user_id, user_email, user_regdate
FROM #{TABLE_PREFIX}users u
WHERE user_id > #{@last_imported_user_id}
ORDER BY user_id
SQL
create_user_emails(users) do |row|
{
imported_id: row["user_id"],
imported_user_id: row["user_id"],
email: row["user_email"],
created_at: Time.zone.at(row["user_regdate"].to_i),
}
end
end
def import_group_users
puts "Importing group users..."
group_users = psql_query <<-SQL
SELECT user_id, group_id
FROM #{TABLE_PREFIX}users u
WHERE user_id > #{@last_imported_user_id}
SQL
create_group_users(group_users) do |row|
{
group_id: group_id_from_imported_id(row["group_id"]),
user_id: user_id_from_imported_id(row["user_id"]),
}
end
end
def import_user_profiles
puts "Importing user profiles..."
user_profiles = psql_query <<-SQL
SELECT user_id, user_website, user_from
FROM #{TABLE_PREFIX}users
WHERE user_id > #{@last_imported_user_id}
ORDER BY user_id
SQL
create_user_profiles(user_profiles) do |row|
{
user_id: user_id_from_imported_id(row["user_id"]),
website:
(
begin
URI.parse(row["user_website"]).to_s
rescue StandardError
nil
end
),
location: row["user_from"],
}
end
end
def import_categories
puts "Importing categories..."
categories = psql_query(<<-SQL).to_a
SELECT forum_id, parent_id, forum_name, forum_desc
FROM #{TABLE_PREFIX}forums
WHERE forum_id > #{@last_imported_category_id}
ORDER BY parent_id, left_id
SQL
return if categories.empty?
parent_categories = categories.select { |c| c["parent_id"].to_i == 0 }
children_categories = categories.select { |c| c["parent_id"].to_i != 0 }
puts "Importing parent categories..."
create_categories(parent_categories) do |row|
{
imported_id: row["forum_id"],
name: normalize_text(row["forum_name"]),
description: normalize_text(row["forum_desc"]),
}
end
puts "Importing children categories..."
create_categories(children_categories) do |row|
{
imported_id: row["forum_id"],
name: normalize_text(row["forum_name"]),
description: normalize_text(row["forum_desc"]),
parent_category_id: category_id_from_imported_id(row["parent_id"]),
}
end
end
def import_topics
puts "Importing topics..."
topics = psql_query <<-SQL
SELECT topic_id, topic_title, forum_id, topic_poster, topic_time, topic_views
FROM #{TABLE_PREFIX}topics
WHERE topic_id > #{@last_imported_topic_id}
AND EXISTS (SELECT 1 FROM #{TABLE_PREFIX}posts WHERE #{TABLE_PREFIX}posts.topic_id = #{TABLE_PREFIX}topics.topic_id)
ORDER BY topic_id
SQL
create_topics(topics) do |row|
{
imported_id: row["topic_id"],
title: normalize_text(row["topic_title"]),
category_id: category_id_from_imported_id(row["forum_id"]),
user_id: user_id_from_imported_id(row["topic_poster"]),
created_at: Time.zone.at(row["topic_time"].to_i),
views: row["topic_views"],
}
end
end
def import_posts
puts "Importing posts..."
posts = psql_query <<-SQL
SELECT p.post_id, p.topic_id, p.poster_id, p.post_time, p.post_text
FROM #{TABLE_PREFIX}posts p
JOIN #{TABLE_PREFIX}topics t ON t.topic_id = p.topic_id
WHERE p.post_id > #{@last_imported_post_id}
ORDER BY p.post_id
SQL
create_posts(posts) do |row|
{
imported_id: row["post_id"],
topic_id: topic_id_from_imported_id(row["topic_id"]),
user_id: user_id_from_imported_id(row["poster_id"]),
created_at: Time.zone.at(row["post_time"].to_i),
raw: process_raw_text(row["post_text"]),
}
end
end
def import_private_topics
puts "Importing private topics..."
@imported_topics = {}
topics = psql_query <<-SQL
SELECT msg_id, message_subject, author_id, to_address, message_time
FROM #{TABLE_PREFIX}privmsgs
WHERE msg_id > (#{@last_imported_private_topic_id - PRIVATE_OFFSET})
ORDER BY msg_id
SQL
create_topics(topics) do |row|
user_ids = get_message_recipients(row["author_id"], row["to_address"])
title = extract_pm_title(row["message_subject"])
key = [title, user_ids]
next if @imported_topics.has_key?(key) || title.blank?
@imported_topics[key] = row["msg_id"].to_i + PRIVATE_OFFSET
{
archetype: Archetype.private_message,
imported_id: row["msg_id"].to_i + PRIVATE_OFFSET,
title: normalize_text(title),
user_id: user_id_from_imported_id(row["author_id"].to_i),
created_at: Time.zone.at(row["message_time"].to_i),
}
end
end
def import_topic_allowed_users
puts "Importing topic allowed users..."
allowed_users = []
psql_query(<<-SQL).each do |row|
SELECT msg_id, author_id, to_address
FROM #{TABLE_PREFIX}privmsgs
WHERE msg_id > (#{@last_imported_private_topic_id - PRIVATE_OFFSET})
ORDER BY msg_id
SQL
next unless topic_id = topic_id_from_imported_id(row["msg_id"].to_i + PRIVATE_OFFSET)
user_ids = get_message_recipients(row["author_id"], row["to_address"])
user_ids.each do |id|
next unless user_id = user_id_from_imported_id(id.to_i)
allowed_users << [topic_id, user_id]
end
end
create_topic_allowed_users(allowed_users) { |row| { topic_id: row[0], user_id: row[1] } }
end
def import_private_posts
puts "Importing private posts..."
posts = psql_query <<-SQL
SELECT msg_id, message_subject, author_id, to_address, message_time, message_text
FROM #{TABLE_PREFIX}privmsgs
WHERE msg_id > (#{@last_imported_private_topic_id - PRIVATE_OFFSET})
ORDER BY msg_id
SQL
create_posts(posts) do |row|
user_ids = get_message_recipients(row["author_id"], row["to_address"])
title = extract_pm_title(row["message_subject"])
key = [title, user_ids]
next unless topic_id = topic_id_from_imported_id(@imported_topics[key])
{
imported_id: row["msg_id"].to_i + PRIVATE_OFFSET,
topic_id: topic_id,
user_id: user_id_from_imported_id(row["author_id"].to_i),
created_at: Time.zone.at(row["message_time"].to_i),
raw: process_raw_text(row["message_text"]),
}
end
end
def get_message_recipients(from, to)
user_ids = to.split(":")
user_ids.map! { |u| u[2..-1].to_i }
user_ids.push(from.to_i)
user_ids.uniq!
user_ids = user_ids.flatten.map(&:to_i).sort
user_ids
end
def extract_pm_title(title)
pm_title = CGI.unescapeHTML(title)
pm_title =
begin
title.gsub(/^Re\s*:\s*/i, "")
rescue StandardError
nil
end
pm_title
end
def parse_birthday(birthday)
return if birthday.blank?
date_of_birth =
begin
Date.strptime(birthday.gsub(/[^\d-]+/, ""), "%m-%d-%Y")
rescue StandardError
nil
end
return if date_of_birth.nil?
if date_of_birth.year < 1904
Date.new(1904, date_of_birth.month, date_of_birth.day)
else
date_of_birth
end
end
def psql_query(sql)
@client.query(sql)
end
def process_raw_text(raw)
return "" if raw.blank?
text = raw.dup
text = CGI.unescapeHTML(text)
text.gsub!(/:(?:\w{8})\]/, "]")
# Some links look like this: http://www.onegameamonth.com
text.gsub!(%r{(.+)}i, '[\2](\1)')
# phpBB shortens link text like this, which breaks our markdown processing:
# [http://answers.yahoo.com/question/index ... 223AAkkPli](http://answers.yahoo.com/question/index?qid=20070920134223AAkkPli)
#
# Work around it for now:
text.gsub!(%r{\[http(s)?://(www\.)?}i, "[")
# convert list tags to ul and list=1 tags to ol
# list=a is not supported, so handle it like list=1
# list=9 and list=x have the same result as list=1 and list=a
text.gsub!(%r{\[list\](.*?)\[/list:u\]}mi, '[ul]\1[/ul]')
text.gsub!(%r{\[list=.*?\](.*?)\[/list:o\]}mi, '[ol]\1[/ol]')
# convert *-tags to li-tags so bbcode-to-md can do its magic on phpBB's lists:
text.gsub!(%r{\[\*\](.*?)\[/\*:m\]}mi, '[li]\1[/li]')
# [QUOTE=""] -- add newline
text.gsub!(/(\[quote="[a-zA-Z\d]+"\])/i) { "#{$1}\n" }
# [/QUOTE] -- add newline
text.gsub!(%r{(\[/quote\])}i) { "\n#{$1}" }
# :) is encoded as
text.gsub!(
//,
) do
smiley = $1
@smiley_map.fetch(smiley) do
# upload_smiley(smiley, $2, $3, $4) || smiley_as_text(smiley)
@smiley_map[smiley] = smiley
end
end
text = bbcode_to_md(text)
text
end
protected
def bbcode_to_md(text)
begin
text.bbcode_to_md(false)
rescue => e
puts "Problem converting \n#{text}\n using ruby-bbcode-to-md"
text
end
end
def add_default_smilies
{
%w[:D :-D :grin:] => ":smiley:",
%w[:) :-) :smile:] => ":slight_smile:",
%w[;) ;-) :wink:] => ":wink:",
%w[:( :-( :sad:] => ":frowning:",
%w[:o :-o :eek:] => ":astonished:",
[":shock:"] => ":open_mouth:",
%w[:? :-? :???:] => ":confused:",
%w[8-) :cool:] => ":sunglasses:",
[":lol:"] => ":laughing:",
%w[:x :-x :mad:] => ":angry:",
%w[:P :-P :razz:] => ":stuck_out_tongue:",
[":oops:"] => ":blush:",
[":cry:"] => ":cry:",
[":evil:"] => ":imp:",
[":twisted:"] => ":smiling_imp:",
[":roll:"] => ":unamused:",
[":!:"] => ":exclamation:",
[":?:"] => ":question:",
[":idea:"] => ":bulb:",
[":arrow:"] => ":arrow_right:",
%w[:| :-|] => ":neutral_face:",
[":geek:"] => ":nerd:",
}.each { |smilies, emoji| smilies.each { |smiley| @smiley_map[smiley] = emoji } }
end
end
BulkImport::PhpBB.new.run