github-mirror/discourse
github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-03-22 18:27:55 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Improve IPBoard3 importer

Browse Source 
- fix encoding issues
- close topics
- don't import queued & not approved topics
- don't import deleted posts
This commit is contained in:
Régis Hanol 2018-03-05 10:24:42 +01:00
parent 2f65393706
commit 49400337b6
1 changed files with 58 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
script/import_scripts/ipboard3.rb
View File

@ -16,6 +16,8 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
password: ENV["DB_PW"],
database: ENV["DB_NAME"],
)
@client.query("SET character_set_results = binary")
end
def execute
@ -23,6 +25,7 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
import_categories
import_topics
import_posts
close_topics
import_personal_topics
import_personal_posts
end
@ -59,11 +62,10 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
break if users.empty?
last_user_id = users[-1]["id"]
user_ids = users.map { |u| u["id"] }
next if all_records_exist?(:users, user_ids)
create_users(users, total: total_users, offset: offset) do |u|
next if user_id_from_imported_user_id(u["id"])
%W{name email title pp_about_me}.each { |k| u[k]&.encode!("utf-8", "utf-8", invalid: :replace, undef: :replace, replace: "") }
next if u["name"].blank? && !Email.is_valid?(u["email"])
{
@ -111,19 +113,21 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
child_categories = categories.select { |c| c["parent_id"] != -1 }
create_categories(parent_categories) do |c|
next if category_id_from_imported_category_id(c["id"])
{
id: c["id"],
name: c["name"],
name: c["name"].encode("utf-8", "utf-8"),
description: clean_up(c["description"]),
position: c["position"],
}
end
create_categories(child_categories) do |c|
next if category_id_from_imported_category_id(c["id"])
{
id: c["id"],
parent_category_id: category_id_from_imported_category_id(c["parent_id"]),
name: c["name"],
name: c["name"].encode("utf-8", "utf-8"),
description: clean_up(c["description"]),
position: c["position"],
}
@ -133,13 +137,18 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
def import_topics
puts "", "importing topics..."
@closed_topic_ids = []
last_topic_id = -1
total_topics = mysql_query(<<~SQL
SELECT COUNT(*) count
FROM topics
JOIN posts ON tid = topic_id
WHERE tdelete_time = 0
AND pdelete_time = 0
AND new_topic = 1
AND approved = 1
AND queued = 0
SQL
).first["count"]
@ -147,6 +156,7 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
topics = mysql_query(<<~SQL
SELECT tid id
, title
, state
, starter_id
, start_date
, views
@ -156,7 +166,10 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
FROM topics
JOIN posts ON tid = topic_id
WHERE tdelete_time = 0
AND pdelete_time = 0
AND new_topic = 1
AND approved = 1
AND queued = 0
AND tid > #{last_topic_id}
ORDER BY tid
LIMIT #{BATCH_SIZE}
@ -166,17 +179,16 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
break if topics.empty?
last_topic_id = topics[-1]["id"]
topic_ids = topics.map { |t| "t-#{t["id"]}" }
next if all_records_exist?(:posts, topic_ids)
create_posts(topics, total: total_topics, offset: offset) do |t|
@closed_topic_ids << "t-#{t["id"]}" if t["state"] != "open"
next if post_id_from_imported_post_id("t-#{t["id"]}")
created_at = Time.zone.at(t["start_date"])
user_id = user_id_from_imported_user_id(t["starter_id"]) || -1
{
id: "t-#{t["id"]}",
title: CGI.unescapeHTML(t["title"]),
title: CGI.unescapeHTML(t["title"].encode("utf-8", "utf-8")),
user_id: user_id,
created_at: created_at,
views: t["views"],
@ -192,7 +204,14 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
puts "", "importing posts..."
last_post_id = -1
total_posts = mysql_query("SELECT COUNT(*) count FROM posts WHERE new_topic = 0").first["count"]
total_posts = mysql_query(<<~SQL
SELECT COUNT(*) count
FROM posts
WHERE new_topic = 0
AND pdelete_time = 0
AND queued = 0
SQL
).first["count"]
batches(BATCH_SIZE) do |offset|
posts = mysql_query(<<~SQL
@ -201,9 +220,10 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
, post_date
, post
, topic_id
, pdelete_time
FROM posts
WHERE new_topic = 0
AND pdelete_time = 0
AND queued = 0
AND pid > #{last_post_id}
ORDER BY pid
LIMIT #{BATCH_SIZE}
@ -213,11 +233,9 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
break if posts.empty?
last_post_id = posts[-1]["id"]
post_ids = posts.map { |p| p["id"] }
next if all_records_exist?(:posts, post_ids)
create_posts(posts, total: total_posts, offset: offset) do |p|
next if post_id_from_imported_post_id(p["id"])
next unless t = topic_lookup_from_imported_post_id("t-#{p["topic_id"]}")
user_id = user_id_from_imported_user_id(p["author_id"]) || -1
@ -227,12 +245,31 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
created_at: Time.zone.at(p["post_date"]),
raw: clean_up(p["post"], user_id),
topic_id: t[:topic_id],
deleted_at: p["pdelete_time"] > 0 ? Time.zone.at(p["pdelete_time"]) : nil,
}
end
end
end
def close_topics
puts "", "closing #{@closed_topic_ids.size} topics..."
sql = <<~SQL
WITH closed_topic_ids AS (
SELECT t.id AS topic_id
FROM post_custom_fields pcf
JOIN posts p ON p.id = pcf.post_id
JOIN topics t ON t.id = p.topic_id
WHERE pcf.name = 'import_id'
AND pcf.value IN (?)
)
UPDATE topics
SET closed = true
WHERE id IN (SELECT topic_id FROM closed_topic_ids)
SQL
Topic.exec_sql(sql, @closed_topic_ids)
end
def import_personal_topics
puts "", "import personal topics..."
@ -268,11 +305,9 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
break if personal_topics.empty?
last_personal_topic_id = personal_topics[-1]["id"]
personal_topic_ids = personal_topics.map { |pt| "pt-#{pt["id"]}" }
next if all_records_exist?(:posts, personal_topic_ids)
create_posts(personal_topics, total: total_personal_topics, offset: offset) do |pt|
next if post_id_from_imported_post_id("pt-#{pt["id"]}")
user_id = user_id_from_imported_user_id(pt["mt_starter_id"]) || -1
user_ids = [pt["mt_to_member_id"]] + pt["mt_invited_members"].scan(/i:(\d+);/).flatten.map(&:to_i)
@ -284,7 +319,7 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
archetype: Archetype.private_message,
id: "pt-#{pt["id"]}",
created_at: Time.zone.at(pt["mt_date"]),
title: CGI.unescapeHTML(pt["mt_title"]),
title: CGI.unescapeHTML(pt["mt_title"].encode("utf-8", "utf-8")),
user_id: user_id,
target_usernames: User.where(id: user_ids).pluck(:username),
raw: clean_up(pt["msg_post"], user_id),
@ -317,11 +352,9 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
break if personal_posts.empty?
last_personal_post_id = personal_posts[-1]["id"]
personal_post_ids = personal_posts.map { |pp| "pp-#{pp["id"]}" }
next if all_records_exist?(:posts, personal_post_ids)
create_posts(personal_posts, total: total_personal_posts, offset: offset) do |pp|
next if post_id_from_imported_post_id("pp-#{pp["id"]}")
next unless t = topic_lookup_from_imported_post_id("pt-#{pp["msg_topic_id"]}")
user_id = user_id_from_imported_user_id(pp["msg_author_id"]) || -1
@ -337,6 +370,8 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
end
def clean_up(raw, user_id = -1)
raw.encode!("utf-8", "utf-8", invalid: :replace, undef: :replace, replace: "")
raw.gsub!(/<(.+)>&nbsp;<\/\1>/, "\n\n")
doc = Nokogiri::HTML.fragment(raw)
@ -344,7 +379,7 @@ class ImportScripts::IPBoard3 < ImportScripts::Base
doc.css("blockquote.ipsBlockquote").each do |bq|
post_id = post_id_from_imported_post_id(bq["data-cid"])
if post = Post.find_by(id: post_id)
bq.replace %{<p>[quote="#{post.user.username},post:#{post.post_number},topic:#{post.topic_id}"]\n#{bq.inner_html}\n[/quote]</p>}
bq.replace %{<br>[quote="#{post.user.username},post:#{post.post_number},topic:#{post.topic_id}"]\n#{bq.inner_html}\n[/quote]<br>}
end
end