mirror of
https://github.com/discourse/discourse.git
synced 2024-12-15 23:33:57 +08:00
361 lines
8.6 KiB
Ruby
361 lines
8.6 KiB
Ruby
# getsatisfaction importer
|
|
#
|
|
# pre-req: you will get a bunch of CSV files, be sure to rename them all so
|
|
#
|
|
# - users.csv is the users table export (it may come from getsatisfaction as Users-Table 1.csv
|
|
# - replies.csv is the reply table export
|
|
# - topics.csv is the topics table export
|
|
#
|
|
#
|
|
# note, the importer will import all topics into a new category called 'Old Forum' and optionally close all the topics
|
|
#
|
|
require 'csv'
|
|
require File.expand_path(File.dirname(__FILE__) + "/base.rb")
|
|
require 'reverse_markdown' # gem 'reverse_markdown'
|
|
|
|
# Call it like this:
|
|
# RAILS_ENV=production bundle exec ruby script/import_scripts/getsatisfaction.rb
|
|
class ImportScripts::GetSatisfaction < ImportScripts::Base
|
|
|
|
BATCH_SIZE = 1000
|
|
|
|
def initialize(path)
|
|
@path = path
|
|
super()
|
|
@bbcode_to_md = true
|
|
@topic_slug = {}
|
|
|
|
puts "loading post mappings..."
|
|
@post_number_map = {}
|
|
Post.pluck(:id, :post_number).each do |post_id, post_number|
|
|
@post_number_map[post_id] = post_number
|
|
end
|
|
end
|
|
|
|
def created_post(post)
|
|
@post_number_map[post.id] = post.post_number
|
|
super
|
|
end
|
|
|
|
|
|
def execute
|
|
c = Category.find_by(name: 'Old Forum') ||
|
|
Category.create!(name: 'Old Forum', user: Discourse.system_user)
|
|
|
|
import_users
|
|
import_posts(c)
|
|
|
|
create_permalinks
|
|
|
|
# uncomment if you want to close all the topics
|
|
# Topic.where(category: c).update_all(closed: true)
|
|
end
|
|
|
|
class RowResolver
|
|
def load(row)
|
|
@row = row
|
|
end
|
|
|
|
def self.create(cols)
|
|
Class.new(RowResolver).new(cols)
|
|
end
|
|
|
|
def initialize(cols)
|
|
cols.each_with_index do |col,idx|
|
|
self.class.send(:define_method, col) do
|
|
@row[idx]
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
def load_user_batch!(users, offset, total)
|
|
if users.length > 0
|
|
create_users(users, offset: offset, total: total) do |user|
|
|
user
|
|
end
|
|
users.clear
|
|
end
|
|
end
|
|
|
|
def csv_parse(name)
|
|
filename = "#{@path}/#{name}.csv"
|
|
first = true
|
|
row = nil
|
|
|
|
current_row = "";
|
|
double_quote_count = 0
|
|
|
|
# In case of Excel export file, I converted it to CSV and used:
|
|
# CSV.open(filename, encoding:'iso-8859-1:utf-8').each do |raw|
|
|
File.open(filename).each_line do |line|
|
|
|
|
line.strip!
|
|
|
|
current_row << "\n" unless current_row.empty?
|
|
current_row << line
|
|
|
|
raw = begin
|
|
CSV.parse(current_row, col_sep: ";")
|
|
rescue CSV::MalformedCSVError => e
|
|
puts e.message
|
|
puts "*" * 100
|
|
puts "Bad row skipped, line is: #{line}"
|
|
puts
|
|
puts current_row
|
|
puts
|
|
puts "double quote count is : #{double_quote_count}"
|
|
puts "*" * 100
|
|
|
|
current_row = ""
|
|
double_quote_count = 0
|
|
|
|
next
|
|
end[0]
|
|
|
|
if first
|
|
row = RowResolver.create(raw)
|
|
|
|
current_row = ""
|
|
double_quote_count = 0
|
|
first = false
|
|
next
|
|
end
|
|
|
|
row.load(raw)
|
|
|
|
yield row
|
|
|
|
current_row = ""
|
|
double_quote_count = 0
|
|
end
|
|
end
|
|
|
|
def total_rows(table)
|
|
# In case of Excel export file, I converted it to CSV and used:
|
|
# CSV.foreach("#{@path}/#{table}.csv", encoding:'iso-8859-1:utf-8').inject(0) {|c, line| c+1} - 1
|
|
File.foreach("#{@path}/#{table}.csv").inject(0) {|c, line| c+1} - 1
|
|
end
|
|
|
|
def import_users
|
|
puts "", "creating users"
|
|
|
|
count = 0
|
|
users = []
|
|
|
|
total = total_rows("users")
|
|
|
|
csv_parse("users") do |row|
|
|
|
|
if row.suspended_at
|
|
puts "skipping suspended user"
|
|
p row
|
|
next
|
|
end
|
|
|
|
id = row.user_id
|
|
email = row.email
|
|
|
|
# fake it
|
|
if row.email.blank? || row.email !~ /@/
|
|
email = SecureRandom.hex << "@domain.com"
|
|
end
|
|
|
|
name = row.real_name
|
|
username = row.nick
|
|
created_at = DateTime.parse(row.m_created)
|
|
|
|
username = name if username == "NULL"
|
|
username = email.split("@")[0] if username.blank?
|
|
name = email.split("@")[0] if name.blank?
|
|
|
|
users << {
|
|
id: id,
|
|
email: email,
|
|
name: name,
|
|
username: username,
|
|
created_at: created_at,
|
|
active: false
|
|
}
|
|
|
|
count += 1
|
|
if count % BATCH_SIZE == 0
|
|
load_user_batch! users, count - users.length, total
|
|
end
|
|
|
|
end
|
|
|
|
load_user_batch! users, count, total
|
|
end
|
|
|
|
def import_categories
|
|
rows = []
|
|
csv_parse("categories") do |row|
|
|
rows << {id: row.id, name: row.name, description: row.description}
|
|
end
|
|
|
|
create_categories(rows) do |row|
|
|
row
|
|
end
|
|
end
|
|
|
|
def normalize_raw!(raw)
|
|
return "<missing>" if raw.nil?
|
|
raw = raw.dup
|
|
|
|
# hoist code
|
|
hoisted = {}
|
|
raw.gsub!(/(<pre>\s*)?<code>(.*?)<\/code>(\s*<\/pre>)?/mi) do
|
|
code = $2
|
|
hoist = SecureRandom.hex
|
|
# tidy code, wow, this is impressively crazy
|
|
code.gsub!(/ (\s*)/,"\n\\1")
|
|
code.gsub!(/^\s*\n$/, "\n")
|
|
code.gsub!(/\n+/m, "\n")
|
|
code.strip!
|
|
hoisted[hoist] = code
|
|
hoist
|
|
end
|
|
|
|
# impressive seems to be using tripple space as a <p> unless hoisted
|
|
# in this case double space works best ... so odd
|
|
raw.gsub!(" ", "\n\n")
|
|
|
|
hoisted.each do |hoist, code|
|
|
raw.gsub!(hoist, "\n```\n" << code << "\n```\n")
|
|
end
|
|
|
|
raw = CGI.unescapeHTML(raw)
|
|
raw = ReverseMarkdown.convert(raw)
|
|
raw
|
|
end
|
|
|
|
def import_post_batch!(posts, topics, offset, total)
|
|
create_posts(posts, total: total, offset: offset) do |post|
|
|
|
|
mapped = {}
|
|
|
|
mapped[:id] = post[:id]
|
|
mapped[:user_id] = user_id_from_imported_user_id(post[:user_id]) || -1
|
|
mapped[:raw] = post[:body]
|
|
mapped[:created_at] = post[:created_at]
|
|
|
|
topic = topics[post[:topic_id]]
|
|
|
|
unless topic
|
|
p "MISSING TOPIC #{post[:topic_id]}"
|
|
p post
|
|
next
|
|
end
|
|
|
|
|
|
unless topic[:post_id]
|
|
mapped[:title] = post[:title] || "Topic title missing"
|
|
topic[:post_id] = post[:id]
|
|
mapped[:category] = post[:category]
|
|
else
|
|
parent = topic_lookup_from_imported_post_id(topic[:post_id])
|
|
next unless parent
|
|
|
|
mapped[:topic_id] = parent[:topic_id]
|
|
|
|
reply_to_post_id = post_id_from_imported_post_id(post[:reply_id])
|
|
if reply_to_post_id
|
|
reply_to_post_number = @post_number_map[reply_to_post_id]
|
|
if reply_to_post_number && reply_to_post_number > 1
|
|
mapped[:reply_to_post_number] = reply_to_post_number
|
|
end
|
|
end
|
|
end
|
|
|
|
next if topic[:deleted] or post[:deleted]
|
|
|
|
mapped
|
|
end
|
|
|
|
posts.clear
|
|
end
|
|
|
|
def import_posts(category)
|
|
puts "", "creating topics and posts"
|
|
|
|
topic_map = {}
|
|
|
|
csv_parse("topics") do |topic|
|
|
@topic_slug[topic.id.to_i] = topic.url
|
|
|
|
topic_map[topic.id] = {
|
|
id: topic.id,
|
|
topic_id: topic.id,
|
|
title: topic.subject,
|
|
deleted: topic.removed == "1",
|
|
closed: true,
|
|
body: normalize_raw!(topic.additional_detail || topic.subject || "<missing>"),
|
|
created_at: DateTime.parse(topic.created_at),
|
|
user_id: topic.UserId,
|
|
category: category.name
|
|
}
|
|
end
|
|
|
|
total = total_rows("replies")
|
|
|
|
posts = []
|
|
count = 0
|
|
|
|
topic_map.each do |_, topic|
|
|
# a bit lazy
|
|
posts << topic if topic[:body]
|
|
end
|
|
|
|
csv_parse("replies") do |row|
|
|
|
|
unless row.created_at
|
|
puts "NO CREATION DATE FOR POST"
|
|
p row
|
|
next
|
|
end
|
|
|
|
row = {
|
|
id: row.id,
|
|
topic_id: row.topic_id,
|
|
reply_id: row.parent_id,
|
|
user_id: row.UserId,
|
|
body: normalize_raw!(row.content),
|
|
created_at: DateTime.parse(row.created_at)
|
|
}
|
|
posts << row
|
|
count+=1
|
|
|
|
if posts.length > 0 && posts.length % BATCH_SIZE == 0
|
|
import_post_batch!(posts, topic_map, count - posts.length, total)
|
|
end
|
|
end
|
|
|
|
import_post_batch!(posts, topic_map, count - posts.length, total) if posts.length > 0
|
|
end
|
|
|
|
def create_permalinks
|
|
puts '', 'Creating Permalinks...', ''
|
|
|
|
topic_mapping = []
|
|
|
|
Topic.listable_topics.find_each do |topic|
|
|
tcf = topic.first_post.custom_fields
|
|
if tcf && tcf["import_id"]
|
|
slug = @topic_slug[tcf["import_id"].to_i]
|
|
# TODO: replace "http://community.example.com/" with the URL of your community
|
|
slug = slug.gsub("http://community.example.com/", "")
|
|
Permalink.create(url: slug, topic_id: topic.id)
|
|
end
|
|
end
|
|
end
|
|
|
|
end
|
|
|
|
unless ARGV[0] && Dir.exist?(ARGV[0])
|
|
puts "", "Usage:", "", "bundle exec ruby script/import_scripts/getsatisfaction.rb DIRNAME", ""
|
|
exit 1
|
|
end
|
|
|
|
ImportScripts::GetSatisfaction.new(ARGV[0]).perform
|