mirror of
https://github.com/discourse/discourse.git
synced 2024-11-22 13:41:31 +08:00
471 lines
13 KiB
Ruby
Executable File
471 lines
13 KiB
Ruby
Executable File
require 'sqlite3'
|
|
require File.expand_path(File.dirname(__FILE__) + "/base.rb")
|
|
|
|
# TODO: ignore ~ emacs backup files
|
|
# DONE: sort filenames before processing
|
|
|
|
# Paste these lines into your shell before running this:
|
|
|
|
=begin
|
|
export MBOX_SUBDIR="messages" # subdirectory with mbox files
|
|
export LIST_NAME=LIST_NAME
|
|
export DEFAULT_TRUST_LEVEL=1
|
|
export DATA_DIR=~/data/import
|
|
export SPLIT_AT="^From " # or "^From (.*)"
|
|
=end
|
|
|
|
class ImportScripts::Mbox < ImportScripts::Base
|
|
include ActiveModel::Validations
|
|
|
|
# CHANGE THESE BEFORE RUNNING THE IMPORTER
|
|
|
|
MBOX_SUBDIR = ENV['MBOX_SUBDIR'] || "messages" # subdirectory with mbox files
|
|
LIST_NAME = ENV['LIST_NAME'] || "" # Will remove [LIST_NAME] from Subjects
|
|
DEFAULT_TRUST_LEVEL = ENV['DEFAULT_TRUST_LEVEL'] || 1
|
|
DATA_DIR = ENV['DATA_DIR'] || "~/data/import"
|
|
MBOX_DIR = File.expand_path(DATA_DIR) # where index.db will be created
|
|
BATCH_SIZE = 1000
|
|
|
|
# Site settings
|
|
SiteSetting.disable_emails = true
|
|
|
|
# Comment out if each file contains a single message
|
|
# Use formail to split yourself: http://linuxcommand.org/man_pages/formail1.html
|
|
# SPLIT_AT = /^From (.*) at/ # for Google Groups?
|
|
SPLIT_AT = /#{ENV['SPLIT_AT']}/ || /^From / # for standard MBOX files
|
|
|
|
# Will create a category if it doesn't exist
|
|
# create subdirectories in MBOX_SUBDIR with categories
|
|
CATEGORY_MAPPINGS = {
|
|
"default" => "uncategorized",
|
|
# ex: "jobs-folder" => "jobs"
|
|
}
|
|
|
|
def execute
|
|
import_categories
|
|
create_email_indices
|
|
create_user_indices
|
|
massage_indices
|
|
import_users
|
|
create_forum_topics
|
|
import_replies
|
|
end
|
|
|
|
def import_categories
|
|
mappings = CATEGORY_MAPPINGS.values - ['uncategorized']
|
|
|
|
create_categories(mappings) do |c|
|
|
{id: c, name: c}
|
|
end
|
|
end
|
|
|
|
def open_db
|
|
SQLite3::Database.new("#{MBOX_DIR}/index.db")
|
|
end
|
|
|
|
def each_line(f)
|
|
infile = File.open(f, 'r')
|
|
if f.ends_with?('.gz')
|
|
gz = Zlib::GzipReader.new(infile)
|
|
gz.each_line do |line|
|
|
yield line
|
|
end
|
|
else
|
|
infile.each_line do |line|
|
|
yield line
|
|
end
|
|
end
|
|
ensure
|
|
infile.close
|
|
end
|
|
|
|
def all_messages
|
|
files = Dir["#{MBOX_DIR}/#{MBOX_SUBDIR}/*"]
|
|
|
|
CATEGORY_MAPPINGS.keys.each do |k|
|
|
files << Dir["#{MBOX_DIR}/#{k}/*"]
|
|
end
|
|
|
|
files.flatten!
|
|
|
|
files.sort!
|
|
|
|
files.each_with_index do |f, idx|
|
|
if SPLIT_AT.present?
|
|
msg = ""
|
|
|
|
each_line(f) do |line|
|
|
line = line.scrub
|
|
if line =~ SPLIT_AT
|
|
if !msg.empty?
|
|
mail = Mail.read_from_string(msg)
|
|
yield mail, f
|
|
print_status(idx, files.size)
|
|
msg = ""
|
|
end
|
|
end
|
|
msg << line
|
|
end
|
|
|
|
if !msg.empty?
|
|
mail = Mail.read_from_string(msg)
|
|
yield mail, f
|
|
print_status(idx, files.size)
|
|
msg = ""
|
|
end
|
|
else
|
|
raw = File.read(f)
|
|
mail = Mail.read_from_string(raw)
|
|
yield mail, f
|
|
print_status(idx, files.size)
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def massage_indices
|
|
db = open_db
|
|
db.execute "UPDATE emails SET reply_to = null WHERE reply_to = ''"
|
|
|
|
rows = db.execute "SELECT msg_id, title, reply_to FROM emails ORDER BY datetime(email_date) ASC"
|
|
|
|
msg_ids = {}
|
|
titles = {}
|
|
rows.each do |row|
|
|
msg_ids[row[0]] = true
|
|
titles[row[1]] = row[0]
|
|
end
|
|
|
|
# First, any replies where the parent doesn't exist should have that field cleared
|
|
not_found = []
|
|
rows.each do |row|
|
|
msg_id, _, reply_to = row
|
|
|
|
if reply_to.present?
|
|
not_found << msg_id if msg_ids[reply_to].blank?
|
|
end
|
|
end
|
|
|
|
puts "#{not_found.size} records couldn't be associated with parents"
|
|
if not_found.present?
|
|
db.execute "UPDATE emails SET reply_to = NULL WHERE msg_id IN (#{not_found.map {|nf| "'#{nf}'"}.join(',')})"
|
|
end
|
|
|
|
dupe_titles = db.execute "SELECT title, COUNT(*) FROM emails GROUP BY title HAVING count(*) > 1"
|
|
puts "#{dupe_titles.size} replies to wire up"
|
|
dupe_titles.each do |t|
|
|
title = t[0]
|
|
first = titles[title]
|
|
db.execute "UPDATE emails SET reply_to = ? WHERE title = ? and msg_id <> ?", [first, title, first]
|
|
end
|
|
|
|
ensure
|
|
db.close
|
|
end
|
|
|
|
def extract_name(mail)
|
|
from_name = nil
|
|
from = mail[:from]
|
|
|
|
from_email = nil
|
|
if mail.from.present?
|
|
from_email = mail.from.dup
|
|
if from_email.kind_of?(Array)
|
|
from_email = from_email.first.dup
|
|
end
|
|
|
|
from_email.gsub!(/ at /, '@')
|
|
from_email.gsub!(/ \(.*$/, '')
|
|
end
|
|
|
|
display_names = from.try(:display_names)
|
|
if display_names.present?
|
|
from_name = display_names.first
|
|
end
|
|
|
|
if from_name.blank? && from.to_s =~ /\(([^\)]+)\)/
|
|
from_name = Regexp.last_match[1]
|
|
end
|
|
from_name = from.to_s if from_name.blank?
|
|
|
|
[from_email, from_name]
|
|
end
|
|
|
|
def create_email_indices
|
|
db = open_db
|
|
db.execute "DROP TABLE IF EXISTS emails"
|
|
db.execute <<-SQL
|
|
CREATE TABLE emails (
|
|
msg_id VARCHAR(995) PRIMARY KEY,
|
|
from_email VARCHAR(255) NOT NULL,
|
|
from_name VARCHAR(255) NOT NULL,
|
|
title VARCHAR(255) NOT NULL,
|
|
reply_to VARCHAR(955) NULL,
|
|
email_date DATETIME NOT NULL,
|
|
message TEXT NOT NULL,
|
|
category VARCHAR(255) NOT NULL
|
|
);
|
|
SQL
|
|
|
|
db.execute "CREATE INDEX by_title ON emails (title)"
|
|
db.execute "CREATE INDEX by_email ON emails (from_email)"
|
|
|
|
puts "", "creating indices"
|
|
|
|
all_messages do |mail, filename|
|
|
|
|
directory = filename.sub("#{MBOX_DIR}/", '').split("/")[0]
|
|
|
|
category = CATEGORY_MAPPINGS[directory] || CATEGORY_MAPPINGS['default'] || 'uncategorized'
|
|
|
|
msg_id = mail['Message-ID'].to_s
|
|
|
|
# Many ways to get a name
|
|
from_email, from_name = extract_name(mail)
|
|
|
|
title = clean_title(mail['Subject'].to_s)
|
|
reply_to = mail['In-Reply-To'].to_s
|
|
email_date = mail['date'].to_s
|
|
email_date = DateTime.parse(email_date).to_s unless email_date.blank?
|
|
|
|
db.execute "INSERT OR IGNORE INTO emails (msg_id,
|
|
from_email,
|
|
from_name,
|
|
title,
|
|
reply_to,
|
|
email_date,
|
|
message,
|
|
category)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
|
[msg_id, from_email, from_name, title, reply_to, email_date, mail.to_s, category]
|
|
end
|
|
ensure
|
|
db.close
|
|
end
|
|
|
|
def create_user_indices
|
|
db = open_db
|
|
db.execute "DROP TABLE IF EXISTS users"
|
|
db.execute <<-SQL
|
|
CREATE TABLE users (
|
|
email VARCHAR(995) PRIMARY KEY,
|
|
name VARCHAR(255) NOT NULL
|
|
);
|
|
SQL
|
|
|
|
db.execute "INSERT OR IGNORE INTO users (email, name) SELECT from_email, from_name FROM emails"
|
|
ensure
|
|
db.close
|
|
end
|
|
|
|
def clean_title(title)
|
|
title ||= ""
|
|
#Strip mailing list name from subject
|
|
title = title.gsub(/\[#{Regexp.escape(LIST_NAME)}\]/, '').strip
|
|
|
|
original_length = title.length
|
|
|
|
#Strip Reply prefix from title (Standard and localized)
|
|
title = title.gsub(/^Re: */i, '')
|
|
title = title.gsub(/^R: */i, '') #Italian
|
|
title = title.gsub(/^RIF: */i, '') #Italian
|
|
|
|
#Strip Forward prefix from title (Standard and localized)
|
|
title = title.gsub(/^Fwd: */i, '')
|
|
title = title.gsub(/^I: */i, '') #Italian
|
|
|
|
title.strip
|
|
|
|
#In case of mixed localized prefixes there could be many of them if the mail client didn't strip the localized ones
|
|
if original_length > title.length
|
|
clean_title(title)
|
|
else
|
|
title
|
|
end
|
|
end
|
|
|
|
def clean_raw(input)
|
|
|
|
raw = input.dup
|
|
raw.gsub!(/-- \nYou received this message because you are subscribed to the Google Groups "[^"]*" group.\nTo unsubscribe from this group and stop receiving emails from it, send an email to [^+@]+\+unsubscribe@googlegroups.com\.\nFor more options, visit https:\/\/groups\.google\.com\/groups\/opt_out\./, '')
|
|
|
|
raw
|
|
end
|
|
|
|
def import_users
|
|
puts "", "importing users"
|
|
db = open_db
|
|
|
|
all_users = db.execute("SELECT name, email FROM users")
|
|
total_count = all_users.size
|
|
|
|
batches(BATCH_SIZE) do |offset|
|
|
users = all_users[offset..offset+BATCH_SIZE-1]
|
|
break if users.nil?
|
|
next if all_records_exist? :users, users.map {|u| u[1]}
|
|
|
|
create_users(users, total: total_count, offset: offset) do |u|
|
|
{
|
|
id: u[1],
|
|
email: u[1],
|
|
name: u[0],
|
|
trust_level: DEFAULT_TRUST_LEVEL,
|
|
}
|
|
end
|
|
end
|
|
ensure
|
|
db.close
|
|
end
|
|
|
|
def parse_email(msg)
|
|
receiver = Email::Receiver.new(msg)
|
|
mail = Mail.read_from_string(msg)
|
|
mail.body
|
|
|
|
selected = receiver.select_body
|
|
selected.force_encoding(selected.encoding).encode("UTF-8")
|
|
end
|
|
|
|
def create_forum_topics
|
|
puts "", "creating forum topics"
|
|
|
|
db = open_db
|
|
all_topics = db.execute("SELECT msg_id,
|
|
from_email,
|
|
from_name,
|
|
title,
|
|
email_date,
|
|
message,
|
|
category
|
|
FROM emails
|
|
WHERE reply_to IS NULL
|
|
ORDER BY DATE(email_date)")
|
|
|
|
topic_count = all_topics.size
|
|
|
|
batches(BATCH_SIZE) do |offset|
|
|
topics = all_topics[offset..offset+BATCH_SIZE-1]
|
|
break if topics.nil?
|
|
|
|
next if all_records_exist? :posts, topics.map {|t| t[0]}
|
|
|
|
create_posts(topics, total: topic_count, offset: offset) do |t|
|
|
raw_email = t[5]
|
|
receiver = Email::Receiver.new(raw_email)
|
|
mail = Mail.read_from_string(raw_email)
|
|
mail.body
|
|
|
|
from_email, _ = extract_name(mail)
|
|
selected = receiver.select_body
|
|
next unless selected
|
|
selected = selected.join('') if selected.kind_of?(Array)
|
|
|
|
raw = selected.force_encoding(selected.encoding).encode("UTF-8")
|
|
|
|
title = mail.subject
|
|
|
|
# import the attachments
|
|
mail.attachments.each do |attachment|
|
|
tmp = Tempfile.new("discourse-email-attachment")
|
|
begin
|
|
# read attachment
|
|
File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded }
|
|
# create the upload for the user
|
|
upload = Upload.create_for(user_id_from_imported_user_id(from_email) || Discourse::SYSTEM_USER_ID, tmp, attachment.filename, tmp.size )
|
|
if upload && upload.errors.empty?
|
|
raw << "\n\n#{receiver.attachment_markdown(upload)}\n\n"
|
|
end
|
|
ensure
|
|
tmp.try(:close!) rescue nil
|
|
end
|
|
end
|
|
|
|
{ id: t[0],
|
|
title: clean_title(title),
|
|
user_id: user_id_from_imported_user_id(from_email) || Discourse::SYSTEM_USER_ID,
|
|
created_at: mail.date,
|
|
category: t[6],
|
|
raw: clean_raw(raw),
|
|
cook_method: Post.cook_methods[:email] }
|
|
end
|
|
end
|
|
ensure
|
|
db.close
|
|
end
|
|
|
|
def import_replies
|
|
puts "", "creating topic replies"
|
|
|
|
db = open_db
|
|
replies = db.execute("SELECT msg_id,
|
|
from_email,
|
|
from_name,
|
|
title,
|
|
email_date,
|
|
message,
|
|
reply_to
|
|
FROM emails
|
|
WHERE reply_to IS NOT NULL
|
|
ORDER BY DATE(email_date)
|
|
")
|
|
|
|
post_count = replies.size
|
|
|
|
batches(BATCH_SIZE) do |offset|
|
|
posts = replies[offset..offset+BATCH_SIZE-1]
|
|
break if posts.nil?
|
|
|
|
next if all_records_exist? :posts, posts.map {|p| p[0]}
|
|
|
|
create_posts(posts, total: post_count, offset: offset) do |p|
|
|
parent_id = p[6]
|
|
id = p[0]
|
|
|
|
topic = topic_lookup_from_imported_post_id(parent_id)
|
|
topic_id = topic[:topic_id] if topic
|
|
next unless topic_id
|
|
|
|
raw_email = p[5]
|
|
receiver = Email::Receiver.new(raw_email)
|
|
mail = Mail.read_from_string(raw_email)
|
|
mail.body
|
|
|
|
from_email, _ = extract_name(mail)
|
|
|
|
selected = receiver.select_body
|
|
selected = selected.join('') if selected.kind_of?(Array)
|
|
next unless selected
|
|
|
|
raw = selected.force_encoding(selected.encoding).encode("UTF-8")
|
|
|
|
# import the attachments
|
|
mail.attachments.each do |attachment|
|
|
tmp = Tempfile.new("discourse-email-attachment")
|
|
begin
|
|
# read attachment
|
|
File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded }
|
|
# create the upload for the user
|
|
upload = Upload.create_for(user_id_from_imported_user_id(from_email) || Discourse::SYSTEM_USER_ID, tmp, attachment.filename, tmp.size )
|
|
if upload && upload.errors.empty?
|
|
raw << "\n\n#{receiver.attachment_markdown(upload)}\n\n"
|
|
end
|
|
ensure
|
|
tmp.try(:close!) rescue nil
|
|
end
|
|
end
|
|
|
|
{ id: id,
|
|
topic_id: topic_id,
|
|
user_id: user_id_from_imported_user_id(from_email) || Discourse::SYSTEM_USER_ID,
|
|
created_at: mail.date,
|
|
raw: clean_raw(raw),
|
|
cook_method: Post.cook_methods[:email] }
|
|
end
|
|
end
|
|
ensure
|
|
db.close
|
|
end
|
|
end
|
|
|
|
ImportScripts::Mbox.new.perform
|