From efff3351bf6c453387bedd217b9b8830c8fe4a7a Mon Sep 17 00:00:00 2001
From: Guo Xiang Tan <tgx_world@hotmail.com>
Date: Wed, 1 Jul 2015 09:16:52 +0800
Subject: [PATCH] Update Disqus importer.
---
.../import_scripts/disqus.rb | 172 +++++++++---------
1 file changed, 91 insertions(+), 81 deletions(-)
rename lib/tasks/disqus.thor => script/import_scripts/disqus.rb (61%)
diff --git a/lib/tasks/disqus.thor b/script/import_scripts/disqus.rb
similarity index 61%
rename from lib/tasks/disqus.thor
rename to script/import_scripts/disqus.rb
index 95a519a4b5f..be2418dc8bc 100644
--- a/lib/tasks/disqus.thor
+++ b/script/import_scripts/disqus.rb
@@ -1,17 +1,84 @@
require 'nokogiri'
+require 'optparse'
+require File.expand_path(File.dirname(__FILE__) + "/base")
+
+class ImportScripts::Disqus < ImportScripts::Base
+ def initialize(options)
+ verify_file(options[:file])
+ @post_as_user = get_post_as_user(options[:post_as])
+ @dry_run = options[:dry_run]
+ @parser = DisqusSAX.new(options[:strip])
+ doc = Nokogiri::XML::SAX::Parser.new(@parser)
+ doc.parse_file(options[:file])
+ @parser.normalize
+ super()
+ end
+
+ def execute
+ @parser.threads.each do |id, t|
+ puts "Creating #{t[:title]}... (#{t[:posts].size} posts)"
+
+ if !@dry_run
+ post = TopicEmbed.import_remote(@post_as_user, t[:link], title: t[:title])
+
+ if post.present?
+ t[:posts].each do |p|
+ post_user = @post_as_user
+
+ if p[:author_email]
+ post_user = create_user({ id: nil, email: p[:author_email] }, nil)
+ end
+
+ attrs = {
+ user_id: post_user.id,
+ topic_id: post.topic_id,
+ raw: p[:cooked],
+ cooked: p[:cooked],
+ created_at: Date.parse(p[:created_at])
+ }
+
+ if p[:parent_id]
+ parent = @parser.posts[p[:parent_id]]
+
+ if parent && parent[:discourse_number]
+ attrs[:reply_to_post_number] = parent[:discourse_number]
+ end
+ end
+
+ post = create_post(attrs, p[:id])
+ p[:discourse_number] = post.post_number
+ end
+
+ TopicFeaturedUsers.new(post.topic).choose
+ end
+ end
+ end
+ end
+
+ private
+
+ def verify_file(file)
+ abort("File '#{file}' not found") if !File.exist?(file)
+ end
+
+ def get_post_as_user(username)
+ user = User.find_by_username_lower(username.downcase)
+ abort("No user found named: '#{username}'") if user.nil?
+ user
+ end
+end
class DisqusSAX < Nokogiri::XML::SAX::Document
attr_accessor :posts, :threads
- def initialize(options=nil)
+ def initialize(strip)
@inside = {}
@posts = {}
@threads = {}
- @options = options || {}
+ @strip = strip
end
def start_element(name, attrs = [])
-
case name
when 'post'
@post = {}
@@ -71,7 +138,6 @@ class DisqusSAX < Nokogiri::XML::SAX::Document
target[sym] ||= ""
target[sym] << str
end
-
end
def inside?(*params)
@@ -79,14 +145,13 @@ class DisqusSAX < Nokogiri::XML::SAX::Document
end
def normalize
-
@threads.each do |id, t|
if t[:posts].size == 0
# Remove any threads that have no posts
@threads.delete(id)
else
# Normalize titles
- t[:title].gsub!(@options[:strip], '') if @options[:strip].present?
+ t[:title].gsub!(@strip, '') if @strip.present?
t[:title].strip!
end
end
@@ -103,86 +168,31 @@ class DisqusSAX < Nokogiri::XML::SAX::Document
@threads.delete(t[:id])
end
end
-
-
end
end
-class Disqus < Thor
- desc "import", "Imports posts from a Disqus XML export"
- method_option :file, aliases: '-f', required: true, desc: "The disqus XML file to import"
- method_option :dry_run, required: false, desc: "Just output what will be imported rather than doing it"
- method_option :post_as, aliases: '-p', required: true, desc: "The Discourse username to post as"
- method_option :strip, aliases: '-s', required: false, desc: "Text to strip from titles"
+options = {
+ dry_run: false
+}
- def import
- require './config/environment'
+OptionParser.new do |opts|
+ opts.banner = 'Usage: RAILS_ENV=production ruby disqus.rb [OPTIONS]'
- email_blacklist = SiteSetting.email_domains_blacklist
-
- user = User.where(username_lower: options[:post_as].downcase).first
- if user.nil?
- puts "No user found named: '#{options[:post_as]}'"
- exit 1
- end
-
- unless File.exist?(options[:file])
- puts "File '#{options[:file]}' not found"
- exit 1
- end
-
- parser = DisqusSAX.new(options)
- doc = Nokogiri::XML::SAX::Parser.new(parser)
- doc.parse_file(options[:file])
- parser.normalize
-
- RateLimiter.disable
-
- SiteSetting.email_domains_blacklist = ""
-
- parser.threads.each do |id, t|
- puts "Creating #{t[:title]}... (#{t[:posts].size} posts)"
-
- if options[:dry_run].blank?
-
- post = TopicEmbed.import_remote(user, t[:link], title: t[:title])
- if post.present?
- t[:posts].each do |p|
- post_user = user
- if p[:author_email]
- email = Email.downcase(p[:author_email])
- post_user = User.where(email: email).first
- if post_user.blank?
- post_user = User.create!(email: email, username: UserNameSuggester.suggest(email))
- end
- end
-
- attrs = {
- topic_id: post.topic_id,
- raw: p[:cooked],
- cooked: p[:cooked],
- created_at: Date.parse(p[:created_at])
- }
-
- if p[:parent_id]
- parent = parser.posts[p[:parent_id]]
- if parent && parent[:discourse_number]
- attrs[:reply_to_post_number] = parent[:discourse_number]
- end
- end
-
- post = PostCreator.new(post_user, attrs).create
- p[:discourse_number] = post.post_number
- end
- TopicFeaturedUsers.new(post.topic).choose
- end
- end
- end
-
- ensure
- RateLimiter.enable
- SiteSetting.email_domains_blacklist = email_blacklist
+ opts.on('-f', '--file=FILE_PATH', 'The disqus XML file to import') do |value|
+ options[:file] = value
end
-end
+ opts.on('-d', '--dry_run', 'Just output what will be imported rather than doing it') do
+ options[:dry_run] = true
+ end
+ opts.on('-p', '--post_as=USERNAME', 'The Discourse username to post as') do |value|
+ options[:post_as] = value
+ end
+
+ opts.on('-s', '--strip=TEXT', 'Text to strip from titles') do |value|
+ options[:strip] = value
+ end
+end.parse!
+
+ImportScripts::Disqus.new(options).perform