github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-02-20 05:40:43 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

FIX: JIVE API Importer

Browse Source 
- Added the 'hierarchical=false' query parameter to retrieve comments (replies) chronologically
- Calling 'create_post' (singular) wasn't updating the LookupContainer and thus we couldn't populate the 'reply_to_post_number' column
This commit is contained in:
Régis Hanol 2017-04-26 15:36:35 +02:00
parent fe0a7d97ca
commit e155cb6db1
2 changed files with 32 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
script/import_scripts/base.rb
View File

@ -106,32 +106,16 @@ class ImportScripts::Base
raise NotImplementedError raise NotImplementedError
end end
def post_id_from_imported_post_id(import_id) %i{ post_id_from_imported_post_id
@lookup.post_id_from_imported_post_id(import_id) topic_lookup_from_imported_post_id
end group_id_from_imported_group_id
find_group_by_import_id
def topic_lookup_from_imported_post_id(import_id) user_id_from_imported_user_id
@lookup.topic_lookup_from_imported_post_id(import_id) find_user_by_import_id
end category_id_from_imported_category_id
add_group add_user add_category add_topic add_post
def group_id_from_imported_group_id(import_id) }.each do |method_name|
@lookup.group_id_from_imported_group_id(import_id) delegate method_name, to: :@lookup
end
def find_group_by_import_id(import_id)
@lookup.find_group_by_import_id(import_id)
end
def user_id_from_imported_user_id(import_id)
@lookup.user_id_from_imported_user_id(import_id)
end
def find_user_by_import_id(import_id)
@lookup.find_user_by_import_id(import_id)
end
def category_id_from_imported_category_id(import_id)
@lookup.category_id_from_imported_category_id(import_id)
end end
def create_admin(opts={}) def create_admin(opts={})
@ -165,14 +149,14 @@ class ImportScripts::Base
results.each do |result| results.each do |result|
g = yield(result) g = yield(result)
if @lookup.group_id_from_imported_group_id(g[:id]) if group_id_from_imported_group_id(g[:id])
skipped += 1 skipped += 1
else else
new_group = create_group(g, g[:id]) new_group = create_group(g, g[:id])
created_group(new_group) created_group(new_group)
if new_group.valid? if new_group.valid?
@lookup.add_group(g[:id].to_s, new_group) add_group(g[:id].to_s, new_group)
created += 1 created += 1
else else
failed += 1 failed += 1
@ -247,14 +231,14 @@ class ImportScripts::Base
else else
import_id = u[:id] import_id = u[:id]
if @lookup.user_id_from_imported_user_id(import_id) if user_id_from_imported_user_id(import_id)
skipped += 1 skipped += 1
elsif u[:email].present? elsif u[:email].present?
new_user = create_user(u, import_id) new_user = create_user(u, import_id)
created_user(new_user) created_user(new_user)
if new_user && new_user.valid? && new_user.user_profile && new_user.user_profile.valid? if new_user && new_user.valid? && new_user.user_profile && new_user.user_profile.valid?
@lookup.add_user(import_id.to_s, new_user) add_user(import_id.to_s, new_user)
created += 1 created += 1
else else
failed += 1 failed += 1
@ -375,7 +359,7 @@ class ImportScripts::Base
params = yield(c) params = yield(c)
# block returns nil to skip # block returns nil to skip
if params.nil? || @lookup.category_id_from_imported_category_id(params[:id]) if params.nil? || category_id_from_imported_category_id(params[:id])
skipped += 1 skipped += 1
else else
# Basic massaging on the category name # Basic massaging on the category name
@ -422,7 +406,7 @@ class ImportScripts::Base
new_category.custom_fields["import_id"] = import_id if import_id new_category.custom_fields["import_id"] = import_id if import_id
new_category.save! new_category.save!
@lookup.add_category(import_id, new_category) add_category(import_id, new_category)
post_create_action.try(:call, new_category) post_create_action.try(:call, new_category)
@ -453,14 +437,14 @@ class ImportScripts::Base
else else
import_id = params.delete(:id).to_s import_id = params.delete(:id).to_s
if @lookup.post_id_from_imported_post_id(import_id) if post_id_from_imported_post_id(import_id)
skipped += 1 # already imported this post skipped += 1 # already imported this post
else else
begin begin
new_post = create_post(params, import_id) new_post = create_post(params, import_id)
if new_post.is_a?(Post) if new_post.is_a?(Post)
@lookup.add_post(import_id, new_post) add_post(import_id, new_post)
@lookup.add_topic(new_post) add_topic(new_post)
created_post(new_post) created_post(new_post)
@ -533,8 +517,8 @@ class ImportScripts::Base
if params.nil? if params.nil?
skipped += 1 skipped += 1
else else
user.id = @lookup.user_id_from_imported_user_id(params[:user_id]) user.id = user_id_from_imported_user_id(params[:user_id])
post.id = @lookup.post_id_from_imported_post_id(params[:post_id]) post.id = post_id_from_imported_post_id(params[:post_id])
if user.id.nil? || post.id.nil? if user.id.nil? || post.id.nil?
skipped += 1 skipped += 1

19
script/import_scripts/jive_api.rb
View File

@ -69,9 +69,6 @@ class ImportScripts::JiveApi < ImportScripts::Base
# category: discussion["question"] ? 5 : 21, # category: discussion["question"] ? 5 : 21,
views: discussion["viewCount"], views: discussion["viewCount"],
custom_fields: { import_id: discussion["contentID"] }, custom_fields: { import_id: discussion["contentID"] },
post_create_action: proc do |post|
DiscourseTagging.tag_topic_by_names(post.topic, STAFF_GUARDIAN, ["legacy"])
end
} }
post_id = post_id_from_imported_post_id(topic[:id]) post_id = post_id_from_imported_post_id(topic[:id])
@ -90,7 +87,7 @@ class ImportScripts::JiveApi < ImportScripts::Base
fields = "fields=published,author.id,content.text,parent,answer,-resources,-author.resources" fields = "fields=published,author.id,content.text,parent,answer,-resources,-author.resources"
loop do loop do
comments = get("messages/contents/#{discussion_id}?#{fields}&count=#{POST_COUNT}&startIndex=#{start_index}") comments = get("messages/contents/#{discussion_id}?#{fields}&hierarchical=false&count=#{POST_COUNT}&startIndex=#{start_index}")
comments["list"].each do |comment| comments["list"].each do |comment|
next if post_id_from_imported_post_id(comment["id"]) next if post_id_from_imported_post_id(comment["id"])
@ -138,9 +135,6 @@ class ImportScripts::JiveApi < ImportScripts::Base
category: 7, category: 7,
views: post["viewCount"], views: post["viewCount"],
custom_fields: { import_id: post["contentID"], import_permalink: post["permalink"] }, custom_fields: { import_id: post["contentID"], import_permalink: post["permalink"] },
post_create_action: proc do |p|
DiscourseTagging.tag_topic_by_names(p.topic, STAFF_GUARDIAN, ["legacy"])
end
} }
create_post(pp, pp[:id]) create_post(pp, pp[:id])
@ -151,6 +145,15 @@ class ImportScripts::JiveApi < ImportScripts::Base
end end
end end
def create_post(options, import_id)
post = super(options, import_id)
if Post === post
add_post(import_id, post)
add_topic(post)
end
post
end
def process_raw(raw) def process_raw(raw)
doc = Nokogiri::HTML.fragment(raw) doc = Nokogiri::HTML.fragment(raw)
@ -199,7 +202,7 @@ class ImportScripts::JiveApi < ImportScripts::Base
puts command.join(" ") puts command.join(" ")
JSON.parse `#{command.join(" ")}` JSON.parse `#{command.join(" ")}`
rescue => e rescue
retry if (tries -= 1) >= 0 retry if (tries -= 1) >= 0
end end