From 4c4ce05964c1efd6c07baae411cf2f1a430afc9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Hanol?= Date: Wed, 13 Aug 2014 22:17:16 +0200 Subject: [PATCH] FIX & PERF: vanilla import PERF: disabled refresh_avatar callback when importing users PERF: avoid using UsernameSuggester when not needed FIX: categories wasn't working FIX: posts from deleted users are now from the system user --- app/models/user.rb | 16 ++++++++++----- script/import_scripts/base.rb | 34 ++++++++++++++++++-------------- script/import_scripts/vanilla.rb | 24 ++++++++++++++-------- 3 files changed, 46 insertions(+), 28 deletions(-) diff --git a/app/models/user.rb b/app/models/user.rb index 62f8537b492..1e982826e39 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -67,19 +67,20 @@ class User < ActiveRecord::Base validate :password_validator validates :ip_address, allowed_ip_address: {on: :create, message: :signup_not_allowed} - before_save :update_username_lower - before_save :ensure_password_is_hashed after_initialize :add_trust_level after_initialize :set_default_email_digest after_initialize :set_default_external_links_in_new_tab - after_save :update_tracked_topics - after_save :clear_global_notice_if_needed - after_create :create_email_token after_create :create_user_stat after_create :create_user_profile after_create :ensure_in_trust_level_group + + before_save :update_username_lower + before_save :ensure_password_is_hashed + + after_save :update_tracked_topics + after_save :clear_global_notice_if_needed after_save :refresh_avatar after_save :badge_grant @@ -95,6 +96,9 @@ class User < ActiveRecord::Base # This is just used to pass some information into the serializer attr_accessor :notification_channel_position + # set to true to optimize creation and save for imports + attr_accessor :import_mode + scope :blocked, -> { where(blocked: true) } # no index scope :not_blocked, -> { where(blocked: false) } # no index scope :suspended, -> { where('suspended_till IS NOT NULL AND suspended_till > ?', Time.zone.now) } # no index @@ -594,6 +598,8 @@ class User < ActiveRecord::Base end def refresh_avatar + return if @import_mode + avatar = user_avatar || create_user_avatar gravatar_downloaded = false diff --git a/script/import_scripts/base.rb b/script/import_scripts/base.rb index 1115b0ade25..feb5fa69e92 100644 --- a/script/import_scripts/base.rb +++ b/script/import_scripts/base.rb @@ -17,6 +17,7 @@ class ImportScripts::Base def initialize require File.expand_path(File.dirname(__FILE__) + "/../../config/environment") + preload_i18n @bbcode_to_md = true if ARGV.include?('bbcode-to-md') @existing_groups = {} @@ -48,6 +49,11 @@ class ImportScripts::Base end end + def preload_i18n + I18n.t("test") + ActiveSupport::Inflector.transliterate("test") + end + def perform Rails.logger.level = 3 # :error, so that we don't create log files that are many GB @@ -62,12 +68,14 @@ class ImportScripts::Base execute + puts "" + update_bumped_at update_feature_topic_users update_category_featured_topics update_topic_count_replies - puts '', 'Done' + puts "", "Done" ensure RateLimiter.enable @@ -132,8 +140,6 @@ class ImportScripts::Base # group in the original datasource. The given id will not be used # to create the Discourse group record. def create_groups(results, opts={}) - puts "", "creating groups" - groups_created = 0 groups_skipped = 0 total = opts[:total] || results.size @@ -182,8 +188,6 @@ class ImportScripts::Base # user in the original datasource. The given id will not be used to # create the Discourse user record. def create_users(results, opts={}) - puts "", "creating users" - num_users_before = User.count users_created = 0 users_skipped = 0 @@ -225,17 +229,21 @@ class ImportScripts::Base opts.delete(:id) post_create_action = opts.delete(:post_create_action) existing = User.where(email: opts[:email].downcase, username: opts[:username]).first - return existing if existing and existing.custom_fields["import_id"].to_i == import_id.to_i + return existing if existing && existing.custom_fields["import_id"].to_i == import_id.to_i bio_raw = opts.delete(:bio_raw) - opts[:name] = User.suggest_name(opts[:name]) if opts[:name] - opts[:username] = UserNameSuggester.suggest((opts[:username].present? ? opts[:username] : nil) || opts[:name] || opts[:email]) + opts[:name] = User.suggest_name(opts[:email]) unless opts[:name] + if opts[:username].blank? || !User.username_available?(opts[:username]) + opts[:username] = UserNameSuggester.suggest(opts[:username] || opts[:name] || opts[:email]) + end opts[:email] = opts[:email].downcase opts[:trust_level] = TrustLevel.levels[:basic] unless opts[:trust_level] + opts[:import_mode] = true u = User.new(opts) u.custom_fields["import_id"] = import_id u.custom_fields["import_username"] = opts[:username] if opts[:username].present? + u.custom_fields["import_avatar_url"] = opts[:avatar_url] if opts[:avatar_url].present? begin User.transaction do @@ -266,8 +274,6 @@ class ImportScripts::Base # create the Discourse category record. # Optional attributes are position, description, and parent_category_id. def create_categories(results) - puts "", "creating categories" - results.each do |c| params = yield(c) puts " #{params[:name]}" @@ -308,8 +314,6 @@ class ImportScripts::Base # Topics should give attributes title and category. # Replies should provide topic_id. Use topic_lookup_from_imported_post_id to find the topic. def create_posts(results, opts={}) - puts "", "creating posts" - skipped = 0 created = 0 total = opts[:total] || results.size @@ -389,8 +393,8 @@ class ImportScripts::Base end def close_inactive_topics(opts={}) + puts "", "Closing topics that have been inactive for more than #{num_days} days." num_days = opts[:days] || 30 - puts '', "Closing topics that have been inactive for more than #{num_days} days." query = Topic.where('last_posted_at < ?', num_days.days.ago).where(closed: false) total_count = query.count @@ -404,7 +408,7 @@ class ImportScripts::Base end def update_bumped_at - puts '', "updating bumped_at on topics" + puts "updating bumped_at on topics" Post.exec_sql("update topics t set bumped_at = (select max(created_at) from posts where topic_id = t.id and post_type != #{Post.types[:moderator_action]})") end @@ -422,7 +426,7 @@ class ImportScripts::Base end def update_category_featured_topics - puts '', "updating featured topics in categories" + puts "updating featured topics in categories" Category.find_each do |category| CategoryFeaturedTopic.feature_topics_for(category) end diff --git a/script/import_scripts/vanilla.rb b/script/import_scripts/vanilla.rb index 2bcbcc774fa..e5127548edc 100644 --- a/script/import_scripts/vanilla.rb +++ b/script/import_scripts/vanilla.rb @@ -79,15 +79,22 @@ class ImportScripts::Vanilla < ImportScripts::Base create_users(@users) do |user| next if user[:name] == "[Deleted User]" - { + u = { id: user[:user_id], email: user[:email], name: user[:name], created_at: parse_date(user[:date_inserted]), bio_raw: clean_up(user[:discovery_text]), + avatar_url: user[:photo], moderator: @user_roles.select { |ur| ur[:user_id] == user[:user_id] }.map { |ur| ur[:role_id] }.include?(moderator_role_id), admin: @user_roles.select { |ur| ur[:user_id] == user[:user_id] }.map { |ur| ur[:role_id] }.include?(admin_role_id), } + + # if @comments.select { |c| c[:insert_user_id] == user[:user_id] }.map { |c| c[:discussion_id] }.uniq.count > 3 + # u[:trust_level] = TrustLevel.levels[:regular] + # end + + u end end @@ -118,14 +125,14 @@ class ImportScripts::Vanilla < ImportScripts::Base c = { id: category[:category_id], name: category[:name], - user_id: user_id_from_imported_user_id(category[:insert_user_id]), + user_id: user_id_from_imported_user_id(category[:insert_user_id]) || Discourse::SYSTEM_USER_ID, position: category[:sort].to_i, created_at: parse_category_date(category[:date_inserted]), description: clean_up(category[:description]), } if category[:parent_category_id] != "-1" - parent_category = category_from_imported_category_id(category[:parent_category_id].to_i) - c[:parent_category_id] = parent_category[:id] if parent_category + parent_category = category_from_imported_category_id(category[:parent_category_id]) + c[:parent_category_id] = parent_category.id if parent_category end c end @@ -140,9 +147,9 @@ class ImportScripts::Vanilla < ImportScripts::Base create_posts(@discussions) do |discussion| { id: "discussion#" + discussion[:discussion_id], - user_id: user_id_from_imported_user_id(discussion[:insert_user_id]), + user_id: user_id_from_imported_user_id(discussion[:insert_user_id]) || Discourse::SYSTEM_USER_ID, title: discussion[:name], - category_id: category_from_imported_category_id(discussion[:category_id]).try(:id), + category: category_from_imported_category_id(discussion[:category_id]).try(:name), raw: clean_up(discussion[:body]), created_at: parse_date(discussion[:date_inserted]), } @@ -157,7 +164,7 @@ class ImportScripts::Vanilla < ImportScripts::Base { id: "comment#" + comment[:comment_id], - user_id: user_id_from_imported_user_id(comment[:insert_user_id]), + user_id: user_id_from_imported_user_id(comment[:insert_user_id]) || Discourse::SYSTEM_USER_ID, topic_id: t[:topic_id], raw: clean_up(comment[:body]), created_at: parse_date(comment[:date_inserted]), @@ -207,7 +214,7 @@ class ImportScripts::Vanilla < ImportScripts::Base { archetype: Archetype.private_message, id: "message#" + message[:message_id], - user_id: user_id_from_imported_user_id(message[:insert_user_id]), + user_id: user_id_from_imported_user_id(message[:insert_user_id]) || Discourse::SYSTEM_USER_ID, topic_id: t[:topic_id], raw: clean_up(message[:body]), created_at: parse_date(message[:date_inserted]), @@ -225,6 +232,7 @@ class ImportScripts::Vanilla < ImportScripts::Base .gsub(/<\/?code\s*>/i, "`") .gsub("<", "<") .gsub(">", ">") + # .gsub("*", "\\*") end end