DEV: Improve phpBB3 import script (#15956)

* Optional import of custom user fields from phpBB 3.1+
* Optional import of likes from phpBB3
  Requires the phpBB "Thanks for posts" extension
* Fix import of bookmarks from phpBB3
* Update `created_at` of existing user
* Support mapping of phpBB forums to existing Discourse categories
  This is in addition to the ability of merging phpBB forums and importing into newly created Discourse categories.

script/import_scripts/base.rb

@@ -366,6 +366,7 @@ class ImportScripts::Base
       # try based on email
       if e.try(:record).try(:errors).try(:messages).try(:[], :primary_email).present?
         if existing = User.find_by_email(opts[:email].downcase)
+          existing.created_at = opts[:created_at] if opts[:created_at]
           existing.custom_fields["import_id"] = import_id
           existing.save!
           u = existing
@@ -630,6 +631,35 @@ class ImportScripts::Base
     [created, skipped]
   end
 
+  def create_likes(results, opts = {})
+    created = 0
+    skipped = 0
+    total = opts[:total] || results.count
+
+    results.each do |result|
+      params = yield(result)
+
+      if params.nil?
+        skipped += 1
+      else
+        created_by = User.find_by(id: user_id_from_imported_user_id(params[:user_id]))
+        post = Post.find_by(id: post_id_from_imported_post_id(params[:post_id]))
+
+        if created_by && post
+          PostActionCreator.create(created_by, post, :like, created_at: params[:created_at])
+          created += 1
+        else
+          skipped += 1
+          puts "Skipping like for user id #{params[:user_id]} and post id #{params[:post_id]}"
+        end
+      end
+
+      print_status(created + skipped + (opts[:offset] || 0), total, get_start_time("likes"))
+    end
+
+    [created, skipped]
+  end
+
   def close_inactive_topics(opts = {})
     num_days = opts[:days] || 30
     puts '', "Closing topics that have been inactive for more than #{num_days} days."

script/import_scripts/phpbb3/database/database_3_0.rb

@@ -13,7 +13,7 @@ module ImportScripts::PhpBB3
       SQL
     end
 
-    def fetch_users(last_user_id)
+    def fetch_users(last_user_id, _profile_fields)
       query(<<-SQL, :user_id)
         SELECT u.user_id, u.user_email, u.username, u.user_password, u.user_regdate, u.user_lastvisit, u.user_ip,
           u.user_type, u.user_inactive_reason, g.group_name, b.ban_start, b.ban_end, b.ban_reason,
@@ -227,12 +227,31 @@ module ImportScripts::PhpBB3
         SELECT b.user_id, t.topic_first_post_id
         FROM #{@table_prefix}bookmarks b
           JOIN #{@table_prefix}topics t ON (b.topic_id = t.topic_id)
-        WHERE b.user_id > #{last_user_id}
+        WHERE (b.user_id, b.topic_id) > (#{last_user_id}, #{last_topic_id})
         ORDER BY b.user_id, b.topic_id
         LIMIT #{@batch_size}
       SQL
     end
 
+    def count_likes
+      count(<<-SQL)
+        SELECT COUNT(*) AS count
+        FROM #{@table_prefix}thanks
+        WHERE user_id <> poster_id
+      SQL
+    end
+
+    def fetch_likes(last_post_id, last_user_id)
+      query(<<-SQL, :post_id, :user_id)
+        SELECT post_id, user_id, thanks_time
+        FROM #{@table_prefix}thanks
+        WHERE user_id <> poster_id
+          AND (post_id, user_id) > (#{last_post_id}, #{last_user_id})
+        ORDER BY post_id, user_id
+        LIMIT #{@batch_size}
+      SQL
+    end
+
     def get_smiley(smiley_code)
       query(<<-SQL).first
         SELECT emotion, smiley_url

script/import_scripts/phpbb3/database/database_3_1.rb

@@ -5,7 +5,7 @@ require_relative '../support/constants'
 
 module ImportScripts::PhpBB3
   class Database_3_1 < Database_3_0
-    def fetch_users(last_user_id)
+    def fetch_users(last_user_id, profile_fields)
       query(<<-SQL, :user_id)
         SELECT u.user_id, u.user_email, u.username,
           CASE WHEN u.user_password LIKE '$2y$%'
@@ -15,6 +15,7 @@ module ImportScripts::PhpBB3
           u.user_type, u.user_inactive_reason, g.group_name, b.ban_start, b.ban_end, b.ban_reason,
           u.user_posts, f.pf_phpbb_website AS user_website, f.pf_phpbb_location AS user_from,
           u.user_birthday, u.user_avatar_type, u.user_avatar
+          #{profile_fields_query(profile_fields)}
         FROM #{@table_prefix}users u
           LEFT OUTER JOIN #{@table_prefix}profile_fields_data f ON (u.user_id = f.user_id)
           JOIN #{@table_prefix}groups g ON (g.group_id = u.group_id)
@@ -27,5 +28,18 @@ module ImportScripts::PhpBB3
         LIMIT #{@batch_size}
       SQL
     end
+
+    private
+
+    def profile_fields_query(profile_fields)
+      @profile_fields_query ||= begin
+        if profile_fields.present?
+          columns = profile_fields.map { |field| "pf_#{field[:phpbb_field_name]}" }
+          ", #{columns.join(', ')}"
+        else
+          ""
+        end
+      end
+    end
   end
 end

script/import_scripts/phpbb3/importer.rb

@@ -38,6 +38,7 @@ module ImportScripts::PhpBB3
       import_posts
       import_private_messages if @settings.import_private_messages
       import_bookmarks if @settings.import_bookmarks
+      import_likes if @settings.import_likes
     end
 
     def change_site_settings
@@ -71,7 +72,7 @@ module ImportScripts::PhpBB3
       last_user_id = 0
 
       batches do |offset|
-        rows, last_user_id = @database.fetch_users(last_user_id)
+        rows, last_user_id = @database.fetch_users(last_user_id, @settings.custom_fields)
         rows = rows.to_a.uniq { |row| row[:user_id] }
         break if rows.size < 1
 
@@ -173,7 +174,7 @@ module ImportScripts::PhpBB3
       importer = @importers.category_importer
 
       create_categories(rows) do |row|
-        next if @settings.category_mappings[row[:forum_id].to_s] == 'SKIP'
+        next if @settings.category_mappings.dig(row[:forum_id].to_s, :skip)
 
         importer.map_category(row)
       end
@@ -241,6 +242,25 @@ module ImportScripts::PhpBB3
       end
     end
 
+    def import_likes
+      puts '', 'importing likes'
+      total_count = @database.count_likes
+      last_post_id = last_user_id = 0
+
+      batches do |offset|
+        rows, last_post_id, last_user_id = @database.fetch_likes(last_post_id, last_user_id)
+        break if rows.size < 1
+
+        create_likes(rows, total: total_count, offset: offset) do |row|
+          {
+            post_id: @settings.prefix(row[:post_id]),
+            user_id: @settings.prefix(row[:user_id]),
+            created_at: Time.zone.at(row[:thanks_time])
+          }
+        end
+      end
+    end
+
     def update_last_seen_at
       # no need for this since the importer sets last_seen_at for each user during the import
     end

script/import_scripts/phpbb3/importers/category_importer.rb

@@ -17,7 +17,7 @@ module ImportScripts::PhpBB3
       return if @settings.category_mappings[row[:forum_id].to_s]
 
       if row[:parent_id] && @settings.category_mappings[row[:parent_id].to_s]
-        puts "parent category (#{row[:parent_id]}) was mapped, but children was not (#{row[:forum_id]})"
+        puts "parent category (#{row[:parent_id]}) was mapped, but child was not (#{row[:forum_id]})"
       end
 
       {

script/import_scripts/phpbb3/importers/post_importer.rb

@@ -22,7 +22,7 @@ module ImportScripts::PhpBB3
     end
 
     def map_post(row)
-      return if @settings.category_mappings[row[:forum_id].to_s] == 'SKIP'
+      return if @settings.category_mappings.dig(row[:forum_id].to_s, :skip)
 
       imported_user_id = @settings.prefix(row[:post_username].blank? ? row[:poster_id] : row[:post_username])
       user_id = @lookup.user_id_from_imported_user_id(imported_user_id) || -1
@@ -56,8 +56,13 @@ module ImportScripts::PhpBB3
     def map_first_post(row, mapped)
       poll_data = add_poll(row, mapped) if @settings.import_polls
 
-      mapped[:category] = @lookup.category_id_from_imported_category_id(@settings.prefix(@settings.category_mappings[row[:forum_id].to_s])) ||
-                          @lookup.category_id_from_imported_category_id(@settings.prefix(row[:forum_id]))
+      mapped[:category] = if category_mapping = @settings.category_mappings[row[:forum_id].to_s]
+        category_mapping[:discourse_category_id] ||
+          @lookup.category_id_from_imported_category_id(@settings.prefix(category_mapping[:target_category_id]))
+      else
+        @lookup.category_id_from_imported_category_id(@settings.prefix(row[:forum_id]))
+      end
+
       mapped[:title] = CGI.unescapeHTML(row[:topic_title]).strip[0...255]
       mapped[:pinned_at] = mapped[:created_at] unless row[:topic_type] == Constants::POST_NORMAL
       mapped[:pinned_globally] = row[:topic_type] == Constants::POST_GLOBAL

script/import_scripts/phpbb3/importers/user_importer.rb

@@ -42,6 +42,7 @@ module ImportScripts::PhpBB3
         website: row[:user_website],
         location: row[:user_from],
         date_of_birth: parse_birthdate(row),
+        custom_fields: custom_fields(row),
         post_create_action: proc do |user|
           suspend_user(user, row)
           @avatar_importer.import_avatar(user, row) if row[:user_avatar_type].present?
@@ -83,6 +84,45 @@ module ImportScripts::PhpBB3
       birthdate && birthdate.year > 0 ? birthdate : nil
     end
 
+    def user_fields
+      @user_fields ||= begin
+        Hash[UserField.all.map { |field| [field.name, field] }]
+      end
+    end
+
+    def field_mappings
+      @field_mappings ||= begin
+        @settings.custom_fields.map do |field|
+          {
+            phpbb_field_name: "pf_#{field[:phpbb_field_name]}".to_sym,
+            discourse_user_field: user_fields[field[:discourse_field_name]]
+          }
+        end
+      end
+    end
+
+    def custom_fields(row)
+      return nil if @settings.custom_fields.blank?
+
+      custom_fields = {}
+
+      field_mappings.each do |field|
+        value = row[field[:phpbb_field_name]]
+        user_field = field[:discourse_user_field]
+
+        case user_field.field_type
+        when "confirm"
+          value = value == 1 ? true : nil
+        when "dropdown"
+          value = user_field.user_field_options.find { |option| option.value == value } ? value : nil
+        end
+
+        custom_fields["user_field_#{user_field.id}"] = value if value.present?
+      end
+
+      custom_fields
+    end
+
     # Suspends the user if it is currently banned.
     def suspend_user(user, row, disable_email = false)
       if row[:user_inactive_reason] == Constants::INACTIVE_MANUAL

script/import_scripts/phpbb3/settings.yml

@@ -36,20 +36,25 @@ import:
 
   # Category mappings
   #
-  # For example, topics from phpBB category 1 and 2 will be imported
-  # in the new "Foo Category" category, topics from phpBB category 3
-  # will be imported in subcategory "Bar category", topics from phpBB
-  # category 4 will be merged into category 5 and category 6 will be
-  # skipped.
+  # * "source_category_id" is the forum ID in phpBB3
+  # * "target_category_id" is either a forum ID from phpBB3 or a "forum_id"
+  #   from the "new_categories" setting (see above)
+  # * "discourse_category_id" is a category ID from Discourse
+  # * "skip" allows you to ignore a category during import
   #
-  # category_mappings:
-  #   1: foo
-  #   2: foo
-  #   3: bar
-  #   4: 5
-  #   6: SKIP
+  # Use "target_category_id" if you want to merge categories and use
+  # "discourse_category_id" if you want to import a forum into an existing
+  # category in Discourse.
   #
-  category_mappings: {}
+  #  category_mappings:
+  #    - source_category_id: 1
+  #      target_category_id: foo
+  #    - source_category_id: 2
+  #      discourse_category_id: 42
+  #    - source_category_id: 6
+  #      skip: true
+  #
+  category_mappings: []
 
   # Tag mappings
   #
@@ -122,6 +127,9 @@ import:
   private_messages: true
   polls: true
 
+  # Import likes from the phpBB's "Thanks for posts" extension
+  likes: false
+
   # When true: each imported user will have the original username from phpBB as its name
   # When false: the name of each imported user will be blank unless the username was changed during import
   username_as_name: false
@@ -134,3 +142,12 @@ import:
     # here are two example mappings...
     smiley: [':D', ':-D', ':grin:']
     heart: ':love:'
+
+  # Map custom profile fields from phpBB to custom user fields in Discourse (works for phpBB 3.1+)
+  #
+  #  custom_fields:
+  #    - phpbb_field_name: "company_name"
+  #      discourse_field_name: "Company"
+  #    - phpbb_field_name: "facebook"
+  #      discourse_field_name: "Facebook"
+  custom_fields: []

script/import_scripts/phpbb3/support/settings.rb

@@ -24,6 +24,7 @@ module ImportScripts::PhpBB3
     attr_reader :import_polls
     attr_reader :import_bookmarks
     attr_reader :import_passwords
+    attr_reader :import_likes
 
     attr_reader :import_uploaded_avatars
     attr_reader :import_remote_avatars
@@ -38,6 +39,7 @@ module ImportScripts::PhpBB3
 
     attr_reader :username_as_name
     attr_reader :emojis
+    attr_reader :custom_fields
 
     attr_reader :database
 
@@ -47,7 +49,7 @@ module ImportScripts::PhpBB3
       @site_name = import_settings['site_name']
 
       @new_categories = import_settings['new_categories']
-      @category_mappings = import_settings['category_mappings']
+      @category_mappings = import_settings.fetch('category_mappings', []).to_h { |m| [m[:source_category_id].to_s, m] }
       @tag_mappings = import_settings['tag_mappings']
       @rank_mapping = import_settings['rank_mapping']
 
@@ -57,6 +59,7 @@ module ImportScripts::PhpBB3
       @import_polls = import_settings['polls']
       @import_bookmarks = import_settings['bookmarks']
       @import_passwords = import_settings['passwords']
+      @import_likes = import_settings['likes']
 
       avatar_settings = import_settings['avatars']
       @import_uploaded_avatars = avatar_settings['uploaded']
@@ -72,6 +75,7 @@ module ImportScripts::PhpBB3
 
       @username_as_name = import_settings['username_as_name']
       @emojis = import_settings.fetch('emojis', [])
+      @custom_fields = import_settings.fetch('custom_fields', [])
 
       @database = DatabaseSettings.new(yaml['database'])
     end