FEATURE: Import script for AnswerBase

Improves the generic database used by some import scripts:
* Adds additional columns for users
* Adds support for attachments
* Allows setting the data type for keys (numeric or string) to ensure correct sorting

Gemfile

@@ -202,10 +202,11 @@ gem 'rchardet', require: false
 if ENV["IMPORT"] == "1"
   gem 'mysql2'
   gem 'redcarpet'
-  gem 'sqlite3', '~> 1.3.13'
+  gem 'sqlite3', '~> 1.3', '>= 1.3.13'
   gem 'ruby-bbcode-to-md', git: 'https://github.com/nlalonde/ruby-bbcode-to-md'
   gem 'reverse_markdown'
   gem 'tiny_tds'
+  gem 'csv', '~> 3.0'
 end
 
 gem 'webpush', require: false

script/import_scripts/answerbase.rb

@@ -0,0 +1,341 @@
+require 'csv'
+require 'reverse_markdown'
+require_relative 'base'
+require_relative 'base/generic_database'
+
+# Call it like this:
+#   RAILS_ENV=production bundle exec ruby script/import_scripts/answerbase.rb DIRNAME
+class ImportScripts::Answerbase < ImportScripts::Base
+  OLD_DOMAIN = "http://answerbase.example.com" # without trailing slash
+  NEW_DOMAIN = "https://discourse.example.com"
+  AVATAR_DIRECTORY = "User Images"
+  ANSWER_ATTACHMENT_DIRECTORY = "Answer Attachments"
+  ANSWER_IMAGE_DIRECTORY = "Answer Images"
+  QUESTION_ATTACHMENT_DIRECTORY = "Question Attachments"
+  QUESTION_IMAGE_DIRECTORY = "Question Images"
+  EMBEDDED_IMAGE_REGEX = /<a[^>]*href="[^"]*relativeUrl=(?<path>[^"\&]*)[^"]*"[^>]*>\s*<img[^>]*>\s*<\/a>/i
+  QUESTION_LINK_REGEX = /<a[^>]*?href="#{Regexp.escape(OLD_DOMAIN)}\/[^"]*?(?:q|questionid=)(?<id>\d+)[^"]*?"[^>]*>(?<text>.*?)<\/a>/i
+  TOPIC_LINK_NORMALIZATION = '/.*?-(q\d+).*/\1'
+  BATCH_SIZE = 1000
+
+  def initialize(path)
+    super()
+
+    @path = path
+    @db = ImportScripts::GenericDatabase.new(
+      @path,
+      batch_size: BATCH_SIZE,
+      recreate: true,
+      numeric_keys: true
+    )
+  end
+
+  def execute
+    read_csv_files
+
+    add_permalink_normalizations
+    import_categories
+    import_users
+    import_topics
+    import_posts
+  end
+
+  def read_csv_files
+    puts "", "reading CSV files..."
+
+    category_position = 0
+    csv_parse("categories") do |row|
+      @db.insert_category(
+        id: row[:id],
+        name: row[:name],
+        position: category_position += 1
+      )
+    end
+
+    csv_parse("users") do |row|
+      @db.insert_user(
+        id: row[:id],
+        email: row[:email],
+        username: row[:username],
+        bio: row[:description],
+        avatar_path: row[:profile_image],
+        created_at: parse_date(row[:createtime]),
+        active: true
+      )
+    end
+
+    last_topic_id = nil
+    csv_parse("questions-answers-comments") do |row|
+      next if row[:published] == "No"
+      user_id = @db.get_user_id(row[:username])
+      created_at = parse_datetime(row[:createtime])
+
+      begin
+        if row[:type] == "Question"
+          attachments = parse_filenames(row[:attachments], QUESTION_ATTACHMENT_DIRECTORY) +
+            parse_filenames(row[:images], QUESTION_IMAGE_DIRECTORY)
+
+          @db.insert_topic(
+            id: row[:id],
+            title: row[:title],
+            raw: row[:text],
+            category_id: row[:categorylist],
+            user_id: user_id,
+            created_at: created_at,
+            attachments: attachments
+          )
+          last_topic_id = row[:id]
+        else
+          attachments = parse_filenames(row[:attachments], ANSWER_ATTACHMENT_DIRECTORY) +
+            parse_filenames(row[:images], ANSWER_IMAGE_DIRECTORY)
+
+          @db.insert_post(
+            id: row[:id],
+            raw: row[:text],
+            topic_id: last_topic_id,
+            user_id: user_id,
+            created_at: created_at,
+            attachments: attachments
+          )
+        end
+      rescue
+        p row
+        raise
+      end
+    end
+  end
+
+  def parse_filenames(text, directory)
+    return [] if text.blank?
+
+    text
+      .split(';')
+      .map { |filename| File.join(@path, directory, filename.strip) }
+  end
+
+  def parse_date(text)
+    return nil if text.blank?
+    DateTime.strptime(text, "%m/%d/%y")
+  end
+
+  def parse_datetime(text)
+    return nil if text.blank?
+    # DateTime.strptime(text, "%m/%d/%Y %H:%M")
+    DateTime.parse(text).utc.to_datetime
+  end
+
+  def import_categories
+    puts "", "creating categories"
+    rows = @db.fetch_categories
+
+    create_categories(rows) do |row|
+      {
+        id: row['id'],
+        name: row['name'],
+        description: row['description'],
+        position: row['position']
+      }
+    end
+  end
+
+  def batches
+    super(BATCH_SIZE)
+  end
+
+  def import_users
+    puts "", "creating users"
+    total_count = @db.count_users
+    last_id = 0
+
+    batches do |offset|
+      rows, last_id = @db.fetch_users(last_id)
+      break if rows.empty?
+
+      next if all_records_exist?(:users, rows.map { |row| row['id'] })
+
+      create_users(rows, total: total_count, offset: offset) do |row|
+        {
+          id: row['id'],
+          email: row['email'],
+          username: row['username'],
+          bio_raw: row['bio'],
+          created_at: row['created_at'],
+          active: row['active'] == 1,
+          post_create_action: proc do |user|
+            create_avatar(user, row['avatar_path'])
+          end
+        }
+      end
+    end
+  end
+
+  def create_avatar(user, avatar_path)
+    return if avatar_path.blank?
+    avatar_path = File.join(@path, AVATAR_DIRECTORY, avatar_path)
+
+    if File.exist?(avatar_path)
+      @uploader.create_avatar(user, avatar_path)
+    else
+      STDERR.puts "Could not find avatar: #{avatar_path}"
+    end
+  end
+
+  def import_topics
+    puts "", "creating topics"
+    total_count = @db.count_topics
+    last_id = 0
+
+    batches do |offset|
+      rows, last_id = @db.fetch_topics(last_id)
+      break if rows.empty?
+
+      next if all_records_exist?(:posts, rows.map { |row| row['id'] })
+
+      create_posts(rows, total: total_count, offset: offset) do |row|
+        attachments = @db.fetch_topic_attachments(row['id']) if row['upload_count'] > 0
+        user_id = user_id_from_imported_user_id(row['user_id']) || Discourse.system_user.id
+
+        {
+          id: row['id'],
+          title: row['title'],
+          raw: raw_with_attachments(row['raw'].presence || row['title'], attachments, user_id),
+          category: category_id_from_imported_category_id(row['category_id']),
+          user_id: user_id,
+          created_at: row['created_at'],
+          closed: row['closed'] == 1,
+          post_create_action: proc do |post|
+            url = "q#{row['id']}"
+            Permalink.create(url: url, topic_id: post.topic.id) unless permalink_exists?(url)
+          end
+        }
+      end
+    end
+  end
+
+  def import_posts
+    puts "", "creating posts"
+    total_count = @db.count_posts
+    last_row_id = 0
+
+    batches do |offset|
+      rows, last_row_id = @db.fetch_posts(last_row_id)
+      break if rows.empty?
+
+      next if all_records_exist?(:posts, rows.map { |row| row['id'] })
+
+      create_posts(rows, total: total_count, offset: offset) do |row|
+        topic = topic_lookup_from_imported_post_id(row['topic_id'])
+        attachments = @db.fetch_post_attachments(row['id']) if row['upload_count'] > 0
+        user_id = user_id_from_imported_user_id(row['user_id']) || Discourse.system_user.id
+
+        {
+          id: row['id'],
+          raw: raw_with_attachments(row['raw'], attachments, user_id),
+          user_id: user_id,
+          topic_id: topic[:topic_id],
+          created_at: row['created_at']
+        }
+      end
+    end
+  end
+
+  def raw_with_attachments(raw, attachments, user_id)
+    raw, embedded_paths, upload_ids = replace_embedded_attachments(raw, user_id)
+    raw = replace_question_links(raw)
+    raw = ReverseMarkdown.convert(raw) || ""
+
+    attachments&.each do |attachment|
+      path = attachment['path']
+      next if embedded_paths.include?(path)
+
+      if File.exist?(path)
+        filename = File.basename(path)
+        upload = @uploader.create_upload(user_id, path, filename)
+
+        if upload.present? && upload.persisted? && !upload_ids.include?(upload.id)
+          raw << "\n" << @uploader.html_for_upload(upload, filename)
+        end
+      else
+        STDERR.puts "Could not find file: #{path}"
+      end
+    end
+
+    raw
+  end
+
+  def replace_embedded_attachments(raw, user_id)
+    paths = []
+    upload_ids = []
+
+    raw = raw.gsub(EMBEDDED_IMAGE_REGEX) do
+      path = File.join(@path, Regexp.last_match['path'])
+      filename = File.basename(path)
+      path = find_image_path(filename)
+
+      if path
+        upload = @uploader.create_upload(user_id, path, filename)
+
+        if upload.present? && upload.persisted?
+          paths << path
+          upload_ids << upload.id
+          @uploader.html_for_upload(upload, filename)
+        end
+      else
+        STDERR.puts "Could not find file: #{path}"
+      end
+    end
+
+    [raw, paths, upload_ids]
+  end
+
+  def find_image_path(filename)
+    [QUESTION_IMAGE_DIRECTORY, ANSWER_IMAGE_DIRECTORY].each do |directory|
+      path = File.join(@path, directory, filename)
+      return path if File.exist?(path)
+    end
+  end
+
+  def replace_question_links(raw)
+    raw.gsub(QUESTION_LINK_REGEX) do
+      topic_id = Regexp.last_match("id")
+      topic = topic_lookup_from_imported_post_id(topic_id)
+      return Regexp.last_match.to_s unless topic
+
+      url = File.join(NEW_DOMAIN, topic[:url])
+      text = Regexp.last_match("text")
+      text.include?(OLD_DOMAIN) ? url : "<a href='#{url}'>#{text}</a>"
+    end
+  end
+
+  def add_permalink_normalizations
+    normalizations = SiteSetting.permalink_normalizations
+    normalizations = normalizations.blank? ? [] : normalizations.split('|')
+
+    add_normalization(normalizations, TOPIC_LINK_NORMALIZATION)
+
+    SiteSetting.permalink_normalizations = normalizations.join('|')
+  end
+
+  def add_normalization(normalizations, normalization)
+    normalizations << normalization unless normalizations.include?(normalization)
+  end
+
+  def permalink_exists?(url)
+    Permalink.find_by(url: url)
+  end
+
+  def csv_parse(table_name)
+    CSV.foreach(File.join(@path, "#{table_name}.csv"),
+                headers: true,
+                header_converters: :symbol,
+                skip_blanks: true,
+                encoding: 'bom|utf-8') { |row| yield row }
+  end
+end
+
+unless ARGV[0] && Dir.exist?(ARGV[0])
+  puts "", "Usage:", "", "bundle exec ruby script/import_scripts/answerbase.rb DIRNAME", ""
+  exit 1
+end
+
+ImportScripts::Answerbase.new(ARGV[0]).perform

script/import_scripts/base/generic_database.rb

@@ -2,12 +2,13 @@ require 'sqlite3'
 
 module ImportScripts
   class GenericDatabase
-    def initialize(directory, batch_size:, recreate: false)
+    def initialize(directory, batch_size:, recreate: false, numeric_keys: false)
       filename = "#{directory}/index.db"
       File.delete(filename) if recreate && File.exists?(filename)
 
       @db = SQLite3::Database.new(filename, results_as_hash: true)
       @batch_size = batch_size
+      @numeric_keys = numeric_keys
 
       configure_database
       create_category_table
@@ -25,36 +26,72 @@ module ImportScripts
 
     def insert_user(user)
       @db.execute(<<-SQL, prepare(user))
-        INSERT OR REPLACE INTO user (id, email, username, name, created_at, last_seen_at, active)
+        INSERT OR REPLACE
-        VALUES (:id, :email, :username, :name, :created_at, :last_seen_at, :active)
+        INTO user (id, email, username, name, bio, avatar_path, created_at, last_seen_at, active)
+        VALUES (:id, :email, :username, :name, :bio, :avatar_path, :created_at, :last_seen_at, :active)
       SQL
     end
 
     def insert_topic(topic)
+      attachments = topic.delete(:attachments)
+      topic[:upload_count] = attachments&.size || 0
+
       @db.execute(<<-SQL, prepare(topic))
-        INSERT OR REPLACE INTO topic (id, title, raw, category_id, closed, user_id, created_at, url)
+        INSERT OR REPLACE INTO topic (id, title, raw, category_id, closed, user_id, created_at, url, upload_count)
-        VALUES (:id, :title, :raw, :category_id, :closed, :user_id, :created_at, :url)
+        VALUES (:id, :title, :raw, :category_id, :closed, :user_id, :created_at, :url, :upload_count)
       SQL
+
+      attachments&.each do |attachment|
+        @db.execute(<<-SQL, topic_id: topic[:id], path: attachment)
+          INSERT OR REPLACE INTO topic_upload (topic_id, path)
+          VALUES (:topic_id, :path)
+        SQL
+      end
     end
 
     def insert_post(post)
+      attachments = post.delete(:attachments)
+      post[:upload_count] = attachments&.size || 0
+
       @db.execute(<<-SQL, prepare(post))
-        INSERT OR REPLACE INTO post (id, raw, topic_id, user_id, created_at, reply_to_post_id, url)
+        INSERT OR REPLACE INTO post (id, raw, topic_id, user_id, created_at, reply_to_post_id, url, upload_count)
-        VALUES (:id, :raw, :topic_id, :user_id, :created_at, :reply_to_post_id, :url)
+        VALUES (:id, :raw, :topic_id, :user_id, :created_at, :reply_to_post_id, :url, :upload_count)
       SQL
+
+      attachments&.each do |attachment|
+        @db.execute(<<-SQL, post_id: post[:id], path: attachment)
+          INSERT OR REPLACE INTO post_upload (post_id, path)
+          VALUES (:post_id, :path)
+        SQL
+      end
     end
 
     def sort_posts_by_created_at
       @db.execute 'DELETE FROM post_order'
 
       @db.execute <<-SQL
-        INSERT INTO post_order (id)
+        INSERT INTO post_order (post_id)
         SELECT id
         FROM post
         ORDER BY created_at, topic_id, id
       SQL
     end
 
+    def delete_unused_users
+      @db.execute <<~SQL
+        DELETE FROM user
+        WHERE NOT EXISTS(
+            SELECT 1
+            FROM topic
+            WHERE topic.user_id = user.id
+        ) AND NOT EXISTS(
+            SELECT 1
+            FROM post
+            WHERE post.user_id = user.id
+        )
+      SQL
+    end
+
     def fetch_categories
       @db.execute(<<-SQL)
         SELECT *
@@ -82,6 +119,14 @@ module ImportScripts
       add_last_column_value(rows, 'id')
     end
 
+    def get_user_id(username)
+      @db.get_first_value(<<-SQL, username)
+        SELECT id
+        FROM user
+        WHERE username = :username
+      SQL
+    end
+
     def count_topics
       @db.get_first_value(<<-SQL)
         SELECT COUNT(*)
@@ -101,6 +146,14 @@ module ImportScripts
       add_last_column_value(rows, 'id')
     end
 
+    def fetch_topic_attachments(topic_id)
+      @db.execute(<<-SQL, topic_id)
+        SELECT path
+        FROM topic_upload
+        WHERE topic_id = :topic_id
+      SQL
+    end
+
     def count_posts
       @db.get_first_value(<<-SQL)
         SELECT COUNT(*)
@@ -110,9 +163,21 @@ module ImportScripts
 
     def fetch_posts(last_row_id)
       rows = @db.execute(<<-SQL, last_row_id)
-        SELECT o.ROWID, p.*
+        SELECT ROWID AS rowid, *
+        FROM post
+        WHERE ROWID > :last_row_id
+        ORDER BY ROWID
+        LIMIT #{@batch_size}
+      SQL
+
+      add_last_column_value(rows, 'rowid')
+    end
+
+    def fetch_sorted_posts(last_row_id)
+      rows = @db.execute(<<-SQL, last_row_id)
+        SELECT o.ROWID AS rowid, p.*
         FROM post p
-          JOIN post_order o USING (id)
+          JOIN post_order o ON (p.id = o.post_id)
         WHERE o.ROWID > :last_row_id
         ORDER BY o.ROWID
         LIMIT #{@batch_size}
@@ -121,6 +186,14 @@ module ImportScripts
       add_last_column_value(rows, 'rowid')
     end
 
+    def fetch_post_attachments(post_id)
+      @db.execute(<<-SQL, post_id)
+        SELECT path
+        FROM post_upload
+        WHERE post_id = :post_id
+      SQL
+    end
+
     def execute_sql(sql)
       @db.execute(sql)
     end
@@ -136,10 +209,14 @@ module ImportScripts
       @db.execute 'PRAGMA locking_mode = EXCLUSIVE'
     end
 
+    def key_data_type
+      @numeric_keys ? 'INTEGER' : 'TEXT'
+    end
+
     def create_category_table
       @db.execute <<-SQL
         CREATE TABLE IF NOT EXISTS category (
-          id TEXT NOT NULL PRIMARY KEY,
+          id #{key_data_type} NOT NULL PRIMARY KEY,
           name TEXT NOT NULL,
           description TEXT,
           position INTEGER,
@@ -151,44 +228,59 @@ module ImportScripts
     def create_user_table
       @db.execute <<-SQL
         CREATE TABLE IF NOT EXISTS user (
-          id TEXT NOT NULL PRIMARY KEY,
+          id #{key_data_type} NOT NULL PRIMARY KEY,
           email TEXT,
           username TEXT,
           name TEXT,
+          bio TEXT,
+          avatar_path TEXT,
           created_at DATETIME,
           last_seen_at DATETIME,
           active BOOLEAN NOT NULL DEFAULT true
         )
       SQL
+
+      @db.execute 'CREATE INDEX IF NOT EXISTS user_by_username ON user (username)'
     end
 
     def create_topic_table
       @db.execute <<-SQL
         CREATE TABLE IF NOT EXISTS topic (
-          id TEXT NOT NULL PRIMARY KEY,
+          id #{key_data_type} NOT NULL PRIMARY KEY,
           title TEXT,
           raw TEXT,
-          category_id TEXT NOT NULL,
+          category_id #{key_data_type},
           closed BOOLEAN NOT NULL DEFAULT false,
-          user_id TEXT NOT NULL,
+          user_id #{key_data_type} NOT NULL,
           created_at DATETIME,
-          url TEXT
+          url TEXT,
+          upload_count INTEGER DEFAULT 0
         )
       SQL
 
       @db.execute 'CREATE INDEX IF NOT EXISTS topic_by_user_id ON topic (user_id)'
+
+      @db.execute <<-SQL
+        CREATE TABLE IF NOT EXISTS topic_upload (
+          topic_id #{key_data_type} NOT NULL,
+          path TEXT NOT NULL
+        )
+      SQL
+
+      @db.execute 'CREATE UNIQUE INDEX IF NOT EXISTS topic_upload_unique ON topic_upload(topic_id, path)'
     end
 
     def create_post_table
       @db.execute <<-SQL
         CREATE TABLE IF NOT EXISTS post (
-          id TEXT NOT NULL PRIMARY KEY,
+          id #{key_data_type} NOT NULL PRIMARY KEY,
           raw TEXT,
-          topic_id TEXT NOT NULL,
+          topic_id #{key_data_type} NOT NULL,
-          user_id TEXT NOT NULL,
+          user_id #{key_data_type} NOT NULL,
           created_at DATETIME,
-          reply_to_post_id TEXT,
+          reply_to_post_id #{key_data_type},
-          url TEXT
+          url TEXT,
+          upload_count INTEGER DEFAULT 0
         )
       SQL
 
@@ -196,9 +288,18 @@ module ImportScripts
 
       @db.execute <<-SQL
         CREATE TABLE IF NOT EXISTS post_order (
-          id TEXT NOT NULL PRIMARY KEY
+          post_id #{key_data_type} NOT NULL PRIMARY KEY
         )
       SQL
+
+      @db.execute <<-SQL
+        CREATE TABLE IF NOT EXISTS post_upload (
+          post_id #{key_data_type} NOT NULL,
+          path TEXT NOT NULL
+        )
+      SQL
+
+      @db.execute 'CREATE UNIQUE INDEX IF NOT EXISTS post_upload_unique ON post_upload(post_id, path)'
     end
 
     def prepare(hash)

script/import_scripts/zendesk.rb

@@ -82,19 +82,7 @@ class ImportScripts::Zendesk < ImportScripts::Base
       )
     end
 
-    @db.execute_sql(<<~SQL)
+    @db.delete_unused_users
-      DELETE FROM user
-      WHERE NOT EXISTS(
-          SELECT 1
-          FROM topic
-          WHERE topic.user_id = user.id
-      ) AND NOT EXISTS(
-          SELECT 1
-          FROM post
-          WHERE post.user_id = user.id
-      )
-    SQL
-
     @db.sort_posts_by_created_at
   end
 
@@ -188,7 +176,7 @@ class ImportScripts::Zendesk < ImportScripts::Base
     last_row_id = 0
 
     batches do |offset|
-      rows, last_row_id = @db.fetch_posts(last_row_id)
+      rows, last_row_id = @db.fetch_sorted_posts(last_row_id)
       break if rows.empty?
 
       next if all_records_exist?(:posts, rows.map { |row| row['id'] })

script/import_scripts/zendesk_api.rb

@@ -8,7 +8,7 @@ require_relative 'base/generic_database'
 
 # Call it like this:
 #   RAILS_ENV=production bundle exec ruby script/import_scripts/zendesk_api.rb SOURCE_URL DIRNAME AUTH_EMAIL AUTH_TOKEN
-class ImportScripts::Zendesk < ImportScripts::Base
+class ImportScripts::ZendeskApi < ImportScripts::Base
   BATCH_SIZE = 1000
 
   def initialize(source_url, path, auth_email, auth_token)
@@ -193,7 +193,7 @@ class ImportScripts::Zendesk < ImportScripts::Base
     last_row_id = 0
 
     batches do |offset|
-      rows, last_row_id = @db.fetch_posts(last_row_id)
+      rows, last_row_id = @db.fetch_sorted_posts(last_row_id)
       break if rows.empty?
 
       create_posts(rows, total: total_count, offset: offset) do |row|
@@ -303,4 +303,4 @@ unless ARGV.length == 4 && Dir.exist?(ARGV[1])
   exit 1
 end
 
-ImportScripts::Zendesk.new(ARGV[0], ARGV[1], ARGV[2], ARGV[3]).perform
+ImportScripts::ZendeskApi.new(ARGV[0], ARGV[1], ARGV[2], ARGV[3]).perform