discourse/script/import_scripts/askbot.rb

require File.expand_path(File.dirname(__FILE__) + "/base.rb")
require 'pg'

class ImportScripts::MyAskBot < ImportScripts::Base
  # CHANGE THESE BEFORE RUNNING THE IMPORTER

  BATCH_SIZE = 1000

  OLD_SITE   = "ask.cvxr.com"
  DB_NAME    = "cvxforum"
  DB_USER    = "cvxforum"
  DB_PORT    = 5432
  DB_HOST    = "ask.cvxr.com"
  DB_PASS    = 'yeah, right'

  # A list of categories to create. Any post with one of these tags will be
  # assigned to that category. Ties are broken by list orer.
  CATEGORIES = [ 'Nonconvex', 'TFOCS', 'MIDCP', 'FAQ' ]

  def initialize
    super

    @thread_parents = {}
    @tagmap = []
    @td = PG::TextDecoder::TimestampWithTimeZone.new
    @client = PG.connect(
      :dbname   => DB_NAME,
      :host     => DB_HOST,
      :port     => DB_PORT,
      :user     => DB_USER,
      :password => DB_PASS
    )
  end

  def execute
    create_cats
    import_users
    read_tags
    import_posts
    import_replies
    post_process_posts
  end

  def create_cats
    puts "", "creating categories"
    CATEGORIES.each do |cat|
      unless Category.where("LOWER(name) = ?", cat.downcase).first
        Category.new(name: cat, user_id: -1).save!
      end
    end
  end

  def read_tags
    puts "", "reading thread tags..."

    tag_count = @client.exec(<<-SQL
          SELECT COUNT(A.id)
          FROM askbot_thread_tags A
          JOIN tag B
          ON A.tag_id = B.id
          WHERE A.tag_id > 0
      SQL
    )[0]["count"]

    tags_done = 0
    batches(BATCH_SIZE) do |offset|
      tags = @client.exec(<<-SQL
        SELECT A.thread_id, B.name
        FROM askbot_thread_tags A
        JOIN tag B
        ON A.tag_id = B.id
        WHERE A.tag_id > 0
        LIMIT #{BATCH_SIZE}
        OFFSET #{offset}
      SQL
      )
      break if tags.ntuples() < 1
      tags.each do |tag|
        tid = tag["thread_id"].to_i
        tnm = tag["name"].downcase
        if @tagmap[tid]
          @tagmap[tid].push( tnm )
        else
          @tagmap[tid] = [ tnm ]
        end
        tags_done += 1
        print_status tags_done, tag_count
      end
    end
  end

  def import_users
    puts "", "importing users"

    total_count = @client.exec(<<-SQL
          SELECT COUNT(id)
            FROM auth_user
      SQL
    )[0]["count"]

    batches(BATCH_SIZE) do |offset|
      users = @client.query(<<-SQL
          SELECT id, username, email, is_staff, date_joined, last_seen, real_name, website, location, about
            FROM auth_user
        ORDER BY date_joined
           LIMIT #{BATCH_SIZE}
          OFFSET #{offset}
      SQL
      )

      break if users.ntuples() < 1

      next if all_records_exist? :users, users.map {|u| u["id"].to_i}

      create_users(users, total: total_count, offset: offset) do |user|
        {
          id:           user["id"],
          username:     user["username"],
          email:        user["email"] || (SecureRandom.hex << "@domain.com"),
          admin:        user["is_staff"],
          created_at:   Time.zone.at(@td.decode(user["date_joined"])),
          last_seen_at: Time.zone.at(@td.decode(user["last_seen"])),
          name:         user["real_name"],
          website:      user["website"],
          location:     user["location"],
        }
      end
    end
  end

  def import_posts
    puts "", "importing questions..."

    post_count = @client.exec(<<-SQL
          SELECT COUNT(A.id)
            FROM askbot_post A
            JOIN askbot_thread B
              ON A.thread_id = B.id
           WHERE NOT B.closed AND A.post_type='question'
      SQL
    )[0]["count"]

    batches(BATCH_SIZE) do |offset|
      posts = @client.exec(<<-SQL
          SELECT A.id, A.author_id, A.added_at, A.text, A.thread_id, B.title
            FROM askbot_post A
            JOIN askbot_thread B
              ON A.thread_id = B.id
           WHERE NOT B.closed AND A.post_type = 'question'
        ORDER BY A.added_at
          LIMIT #{BATCH_SIZE}
          OFFSET #{offset}
      SQL
      )

      break if posts.ntuples() < 1

      next if all_records_exist? :posts, posts.map {|p| p["id"].to_i}

      create_posts(posts, total: post_count, offset: offset) do |post|
        pid = post["id"]
        tid = post["thread_id"].to_i
        tags = @tagmap[tid]
        cat = nil
        if tags
          CATEGORIES.each do |cname|
            next unless tags.include?(cname.downcase)
            cat = cname
            break
          end
        end
        @thread_parents[tid] = pid
        {
          id: pid,
          title: post["title"],
          category: cat,
          custom_fields: {import_id: pid, import_thread_id: tid, import_tags: tags},
          user_id: user_id_from_imported_user_id(post["author_id"]) || Discourse::SYSTEM_USER_ID,
          created_at: Time.zone.at(@td.decode(post["added_at"])),
          raw: post["text"],
        }
      end
    end
  end

  def import_replies
    puts "", "importing answers and comments..."

    post_count = @client.exec(<<-SQL
          SELECT COUNT(A.id)
            FROM askbot_post A
            JOIN askbot_thread B
              ON A.thread_id = B.id
           WHERE NOT B.closed AND A.post_type<>'question'
      SQL
    )[0]["count"]

    batches(BATCH_SIZE) do |offset|
      posts = @client.exec(<<-SQL
          SELECT A.id, A.author_id, A.added_at, A.text, A.thread_id, B.title
            FROM askbot_post A
            JOIN askbot_thread B
              ON A.thread_id = B.id
           WHERE NOT B.closed AND A.post_type <> 'question'
        ORDER BY A.added_at
          LIMIT #{BATCH_SIZE}
          OFFSET #{offset}
      SQL
      )

      break if posts.ntuples() < 1

      next if all_records_exist? :posts, posts.map {|p| p["id"].to_i}

      create_posts(posts, total: post_count, offset: offset) do |post|
        tid = post["thread_id"].to_i
        next unless thread = @thread_parents[tid]
        next unless parent = topic_lookup_from_imported_post_id(thread)
        pid = post["id"]
        {
          id: pid,
          topic_id: parent[:topic_id],
          custom_fields: {import_id: pid},
          user_id: user_id_from_imported_user_id(post["author_id"]) || Discourse::SYSTEM_USER_ID,
          created_at: Time.zone.at(@td.decode(post["added_at"])),
          raw: post["text"]
        }
      end
    end
  end

  def post_process_posts
      puts "", "Postprocessing posts..."
      current = 0
      max = Post.count
      # Rewrite internal links; e.g.
      # ask.cvxr.com/question/(\d+)/[^'"}]*
      # I am sure this is incomplete, but we didn't make heavy use of internal
      # links on our site.
      tmp = Regexp.quote("http://" << OLD_SITE)
      r1 = /"(#{tmp})?\/question\/(\d+)\/[a-zA-Z-]*\/?"/
      r2 = /\((#{tmp})?\/question\/(\d+)\/[a-zA-Z-]*\/?\)/
      r3 = /<?#tmp\/question\/(\d+)\/[a-zA-Z-]*\/?>?/
      Post.find_each do |post|
        raw = post.raw.gsub(r1) do
          if topic = topic_lookup_from_imported_post_id($2)
            "\"#{topic[:url]}\""
          else
            $&
          end
        end
        raw = raw.gsub(r2) do
          if topic = topic_lookup_from_imported_post_id($2)
            "(#{topic[:url]})"
          else
            $&
          end
        end
        raw = raw.gsub(r3) do
           if topic = topic_lookup_from_imported_post_id($1)
            trec = Topic.find_by(id: topic[:topic_id])
            "[#{trec.title}](#{topic[:url]})"
          else
            $&
          end
        end
        if raw != post.raw
          post.raw = raw
          post.save
        end
        print_status(current += 1, max)
      end
    end
  end

ImportScripts::MyAskBot.new.perform