discourse/script/import_scripts/yahoogroup.rb

# frozen_string_literal: true

require File.expand_path(File.dirname(__FILE__) + "/base.rb")
require "mongo"

# Import YahooGroups data as exported into MongoDB by:
#   https://github.com/jonbartlett/yahoo-groups-export
#
#   Optionally paste these lines into your shell before running this:
#
#   =begin
#   export CATEGORY_ID=<CATEGORY_ID>
#   =end

class ImportScripts::YahooGroup < ImportScripts::Base
  MONGODB_HOST = "192.168.10.1:27017"
  MONGODB_DB = "syncro"

  def initialize
    super

    client = Mongo::Client.new([MONGODB_HOST], database: MONGODB_DB)
    db = client.database
    Mongo::Logger.logger.level = Logger::FATAL
    puts "connected to db...."

    @collection = client[:posts]

    @user_profile_map = {}
  end

  def execute
    puts "", "Importing from Mongodb...."

    import_users
    import_discussions

    puts "", "Done"
  end

  def import_users
    puts "", "Importing users"

    # fetch distinct list of Yahoo "profile" names
    profiles = @collection.aggregate([{ "$group": { _id: { profile: "$ygData.profile" } } }])

    user_id = 0

    create_users(profiles.to_a) do |u|
      user_id = user_id + 1

      # fetch last message for profile to pickup latest user info as this may have changed
      user_info =
        @collection
          .find("ygData.profile": u["_id"]["profile"])
          .sort("ygData.msgId": -1)
          .limit(1)
          .to_a[
          0
        ]

      # Store user_id to profile lookup
      @user_profile_map.store(user_info["ygData"]["profile"], user_id)

      puts "User created: #{user_info["ygData"]["profile"]}"

      user = {
        id: user_id, # yahoo "userId" sequence appears to have changed mid forum life so generate this
        username: user_info["ygData"]["profile"],
        name: user_info["ygData"]["authorName"],
        email: user_info["ygData"]["from"], # mandatory
        created_at: Time.now,
      }
      user
    end

    puts "#{user_id} users created"
  end

  def import_discussions
    puts "", "Importing discussions"

    topics_count = 0
    posts_count = 0

    topics = @collection.aggregate([{ "$group": { _id: { topicId: "$ygData.topicId" } } }]).to_a

    # for each distinct topicId found
    topics.each_with_index do |t, tidx|
      # create "topic" post first.
      # fetch topic document
      topic_post = @collection.find("ygData.msgId": t["_id"]["topicId"]).to_a[0]
      next if topic_post.nil?

      puts "Topic: #{tidx + 1} / #{topics.count()}  (#{sprintf("%.2f", ((tidx + 1).to_f / topics.count().to_f) * 100)}%)  Subject: #{topic_post["ygData"]["subject"]}"

      if topic_post["ygData"]["subject"].to_s.empty?
        topic_title = "No Subject"
      else
        topic_title = topic_post["ygData"]["subject"]
      end

      topic = {
        id: tidx + 1,
        user_id: @user_profile_map[topic_post["ygData"]["profile"]] || -1,
        raw: topic_post["ygData"]["messageBody"],
        created_at: Time.at(topic_post["ygData"]["postDate"].to_i),
        cook_method: Post.cook_methods[:raw_html],
        title: topic_title,
        category: ENV["CATEGORY_ID"],
        custom_fields: {
          import_id: topic_post["ygData"]["msgId"],
        },
      }

      topics_count += 1

      # create topic post
      parent_post = create_post(topic, topic[:id])

      # find all posts for topic id
      posts = @collection.find("ygData.topicId": topic_post["ygData"]["topicId"]).to_a

      posts.each_with_index do |p, pidx|
        # skip over first post as this is created by topic above
        next if p["ygData"]["msgId"] == topic_post["ygData"]["topicId"]

        puts "  Post: #{pidx + 1} / #{posts.count()}"

        post = {
          id: pidx + 1,
          topic_id: parent_post[:topic_id],
          user_id: @user_profile_map[p["ygData"]["profile"]] || -1,
          raw: p["ygData"]["messageBody"],
          created_at: Time.at(p["ygData"]["postDate"].to_i),
          cook_method: Post.cook_methods[:raw_html],
          custom_fields: {
            import_id: p["ygData"]["msgId"],
          },
        }

        child_post = create_post(post, post[:id])

        posts_count += 1
      end
    end

    puts "", "Imported #{topics_count} topics with #{topics_count + posts_count} posts."
  end
end

ImportScripts::YahooGroup.new.perform