FEATURE: Import Script for Fusionforge (#22281)

This is an import script for the forum/development platform https://www.fusionforge.org/projects/fusionforge
imports users, forums and posts including attachments

script/import_scripts/fusionforge.rb

@@ -0,0 +1,237 @@
+# frozen_string_literal: true
+# (c) 2023 Intevation GmbH
+
+require "pg"
+
+require File.expand_path(File.dirname(__FILE__) + "/base.rb")
+
+# Call it like this:
+#   RAILS_ENV=production bundle exec ruby script/import_scripts/fusionforge.rb
+class ImportScripts::FusionForge < ImportScripts::Base
+  FUSIONFORGE = "fusionforge"
+  BATCH_SIZE = 1000
+
+  def initialize
+    super
+
+    @client =
+      PG.connect(
+        host: "localhost",
+        user: "fusionforge",
+        password: "fusionforge",
+        dbname: FUSIONFORGE,
+      )
+  end
+
+  def execute
+    import_users
+    import_categories
+    import_posts
+    import_attachments
+  end
+
+  def import_users
+    puts "", "creating users"
+
+    total_count = @client.exec("
+        WITH relevant_posts AS (
+          SELECT DISTINCT posted_by FROM forum
+        )
+        SELECT
+          COUNT(DISTINCT user_id) AS count
+        FROM users u
+        JOIN relevant_posts f on u.user_id = f.posted_by
+        ").first["count"]
+
+    batches(BATCH_SIZE) do |offset|
+      results =
+        @client.exec(
+          # Only select users which have some content
+          "WITH relevant_posts AS (
+            SELECT DISTINCT posted_by FROM forum
+            )
+          SELECT
+            DISTINCT user_id, email, user_name, add_date, status, unix_pw
+          FROM users u
+          JOIN relevant_posts f on u.user_id = f.posted_by
+          LIMIT #{BATCH_SIZE}
+          OFFSET #{offset};",
+      )
+
+      break if results.ntuples < 1
+
+      next if all_records_exist? :users, results.map { |u| u["user_id"].to_i }
+      puts "Creating users"
+
+      create_users(results, total: total_count, offset: offset) do |user|
+        {
+          id: user["user_id"],
+          email: user["email"],
+          username: user["user_name"],
+          name: user["name"],
+          active: user["status"] == 'A' && user["unix_pw"] != 'deleted',
+          created_at: Time.zone.at(user["add_date"].to_i),
+          last_emailed_at: nil,  # default is "now", which is not true
+          approved: true,
+          # for https://github.com/communiteq/discourse-migratepassword/
+          # this field results in custom_fields['import_pass']. This also activates the accounts, see base.rb on `u.activate`.
+          password: user["unix_pw"] != 'deleted' ? user["unix_pw"] : nil,
+        }
+      end
+    end
+  end
+
+  def import_categories
+    puts "", "importing groups..."
+
+    categories =
+      @client.exec(
+        "
+          SELECT group_id, group_name
+          FROM groups
+          WHERE use_forum = 1 AND (SELECT COUNT(*) FROM forum_group_list WHERE forum_group_list.group_id = groups.group_id) > 0
+          ORDER BY group_id ASC
+        ",
+      ).to_a
+
+    create_categories(categories) { |category| { id: category["group_id"], name: category["group_name"] } }
+
+    puts "", "importing forums..."
+
+    children_categories =
+      @client.exec(
+        "
+          SELECT group_forum_id, group_id, forum_name, description
+          FROM forum_group_List
+          ORDER BY group_id, group_forum_id
+        ",
+      ).to_a
+
+    create_categories(children_categories) do |category|
+      {
+        id: "child##{category["group_forum_id"]}",
+        name: category["forum_name"],
+        description: category["description"],
+        parent_category_id: category_id_from_imported_category_id(category["group_id"]),
+      }
+    end
+  end
+
+  def import_posts
+    puts "", "creating topics and posts"
+
+    total_count = @client.exec("SELECT count(*) as count from forum").first["count"]
+
+    batches(BATCH_SIZE) do |offset|
+      results =
+        @client.exec(
+          "
+        SELECT msg_id,
+               group_forum_id,
+               subject,
+               thread_id,
+               posted_by,
+               body,
+               post_date,
+               is_followup_to,
+               has_followups
+        FROM forum
+        ORDER BY thread_id, post_date
+        LIMIT #{BATCH_SIZE}
+        OFFSET #{offset};
+        ",
+        ).to_a
+
+      break if results.length < 1
+      next if all_records_exist? :posts, results.map { |m| m["msg_id"].to_i }
+
+      create_posts(results, total: total_count, offset: offset) do |m|
+        skip = false
+        mapped = {}
+
+        mapped[:id] = m["msg_id"]
+        mapped[:user_id] = user_id_from_imported_user_id(m["posted_by"]) || -1
+        mapped[:raw] = CGI.unescapeHTML(m["body"])
+        mapped[:created_at] = Time.zone.at(m["post_date"].to_i)
+
+        if m["is_followup_to"] == "0"
+          # if is not a follow up, then it's a thread
+          mapped[:category] = category_id_from_imported_category_id(m["group_forum_id"])
+          mapped[:title] = CGI.unescapeHTML(m["subject"])
+        else
+          parent = topic_lookup_from_imported_post_id(m["is_followup_to"].to_i)
+          if parent
+            mapped[:topic_id] = parent[:topic_id]
+          else
+            skip = true
+          end
+        end
+        skip ? nil : mapped
+      end
+    end
+  end
+
+  def import_attachments
+    puts "", "importing attachments..."
+
+    uploads = @client.exec("
+      SELECT msg_id, filename, attachmentid
+      FROM forum_attachment
+      order by msg_id
+    ").to_a
+
+    current_count = 0
+    total_count = uploads.count
+
+    uploads.each do |upload|
+      post_id = post_id_from_imported_post_id(upload["msg_id"])
+
+      if post_id.nil?
+        puts "Post #{upload["msg_id"]} for attachment #{upload["attachmentid"]} not found"
+        next
+      end
+
+      post = Post.find(post_id)
+
+      real_filename = upload["filename"]
+      real_filename.prepend SecureRandom.hex if real_filename[0] == "."
+
+      file_hex = sprintf("%x", upload["attachmentid"])
+      prefix = file_hex[-2..-1]
+      if not prefix
+        prefix = file_hex
+      end
+      postfix = file_hex[0..-3].to_s
+      if postfix == ''
+        postfix = '0'
+      end
+      filename = File.join("/tmp/var/lib/fusionforge/forum/", prefix, "/", postfix)
+
+      upl_obj = create_upload(post.user.id, filename, real_filename)
+
+      if upl_obj&.persisted?
+        html = html_for_upload(upl_obj, real_filename)
+        if !post.raw[html]
+          post.raw += "\n\n#{html}\n\n"
+          post.save!
+          if PostUpload.where(post: post, upload: upl_obj).exists?
+            puts "skipping creating uploaded for previously uploaded file #{upload["attachmentid"]}"
+          else
+            PostUpload.create!(post: post, upload: upl_obj)
+          end
+        else
+          puts "Skipping attachment #{upload["attachmentid"]}"
+        end
+      else
+        puts "Failed to upload attachment #{upload["attachmentid"]}"
+        exit
+      end
+
+      current_count += 1
+      print_status(current_count, total_count)
+    end
+  end
+
+end
+
+ImportScripts::FusionForge.new.perform