discourse/app/jobs/regular/export_csv_file.rb

# frozen_string_literal: true

require "csv"

module Jobs
  class ExportCsvFile < ::Jobs::Base
    sidekiq_options retry: false

    attr_accessor :extra
    attr_accessor :current_user
    attr_accessor :entity

    HEADER_ATTRS_FOR =
      HashWithIndifferentAccess.new(
        user_list: %w[
          id
          name
          username
          email
          title
          created_at
          last_seen_at
          last_posted_at
          last_emailed_at
          trust_level
          approved
          suspended_at
          suspended_till
          silenced_till
          active
          admin
          moderator
          ip_address
          staged
          secondary_emails
        ],
        user_stats: %w[
          topics_entered
          posts_read_count
          time_read
          topic_count
          post_count
          likes_given
          likes_received
        ],
        user_profile: %w[location website views],
        user_sso: %w[
          external_id
          external_email
          external_username
          external_name
          external_avatar_url
        ],
        staff_action: %w[staff_user action subject created_at details context],
        screened_email: %w[email action match_count last_match_at created_at ip_address],
        screened_ip: %w[ip_address action match_count last_match_at created_at],
        screened_url: %w[domain action match_count last_match_at created_at],
        report: %w[date value],
      )

    def execute(args)
      @entity = args[:entity]
      @extra = HashWithIndifferentAccess.new(args[:args]) if args[:args]
      @current_user = User.find_by(id: args[:user_id])

      entity = { name: @entity }
      entity[:method] = :"#{entity[:name]}_export"
      raise Discourse::InvalidParameters.new(:entity) unless respond_to?(entity[:method])

      @timestamp ||= Time.now.strftime("%y%m%d-%H%M%S")
      entity[:filename] = if entity[:name] == "report" && @extra[:name].present?
        "#{@extra[:name].dasherize}-#{@timestamp}"
      else
        "#{entity[:name].dasherize}-#{@timestamp}"
      end

      export_title =
        if @entity == "report" && @extra[:name].present?
          I18n.t("reports.#{@extra[:name]}.title")
        else
          @entity.gsub("_", " ").titleize
        end

      filename = entity[:filename]
      user_export = UserExport.create(file_name: filename, user_id: @current_user.id)
      filename = "#{filename}-#{user_export.id}"

      zip_filename = write_to_csv_and_zip(filename, entity)

      # create upload
      upload = nil

      if File.exist?(zip_filename)
        File.open(zip_filename) do |file|
          upload =
            UploadCreator.new(
              file,
              File.basename(zip_filename),
              type: "csv_export",
              for_export: "true",
            ).create_for(@current_user.id)

          if upload.persisted?
            user_export.update_columns(upload_id: upload.id)
          else
            Rails.logger.warn(
              "Failed to upload the file #{zip_filename}: #{upload.errors.full_messages}",
            )
          end
        end

        File.delete(zip_filename)
      end
    ensure
      post = notify_user(upload, export_title)

      if user_export.present? && post.present?
        topic = post.topic
        user_export.update_columns(topic_id: topic.id)
        topic.update_status("closed", true, Discourse.system_user)
      end
    end

    def user_list_export
      user_field_ids = UserField.pluck(:id)

      condition = {}
      if @extra && @extra[:trust_level] &&
           trust_level = TrustLevel.levels[@extra[:trust_level].to_sym]
        condition = { trust_level: trust_level }
      end

      includes = %i[user_profile user_stat groups user_emails]
      includes << [:single_sign_on_record] if SiteSetting.enable_discourse_connect

      User
        .where(condition)
        .includes(*includes)
        .find_each do |user|
          user_info_array = get_base_user_array(user)
          if SiteSetting.enable_discourse_connect
            user_info_array = add_single_sign_on(user, user_info_array)
          end
          user_info_array = add_custom_fields(user, user_info_array, user_field_ids)
          user_info_array = add_group_names(user, user_info_array)
          yield user_info_array
        end
    end

    def staff_action_export
      staff_action_data =
        if @current_user.admin?
          UserHistory.only_staff_actions
        else
          UserHistory.where(admin_only: false).only_staff_actions
        end

      staff_action_data.find_each(order: :desc) do |staff_action|
        yield get_staff_action_fields(staff_action)
      end
    end

    def screened_email_export
      ScreenedEmail.find_each(order: :desc) do |screened_email|
        yield get_screened_email_fields(screened_email)
      end
    end

    def screened_ip_export
      ScreenedIpAddress.find_each(order: :desc) do |screened_ip|
        yield get_screened_ip_fields(screened_ip)
      end
    end

    def screened_url_export
      ScreenedUrl
        .select(
          "domain, sum(match_count) as match_count, max(last_match_at) as last_match_at, min(created_at) as created_at",
        )
        .group(:domain)
        .order("last_match_at DESC")
        .each { |screened_url| yield get_screened_url_fields(screened_url) }
    end

    def report_export
      # If dates are invalid consider then `nil`
      if @extra[:start_date].is_a?(String)
        @extra[:start_date] = begin
          @extra[:start_date].to_date.beginning_of_day
        rescue StandardError
          nil
        end
      end
      if @extra[:end_date].is_a?(String)
        @extra[:end_date] = begin
          @extra[:end_date].to_date.end_of_day
        rescue StandardError
          nil
        end
      end

      @extra[:filters] = {}
      @extra[:filters][:category] = @extra[:category].to_i if @extra[:category].present?
      @extra[:filters][:group] = @extra[:group].to_i if @extra[:group].present?
      @extra[:filters][:include_subcategories] = !!ActiveRecord::Type::Boolean.new.cast(
        @extra[:include_subcategories],
      ) if @extra[:include_subcategories].present?

      report = Report.find(@extra[:name], @extra)

      header = []
      titles = {}

      report.labels.each do |label|
        if label[:type] == :user
          titles[label[:properties][:username]] = label[:title]
          header << label[:properties][:username]
        elsif label[:type] == :topic
          titles[label[:properties][:id]] = label[:title]
          header << label[:properties][:id]
        elsif label[:type] == :post
          titles[label[:properties][:truncated_raw]] = label[:title]
          header << label[:properties][:truncated_raw]
        else
          titles[label[:property]] = label[:title]
          header << label[:property]
        end
      end

      if report.modes == [Report::MODES[:stacked_chart]]
        header = [:x]
        data = {}

        report.data.map do |series|
          header << series[:label]
          series[:data].each do |datapoint|
            data[datapoint[:x]] ||= { x: datapoint[:x] }
            data[datapoint[:x]][series[:label]] = datapoint[:y]
          end
        end

        data = data.values
      else
        data = report.data
      end

      yield header.map { |k| titles[k] || k }
      data.each { |row| yield row.values_at(*header).map(&:to_s) }
    end

    def get_header(entity)
      if entity == "user_list"
        header_array =
          HEADER_ATTRS_FOR["user_list"] + HEADER_ATTRS_FOR["user_stats"] +
            HEADER_ATTRS_FOR["user_profile"]
        header_array.concat(HEADER_ATTRS_FOR["user_sso"]) if SiteSetting.enable_discourse_connect
        user_custom_fields = UserField.all
        if user_custom_fields.present?
          user_custom_fields.each do |custom_field|
            header_array.push("#{custom_field.name} (custom user field)")
          end
        end
        header_array.push("group_names")
      else
        header_array = HEADER_ATTRS_FOR[entity]
      end

      header_array
    end

    private

    def escape_comma(string)
      string&.include?(",") ? %Q|"#{string}"| : string
    end

    def get_base_user_array(user)
      # preloading scopes is hard, do this by hand
      secondary_emails = []
      primary_email = nil

      user.user_emails.each do |user_email|
        if user_email.primary?
          primary_email = user_email.email
        else
          secondary_emails << user_email.email
        end
      end

      [
        user.id,
        escape_comma(user.name),
        user.username,
        primary_email,
        escape_comma(user.title),
        user.created_at,
        user.last_seen_at,
        user.last_posted_at,
        user.last_emailed_at,
        user.trust_level,
        user.approved,
        user.suspended_at,
        user.suspended_till,
        user.silenced_till,
        user.active,
        user.admin,
        user.moderator,
        user.ip_address,
        user.staged,
        secondary_emails.join(";"),
        user.user_stat.topics_entered,
        user.user_stat.posts_read_count,
        user.user_stat.time_read,
        user.user_stat.topic_count,
        user.user_stat.post_count,
        user.user_stat.likes_given,
        user.user_stat.likes_received,
        escape_comma(user.user_profile.location),
        user.user_profile.website,
        user.user_profile.views,
      ]
    end

    def add_single_sign_on(user, user_info_array)
      if user.single_sign_on_record
        user_info_array.push(
          user.single_sign_on_record.external_id,
          user.single_sign_on_record.external_email,
          user.single_sign_on_record.external_username,
          escape_comma(user.single_sign_on_record.external_name),
          user.single_sign_on_record.external_avatar_url,
        )
      else
        user_info_array.push(nil, nil, nil, nil, nil)
      end
      user_info_array
    end

    def add_custom_fields(user, user_info_array, user_field_ids)
      if user_field_ids.present?
        user.user_fields.each { |custom_field| user_info_array << escape_comma(custom_field[1]) }
      end
      user_info_array
    end

    def add_group_names(user, user_info_array)
      group_names = user.groups.map { |g| g.name }.join(";")
      if group_names.present?
        user_info_array << escape_comma(group_names)
      else
        user_info_array << nil
      end
      user_info_array
    end

    def get_staff_action_fields(staff_action)
      staff_action_array = []

      HEADER_ATTRS_FOR["staff_action"].each do |attr|
        data =
          if attr == "action"
            UserHistory.actions.key(staff_action.attributes[attr]).to_s
          elsif attr == "staff_user"
            user = User.find_by(id: staff_action.attributes["acting_user_id"])
            user.username if !user.nil?
          elsif attr == "subject"
            user = User.find_by(id: staff_action.attributes["target_user_id"])
            if user.nil?
              staff_action.attributes[attr]
            else
              "#{user.username} #{staff_action.attributes[attr]}"
            end
          else
            staff_action.attributes[attr]
          end

        staff_action_array.push(data)
      end
      staff_action_array
    end

    def get_screened_email_fields(screened_email)
      screened_email_array = []

      HEADER_ATTRS_FOR["screened_email"].each do |attr|
        data =
          if attr == "action"
            ScreenedEmail.actions.key(screened_email.attributes["action_type"]).to_s
          else
            screened_email.attributes[attr]
          end

        screened_email_array.push(data)
      end

      screened_email_array
    end

    def get_screened_ip_fields(screened_ip)
      screened_ip_array = []

      HEADER_ATTRS_FOR["screened_ip"].each do |attr|
        data =
          if attr == "action"
            ScreenedIpAddress.actions.key(screened_ip.attributes["action_type"]).to_s
          else
            screened_ip.attributes[attr]
          end

        screened_ip_array.push(data)
      end

      screened_ip_array
    end

    def get_screened_url_fields(screened_url)
      screened_url_array = []

      HEADER_ATTRS_FOR["screened_url"].each do |attr|
        data =
          if attr == "action"
            action = ScreenedUrl.actions.key(screened_url.attributes["action_type"]).to_s
            action = "do nothing" if action.blank?
          else
            screened_url.attributes[attr]
          end

        screened_url_array.push(data)
      end

      screened_url_array
    end

    def notify_user(upload, export_title)
      post = nil

      if @current_user
        post =
          if upload&.errors&.empty?
            SystemMessage.create_from_system_user(
              @current_user,
              :csv_export_succeeded,
              download_link: UploadMarkdown.new(upload).attachment_markdown,
              export_title: export_title,
            )
          else
            SystemMessage.create_from_system_user(@current_user, :csv_export_failed)
          end
      end

      post
    end

    def write_to_csv_and_zip(filename, entity)
      dirname = "#{UserExport.base_directory}/#{filename}"
      FileUtils.mkdir_p(dirname) unless Dir.exist?(dirname)
      begin
        CSV.open("#{dirname}/#{entity[:filename]}.csv", "w") do |csv|
          csv << get_header(entity[:name]) if entity[:name] != "report"
          public_send(entity[:method]) { |d| csv << d }
        end

        Compression::Zip.new.compress(UserExport.base_directory, filename)
      ensure
        FileUtils.rm_rf(dirname)
      end
    end
  end
end