discourse/app/jobs/regular/exporter.rb

require_dependency 'export/json_encoder'
require_dependency 'export/export'
require_dependency 'import/import'

module Jobs

  class Exporter < Jobs::Base

    sidekiq_options retry: false

    def execute(args)
      raise Import::ImportInProgressError if Import::is_import_running?
      raise Export::ExportInProgressError if Export::is_export_running?

      @format = args[:format] || :json

      @output_base_filename = File.absolute_path( args[:filename] || File.join( Rails.root, 'tmp', "export-#{Time.now.strftime('%Y-%m-%d-%H%M%S')}" ) )
      @output_base_filename = @output_base_filename[0...-3] if @output_base_filename[-3..-1] == '.gz'
      @output_base_filename = @output_base_filename[0...-4] if @output_base_filename[-4..-1] == '.tar'

      @user = args[:user_id] ? User.where(id: args[:user_id].to_i).first : nil

      start_export
      @encoder.write_schema_info( source: 'discourse', version: Export.current_schema_version )
      ordered_models_for_export.each do |model|
        log "  #{model.table_name}"
        column_info = model.columns
        column_names = model.column_names

        results = model.connection.raw_connection.async_exec("select * from #{model.table_name}").to_enum

        @encoder.write_table(model.table_name, column_info) do |num_rows_written|
          log("#{num_rows_written} rows written") if num_rows_written > 0

          rows = []
          begin
            while rows.count < batch_size
              row = results.next
              rows << column_names.map{|col| row[col]}
            end
          rescue StopIteration
            # we are done
          end

          rows
        end
      end
      "#{@output_base_filename}.tar.gz"
    ensure
      finish_export
    end

    def ordered_models_for_export
      Export.models_included_in_export
    end

    def order_columns_for(model)
      @order_columns_for_hash ||= {
        'CategoryFeaturedTopic' => 'category_id, topic_id',
        'CategorySearchData'    => 'category_id',
        'PostOneboxRender'      => 'post_id, onebox_render_id',
        'PostReply'             => 'post_id, reply_id',
        'PostSearchData'        => 'post_id',
        'PostTiming'            => 'topic_id, post_number, user_id',
        'SiteContent'           => 'content_type',
        'TopicUser'             => 'topic_id, user_id',
        'UserSearchData'        => 'user_id',
        'UserStat'              => 'user_id',
        'View'                  => 'parent_id, parent_type, ip_address, viewed_at'
      }
      @order_columns_for_hash[model.name]
    end

    def batch_size
      1000
    end

    def start_export
      if @format == :json
        @encoder = Export::JsonEncoder.new
      else
        raise Export::FormatInvalidError
      end
      Export.set_export_started
      Discourse.enable_maintenance_mode
    end

    def finish_export
      if @encoder
        @encoder.finish
        create_tar_file
        @encoder.remove_tmp_directory('export')
      end
    ensure
      Export.set_export_is_not_running
      Discourse.disable_maintenance_mode
      send_notification
    end

    def create_tar_file
      filenames = @encoder.filenames
      tar_filename = "#{@output_base_filename}.tar"
      upload_directory = "uploads/" + RailsMultisite::ConnectionManagement.current_db

      FileUtils.cd(File.join(Rails.root, 'public')) do
        `tar cvf #{tar_filename} #{upload_directory}`
      end

      filenames.each do |filename|
        FileUtils.cd(File.dirname(filename)) do
          `tar --append --file=#{tar_filename} #{File.basename(filename)}`
        end
      end


      `gzip #{tar_filename}`

      true
    end

    def send_notification
      SystemMessage.new(@user).create('export_succeeded') if @user
      true
    end

  end

end