2013-02-06 03:16:51 +08:00
|
|
|
require_dependency 'export/json_encoder'
|
|
|
|
require_dependency 'export/export'
|
|
|
|
require_dependency 'import/import'
|
|
|
|
|
|
|
|
module Jobs
|
|
|
|
|
|
|
|
class Exporter < Jobs::Base
|
|
|
|
|
2013-03-23 23:02:59 +08:00
|
|
|
sidekiq_options retry: false
|
2013-02-06 03:16:51 +08:00
|
|
|
|
|
|
|
def execute(args)
|
|
|
|
raise Import::ImportInProgressError if Import::is_import_running?
|
|
|
|
raise Export::ExportInProgressError if Export::is_export_running?
|
|
|
|
|
|
|
|
@format = args[:format] || :json
|
|
|
|
|
|
|
|
@output_base_filename = File.absolute_path( args[:filename] || File.join( Rails.root, 'tmp', "export-#{Time.now.strftime('%Y-%m-%d-%H%M%S')}" ) )
|
|
|
|
@output_base_filename = @output_base_filename[0...-3] if @output_base_filename[-3..-1] == '.gz'
|
|
|
|
@output_base_filename = @output_base_filename[0...-4] if @output_base_filename[-4..-1] == '.tar'
|
|
|
|
|
|
|
|
@user = args[:user_id] ? User.where(id: args[:user_id].to_i).first : nil
|
|
|
|
|
|
|
|
start_export
|
|
|
|
@encoder.write_schema_info( source: 'discourse', version: Export.current_schema_version )
|
|
|
|
ordered_models_for_export.each do |model|
|
|
|
|
log " #{model.table_name}"
|
|
|
|
column_info = model.columns
|
2014-01-29 13:49:01 +08:00
|
|
|
column_names = model.column_names
|
|
|
|
|
|
|
|
results = model.connection.raw_connection.async_exec("select * from #{model.table_name}").to_enum
|
|
|
|
|
2013-02-06 03:16:51 +08:00
|
|
|
@encoder.write_table(model.table_name, column_info) do |num_rows_written|
|
2014-01-29 13:49:01 +08:00
|
|
|
log("#{num_rows_written} rows written") if num_rows_written > 0
|
|
|
|
|
|
|
|
rows = []
|
|
|
|
begin
|
|
|
|
while rows.count < batch_size
|
|
|
|
row = results.next
|
|
|
|
rows << column_names.map{|col| row[col]}
|
|
|
|
end
|
|
|
|
rescue StopIteration
|
|
|
|
# we are done
|
2013-02-06 03:16:51 +08:00
|
|
|
end
|
2014-01-29 13:49:01 +08:00
|
|
|
|
|
|
|
rows
|
2013-02-06 03:16:51 +08:00
|
|
|
end
|
|
|
|
end
|
|
|
|
"#{@output_base_filename}.tar.gz"
|
|
|
|
ensure
|
|
|
|
finish_export
|
|
|
|
end
|
|
|
|
|
|
|
|
def ordered_models_for_export
|
|
|
|
Export.models_included_in_export
|
|
|
|
end
|
|
|
|
|
|
|
|
def order_columns_for(model)
|
|
|
|
@order_columns_for_hash ||= {
|
|
|
|
'CategoryFeaturedTopic' => 'category_id, topic_id',
|
2013-07-22 21:49:17 +08:00
|
|
|
'CategorySearchData' => 'category_id',
|
2013-02-06 03:16:51 +08:00
|
|
|
'PostOneboxRender' => 'post_id, onebox_render_id',
|
|
|
|
'PostReply' => 'post_id, reply_id',
|
2013-07-22 21:49:17 +08:00
|
|
|
'PostSearchData' => 'post_id',
|
2013-02-06 03:16:51 +08:00
|
|
|
'PostTiming' => 'topic_id, post_number, user_id',
|
2013-07-22 21:49:17 +08:00
|
|
|
'SiteContent' => 'content_type',
|
2013-02-06 03:16:51 +08:00
|
|
|
'TopicUser' => 'topic_id, user_id',
|
2013-07-22 21:49:17 +08:00
|
|
|
'UserSearchData' => 'user_id',
|
2013-11-11 23:32:31 +08:00
|
|
|
'UserStat' => 'user_id',
|
2013-07-10 02:58:49 +08:00
|
|
|
'View' => 'parent_id, parent_type, ip_address, viewed_at'
|
2013-02-06 03:16:51 +08:00
|
|
|
}
|
|
|
|
@order_columns_for_hash[model.name]
|
|
|
|
end
|
|
|
|
|
|
|
|
def batch_size
|
|
|
|
1000
|
|
|
|
end
|
|
|
|
|
|
|
|
def start_export
|
|
|
|
if @format == :json
|
|
|
|
@encoder = Export::JsonEncoder.new
|
|
|
|
else
|
|
|
|
raise Export::FormatInvalidError
|
|
|
|
end
|
|
|
|
Export.set_export_started
|
|
|
|
Discourse.enable_maintenance_mode
|
|
|
|
end
|
|
|
|
|
|
|
|
def finish_export
|
|
|
|
if @encoder
|
|
|
|
@encoder.finish
|
|
|
|
create_tar_file
|
2013-07-01 02:54:45 +08:00
|
|
|
@encoder.remove_tmp_directory('export')
|
2013-02-06 03:16:51 +08:00
|
|
|
end
|
|
|
|
ensure
|
|
|
|
Export.set_export_is_not_running
|
|
|
|
Discourse.disable_maintenance_mode
|
|
|
|
send_notification
|
|
|
|
end
|
|
|
|
|
|
|
|
def create_tar_file
|
|
|
|
filenames = @encoder.filenames
|
2014-01-03 00:47:19 +08:00
|
|
|
tar_filename = "#{@output_base_filename}.tar"
|
|
|
|
upload_directory = "uploads/" + RailsMultisite::ConnectionManagement.current_db
|
2013-02-06 03:16:51 +08:00
|
|
|
|
2014-01-03 00:47:19 +08:00
|
|
|
FileUtils.cd(File.join(Rails.root, 'public')) do
|
|
|
|
`tar cvf #{tar_filename} #{upload_directory}`
|
2013-02-06 03:16:51 +08:00
|
|
|
end
|
|
|
|
|
2014-01-29 13:49:01 +08:00
|
|
|
filenames.each do |filename|
|
|
|
|
FileUtils.cd(File.dirname(filename)) do
|
|
|
|
`tar --append --file=#{tar_filename} #{File.basename(filename)}`
|
|
|
|
end
|
2014-01-17 09:33:19 +08:00
|
|
|
end
|
|
|
|
|
|
|
|
|
2014-01-03 00:47:19 +08:00
|
|
|
`gzip #{tar_filename}`
|
2013-02-06 03:16:51 +08:00
|
|
|
|
|
|
|
true
|
|
|
|
end
|
|
|
|
|
|
|
|
def send_notification
|
|
|
|
SystemMessage.new(@user).create('export_succeeded') if @user
|
|
|
|
true
|
|
|
|
end
|
|
|
|
|
|
|
|
end
|
|
|
|
|
|
|
|
end
|