# frozen_string_literal: true require "csv" RSpec.describe Jobs::ExportUserArchive do fab!(:user) { Fabricate(:user, username: "john_doe", refresh_auto_groups: true) } fab!(:user2) { Fabricate(:user) } let(:extra) { {} } let(:job) do j = Jobs::ExportUserArchive.new j.current_user = user j.extra = extra j end let(:component) { raise "component not set" } fab!(:admin) fab!(:category) { Fabricate(:category_with_definition, name: "User Archive Category") } fab!(:subcategory) { Fabricate(:category_with_definition, parent_category_id: category.id) } fab!(:topic) { Fabricate(:topic, category: category) } let(:post) { Fabricate(:post, user: user, topic: topic) } def make_component_csv data_rows = [] csv_out = CSV.generate do |csv| csv << job.get_header(component) job.public_send(:"#{component}_export") do |row| csv << row data_rows << Jobs::ExportUserArchive::HEADER_ATTRS_FOR[component] .zip(row.map(&:to_s)) .to_h .with_indifferent_access end end [data_rows, csv_out] end def make_component_json JSON.parse(MultiJson.dump(job.public_send(:"#{component}_export"))) end describe "#execute" do before do _ = post user.user_profile.website = "https://doe.example.com/john" user.user_profile.save # force a UserAuthTokenLog entry env = create_request_env.merge( "HTTP_USER_AGENT" => "MyWebBrowser", "REQUEST_PATH" => "/some_path/456852", ) cookie_jar = ActionDispatch::Request.new(env).cookie_jar Discourse.current_user_provider.new(env).log_on_user(user, {}, cookie_jar) # force a nonstandard post action PostAction.new(user: user, post: post, post_action_type_id: 5).save end after { user.uploads.each(&:destroy!) } it "raises an error when the user is missing" do expect { Jobs::ExportCsvFile.new.execute(user_id: user.id + (1 << 20)) }.to raise_error( Discourse::InvalidParameters, ) end it "works" do expect do Jobs::ExportUserArchive.new.execute(user_id: user.id) end.to change { Upload.count }.by(1) system_message = user.topics_allowed.last expect(system_message.title).to eq( I18n.t( "system_messages.csv_export_succeeded.subject_template", export_title: "User Archive", ), ) upload = system_message.first_post.uploads.first expect(system_message.first_post.raw).to eq( I18n.t( "system_messages.csv_export_succeeded.text_body_template", download_link: "[#{upload.original_filename}|attachment](#{upload.short_url}) (#{upload.human_filesize})", ).chomp, ) expect(system_message.id).to eq(UserExport.last.topic_id) expect(system_message.closed).to eq(true) files = [] Zip::File.open(Discourse.store.path_for(upload)) do |zip_file| zip_file.each { |entry| files << entry.name } end expect(files.size).to eq(Jobs::ExportUserArchive::COMPONENTS.length) expect(files.find { |f| f == "user_archive.csv" }).to_not be_nil expect(files.find { |f| f == "category_preferences.csv" }).to_not be_nil end it "sends a message if it fails" do SiteSetting.max_export_file_size_kb = 1 expect do Jobs::ExportUserArchive.new.execute(user_id: user.id) end.not_to change { Upload.count } system_message = user.topics_allowed.last expect(system_message.title).to eq( I18n.t("system_messages.csv_export_failed.subject_template"), ) end end describe "user_archive posts" do let(:component) { "user_archive" } let(:subsubcategory) do Fabricate(:category_with_definition, parent_category_id: subcategory.id) end let(:subsubtopic) { Fabricate(:topic, category: subsubcategory) } let(:subsubpost) { Fabricate(:post, user: user, topic: subsubtopic) } let(:normal_post) { Fabricate(:post, user: user, topic: topic) } let(:reply) do PostCreator.new( user2, raw: "asdf1234qwert7896", topic_id: topic.id, reply_to_post_number: normal_post.post_number, ).create end let(:message) { Fabricate(:private_message_topic) } let(:message_post) { Fabricate(:post, user: user, topic: message) } it "properly exports posts" do SiteSetting.max_category_nesting = 3 [reply, subsubpost, message_post] PostActionCreator.like(user2, normal_post) rows = [] job.user_archive_export do |row| rows << Jobs::ExportUserArchive::HEADER_ATTRS_FOR["user_archive"].zip(row).to_h end expect(rows.length).to eq(3) post1 = rows.find { |r| r["topic_title"] == topic.title } post2 = rows.find { |r| r["topic_title"] == subsubtopic.title } post3 = rows.find { |r| r["topic_title"] == message.title } expect(post1["categories"]).to eq("#{category.name}") expect(post2["categories"]).to eq( "#{category.name}|#{subcategory.name}|#{subsubcategory.name}", ) expect(post3["categories"]).to eq("-") expect(post1["is_pm"]).to eq(I18n.t("csv_export.boolean_no")) expect(post2["is_pm"]).to eq(I18n.t("csv_export.boolean_no")) expect(post3["is_pm"]).to eq(I18n.t("csv_export.boolean_yes")) expect(post1["post_raw"]).to eq(normal_post.raw) expect(post2["post_raw"]).to eq(subsubpost.raw) expect(post3["post_raw"]).to eq(message_post.raw) expect(post1["post_cooked"]).to eq(normal_post.cooked) expect(post2["post_cooked"]).to eq(subsubpost.cooked) expect(post3["post_cooked"]).to eq(message_post.cooked) expect(post1["like_count"]).to eq(1) expect(post2["like_count"]).to eq(0) expect(post1["reply_count"]).to eq(1) expect(post2["reply_count"]).to eq(0) end it "can export a post from a deleted category" do cat2 = Fabricate(:category) topic2 = Fabricate(:topic, category: cat2, user: user) _post2 = Fabricate(:post, topic: topic2, user: user) cat2_id = cat2.id cat2.destroy! _, csv_out = make_component_csv expect(csv_out).to match cat2_id.to_s end it "can export a post from a secure category, obscuring the category name" do cat2 = Fabricate(:private_category, group: Fabricate(:group), name: "Secret Cat") topic2 = Fabricate(:topic, category: cat2, user: user, title: "This is a test secure topic") _post2 = Fabricate(:post, topic: topic2, user: user) data, csv_out = make_component_csv expect(csv_out).not_to match "Secret Cat" expect(data.length).to eq(1) expect(data[0][:topic_title]).to eq("This is a test secure topic") expect(data[0][:categories]).to eq("-") end end describe "preferences" do let(:component) { "preferences" } before do user.user_profile.website = "https://doe.example.com/john" user.user_profile.bio_raw = "I am John Doe\n\nHere I am" user.user_profile.save user.user_option.text_size = :smaller user.user_option.automatically_unpin_topics = false user.user_option.save end it "properly includes the profile fields" do _serializer = job.preferences_export # puts MultiJson.dump(serializer, indent: 4) output = make_component_json payload = output["user"] expect(payload["website"]).to match("doe.example.com") expect(payload["bio_raw"]).to match("Doe\n\nHere") expect(payload["user_option"]["automatically_unpin_topics"]).to eq(false) expect(payload["user_option"]["text_size"]).to eq("smaller") end end describe "auth tokens" do let(:component) { "auth_tokens" } before do env = create_request_env.merge( "HTTP_USER_AGENT" => "MyWebBrowser", "REQUEST_PATH" => "/some_path/456852", ) cookie_jar = ActionDispatch::Request.new(env).cookie_jar Discourse.current_user_provider.new(env).log_on_user(user, {}, cookie_jar) end it "properly includes session records" do data, _csv_out = make_component_csv expect(data.length).to eq(1) expect(data[0]["user_agent"]).to eq("MyWebBrowser") end context "with auth token logs" do let(:component) { "auth_token_logs" } it "includes details such as the path" do data, _csv_out = make_component_csv expect(data.length).to eq(1) expect(data[0]["action"]).to eq("generate") expect(data[0]["path"]).to eq("/some_path/456852") end end end describe "badges" do let(:component) { "badges" } let(:badge1) { Fabricate(:badge) } let(:badge2) { Fabricate(:badge, multiple_grant: true) } let(:badge3) { Fabricate(:badge, multiple_grant: true) } let(:day_ago) { 1.day.ago } it "properly includes badge records" do grant_start = Time.now.utc BadgeGranter.grant(badge1, user) BadgeGranter.grant(badge2, user) BadgeGranter.grant(badge2, user, granted_by: admin) BadgeGranter.grant(badge3, user, post_id: Fabricate(:post).id) BadgeGranter.grant(badge3, user, post_id: Fabricate(:post).id) BadgeGranter.grant(badge3, user, post_id: Fabricate(:post).id) data, _csv_out = make_component_csv expect(data.length).to eq(6) expect(data[0]["badge_id"]).to eq(badge1.id.to_s) expect(data[0]["badge_name"]).to eq(badge1.display_name) expect(data[0]["featured_rank"]).to_not eq("") expect(DateTime.parse(data[0]["granted_at"])).to be >= DateTime.parse(grant_start.to_s) expect(data[2]["granted_manually"]).to eq("true") expect(Post.find(data[3]["post_id"])).to_not be_nil end end describe "bookmarks" do let(:component) { "bookmarks" } let(:name) { "Collect my thoughts on this" } let(:manager) { BookmarkManager.new(user) } let(:topic1) { Fabricate(:topic) } let(:post1) { Fabricate(:post, topic: topic1, post_number: 5) } let(:post2) { Fabricate(:post) } let(:post3) { Fabricate(:post) } let(:private_message_topic) { Fabricate(:private_message_topic) } let(:post4) { Fabricate(:post, topic: private_message_topic) } let(:reminder_at) { 1.day.from_now } it "properly includes bookmark records" do now = freeze_time "2017-03-01 12:00" bookmark1 = manager.create_for(bookmarkable_id: post1.id, bookmarkable_type: "Post", name: name) update1_at = now + 1.hours bookmark1.update(name: "great food recipe", updated_at: update1_at) manager.create_for( bookmarkable_id: post2.id, bookmarkable_type: "Post", name: name, reminder_at: reminder_at, options: { auto_delete_preference: Bookmark.auto_delete_preferences[:when_reminder_sent], }, ) twelve_hr_ago = freeze_time now - 12.hours pending_reminder = manager.create_for( bookmarkable_id: post3.id, bookmarkable_type: "Post", name: name, reminder_at: now - 8.hours, ) freeze_time now tau_record = private_message_topic.topic_allowed_users.create!(user_id: user.id) manager.create_for(bookmarkable_id: post4.id, bookmarkable_type: "Post", name: name) tau_record.destroy! BookmarkReminderNotificationHandler.new(pending_reminder).send_notification data, _csv_out = make_component_csv expect(data.length).to eq(4) expect(data[0]["bookmarkable_id"]).to eq(post1.id.to_s) expect(data[0]["bookmarkable_type"]).to eq("Post") expect(data[0]["link"]).to eq(post1.full_url) expect(DateTime.parse(data[0]["updated_at"])).to eq(DateTime.parse(update1_at.to_s)) expect(data[1]["name"]).to eq(name) expect(DateTime.parse(data[1]["reminder_at"])).to eq(DateTime.parse(reminder_at.to_s)) expect(data[1]["auto_delete_preference"]).to eq("when_reminder_sent") expect(DateTime.parse(data[2]["created_at"])).to eq(DateTime.parse(twelve_hr_ago.to_s)) expect(DateTime.parse(data[2]["reminder_last_sent_at"])).to eq(DateTime.parse(now.to_s)) expect(data[2]["reminder_set_at"]).to eq("") expect(data[3]["bookmarkable_id"]).to eq(post4.id.to_s) expect(data[3]["bookmarkable_type"]).to eq("Post") expect(data[3]["link"]).to eq("") end end describe "category_preferences" do let(:component) { "category_preferences" } let(:subsubcategory) do Fabricate( :category_with_definition, parent_category_id: subcategory.id, name: "User Archive Subcategory", ) end let(:announcements) { Fabricate(:category_with_definition, name: "Announcements") } let(:deleted_category) { Fabricate(:category, name: "Deleted Category") } let(:secure_category_group) { Fabricate(:group) } let(:secure_category) do Fabricate(:private_category, group: secure_category_group, name: "Super Secret Category") end let(:reset_at) { DateTime.parse("2017-03-01 12:00") } before do SiteSetting.max_category_nesting = 3 # TopicsController#reset-new?category_id=&include_subcategories=true category_ids = [subcategory.id, subsubcategory.id] category_ids.each do |category_id| user .category_users .where(category_id: category_id) .first_or_initialize .update!(last_seen_at: reset_at, notification_level: NotificationLevels.all[:regular]) end # Set Watching First Post on announcements, Tracking on subcategory, Muted on deleted, nothing on subsubcategory CategoryUser.set_notification_level_for_category( user, NotificationLevels.all[:watching_first_post], announcements.id, ) CategoryUser.set_notification_level_for_category( user, NotificationLevels.all[:tracking], subcategory.id, ) CategoryUser.set_notification_level_for_category( user, NotificationLevels.all[:muted], deleted_category.id, ) deleted_category.destroy! end it "correctly exports the CategoryUser table, excluding deleted categories" do data, _csv_out = make_component_csv expect(data.find { |r| r["category_id"] == category.id.to_s }).to be_nil expect(data.find { |r| r["category_id"] == deleted_category.id.to_s }).to be_nil expect(data.length).to eq(3) data.sort! { |a, b| a["category_id"].to_i <=> b["category_id"].to_i } expect(data[0][:category_id]).to eq(subcategory.id.to_s) expect(data[0][:notification_level].to_s).to eq("tracking") expect(DateTime.parse(data[0][:dismiss_new_timestamp])).to eq(reset_at) expect(data[1][:category_id]).to eq(subsubcategory.id.to_s) expect(data[1][:category_names]).to eq( "#{category.name}|#{subcategory.name}|#{subsubcategory.name}", ) expect(data[1][:notification_level]).to eq("regular") expect(DateTime.parse(data[1][:dismiss_new_timestamp])).to eq(reset_at) expect(data[2][:category_id]).to eq(announcements.id.to_s) expect(data[2][:category_names]).to eq(announcements.name) expect(data[2][:notification_level]).to eq("watching_first_post") expect(data[2][:dismiss_new_timestamp]).to eq("") end it "does not include any secure categories the user does not have access to, even if the user has a CategoryUser record" do CategoryUser.set_notification_level_for_category( user, NotificationLevels.all[:muted], secure_category.id, ) data, _csv_out = make_component_csv expect(data.any? { |r| r["category_id"] == secure_category.id.to_s }).to eq(false) expect(data.length).to eq(3) end it "does include secure categories that the user has access to" do CategoryUser.set_notification_level_for_category( user, NotificationLevels.all[:muted], secure_category.id, ) GroupUser.create!(user: user, group: secure_category_group) data, _csv_out = make_component_csv expect(data.any? { |r| r["category_id"] == secure_category.id.to_s }).to eq(true) expect(data.length).to eq(4) end end describe "flags" do let(:component) { "flags" } let(:other_post) { Fabricate(:post, user: admin) } let(:post3) { Fabricate(:post) } let(:post4) { Fabricate(:post) } it "correctly exports flags" do result0 = PostActionCreator.notify_moderators(user, other_post, "helping out the admins") PostActionCreator.spam(user, post3) PostActionDestroyer.destroy(user, post3, :spam) PostActionCreator.inappropriate(user, post3) result3 = PostActionCreator.off_topic(user, post4) result3.reviewable.perform(admin, :agree_and_keep) data, _csv_out = make_component_csv expect(data.length).to eq(4) data.sort_by! { |row| row["post_id"].to_i } expect(data[0]["post_id"]).to eq(other_post.id.to_s) expect(data[0]["flag_type"]).to eq("notify_moderators") expect(data[0]["related_post_id"]).to eq(result0.post_action.related_post_id.to_s) expect(data[1]["flag_type"]).to eq("spam") expect(data[2]["flag_type"]).to eq("inappropriate") expect(data[1]["deleted_at"]).to_not be_empty expect(data[1]["deleted_by"]).to eq("self") expect(data[2]["deleted_at"]).to be_empty expect(data[3]["post_id"]).to eq(post4.id.to_s) expect(data[3]["flag_type"]).to eq("off_topic") expect(data[3]["deleted_at"]).to be_empty end end describe "likes" do let(:component) { "likes" } let(:other_post) { Fabricate(:post, user: admin) } let(:post3) { Fabricate(:post) } it "correctly exports likes" do PostActionCreator.like(user, other_post) PostActionCreator.like(user, post3) PostActionCreator.like(admin, post3) PostActionDestroyer.destroy(user, post3, :like) post3.destroy! data, _csv_out = make_component_csv expect(data.length).to eq(2) data.sort_by! { |row| row["post_id"].to_i } expect(data[0]["post_id"]).to eq(other_post.id.to_s) expect(data[1]["post_id"]).to eq(post3.id.to_s) expect(data[1]["deleted_at"]).to_not be_empty expect(data[1]["deleted_by"]).to eq("self") end end describe "queued posts" do let(:component) { "queued_posts" } let(:reviewable_post) do Fabricate(:reviewable_queued_post, topic: topic, target_created_by: user) end let(:reviewable_topic) do Fabricate(:reviewable_queued_post_topic, category: category, target_created_by: user) end it "correctly exports queued posts" do SiteSetting.tagging_enabled = true reviewable_post.perform(admin, :reject_post) reviewable_topic.payload["tags"] = ["example_tag"] result = reviewable_topic.perform(admin, :approve_post) expect(result.success?).to eq(true) data, csv_out = make_component_csv expect(data.length).to eq(2) expect(csv_out).to_not match(admin.username) approved = data.find { |el| el["verdict"] === "approved" } rejected = data.find { |el| el["verdict"] === "rejected" } expect(approved["other_json"]).to match("example_tag") expect(rejected["post_raw"]).to eq("hello world post contents.") expect(rejected["other_json"]).to match("reply_to_post_number") end end describe "visits" do let(:component) { "visits" } it "correctly exports the UserVisit table" do freeze_time "2017-03-01 12:00" UserVisit.create( user_id: user.id, visited_at: 1.minute.ago, posts_read: 1, mobile: false, time_read: 10, ) UserVisit.create( user_id: user.id, visited_at: 2.days.ago, posts_read: 2, mobile: false, time_read: 20, ) UserVisit.create( user_id: user.id, visited_at: 1.week.ago, posts_read: 3, mobile: true, time_read: 30, ) UserVisit.create( user_id: user.id, visited_at: 1.year.ago, posts_read: 4, mobile: false, time_read: 40, ) UserVisit.create( user_id: user2.id, visited_at: 1.minute.ago, posts_read: 1, mobile: false, time_read: 50, ) data, _csv_out = make_component_csv # user2's data is not mixed in expect(data.length).to eq(4) expect(data.find { |r| r["time_read"] == 50 }).to be_nil expect(data[0]["visited_at"]).to eq("2016-03-01") expect(data[0]["posts_read"]).to eq("4") expect(data[0]["time_read"]).to eq("40") expect(data[1]["mobile"]).to eq("true") expect(data[3]["visited_at"]).to eq("2017-03-01") end end end