discourse/spec/components/imap/sync_spec.rb

# frozen_string_literal: true

require 'rails_helper'
require 'imap/sync'
require_relative 'imap_helper'

describe Imap::Sync do

  before do
    SiteSetting.tagging_enabled = true
    SiteSetting.allow_staff_to_tag_pms = true

    SiteSetting.enable_imap = true

    Jobs.run_immediately!
  end

  let(:group) do
    Fabricate(
      :group,
      imap_server: 'imap.gmail.com',
      imap_port: 993,
      email_username: 'discourse@example.com',
      email_password: 'discourse@example.com',
      imap_mailbox_name: '[Gmail]/All Mail'
    )
  end

  let(:sync_handler) { Imap::Sync.new(group) }

  before do
    mocked_imap_provider = MockedImapProvider.new(
      group.imap_server,
      port: group.imap_port,
      ssl: group.imap_ssl,
      username: group.email_username,
      password: group.email_password
    )
    Imap::Providers::Detector.stubs(:init_with_detected_provider).returns(
      mocked_imap_provider
    )
  end

  context 'no previous sync' do
    let(:from) { 'john@free.fr' }
    let(:subject) { 'Testing email post' }
    let(:message_id) { "#{SecureRandom.hex}@example.com" }

    let(:email) do
      EmailFabricator(
        from: from,
        to: group.email_username,
        subject: subject,
        message_id: message_id)
    end

    before do
      provider = MockedImapProvider.any_instance
      provider.stubs(:open_mailbox).returns(uid_validity: 1)
      provider.stubs(:uids).with.returns([100])
      provider.stubs(:uids).with(to: 100).returns([100])
      provider.stubs(:uids).with(from: 101).returns([])
      provider.stubs(:emails).returns(
        [
          {
            'UID' => 100,
            'LABELS' => %w[\\Important test-label],
            'FLAGS' => %i[Seen],
            'RFC822' => email
          }
        ]
      )
    end

    it 'creates a topic from an incoming email' do
      expect { sync_handler.process }
        .to change { Topic.count }.by(1)
        .and change { Post.where(post_type: Post.types[:regular]).count }.by(1)
        .and change { IncomingEmail.count }.by(1)

      expect(group.imap_uid_validity).to eq(1)
      expect(group.imap_last_uid).to eq(100)

      topic = Topic.last
      expect(topic.title).to eq(subject)
      expect(topic.user.email).to eq(from)
      expect(topic.tags.pluck(:name)).to contain_exactly("seen", "important", "test-label")

      post = topic.first_post
      expect(post.raw).to eq('This is an email *body*. :smile:')

      incoming_email = post.incoming_email
      expect(incoming_email.raw.lines.map(&:strip)).to eq(email.lines.map(&:strip))
      expect(incoming_email.message_id).to eq(message_id)
      expect(incoming_email.from_address).to eq(from)
      expect(incoming_email.to_addresses).to eq(group.email_username)
      expect(incoming_email.imap_uid_validity).to eq(1)
      expect(incoming_email.imap_uid).to eq(100)
      expect(incoming_email.imap_sync).to eq(false)
      expect(incoming_email.imap_group_id).to eq(group.id)
    end

    context "when tagging not enabled" do
      before do
        SiteSetting.tagging_enabled = false
        SiteSetting.allow_staff_to_tag_pms = false
      end

      it "creates a topic from an incoming email but with no tags added" do
        expect { sync_handler.process }
          .to change { Topic.count }.by(1)
          .and change { Post.where(post_type: Post.types[:regular]).count }.by(1)
          .and change { IncomingEmail.count }.by(1)

        expect(group.imap_uid_validity).to eq(1)
        expect(group.imap_last_uid).to eq(100)

        topic = Topic.last
        expect(topic.title).to eq(subject)
        expect(topic.user.email).to eq(from)
        expect(topic.tags).to eq([])
      end
    end

    it 'does not duplicate topics' do
      expect { sync_handler.process }
        .to change { Topic.count }.by(1)
        .and change { Post.where(post_type: Post.types[:regular]).count }.by(1)
        .and change { IncomingEmail.count }.by(1)

      expect { sync_handler.process }
        .to change { Topic.count }.by(0)
        .and change { Post.where(post_type: Post.types[:regular]).count }.by(0)
        .and change { IncomingEmail.count }.by(0)
    end

    it 'creates a new incoming email if the message ID does not match the receiver post id regex' do
      incoming_email = Fabricate(:incoming_email, message_id: message_id)

      expect { sync_handler.process }
        .to change { Topic.count }.by(1)
        .and change { Post.where(post_type: Post.types[:regular]).count }.by(1)
        .and change { IncomingEmail.count }.by(1)

      last_incoming = IncomingEmail.where(message_id: message_id).last
      expect(last_incoming.message_id).to eq(message_id)
      expect(last_incoming.imap_uid_validity).to eq(1)
      expect(last_incoming.imap_uid).to eq(100)
      expect(last_incoming.imap_sync).to eq(false)
      expect(last_incoming.imap_group_id).to eq(group.id)
    end

    context "when the message id matches the receiver post id regex" do
      let(:message_id) { "topic/999/324@test.localhost" }
      it 'does not duplicate incoming email' do
        incoming_email = Fabricate(:incoming_email, message_id: message_id)

        expect { sync_handler.process }
          .to change { Topic.count }.by(0)
          .and change { Post.where(post_type: Post.types[:regular]).count }.by(0)
          .and change { IncomingEmail.count }.by(0)

        incoming_email.reload
        expect(incoming_email.message_id).to eq(message_id)
        expect(incoming_email.imap_uid_validity).to eq(1)
        expect(incoming_email.imap_uid).to eq(100)
        expect(incoming_email.imap_sync).to eq(false)
        expect(incoming_email.imap_group_id).to eq(group.id)
      end
    end
  end

  context 'previous sync' do
    let(:subject) { 'Testing email post' }

    let(:first_from) { 'john@free.fr' }
    let(:first_message_id) { SecureRandom.hex }
    let(:first_body) { 'This is the first message of this exchange.' }

    let(:second_from) { 'sam@free.fr' }
    let(:second_message_id) { SecureRandom.hex }
    let(:second_body) { '<p>This is an <b>answer</b> to this message.</p>' }

    it 'continues with new emails' do
      provider = MockedImapProvider.any_instance
      provider.stubs(:open_mailbox).returns(uid_validity: 1)

      provider.stubs(:uids).with.returns([100])
      provider.stubs(:emails).with([100], ['UID', 'FLAGS', 'LABELS', 'RFC822'], anything).returns(
        [
          {
            'UID' => 100,
            'LABELS' => %w[\\Inbox],
            'FLAGS' => %i[Seen],
            'RFC822' => EmailFabricator(
              message_id: first_message_id,
              from: first_from,
              to: group.email_username,
              cc: second_from,
              subject: subject,
              body: first_body
            )
          }
        ]
      )

      expect { sync_handler.process }
        .to change { Topic.count }.by(1)
        .and change { Post.where(post_type: Post.types[:regular]).count }.by(1)
        .and change { IncomingEmail.count }.by(1)

      topic = Topic.last
      expect(topic.title).to eq(subject)
      expect(GroupArchivedMessage.where(topic_id: topic.id).exists?).to eq(false)

      post = Post.where(post_type: Post.types[:regular]).last
      expect(post.user.email).to eq(first_from)
      expect(post.raw).to eq(first_body)
      expect(group.imap_uid_validity).to eq(1)
      expect(group.imap_last_uid).to eq(100)

      provider.stubs(:uids).with(to: 100).returns([100])
      provider.stubs(:uids).with(from: 101).returns([200])
      provider.stubs(:emails).with([100], ['UID', 'FLAGS', 'LABELS', 'ENVELOPE'], anything).returns(
        [
          {
            'UID' => 100,
            'LABELS' => %w[\\Inbox],
            'FLAGS' => %i[Seen]
          }
        ]
      )
      provider.stubs(:emails).with([200], ['UID', 'FLAGS', 'LABELS', 'RFC822'], anything).returns(
        [
          {
            'UID' => 200,
            'LABELS' => %w[\\Inbox],
            'FLAGS' => %i[Recent],
            'RFC822' => EmailFabricator(
              message_id: SecureRandom.hex,
              in_reply_to: first_message_id,
              from: second_from,
              to: group.email_username,
              subject: "Re: #{subject}",
              body: second_body
            )
          }
        ]
      )

      expect { sync_handler.process }
        .to change { Topic.count }.by(0)
        .and change { Post.where(post_type: Post.types[:regular]).count }.by(1)
        .and change { IncomingEmail.count }.by(1)

      post = Post.where(post_type: Post.types[:regular]).last
      expect(post.user.email).to eq(second_from)
      expect(post.raw).to eq(second_body)
      expect(group.imap_uid_validity).to eq(1)
      expect(group.imap_last_uid).to eq(200)

      provider.stubs(:uids).with(to: 200).returns([100, 200])
      provider.stubs(:uids).with(from: 201).returns([])
      provider.stubs(:emails).with([100, 200], ['UID', 'FLAGS', 'LABELS', 'ENVELOPE'], anything).returns(
        [
          {
            'UID' => 100,
            'LABELS' => %w[],
            'FLAGS' => %i[Seen]
          },
          {
            'UID' => 200,
            'LABELS' => %w[],
            'FLAGS' => %i[Recent],
          }
        ]
      )

      expect { sync_handler.process }
        .to change { Topic.count }.by(0)
        .and change { Post.where(post_type: Post.types[:regular]).count }.by(0)
        .and change { IncomingEmail.count }.by(0)

      topic = Topic.last
      expect(topic.title).to eq(subject)
      expect(GroupArchivedMessage.where(topic_id: topic.id).exists?).to eq(true)

      expect(Topic.last.posts.where(post_type: Post.types[:regular]).count).to eq(2)
    end

    describe "detecting deleted emails and deleting the topic in discourse" do
      let(:provider) { MockedImapProvider.any_instance }
      before do
        provider.stubs(:open_mailbox).returns(uid_validity: 1)

        provider.stubs(:uids).with.returns([100])
        provider.stubs(:emails).with([100], ['UID', 'FLAGS', 'LABELS', 'RFC822'], anything).returns(
          [
            {
              'UID' => 100,
              'LABELS' => %w[\\Inbox],
              'FLAGS' => %i[Seen],
              'RFC822' => EmailFabricator(
                message_id: first_message_id,
                from: first_from,
                to: group.email_username,
                cc: second_from,
                subject: subject,
                body: first_body
              )
            }
          ]
        )

      end

      it "detects previously synced UIDs are missing and deletes the posts if they are in the trash mailbox" do
        sync_handler.process
        incoming_100 = IncomingEmail.find_by(imap_uid: 100)
        provider.stubs(:uids).with.returns([])

        provider.stubs(:uids).with(to: 100).returns([])
        provider.stubs(:uids).with(from: 101).returns([])
        provider.stubs(:find_spam_by_message_ids).returns(stub(spam_emails: []))
        provider.stubs(:find_trashed_by_message_ids).returns(
          stub(
            trashed_emails: [
              stub(
                uid: 10,
                message_id: incoming_100.message_id
              )
            ],
            trash_uid_validity: 99
          )
        )
        sync_handler.process

        incoming_100.reload
        expect(incoming_100.imap_uid_validity).to eq(99)
        expect(incoming_100.imap_uid).to eq(10)
        expect(Post.with_deleted.find(incoming_100.post_id).deleted_at).not_to eq(nil)
        expect(Topic.with_deleted.find(incoming_100.topic_id).deleted_at).not_to eq(nil)
      end

      it "detects previously synced UIDs are missing and deletes the posts if they are in the spam/junk mailbox" do
        sync_handler.process
        incoming_100 = IncomingEmail.find_by(imap_uid: 100)
        provider.stubs(:uids).with.returns([])

        provider.stubs(:uids).with(to: 100).returns([])
        provider.stubs(:uids).with(from: 101).returns([])
        provider.stubs(:find_trashed_by_message_ids).returns(stub(trashed_emails: []))
        provider.stubs(:find_spam_by_message_ids).returns(
          stub(
            spam_emails: [
              stub(
                uid: 10,
                message_id: incoming_100.message_id
              )
            ],
            spam_uid_validity: 99
          )
        )
        sync_handler.process

        incoming_100.reload
        expect(incoming_100.imap_uid_validity).to eq(99)
        expect(incoming_100.imap_uid).to eq(10)
        expect(Post.with_deleted.find(incoming_100.post_id).deleted_at).not_to eq(nil)
        expect(Topic.with_deleted.find(incoming_100.topic_id).deleted_at).not_to eq(nil)
      end

      it "marks the incoming email as IMAP missing if it cannot be found in spam or trash" do
        sync_handler.process
        incoming_100 = IncomingEmail.find_by(imap_uid: 100)
        provider.stubs(:uids).with.returns([])

        provider.stubs(:uids).with(to: 100).returns([])
        provider.stubs(:uids).with(from: 101).returns([])
        provider.stubs(:find_trashed_by_message_ids).returns(stub(trashed_emails: []))
        provider.stubs(:find_spam_by_message_ids).returns(stub(spam_emails: []))
        sync_handler.process

        incoming_100.reload
        expect(incoming_100.imap_missing).to eq(true)
      end

      it "detects the topic being deleted on the discourse site and deletes on the IMAP server and
      does not attempt to delete again on discourse site when deleted already by us on the IMAP server" do
        SiteSetting.enable_imap_write = true
        sync_handler.process
        incoming_100 = IncomingEmail.find_by(imap_uid: 100)
        provider.stubs(:uids).with.returns([100])

        provider.stubs(:uids).with(to: 100).returns([100])
        provider.stubs(:uids).with(from: 101).returns([])

        PostDestroyer.new(Discourse.system_user, incoming_100.post).destroy
        provider.stubs(:emails).with([100], ['UID', 'FLAGS', 'LABELS', 'ENVELOPE'], anything).returns(
          [
            {
              'UID' => 100,
              'LABELS' => %w[\\Inbox],
              'FLAGS' => %i[Seen],
              'RFC822' => EmailFabricator(
                message_id: first_message_id,
                from: first_from,
                to: group.email_username,
                cc: second_from,
                subject: subject,
                body: first_body
              )
            }
          ]
        )
        provider.stubs(:emails).with(100, ['FLAGS', 'LABELS']).returns(
          [
            {
              'LABELS' => %w[\\Inbox],
              'FLAGS' => %i[Seen]
            }
          ]
        )

        provider.expects(:trash).with(100)
        sync_handler.process

        provider.stubs(:uids).with.returns([])

        provider.stubs(:uids).with(to: 100).returns([])
        provider.stubs(:uids).with(from: 101).returns([])
        provider.stubs(:find_spam_by_message_ids).returns(stub(spam_emails: []))
        provider.stubs(:find_trashed_by_message_ids).returns(
          stub(
            trashed_emails: [
              stub(
                uid: 10,
                message_id: incoming_100.message_id
              )
            ],
            trash_uid_validity: 99
          )
        )
        PostDestroyer.expects(:new).never

        sync_handler.process

        incoming_100.reload
        expect(incoming_100.imap_uid_validity).to eq(99)
        expect(incoming_100.imap_uid).to eq(10)
      end
    end

    describe "archiving emails" do
      let(:provider) { MockedImapProvider.any_instance }
      before do
        SiteSetting.enable_imap_write = true
        provider.stubs(:open_mailbox).returns(uid_validity: 1)

        provider.stubs(:uids).with.returns([100])
        provider.stubs(:emails).with([100], ['UID', 'FLAGS', 'LABELS', 'RFC822'], anything).returns(
          [
            {
              'UID' => 100,
              'LABELS' => %w[\\Inbox],
              'FLAGS' => %i[Seen],
              'RFC822' => EmailFabricator(
                message_id: first_message_id,
                from: first_from,
                to: group.email_username,
                cc: second_from,
                subject: subject,
                body: first_body
              )
            }
          ]
        )

        sync_handler.process
        @incoming_email = IncomingEmail.find_by(message_id: first_message_id)
        @topic = @incoming_email.topic

        provider.stubs(:uids).with(to: 100).returns([100])
        provider.stubs(:uids).with(from: 101).returns([101])
        provider.stubs(:emails).with([100], ['UID', 'FLAGS', 'LABELS', 'ENVELOPE'], anything).returns(
          [
            {
              'UID' => 100,
              'LABELS' => %w[\\Inbox],
              'FLAGS' => %i[Seen]
            }
          ]
        )
        provider.stubs(:emails).with([101], ['UID', 'FLAGS', 'LABELS', 'RFC822'], anything).returns(
          []
        )
        provider.stubs(:emails).with(100, ['FLAGS', 'LABELS']).returns(
          [
            {
              'LABELS' => %w[\\Inbox],
              'FLAGS' => %i[Seen]
            }
          ]
        )
      end

      it "archives an email on the IMAP server when archived in discourse" do
        GroupArchivedMessage.archive!(group.id, @topic, skip_imap_sync: false)
        @incoming_email.update(imap_sync: true)

        provider.stubs(:store).with(100, 'FLAGS', anything, anything)
        provider.stubs(:store).with(100, 'LABELS', ["\\Inbox"], ["seen"])

        provider.expects(:archive).with(100)
        sync_handler.process
      end

      it "does not archive email if not archived in discourse, it unarchives it instead" do
        @incoming_email.update(imap_sync: true)
        provider.stubs(:store).with(100, 'FLAGS', anything, anything)
        provider.stubs(:store).with(100, 'LABELS', ["\\Inbox"], ["\\Inbox", "seen"])

        provider.expects(:archive).with(100).never
        provider.expects(:unarchive).with(100)
        sync_handler.process
      end
    end

  end

  context 'invaidated previous sync' do
    let(:subject) { 'Testing email post' }

    let(:first_from) { 'john@free.fr' }
    let(:first_message_id) { SecureRandom.hex }
    let(:first_body) { 'This is the first message of this exchange.' }

    let(:second_from) { 'sam@free.fr' }
    let(:second_message_id) { SecureRandom.hex }
    let(:second_body) { '<p>This is an <b>answer</b> to this message.</p>' }

    # TODO: Improve the invalidating flow for mailbox change. This is a destructive
    # action so it should not be done often.
    xit 'is updated' do
      provider = MockedImapProvider.any_instance

      provider.stubs(:open_mailbox).returns(uid_validity: 1)
      provider.stubs(:uids).with.returns([100, 200])
      provider.stubs(:emails).with([100, 200], ['UID', 'FLAGS', 'LABELS', 'RFC822'], anything).returns(
        [
          {
            'UID' => 100,
            'LABELS' => %w[\\Inbox],
            'FLAGS' => %i[Seen],
            'RFC822' => EmailFabricator(
              message_id: first_message_id,
              from: first_from,
              to: group.email_username,
              cc: second_from,
              subject: subject,
              body: first_body
            )
          },
          {
            'UID' => 200,
            'LABELS' => %w[\\Inbox],
            'FLAGS' => %i[Recent],
            'RFC822' => EmailFabricator(
              message_id: second_message_id,
              in_reply_to: first_message_id,
              from: second_from,
              to: group.email_username,
              subject: "Re: #{subject}",
              body: second_body
            )
          }
        ]
      )

      expect { sync_handler.process }
        .to change { Topic.count }.by(1)
        .and change { Post.where(post_type: Post.types[:regular]).count }.by(2)
        .and change { IncomingEmail.count }.by(2)

      imap_data = Topic.last.incoming_email.pluck(:imap_uid_validity, :imap_uid, :imap_group_id)
      expect(imap_data).to contain_exactly([1, 100, group.id], [1, 200, group.id])

      provider.stubs(:open_mailbox).returns(uid_validity: 2)
      provider.stubs(:uids).with.returns([111, 222])
      provider.stubs(:emails).with([111, 222], ['UID', 'FLAGS', 'LABELS', 'RFC822'], anything).returns(
        [
          {
            'UID' => 111,
            'LABELS' => %w[\\Inbox],
            'FLAGS' => %i[Seen],
            'RFC822' => EmailFabricator(
              message_id: first_message_id,
              from: first_from,
              to: group.email_username,
              cc: second_from,
              subject: subject,
              body: first_body
            )
          },
          {
            'UID' => 222,
            'LABELS' => %w[\\Inbox],
            'FLAGS' => %i[Recent],
            'RFC822' => EmailFabricator(
              message_id: second_message_id,
              in_reply_to: first_message_id,
              from: second_from,
              to: group.email_username,
              subject: "Re: #{subject}",
              body: second_body
            )
          }
        ]
      )

      expect { sync_handler.process }
        .to change { Topic.count }.by(0)
        .and change { Post.where(post_type: Post.types[:regular]).count }.by(0)
        .and change { IncomingEmail.count }.by(0)

      imap_data = Topic.last.incoming_email.pluck(:imap_uid_validity, :imap_uid, :imap_group_id)
      expect(imap_data).to contain_exactly([2, 111, group.id], [2, 222, group.id])
    end
  end
end