FEATURE: Create hidden posts for received spam emails (#6010)

* Add possibility to add hidden posts with PostCreator

* FEATURE: Create hidden posts for received spam emails

Spamchecker usually have 3 results: HAM, SPAM and PROBABLY_SPAM
SPAM gets usually directly rejected and needs no further handling.
HAM is good message and usually gets passed unmodified.
PROBABLY_SPAM gets an additional header to allow further processing.
This change addes processing capabilities for such headers and marks
new posts created as hidden when received via email.

app/models/post.rb

@@ -113,7 +113,8 @@ class Post < ActiveRecord::Base
     @hidden_reasons ||= Enum.new(flag_threshold_reached: 1,
                                  flag_threshold_reached_again: 2,
                                  new_user_spam_threshold_reached: 3,
-                                 flagged_by_tl3_user: 4)
+                                 flagged_by_tl3_user: 4,
+                                 email_spam_header_found: 5)
   end
 
   def self.types

config/locales/server.en.yml

@@ -1541,6 +1541,7 @@ en:
     log_mail_processing_failures: "Log all email processing failures to http://yoursitename.com/logs"
     email_in: "Allow users to post new topics via email (requires manual or pop3 polling). Configure the addresses in the \"Settings\" tab of each category."
     email_in_min_trust: "The minimum trust level a user needs to have to be allowed to post new topics via email."
+    email_in_spam_header: "The email header to detect spam."
     email_prefix: "The [label] used in the subject of emails. It will default to 'title' if not set."
     email_site_title: "The title of the site used as the sender of emails from the site. Default to 'title' if not set. If your 'title' contains characters that are not allowed in email sender strings, use this setting."
 

config/site_settings.yml

@@ -790,6 +790,13 @@ email:
   email_in_min_trust:
     default: 2
     enum: 'TrustLevelSetting'
+  email_in_spam_header:
+    type: enum
+    default: 'none'
+    choices:
+     - none
+     - X-Spam-Flag
+     - X-Spam-Status
   email_prefix: ''
   email_site_title: ''
   disable_emails:

lib/email/receiver.rb

@@ -111,6 +111,8 @@ module Email
       raise FromReplyByAddressError if is_from_reply_by_email_address?
       raise ScreenedEmailError if ScreenedEmail.should_block?(@from_email)
 
+      hidden_reason_id = is_spam? ? Post.hidden_reasons[:email_spam_header_found] : nil
+
       user = find_user(@from_email)
 
       if user.present?
@@ -149,6 +151,7 @@ module Email
         create_reply(user: user,
                      raw: body,
                      elided: elided,
+                     hidden_reason_id: hidden_reason_id,
                      post: post,
                      topic: post.topic,
                      skip_validations: user.staged?)
@@ -157,7 +160,7 @@ module Email
 
         destinations.each do |destination|
           begin
-            process_destination(destination, user, body, elided)
+            process_destination(destination, user, body, elided, hidden_reason_id)
           rescue => e
             first_exception ||= e
           else
@@ -243,6 +246,17 @@ module Email
       @mail.header.to_s[/auto[\-_]?(response|submitted|replied|reply|generated|respond)|holidayreply|machinegenerated/i]
     end
 
+    def is_spam?
+      case SiteSetting.email_in_spam_header
+      when 'X-Spam-Flag'
+        @mail[:x_spam_flag].to_s[/YES/i]
+      when 'X-Spam-Status'
+        @mail[:x_spam_status].to_s[/^Yes, /i]
+      else
+        false
+      end
+    end
+
     def select_body
       text = nil
       html = nil
@@ -541,7 +555,7 @@ module Email
       nil
     end
 
-    def process_destination(destination, user, body, elided)
+    def process_destination(destination, user, body, elided, hidden_reason_id)
       return if SiteSetting.enable_forwarded_emails &&
                 has_been_forwarded? &&
                 process_forwarded_email(destination, user)
@@ -549,7 +563,7 @@ module Email
       case destination[:type]
       when :group
         group = destination[:obj]
-        create_group_post(group, user, body, elided)
+        create_group_post(group, user, body, elided, hidden_reason_id)
 
       when :category
         category = destination[:obj]
@@ -560,6 +574,7 @@ module Email
         create_topic(user: user,
                      raw: body,
                      elided: elided,
+                     hidden_reason_id: hidden_reason_id,
                      title: subject,
                      category: category.id,
                      skip_validations: user.staged?)
@@ -574,13 +589,14 @@ module Email
         create_reply(user: user,
                      raw: body,
                      elided: elided,
+                     hidden_reason_id: hidden_reason_id,
                      post: email_log.post,
                      topic: email_log.post.topic,
                      skip_validations: user.staged?)
       end
     end
 
-    def create_group_post(group, user, body, elided)
+    def create_group_post(group, user, body, elided, hidden_reason_id)
       message_ids = Email::Receiver.extract_reply_message_ids(@mail, max_message_id_count: 5)
       post_ids = []
 
@@ -598,6 +614,7 @@ module Email
         create_reply(user: user,
                      raw: body,
                      elided: elided,
+                     hidden_reason_id: hidden_reason_id,
                      post: post,
                      topic: post.topic,
                      skip_validations: true)
@@ -605,6 +622,7 @@ module Email
         create_topic(user: user,
                      raw: body,
                      elided: elided,
+                     hidden_reason_id: hidden_reason_id,
                      title: subject,
                      archetype: Archetype.private_message,
                      target_group_names: [group.name],

lib/post_creator.rb

@@ -36,6 +36,7 @@ class PostCreator
   #                             wrap `PostCreator` in a transaction, as the sidekiq jobs could
   #                             dequeue before the commit finishes. If you do this, be sure to
   #                             call `enqueue_jobs` after the transaction is comitted.
+  #   hidden_reason_id        - Reason for hiding the post (optional)
   #
   #   When replying to a topic:
   #     topic_id              - topic we're replying to
@@ -62,6 +63,7 @@ class PostCreator
     opts[:title] = pg_clean_up(opts[:title]) if opts[:title] && opts[:title].include?("\u0000")
     opts[:raw] = pg_clean_up(opts[:raw]) if opts[:raw] && opts[:raw].include?("\u0000")
     opts.delete(:reply_to_post_number) unless opts[:topic_id]
+    opts[:visible] = false if opts[:visible].nil? && opts[:hidden_reason_id].present?
     @guardian = opts[:guardian] if opts[:guardian]
 
     @spam = false
@@ -454,6 +456,12 @@ class PostCreator
       post.custom_fields = fields
     end
 
+    if @opts[:hidden_reason_id].present?
+      post.hidden = true
+      post.hidden_at = Time.zone.now
+      post.hidden_reason_id = @opts[:hidden_reason_id]
+    end
+
     @post = post
   end
 

spec/components/email/receiver_spec.rb

@@ -622,6 +622,51 @@ describe Email::Receiver do
       expect { process(:new_user) }.to change(Topic, :count)
     end
 
+    it "creates visible topic for ham" do
+      SiteSetting.email_in_spam_header = 'none'
+
+      Fabricate(:user, email: "existing@bar.com", trust_level: SiteSetting.email_in_min_trust)
+      expect { process(:existing_user) }.to change { Topic.count }.by(1) # Topic created
+
+      topic = Topic.last
+      expect(topic.visible).to eq(true)
+
+      post = Post.last
+      expect(post.hidden).to eq(false)
+      expect(post.hidden_at).to eq(nil)
+      expect(post.hidden_reason_id).to eq(nil)
+    end
+
+    it "creates hidden topic for X-Spam-Flag" do
+      SiteSetting.email_in_spam_header = 'X-Spam-Flag'
+
+      Fabricate(:user, email: "existing@bar.com", trust_level: SiteSetting.email_in_min_trust)
+      expect { process(:spam_x_spam_flag) }.to change { Topic.count }.by(1) # Topic created
+
+      topic = Topic.last
+      expect(topic.visible).to eq(false)
+
+      post = Post.last
+      expect(post.hidden).to eq(true)
+      expect(post.hidden_at).not_to eq(nil)
+      expect(post.hidden_reason_id).to eq(Post.hidden_reasons[:email_spam_header_found])
+    end
+
+    it "creates hidden topic for X-Spam-Status" do
+      SiteSetting.email_in_spam_header = 'X-Spam-Status'
+
+      Fabricate(:user, email: "existing@bar.com", trust_level: SiteSetting.email_in_min_trust)
+      expect { process(:spam_x_spam_status) }.to change { Topic.count }.by(1) # Topic created
+
+      topic = Topic.last
+      expect(topic.visible).to eq(false)
+
+      post = Post.last
+      expect(post.hidden).to eq(true)
+      expect(post.hidden_at).not_to eq(nil)
+      expect(post.hidden_reason_id).to eq(Post.hidden_reasons[:email_spam_header_found])
+    end
+
     it "adds the 'elided' part of the original message when always_show_trimmed_content is enabled" do
       SiteSetting.always_show_trimmed_content = true
 

spec/components/post_creator_spec.rb

@@ -38,6 +38,15 @@ describe PostCreator do
       expect(post.wiki).to eq(true)
     end
 
+    it "can be created with a hidden reason" do
+      hri = Post.hidden_reasons[:flag_threshold_reached]
+      post = PostCreator.create(user, basic_topic_params.merge(hidden_reason_id: hri))
+      expect(post.hidden).to eq(true)
+      expect(post.hidden_at).to be_present
+      expect(post.hidden_reason_id).to eq(hri)
+      expect(post.topic.visible).to eq(false)
+    end
+
     it "ensures the user can create the topic" do
       Guardian.any_instance.expects(:can_create?).with(Topic, nil).returns(false)
       expect { creator.create }.to raise_error(Discourse::InvalidAccess)
@@ -71,6 +80,14 @@ describe PostCreator do
     context "success" do
       before { creator }
 
+      it "is not hidden" do
+        p = creator.create
+        expect(p.hidden).to eq(false)
+        expect(p.hidden_at).not_to be_present
+        expect(p.hidden_reason_id).to eq(nil)
+        expect(p.topic.visible).to eq(true)
+      end
+
       it "doesn't return true for spam" do
         creator.create
         expect(creator.spam?).to eq(false)

spec/fixtures/emails/spam_x_spam_flag.eml

@@ -0,0 +1,12 @@
+Return-Path: <existing@bar.com>
+From: Foo Bar <existing@bar.com>
+To: category@bar.com
+Subject: This is a topic from an existing user
+Date: Fri, 15 Jan 2016 00:12:43 +0100
+Message-ID: <32@foo.bar.mail>
+Mime-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: quoted-printable
+X-Spam-Flag: YES
+
+Hey, this is a topic from an existing user ;)

spec/fixtures/emails/spam_x_spam_status.eml

@@ -0,0 +1,12 @@
+Return-Path: <existing@bar.com>
+From: Foo Bar <existing@bar.com>
+To: category@bar.com
+Subject: This is a topic from an existing user
+Date: Fri, 15 Jan 2016 00:12:43 +0100
+Message-ID: <32@foo.bar.mail>
+Mime-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: quoted-printable
+X-Spam-Status: Yes, score=12.3 required=4.5
+
+Hey, this is a topic from an existing user ;)