FEATURE: Add setting to strip whitespaces from incoming emails. (#7375)

Some email clients add leading whitespaces which get are transformed in
code blocks when processed.

config/locales/server.en.yml

@@ -1742,6 +1742,7 @@ en:
     reply_by_email_address: "Template for reply by email incoming email address, for example: %%{reply_key}@reply.example.com or replies+%%{reply_key}@example.com"
     alternative_reply_by_email_addresses: "List of alternative templates for reply by email incoming email addresses. Example: %%{reply_key}@reply.example.com|replies+%%{reply_key}@example.com"
     incoming_email_prefer_html: "Use HTML instead of text for incoming email."
+    strip_incoming_email_lines: "Remove leading and trailing whitespaces from each line of incoming emails."
 
     disable_emails: "Prevent Discourse from sending any kind of emails. Select 'yes' to disable emails for all users. Select 'non-staff' to disable emails for non-staff users only."
 

config/site_settings.yml

@@ -896,6 +896,7 @@ email:
   pop3_polling_delete_from_server: true
   log_mail_processing_failures: false
   incoming_email_prefer_html: true
+  strip_incoming_email_lines: false
   email_in:
     default: false
     client: true

lib/email/receiver.rb

@@ -353,11 +353,32 @@ module Email
         end
       end
 
+      text_format = Receiver::formats[:plaintext]
       if text.blank? || (SiteSetting.incoming_email_prefer_html && markdown.present?)
-        return [markdown, elided_markdown, Receiver::formats[:markdown]]
+        text, elided_text, text_format = markdown, elided_markdown, Receiver::formats[:markdown]
-      else
-        return [text, elided_text, Receiver::formats[:plaintext]]
       end
+
+      if SiteSetting.strip_incoming_email_lines
+        in_code = nil
+
+        text = text.lines.map! do |line|
+          stripped = line.strip << "\n"
+
+          if !in_code && stripped[0..2] == '```'
+            in_code = '```'
+          elsif in_code == '```' && stripped[0..2] == '```'
+            in_code = nil
+          elsif !in_code && stripped[0..4] == '[code'
+            in_code = '[code]'
+          elsif in_code == '[code]' && stripped[0..6] == '[/code]'
+            in_code = nil
+          end
+
+          in_code ? line : stripped
+        end.join
+      end
+
+      [text, elided_text, text_format]
     end
 
     def to_markdown(html, elided_html)

spec/components/email/receiver_spec.rb

@@ -1256,4 +1256,91 @@ describe Email::Receiver do
     expect(email.to_addresses).to eq("foo@bar.com")
     expect(email.cc_addresses).to eq("bob@example.com;carol@example.com")
   end
+
+  context "#select_body" do
+
+    let(:email) {
+      <<~EOF
+      MIME-Version: 1.0
+      Date: Tue, 01 Jan 2019 00:00:00 +0300
+      Subject: An email with whitespaces
+      From: Foo <foo@discourse.org>
+      To: bar@discourse.org
+      Content-Type: text/plain; charset="UTF-8"
+
+          This is a line that will be stripped
+          This is another line that will be stripped
+
+      This is a line that will not be touched.
+      This is another line that will not be touched.
+
+      [code]
+        1.upto(10).each do |i|
+          puts i
+        end
+
+      ```
+        # comment
+      [/code]
+
+          This is going to be stripped too.
+
+      ```
+        1.upto(10).each do |i|
+          puts i
+        end
+
+      [/code]
+        # comment
+      ```
+
+              This is going to be stripped too.
+
+      Bye!
+      EOF
+    }
+
+    let(:stripped_text) {
+      <<~EOF
+      This is a line that will be stripped
+      This is another line that will be stripped
+
+      This is a line that will not be touched.
+      This is another line that will not be touched.
+
+      [code]
+        1.upto(10).each do |i|
+          puts i
+        end
+
+      ```
+        # comment
+      [/code]
+
+      This is going to be stripped too.
+
+      ```
+        1.upto(10).each do |i|
+          puts i
+        end
+
+      [/code]
+        # comment
+      ```
+
+      This is going to be stripped too.
+
+      Bye!
+      EOF
+    }
+
+    it "strips lines if strip_incoming_email_lines is enabled" do
+      SiteSetting.strip_incoming_email_lines = true
+
+      receiver = Email::Receiver.new(email)
+      text, elided, format = receiver.select_body
+      expect(text).to eq(stripped_text)
+    end
+
+  end
 end