Simple "cook" for email imports from mailing lists

Gemfile.lock

@@ -486,4 +486,4 @@ DEPENDENCIES
   unicorn
 
 BUNDLED WITH
-   1.10.2
+   1.10.3

app/models/post.rb

@@ -6,6 +6,7 @@ require_dependency 'enum'
 require_dependency 'post_analyzer'
 require_dependency 'validators/post_validator'
 require_dependency 'plugin/filter'
+require_dependency 'email_cook'
 
 require 'archetype'
 require 'digest/sha1'
@@ -76,7 +77,7 @@ class Post < ActiveRecord::Base
   end
 
   def self.cook_methods
-    @cook_methods ||= Enum.new(:regular, :raw_html)
+    @cook_methods ||= Enum.new(:regular, :raw_html, :email)
   end
 
   def self.find_by_detail(key, value)
@@ -161,16 +162,20 @@ class Post < ActiveRecord::Base
     # case we can skip the rendering pipeline.
     return raw if cook_method == Post.cook_methods[:raw_html]
 
-    # Default is to cook posts
-    cooked = if !self.user || SiteSetting.tl3_links_no_follow || !self.user.has_trust_level?(TrustLevel[3])
-               post_analyzer.cook(*args)
-             else
-               # At trust level 3, we don't apply nofollow to links
-               cloned = args.dup
-               cloned[1] ||= {}
-               cloned[1][:omit_nofollow] = true
-               post_analyzer.cook(*cloned)
-             end
+    cooked = nil
+    if cook_method == Post.cook_methods[:email]
+      cooked = EmailCook.new(raw).cook
+    else
+      cooked = if !self.user || SiteSetting.tl3_links_no_follow || !self.user.has_trust_level?(TrustLevel[3])
+                 post_analyzer.cook(*args)
+               else
+                 # At trust level 3, we don't apply nofollow to links
+                 cloned = args.dup
+                 cloned[1] ||= {}
+                 cloned[1][:omit_nofollow] = true
+                 post_analyzer.cook(*cloned)
+               end
+    end
 
     new_cooked = Plugin::Filter.apply(:after_post_cook, self, cooked)
 

lib/email_cook.rb

@@ -0,0 +1,36 @@
+# A very simple formatter for imported emails
+class EmailCook
+
+  def initialize(raw)
+    @raw = raw
+  end
+
+  def cook
+    result = ""
+
+    in_quote = false
+    quote_buffer = ""
+    @raw.each_line do |l|
+
+      if l =~ /^\s*>/
+        in_quote = true
+        quote_buffer << l.sub(/^[\s>]*/, '') << "<br>"
+      elsif in_quote
+        result << "<blockquote>#{quote_buffer}</blockquote>"
+        quote_buffer = ""
+        in_quote = false
+      else
+        result << l << "<br>"
+      end
+    end
+
+    if in_quote
+      result << "<blockquote>#{quote_buffer}</blockquote>"
+    end
+
+    result.gsub!(/(<br>){3,10}/, '<br><br>')
+
+    result
+  end
+
+end

lib/post_creator.rb

@@ -28,6 +28,7 @@ class PostCreator
   #   cook_method             - Method of cooking the post.
   #                               :regular - Pass through Markdown parser and strip bad HTML
   #                               :raw_html - Perform no processing
+  #                               :raw_email - Imported from an email
   #   via_email               - Mark this post as arriving via email
   #   raw_email               - Full text of arriving email (to store)
   #

script/import_scripts/nabble.rb

@@ -54,7 +54,10 @@ class ImportScripts::MyAskBot < ImportScripts::Base
   def parse_email(msg)
     receiver = Email::Receiver.new(msg, skip_sanity_check: true)
     mail = Mail.read_from_string(msg)
-    receiver.parse_body(mail)
+    mail.body
+
+    selected = receiver.select_body(mail)
+    selected.force_encoding(selected.encoding).encode("UTF-8")
   end
 
   def create_forum_topics
@@ -86,7 +89,8 @@ class ImportScripts::MyAskBot < ImportScripts::Base
           user_id: user_id_from_imported_user_id(t["owner_id"]) || Discourse::SYSTEM_USER_ID,
           created_at: Time.zone.at(@td.decode(t["when_created"])),
           category: CATEGORY_ID,
-          raw: raw }
+          raw: raw,
+          cook_method: Post.cook_methods[:email] }
       end
     end
   end
@@ -137,7 +141,8 @@ class ImportScripts::MyAskBot < ImportScripts::Base
           topic_id: topic_id,
           user_id: user_id_from_imported_user_id(p['owner_id']) || Discourse::SYSTEM_USER_ID,
           created_at: Time.zone.at(@td.decode(p["when_created"])),
-          raw: raw }
+          raw: raw,
+          cook_method: Post.cook_methods[:email] }
       end
     end
   end