# frozen_string_literal: true require "digest" module Email class Receiver # If you add a new error, you need to # * add it to Email::Processor#handle_failure() # * add text to server.en.yml (parent key: "emails.incoming.errors") class ProcessingError < StandardError; end class EmptyEmailError < ProcessingError; end class ScreenedEmailError < ProcessingError; end class UserNotFoundError < ProcessingError; end class AutoGeneratedEmailError < ProcessingError; end class BouncedEmailError < ProcessingError; end class NoBodyDetectedError < ProcessingError; end class NoSenderDetectedError < ProcessingError; end class FromReplyByAddressError < ProcessingError; end class InactiveUserError < ProcessingError; end class SilencedUserError < ProcessingError; end class BadDestinationAddress < ProcessingError; end class StrangersNotAllowedError < ProcessingError; end class ReplyNotAllowedError < ProcessingError; end class InsufficientTrustLevelError < ProcessingError; end class ReplyUserNotMatchingError < ProcessingError; end class TopicNotFoundError < ProcessingError; end class TopicClosedError < ProcessingError; end class InvalidPost < ProcessingError; end class TooShortPost < ProcessingError; end class InvalidPostAction < ProcessingError; end class UnsubscribeNotAllowed < ProcessingError; end class EmailNotAllowed < ProcessingError; end class OldDestinationError < ProcessingError; end class ReplyToDigestError < ProcessingError; end attr_reader :incoming_email attr_reader :raw_email attr_reader :mail attr_reader :message_id COMMON_ENCODINGS ||= [-"utf-8", -"windows-1252", -"iso-8859-1"] def self.formats @formats ||= Enum.new(plaintext: 1, markdown: 2) end def initialize(mail_string, opts = {}) raise EmptyEmailError if mail_string.blank? @staged_users = [] @raw_email = mail_string COMMON_ENCODINGS.each do |encoding| fixed = try_to_encode(mail_string, encoding) break @raw_email = fixed if fixed.present? end @mail = Mail.new(@raw_email) @message_id = @mail.message_id.presence || Digest::MD5.hexdigest(mail_string) @opts = opts @destinations ||= opts[:destinations] end def process! return if is_blocked? id_hash = Digest::SHA1.hexdigest(@message_id) DistributedMutex.synchronize("process_email_#{id_hash}") do begin # If we find an existing incoming email record with the exact same `message_id` # do not create a new `IncomingEmail` record to avoid double ups. return if @incoming_email = find_existing_and_update_imap ensure_valid_address_lists ensure_valid_date @from_email, @from_display_name = parse_from_field @from_user = User.find_by_email(@from_email) @incoming_email = create_incoming_email post = process_internal raise BouncedEmailError if is_bounce? post rescue Exception => e error = e.to_s error = e.class.name if error.blank? @incoming_email.update_columns(error: error) if @incoming_email delete_staged_users raise end end end def find_existing_and_update_imap return unless incoming_email = IncomingEmail.find_by(message_id: @message_id) # If we are not doing this for IMAP purposes just return the record. return incoming_email if @opts[:imap_uid].blank? # If the message_id matches the post id regexp then we # generated the message_id not the imap server, e.g. in GroupSmtpEmail, # so we want to update the incoming email because it will # be missing IMAP details. # # Otherwise the incoming email is a completely new one from the IMAP # server (e.g. a message_id generated by Gmail) and does not need to # be updated, because message_ids from the IMAP server are not guaranteed # to be unique. return unless discourse_generated_message_id?(@message_id) incoming_email.update( imap_uid_validity: @opts[:imap_uid_validity], imap_uid: @opts[:imap_uid], imap_group_id: @opts[:imap_group_id], imap_sync: false ) incoming_email end def ensure_valid_address_lists [:to, :cc, :bcc].each do |field| addresses = @mail[field] if addresses&.errors.present? @mail[field] = addresses.to_s.scan(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i) end end end def ensure_valid_date if @mail.date.nil? raise InvalidPost, I18n.t("system_messages.email_reject_invalid_post_specified.date_invalid") end end def is_blocked? return false if SiteSetting.ignore_by_title.blank? Regexp.new(SiteSetting.ignore_by_title, Regexp::IGNORECASE) =~ @mail.subject end def create_incoming_email IncomingEmail.create( message_id: @message_id, raw: Email::Cleaner.new(@raw_email).execute, subject: subject, from_address: @from_email, to_addresses: @mail.to, cc_addresses: @mail.cc, imap_uid_validity: @opts[:imap_uid_validity], imap_uid: @opts[:imap_uid], imap_group_id: @opts[:imap_group_id], imap_sync: false, created_via: IncomingEmail.created_via_types[@opts[:source] || :unknown] ) end def process_internal handle_bounce if is_bounce? raise NoSenderDetectedError if @from_email.blank? raise FromReplyByAddressError if is_from_reply_by_email_address? raise ScreenedEmailError if ScreenedEmail.should_block?(@from_email) user = @from_user if user.present? log_and_validate_user(user) else raise UserNotFoundError unless SiteSetting.enable_staged_users end body, elided = select_body body ||= "" raise NoBodyDetectedError if body.blank? && attachments.empty? && !is_bounce? if is_auto_generated? && !sent_to_mailinglist_mirror? @incoming_email.update_columns(is_auto_generated: true) if SiteSetting.block_auto_generated_emails? && !is_bounce? && !@opts[:allow_auto_generated] raise AutoGeneratedEmailError end end if action = subscription_action_for(body, subject) raise UnsubscribeNotAllowed if user.nil? send_subscription_mail(action, user) return end if post = find_related_post # Most of the time, it is impossible to **reply** without a reply key, so exit early if user.blank? if sent_to_mailinglist_mirror? || !SiteSetting.find_related_post_with_key user = stage_from_user elsif user.blank? raise BadDestinationAddress end end create_reply(user: user, raw: body, elided: elided, post: post, topic: post.topic, skip_validations: user.staged?, bounce: is_bounce?) else first_exception = nil destinations.each do |destination| begin return process_destination(destination, user, body, elided) rescue => e first_exception ||= e end end raise first_exception if first_exception # We don't stage new users for emails to reply addresses, exit if user is nil raise BadDestinationAddress if user.blank? # We only get here if there are no destinations (the email is not going to # a Category, Group, or PostReplyKey) post = find_related_post(force: true) if post && Guardian.new(user).can_see_post?(post) if destination_too_old?(post) raise OldDestinationError.new("#{Discourse.base_url}/p/#{post.id}") end end raise ReplyToDigestError if EmailLog.where(email_type: "digest", message_id: @mail.in_reply_to).exists? raise BadDestinationAddress end end def log_and_validate_user(user) @incoming_email.update_columns(user_id: user.id) raise InactiveUserError if !user.active && !user.staged raise SilencedUserError if user.silenced? end def is_bounce? @mail.bounced? || bounce_key end def handle_bounce @incoming_email.update_columns(is_bounce: true) if email_log.present? email_log.update_columns(bounced: true) post = email_log.post topic = email_log.topic end if @mail.error_status.present? && Array.wrap(@mail.error_status).any? { |s| s.start_with?("4.") } Email::Receiver.update_bounce_score(@from_email, SiteSetting.soft_bounce_score) else Email::Receiver.update_bounce_score(@from_email, SiteSetting.hard_bounce_score) end if SiteSetting.enable_whispers? && @from_user&.staged? return if email_log.blank? if post.present? && topic.present? && topic.archetype == Archetype.private_message body, elided = select_body body ||= "" create_reply(user: @from_user, raw: body, elided: elided, post: post, topic: topic, skip_validations: true, bounce: true) end end raise BouncedEmailError end def is_from_reply_by_email_address? Email::Receiver.reply_by_email_address_regex.match(@from_email) end def bounce_key @bounce_key ||= begin verp = all_destinations.select { |to| to[/\+verp-\h{32}@/] }.first verp && verp[/\+verp-(\h{32})@/, 1] end end def email_log return nil if bounce_key.blank? @email_log ||= EmailLog.find_by(bounce_key: bounce_key) end def self.update_bounce_score(email, score) if user = User.find_by_email(email) old_bounce_score = user.user_stat.bounce_score new_bounce_score = old_bounce_score + score range = (old_bounce_score + 1..new_bounce_score) user.user_stat.bounce_score = new_bounce_score user.user_stat.reset_bounce_score_after = SiteSetting.reset_bounce_score_after_days.days.from_now user.user_stat.save! if range === SiteSetting.bounce_score_threshold # NOTE: we check bounce_score before sending emails # So log we revoked the email... reason = I18n.t("user.email.revoked", email: user.email, date: user.user_stat.reset_bounce_score_after) StaffActionLogger.new(Discourse.system_user).log_revoke_email(user, reason) # ... and PM the user SystemMessage.create_from_system_user(user, :email_revoked) end end end def is_auto_generated? return false if SiteSetting.auto_generated_allowlist.split('|').include?(@from_email) @mail[:precedence].to_s[/list|junk|bulk|auto_reply/i] || @mail[:from].to_s[/(mailer[\-_]?daemon|post[\-_]?master|no[\-_]?reply)@/i] || @mail[:subject].to_s[/^\s*(Auto:|Automatic reply|Autosvar|Automatisk svar|Automatisch antwoord|Abwesenheitsnotiz|Risposta Non al computer|Automatisch antwoord|Auto Response|Respuesta automática|Fuori sede|Out of Office|Frånvaro|Réponse automatique)/i] || @mail.header.to_s[/auto[\-_]?(response|submitted|replied|reply|generated|respond)|holidayreply|machinegenerated/i] end def is_spam? case SiteSetting.email_in_spam_header when 'X-Spam-Flag' @mail[:x_spam_flag].to_s[/YES/i] when 'X-Spam-Status' @mail[:x_spam_status].to_s[/^Yes, /i] when 'X-SES-Spam-Verdict' @mail[:x_ses_spam_verdict].to_s[/FAIL/i] else false end end def auth_res_action @auth_res_action ||= AuthenticationResults.new(@mail.header[:authentication_results]).action end def select_body text = nil html = nil text_content_type = nil if @mail.multipart? text = fix_charset(@mail.text_part) html = fix_charset(@mail.html_part) text_content_type = @mail.text_part&.content_type elsif @mail.content_type.to_s["text/html"] html = fix_charset(@mail) elsif @mail.content_type.blank? || @mail.content_type["text/plain"] text = fix_charset(@mail) text_content_type = @mail.content_type end return unless text.present? || html.present? if text.present? text = trim_discourse_markers(text) text, elided_text = trim_reply_and_extract_elided(text) if @opts[:convert_plaintext] || sent_to_mailinglist_mirror? text_content_type ||= "" converter_opts = { format_flowed: !!(text_content_type =~ /format\s*=\s*["']?flowed["']?/i), delete_flowed_space: !!(text_content_type =~ /DelSp\s*=\s*["']?yes["']?/i) } text = PlainTextToMarkdown.new(text, converter_opts).to_markdown elided_text = PlainTextToMarkdown.new(elided_text, converter_opts).to_markdown end end markdown, elided_markdown = if html.present? # use the first html extracter that matches if html_extracter = HTML_EXTRACTERS.select { |_, r| html[r] }.min_by { |_, r| html =~ r } doc = Nokogiri::HTML5.fragment(html) self.public_send(:"extract_from_#{html_extracter[0]}", doc) else markdown = HtmlToMarkdown.new(html, keep_img_tags: true, keep_cid_imgs: true).to_markdown markdown = trim_discourse_markers(markdown) trim_reply_and_extract_elided(markdown) end end text_format = Receiver::formats[:plaintext] if text.blank? || (SiteSetting.incoming_email_prefer_html && markdown.present?) text, elided_text, text_format = markdown, elided_markdown, Receiver::formats[:markdown] end if SiteSetting.strip_incoming_email_lines && text.present? in_code = nil text = text.lines.map! do |line| stripped = line.strip << "\n" # Do not strip list items. next line if (stripped[0] == '*' || stripped[0] == '-' || stripped[0] == '+') && stripped[1] == ' ' # Match beginning and ending of code blocks. if !in_code && stripped[0..2] == '```' in_code = '```' elsif in_code == '```' && stripped[0..2] == '```' in_code = nil elsif !in_code && stripped[0..4] == '[code' in_code = '[code]' elsif in_code == '[code]' && stripped[0..6] == '[/code]' in_code = nil end # Strip only lines outside code blocks. in_code ? line : stripped end.join end [text, elided_text, text_format] end def to_markdown(html, elided_html) markdown = HtmlToMarkdown.new(html, keep_img_tags: true, keep_cid_imgs: true).to_markdown elided_markdown = HtmlToMarkdown.new(elided_html, keep_img_tags: true, keep_cid_imgs: true).to_markdown [EmailReplyTrimmer.trim(markdown), elided_markdown] end HTML_EXTRACTERS ||= [ [:gmail, /class="gmail_(signature|extra)/], [:outlook, /id="(divRplyFwdMsg|Signature)"/], [:word, /class="WordSection1"/], [:exchange, /name="message(Body|Reply)Section"/], [:apple_mail, /id="AppleMailSignature"/], [:mozilla, /class="moz-/], [:protonmail, /class="protonmail_/], [:zimbra, /data-marker="__/], [:newton, /(id|class)="cm_/], [:front, /class="front-/], ] def extract_from_gmail(doc) # GMail adds a bunch of 'gmail_' prefixed classes like: gmail_signature, gmail_extra, gmail_quote, gmail_default... elided = doc.css(".gmail_signature, .gmail_extra").remove to_markdown(doc.to_html, elided.to_html) end def extract_from_outlook(doc) # Outlook properly identifies the signature and any replied/forwarded email # Use their id to remove them and anything that comes after elided = doc.css("#Signature, #Signature ~ *, hr, #divRplyFwdMsg, #divRplyFwdMsg ~ *").remove to_markdown(doc.to_html, elided.to_html) end def extract_from_word(doc) # Word (?) keeps the content in the 'WordSection1' class and uses

tags # When there's something else (,
, etc..) there's high chance it's a signature or forwarded email elided = doc.css(".WordSection1 > :not(p):not(ul):first-of-type, .WordSection1 > :not(p):not(ul):first-of-type ~ *").remove to_markdown(doc.at(".WordSection1").to_html, elided.to_html) end def extract_from_exchange(doc) # Exchange is using the 'messageReplySection' class for forwarded emails # And 'messageBodySection' for the actual email elided = doc.css("div[name='messageReplySection']").remove to_markdown(doc.css("div[name='messageReplySection']").to_html, elided.to_html) end def extract_from_apple_mail(doc) # AppleMail is the worst. It adds 'AppleMailSignature' ids (!) to several div/p with no deterministic rules # Our best guess is to elide whatever comes after that. elided = doc.css("#AppleMailSignature:last-of-type ~ *").remove to_markdown(doc.to_html, elided.to_html) end def extract_from_mozilla(doc) # Mozilla (Thunderbird ?) properly identifies signature and forwarded emails # Remove them and anything that comes after elided = doc.css("*[class^='moz-'], *[class^='moz-'] ~ *").remove to_markdown(doc.to_html, elided.to_html) end def extract_from_protonmail(doc) # Removes anything that has a class starting with "protonmail_" and everything after that elided = doc.css("*[class^='protonmail_'], *[class^='protonmail_'] ~ *").remove to_markdown(doc.to_html, elided.to_html) end def extract_from_zimbra(doc) # Removes anything that has a 'data-marker' attribute elided = doc.css("*[data-marker]").remove to_markdown(doc.to_html, elided.to_html) end def extract_from_newton(doc) # Removes anything that has an id or a class starting with 'cm_' elided = doc.css("*[id^='cm_'], *[class^='cm_']").remove to_markdown(doc.to_html, elided.to_html) end def extract_from_front(doc) # Removes anything that has a class starting with 'front-' elided = doc.css("*[class^='front-']").remove to_markdown(doc.to_html, elided.to_html) end def trim_reply_and_extract_elided(text) return [text, ""] if @opts[:skip_trimming] || !SiteSetting.trim_incoming_emails EmailReplyTrimmer.trim(text, true) end def fix_charset(mail_part) return nil if mail_part.blank? || mail_part.body.blank? string = mail_part.body.decoded rescue nil return nil if string.blank? # common encodings encodings = COMMON_ENCODINGS.dup encodings.unshift(mail_part.charset) if mail_part.charset.present? # mail (>=2.5) decodes mails with 8bit transfer encoding to utf-8, so always try UTF-8 first if mail_part.content_transfer_encoding == "8bit" encodings.delete("UTF-8") encodings.unshift("UTF-8") end encodings.uniq.each do |encoding| fixed = try_to_encode(string, encoding) return fixed if fixed.present? end nil end def try_to_encode(string, encoding) encoded = string.encode("UTF-8", encoding) !encoded.nil? && encoded.valid_encoding? ? encoded : nil rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError, Encoding::ConverterNotFoundError nil end def previous_replies_regex @previous_replies_regex ||= /^--[- ]\n\*#{I18n.t("user_notifications.previous_discussion")}\*\n/im end def reply_above_line_regex @reply_above_line_regex ||= /\n#{I18n.t("user_notifications.reply_above_line")}\n/im end def trim_discourse_markers(reply) reply = reply.split(previous_replies_regex)[0] reply.split(reply_above_line_regex)[0] end def parse_from_field(mail = nil) mail ||= @mail if email_log.present? email = email_log.to_address || email_log.user&.email return [email, email_log.user&.username] elsif mail.bounced? Array.wrap(mail.final_recipient).each do |from| return extract_from_address_and_name(from) end end return unless mail[:from] # For now we are only using the Reply-To header if the email has # been forwarded via Google Groups, which is why we are checking the # X-Original-From header too. In future we may want to use the Reply-To # header in more cases. if mail['X-Original-From'].present? if mail[:reply_to] && mail[:reply_to].errors.blank? mail[:reply_to].each do |address_field| from_address = address_field.address from_display_name = address_field.display_name&.to_s next if address_field.to_s != mail['X-Original-From'].to_s next if !from_address&.include?("@") return [from_address&.downcase, from_display_name&.strip] end end end if mail[:from].errors.blank? mail[:from].each do |address_field| from_address = address_field.address from_display_name = address_field.display_name&.to_s next if !from_address&.include?("@") return [from_address&.downcase, from_display_name&.strip] end end return extract_from_address_and_name(mail.from) if mail.from.is_a? String if mail.from.is_a? Mail::AddressContainer mail.from.each do |from| from_address, from_display_name = extract_from_address_and_name(from) return [from_address, from_display_name] if from_address end end nil rescue StandardError nil end def extract_from_address_and_name(value) if value[";"] from_display_name, from_address = value.split(";") return [from_address&.strip&.downcase, from_display_name&.strip] end if value[/<[^>]+>/] from_address = value[/<([^>]+)>/, 1] from_display_name = value[/^([^<]+)/, 1] end if (from_address.blank? || !from_address["@"]) && value[/\[mailto:[^\]]+\]/] from_address = value[/\[mailto:([^\]]+)\]/, 1] from_display_name = value[/^([^\[]+)/, 1] end [from_address&.downcase, from_display_name&.strip] end def subject @subject ||= if mail_subject = @mail.subject mail_subject.delete("\u0000")[0..254] else I18n.t("emails.incoming.default_subject", email: @from_email) end end def find_or_create_user(email, display_name, raise_on_failed_create: false) user = nil User.transaction do user = User.find_by_email(email) if user.nil? && SiteSetting.enable_staged_users raise EmailNotAllowed unless EmailValidator.allowed?(email) username = UserNameSuggester.sanitize_username(display_name) if display_name.present? begin user = User.create!( email: email, username: UserNameSuggester.suggest(username.presence || email), name: display_name.presence || User.suggest_name(email), staged: true ) @staged_users << user rescue PG::UniqueViolation, ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid raise if raise_on_failed_create user = nil end end end user end def find_or_create_user!(email, display_name) find_or_create_user(email, display_name, raise_on_failed_create: true) end def all_destinations @all_destinations ||= [ @mail.destinations, [@mail[:x_forwarded_to]].flatten.compact.map(&:decoded), [@mail[:delivered_to]].flatten.compact.map(&:decoded), ].flatten.select(&:present?).uniq.lazy end def destinations @destinations ||= all_destinations .map { |d| Email::Receiver.check_address(d, is_bounce?) } .reject(&:blank?) end def sent_to_mailinglist_mirror? @sent_to_mailinglist_mirror ||= begin destinations.each do |destination| return true if destination.is_a?(Category) && destination.mailinglist_mirror? end false end end def self.check_address(address, include_verp = false) # only check for a group/category when 'email_in' is enabled if SiteSetting.email_in group = Group.find_by_email(address) return group if group category = Category.find_by_email(address) return category if category end # reply match = Email::Receiver.reply_by_email_address_regex(true, include_verp).match(address) if match && match.captures match.captures.each do |c| next if c.blank? post_reply_key = PostReplyKey.find_by(reply_key: c) return post_reply_key if post_reply_key end end nil end def process_destination(destination, user, body, elided) return if SiteSetting.forwarded_emails_behaviour != "hide" && has_been_forwarded? && process_forwarded_email(destination, user) return if is_bounce? && !destination.is_a?(PostReplyKey) if destination.is_a?(Group) user ||= stage_from_user create_group_post(destination, user, body, elided) elsif destination.is_a?(Category) raise StrangersNotAllowedError if (user.nil? || user.staged?) && !destination.email_in_allow_strangers user ||= stage_from_user raise InsufficientTrustLevelError if !user.has_trust_level?(SiteSetting.email_in_min_trust) && !sent_to_mailinglist_mirror? create_topic(user: user, raw: body, elided: elided, title: subject, category: destination.id, skip_validations: user.staged?) elsif destination.is_a?(PostReplyKey) # We don't stage new users for emails to reply addresses, exit if user is nil raise BadDestinationAddress if user.blank? post = Post.with_deleted.find(destination.post_id) raise ReplyNotAllowedError if !Guardian.new(user).can_create_post?(post&.topic) if destination.user_id != user.id && !forwarded_reply_key?(destination, user) raise ReplyUserNotMatchingError, "post_reply_key.user_id => #{destination.user_id.inspect}, user.id => #{user.id.inspect}" end create_reply(user: user, raw: body, elided: elided, post: post, topic: post&.topic, skip_validations: user.staged?, bounce: is_bounce?) end end def create_group_post(group, user, body, elided) message_ids = Email::Receiver.extract_reply_message_ids(@mail, max_message_id_count: 5) # incoming emails with matching message ids, and then cross references # these with any email addresses for the user vs to/from/cc of the # incoming emails. in effect, any incoming email record for these # message ids where the user is involved in any way will be returned incoming_emails = IncomingEmail.where(message_id: message_ids) if !group.allow_unknown_sender_topic_replies incoming_emails = incoming_emails.addressed_to_user(user) end post_ids = incoming_emails.pluck(:post_id) || [] # if the user is directly replying to an email send to them from discourse, # there will be a corresponding EmailLog record, so we can use that as the # reply post if it exists if discourse_generated_message_id?(mail.in_reply_to) post_id_from_email_log = EmailLog.where(message_id: mail.in_reply_to) .addressed_to_user(user) .order(created_at: :desc) .limit(1) .pluck(:post_id).last post_ids << post_id_from_email_log if post_id_from_email_log end target_post = post_ids.any? && Post.where(id: post_ids).order(:created_at).last too_old_for_group_smtp = (destination_too_old?(target_post) && group.smtp_enabled) if target_post.blank? || too_old_for_group_smtp create_topic(user: user, raw: new_group_topic_body(body, target_post, too_old_for_group_smtp), elided: elided, title: subject, archetype: Archetype.private_message, target_group_names: [group.name], is_group_message: true, skip_validations: true) else # This must be done for the unknown user (who is staged) to # be allowed to post a reply in the topic. if group.allow_unknown_sender_topic_replies target_post.topic.topic_allowed_users.find_or_create_by!(user_id: user.id) end create_reply(user: user, raw: body, elided: elided, post: target_post, topic: target_post.topic, skip_validations: true) end end def new_group_topic_body(body, target_post, too_old_for_group_smtp) return body if !too_old_for_group_smtp body + "\n\n----\n\n" + I18n.t( "emails.incoming.continuing_old_discussion", url: target_post.topic.url, title: target_post.topic.title, count: SiteSetting.disallow_reply_by_email_after_days ) end def forwarded_reply_key?(post_reply_key, user) incoming_emails = IncomingEmail .joins(:post) .where('posts.topic_id = ?', post_reply_key.post.topic_id) .addressed_to(post_reply_key.reply_key) .addressed_to_user(user) .pluck(:to_addresses, :cc_addresses) incoming_emails.each do |to_addresses, cc_addresses| next unless contains_email_address_of_user?(to_addresses, user) || contains_email_address_of_user?(cc_addresses, user) return true if contains_reply_by_email_address(to_addresses, post_reply_key.reply_key) || contains_reply_by_email_address(cc_addresses, post_reply_key.reply_key) end false end def contains_email_address_of_user?(addresses, user) return false if addresses.blank? addresses = addresses.split(";") user.user_emails.any? { |user_email| addresses.include?(user_email.email) } end def contains_reply_by_email_address(addresses, reply_key) return false if addresses.blank? addresses.split(";").each do |address| match = Email::Receiver.reply_by_email_address_regex.match(address) return true if match && match.captures&.include?(reply_key) end false end def has_been_forwarded? subject[/^[[:blank:]]*(fwd?|tr)[[:blank:]]?:/i] && embedded_email_raw.present? end def embedded_email_raw return @embedded_email_raw if @embedded_email_raw text = fix_charset(@mail.multipart? ? @mail.text_part : @mail) @embedded_email_raw, @before_embedded = EmailReplyTrimmer.extract_embedded_email(text) @embedded_email_raw end def process_forwarded_email(destination, user) user ||= stage_from_user case SiteSetting.forwarded_emails_behaviour when "create_replies" forwarded_email_create_replies(destination, user) when "quote" forwarded_email_quote_forwarded(destination, user) else false end end def forwarded_email_create_topic(destination: , user: , raw: , title: , date: nil, embedded_user: nil) if destination.is_a?(Group) topic_user = embedded_user&.call || user create_topic(user: topic_user, raw: raw, title: title, archetype: Archetype.private_message, target_usernames: [user.username], target_group_names: [destination.name], is_group_message: true, skip_validations: true, created_at: date) elsif destination.is_a?(Category) return false if user.staged? && !destination.email_in_allow_strangers return false if !user.has_trust_level?(SiteSetting.email_in_min_trust) topic_user = embedded_user&.call || user create_topic(user: topic_user, raw: raw, title: title, category: destination.id, skip_validations: topic_user.staged?, created_at: date) else false end end def forwarded_email_create_replies(destination, user) embedded = Mail.new(embedded_email_raw) email, display_name = parse_from_field(embedded) return false if email.blank? || !email["@"] post = forwarded_email_create_topic(destination: destination, user: user, raw: try_to_encode(embedded.decoded, "UTF-8").presence || embedded.to_s, title: embedded.subject.presence || subject, date: embedded.date, embedded_user: lambda { find_or_create_user(email, display_name) }) return false unless post if post.topic # mark post as seen for the forwarder PostTiming.record_timing(user_id: user.id, topic_id: post.topic_id, post_number: post.post_number, msecs: 5000) # create reply when available if @before_embedded.present? post_type = Post.types[:regular] post_type = Post.types[:whisper] if post.topic.private_message? && destination.usernames[user.username] create_reply(user: user, raw: @before_embedded, post: post, topic: post.topic, post_type: post_type, skip_validations: user.staged?) else post.topic.add_small_action(user, "forwarded") end end true end def forwarded_email_quote_forwarded(destination, user) embedded = embedded_email_raw raw = <<~EOF #{@before_embedded} [quote] #{PlainTextToMarkdown.new(embedded).to_markdown} [/quote] EOF return true if forwarded_email_create_topic(destination: destination, user: user, raw: raw, title: subject) end def self.reply_by_email_address_regex(extract_reply_key = true, include_verp = false) reply_addresses = [SiteSetting.reply_by_email_address] reply_addresses << (SiteSetting.alternative_reply_by_email_addresses.presence || "").split("|") if include_verp && SiteSetting.reply_by_email_address.present? && SiteSetting.reply_by_email_address["+"] reply_addresses << SiteSetting.reply_by_email_address.sub("%{reply_key}", "verp-%{reply_key}") end reply_addresses.flatten! reply_addresses.select!(&:present?) reply_addresses.map! { |a| Regexp.escape(a) } reply_addresses.map! { |a| a.gsub("\+", "\+?") } reply_addresses.map! { |a| a.gsub(Regexp.escape("%{reply_key}"), "(\\h{32})?") } if reply_addresses.empty? /$a/ # a regex that can never match else /#{reply_addresses.join("|")}/ end end def group_incoming_emails_regex @group_incoming_emails_regex = Regexp.union( DB.query_single(<<~SQL).map { |e| e.split("|") }.flatten.compact_blank.uniq SELECT CONCAT(incoming_email, '|', email_username) FROM groups WHERE incoming_email IS NOT NULL OR email_username IS NOT NULL SQL ) end def category_email_in_regex @category_email_in_regex ||= Regexp.union Category.pluck(:email_in).select(&:present?).map { |e| e.split("|") }.flatten.uniq end def find_related_post(force: false) return if !force && SiteSetting.find_related_post_with_key && !sent_to_mailinglist_mirror? message_ids = Email::Receiver.extract_reply_message_ids(@mail, max_message_id_count: 5) return if message_ids.empty? post_ids = message_ids.map { |message_id| message_id[message_id_post_id_regexp, 1] }.compact.map(&:to_i) post_ids << Post.where(topic_id: message_ids.map { |message_id| message_id[message_id_topic_id_regexp, 1] }.compact, post_number: 1).pluck(:id) post_ids << EmailLog.where(message_id: message_ids).pluck(:post_id) post_ids << IncomingEmail.where(message_id: message_ids).pluck(:post_id) post_ids.flatten! post_ids.compact! post_ids.uniq! return if post_ids.empty? Post.where(id: post_ids).order(:created_at).last end def host @host ||= Email::Sender.host_for(Discourse.base_url) end def discourse_generated_message_id?(message_id) !!(message_id =~ message_id_post_id_regexp) || !!(message_id =~ message_id_topic_id_regexp) end def message_id_post_id_regexp @message_id_post_id_regexp ||= Regexp.new "topic/\\d+/(\\d+)@#{Regexp.escape(host)}" end def message_id_topic_id_regexp @message_id_topic_id_regexp ||= Regexp.new "topic/(\\d+)@#{Regexp.escape(host)}" end def self.extract_reply_message_ids(mail, max_message_id_count:) message_ids = [mail.in_reply_to, Email::Receiver.extract_references(mail.references)] message_ids.flatten! message_ids.select!(&:present?) message_ids.uniq! message_ids.first(max_message_id_count) end def self.extract_references(references) if Array === references references elsif references.present? references.split(/[\s,]/).map do |r| Email.message_id_clean(r) end end end def likes @likes ||= Set.new ["+1", "<3", "❤", I18n.t('post_action_types.like.title').downcase] end def subscription_action_for(body, subject) return unless SiteSetting.unsubscribe_via_email return if sent_to_mailinglist_mirror? if ([subject, body].compact.map(&:to_s).map(&:downcase) & ['unsubscribe']).any? :confirm_unsubscribe end end def post_action_for(body) PostActionType.types[:like] if likes.include?(body.strip.downcase) end def create_topic(options = {}) if options[:archetype] == Archetype.private_message enable_email_pm_setting(options[:user]) end create_post_with_attachments(options) end def notification_level_for(body) # since we are stripping save all this work on long replies return nil if body.length > 40 body = body.strip.downcase case body when "mute" NotificationLevels.topic_levels[:muted] when "track" NotificationLevels.topic_levels[:tracking] when "watch" NotificationLevels.topic_levels[:watching] else nil end end def create_reply(options = {}) raise TopicNotFoundError if options[:topic].nil? || options[:topic].trashed? raise BouncedEmailError if options[:bounce] && options[:topic].archetype != Archetype.private_message options[:post] = nil if options[:post]&.trashed? enable_email_pm_setting(options[:user]) if options[:topic].archetype == Archetype.private_message if post_action_type = post_action_for(options[:raw]) create_post_action(options[:user], options[:post], post_action_type) elsif notification_level = notification_level_for(options[:raw]) TopicUser.change(options[:user].id, options[:post].topic_id, notification_level: notification_level) else raise TopicClosedError if options[:topic].closed? options[:topic_id] = options[:topic].id options[:reply_to_post_number] = options[:post]&.post_number options[:is_group_message] = options[:topic].private_message? && options[:topic].allowed_groups.exists? create_post_with_attachments(options) end end def create_post_action(user, post, type) result = PostActionCreator.new(user, post, type).perform raise InvalidPostAction.new if result.failed? && result.forbidden end def is_allowed?(attachment) attachment.content_type !~ SiteSetting.blocked_attachment_content_types_regex && attachment.filename !~ SiteSetting.blocked_attachment_filenames_regex end def attachments @attachments ||= begin attachments = @mail.attachments.select { |attachment| is_allowed?(attachment) } attachments << @mail if @mail.attachment? && is_allowed?(@mail) @mail.parts.each do |part| attachments << part if part.attachment? && is_allowed?(part) end attachments.uniq! attachments end end def create_post_with_attachments(options = {}) add_elided_to_raw!(options) options[:raw] = add_attachments(options[:raw], options[:user], options) create_post(options) end def add_attachments(raw, user, options = {}) raw = raw.dup rejected_attachments = [] attachments.each do |attachment| tmp = Tempfile.new(["discourse-email-attachment", File.extname(attachment.filename)]) begin # read attachment File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded } # create the upload for the user opts = { for_group_message: options[:is_group_message] } upload = UploadCreator.new(tmp, attachment.filename, opts).create_for(user.id) if upload.errors.empty? # try to inline images if attachment.content_type&.start_with?("image/") if raw[attachment.url] raw.sub!(attachment.url, upload.url) InlineUploads.match_img(raw, uploads: { upload.url => upload }) do |match, src, replacement, _| if src == upload.url raw = raw.sub(match, replacement) end end elsif raw[/\[image:[^\]]*\]/i] raw.sub!(/\[image:[^\]]*\]/i, UploadMarkdown.new(upload).to_markdown) else raw << "\n\n#{UploadMarkdown.new(upload).to_markdown}\n\n" end else raw << "\n\n#{UploadMarkdown.new(upload).to_markdown}\n\n" end else rejected_attachments << upload raw << "\n\n#{I18n.t('emails.incoming.missing_attachment', filename: upload.original_filename)}\n\n" end ensure tmp&.close! end end notify_about_rejected_attachment(rejected_attachments) if rejected_attachments.present? && !user.staged? raw end def notify_about_rejected_attachment(attachments) errors = [] attachments.each do |a| error = a.errors.messages.values[0][0] errors << "#{a.original_filename}: #{error}" end message = Mail::Message.new(@mail) template_args = { former_title: message.subject, destination: message.to, site_name: SiteSetting.title, rejected_errors: errors.join("\n") } client_message = RejectionMailer.send_rejection(:email_reject_attachment, message.from, template_args) Email::Sender.new(client_message, :email_reject_attachment).send end def add_elided_to_raw!(options) is_private_message = options[:archetype] == Archetype.private_message || options[:topic].try(:private_message?) # only add elided part in messages if options[:elided].present? && (SiteSetting.always_show_trimmed_content || is_private_message) options[:raw] << Email::Receiver.elided_html(options[:elided]) options[:elided] = "" end end def create_post(options = {}) options[:import_mode] = @opts[:import_mode] options[:via_email] = true options[:raw_email] = @raw_email options[:created_at] ||= @mail.date options[:created_at] = DateTime.now if options[:created_at] > DateTime.now add_elided_to_raw!(options) if sent_to_mailinglist_mirror? options[:skip_validations] = true options[:skip_guardian] = true else options[:email_spam] = is_spam? options[:first_post_checks] = true if is_spam? options[:email_auth_res_action] = auth_res_action end user = options.delete(:user) if options[:bounce] options[:raw] = I18n.t("system_messages.email_bounced", email: user.email, raw: options[:raw]) user = Discourse.system_user options[:post_type] = Post.types[:whisper] end # To avoid race conditions with the post alerter and Group SMTP # emails, we skip the jobs here and enqueue them only _after_ # the incoming email has been updated with the post and topic. options[:skip_jobs] = true result = NewPostManager.new(user, options).perform errors = result.errors.full_messages if errors.any? do |message| message.include?(I18n.t("activerecord.attributes.post.raw").strip) && message.include?(I18n.t("errors.messages.too_short", count: SiteSetting.min_post_length).strip) end raise TooShortPost end if result.errors.present? raise InvalidPost, errors.join("\n") end if result.post @incoming_email.update_columns(topic_id: result.post.topic_id, post_id: result.post.id) if result.post.topic&.private_message? && !is_bounce? add_other_addresses(result.post, user) end # Alert the people involved in the topic now that the incoming email # has been linked to the post. PostJobsEnqueuer.new(result.post, result.post.topic, options[:topic_id].blank?, import_mode: options[:import_mode], post_alert_options: options[:post_alert_options] ).enqueue_jobs end result.post end def self.elided_html(elided) html = +"\n\n" << "
" << "\n" html << "···" << "\n\n" html << elided << "\n\n" html << "
" << "\n" html end def add_other_addresses(post, sender) %i(to cc bcc).each do |d| next if @mail[d].blank? @mail[d].each do |address_field| begin address_field.decoded email = address_field.address.downcase display_name = address_field.display_name.try(:to_s) next unless email["@"] if should_invite?(email) user = find_or_create_user(email, display_name) if user && can_invite?(post.topic, user) post.topic.topic_allowed_users.create!(user_id: user.id) TopicUser.auto_notification_for_staging(user.id, post.topic_id, TopicUser.notification_reasons[:auto_watch]) post.topic.add_small_action(sender, "invited_user", user.username, import_mode: @opts[:import_mode]) end # cap number of staged users created per email if @staged_users.count > SiteSetting.maximum_staged_users_per_email post.topic.add_moderator_post(sender, I18n.t("emails.incoming.maximum_staged_user_per_email_reached"), import_mode: @opts[:import_mode]) return end end rescue ActiveRecord::RecordInvalid, EmailNotAllowed # don't care if user already allowed or the user's email address is not allowed end end end end def should_invite?(email) email !~ Email::Receiver.reply_by_email_address_regex && email !~ group_incoming_emails_regex && email !~ category_email_in_regex end def can_invite?(topic, user) !topic.topic_allowed_users.where(user_id: user.id).exists? && !topic.topic_allowed_groups.where("group_id IN (SELECT group_id FROM group_users WHERE user_id = ?)", user.id).exists? end def send_subscription_mail(action, user) message = SubscriptionMailer.public_send(action, user) Email::Sender.new(message, :subscription).send end def stage_from_user @from_user ||= find_or_create_user!(@from_email, @from_display_name).tap do |u| log_and_validate_user(u) end end def delete_staged_users @staged_users.each do |user| if @incoming_email.user&.id == user.id @incoming_email.update_columns(user_id: nil) end if user.posts.count == 0 UserDestroyer.new(Discourse.system_user).destroy(user, quiet: true) end end end def enable_email_pm_setting(user) # ensure user PM emails are enabled (since user is posting via email) if !user.staged && user.user_option.email_messages_level == UserOption.email_level_types[:never] user.user_option.update!(email_messages_level: UserOption.email_level_types[:always]) end end def destination_too_old?(post) return false if post.blank? num_of_days = SiteSetting.disallow_reply_by_email_after_days num_of_days > 0 && post.created_at < num_of_days.days.ago end end end