DEV: Add service to validate email settings (#13021)

We have a few places in the code where we need to validate various email related settings, and will have another soon with the improved group email settings UI. This PR introduces a class which can validate POP3, IMAP, and SMTP credentials and also provide a friendly error message for issues if they must be presented to an end user.

This PR does not change any existing code to use the new service. I have added a TODO to change POP3 validation and the email test rake task to use the new validator post-release.

app/services/email_settings_validator.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+require 'net/imap'
+require 'net/smtp'
+require 'net/pop'
+
+# Usage:
+#
+# begin
+#   EmailSettingsValidator.validate_imap(host: "imap.test.com", port: 999, username: "test@test.com", password: "password")
+#
+#   # or for specific host preset
+#   EmailSettingsValidator.validate_imap(**{ username: "test@gmail.com", password: "test" }.merge(Email.gmail_imap_settings))
+#
+# rescue *EmailSettingsValidator::FRIENDLY_EXCEPTIONS => err
+#   EmailSettingsValidator.friendly_exception_message(err)
+# end
+class EmailSettingsValidator
+  EXPECTED_EXCEPTIONS = [
+    Net::POPAuthenticationError,
+    Net::IMAP::NoResponseError,
+    Net::SMTPAuthenticationError,
+    Net::SMTPServerBusy,
+    Net::SMTPSyntaxError,
+    Net::SMTPFatalError,
+    Net::SMTPUnknownError,
+    Net::OpenTimeout,
+    Net::ReadTimeout,
+    SocketError,
+    Errno::ECONNREFUSED
+  ]
+
+  def self.friendly_exception_message(exception)
+    case exception
+    when Net::POPAuthenticationError
+      I18n.t("email_settings.pop3_authentication_error")
+    when Net::IMAP::NoResponseError
+
+      # Most of IMAP's errors are lumped under the NoResponseError, including invalid
+      # credentials errors, because it is raised when a "NO" response is
+      # raised from the IMAP server https://datatracker.ietf.org/doc/html/rfc3501#section-7.1.2
+      #
+      # Generally, it should be fairly safe to just return the error message as is.
+      if exception.message.match(/Invalid credentials/)
+        I18n.t("email_settings.imap_authentication_error")
+      else
+        I18n.t("email_settings.imap_no_response_error", message: exception.message.gsub(" (Failure)", ""))
+      end
+    when Net::SMTPAuthenticationError
+      I18n.t("email_settings.smtp_authentication_error")
+    when Net::SMTPServerBusy
+      I18n.t("email_settings.smtp_server_busy_error")
+    when Net::SMTPSyntaxError, Net::SMTPFatalError, Net::SMTPUnknownError
+      I18n.t("email_settings.smtp_unhandled_error", message: exception.message)
+    when SocketError, Errno::ECONNREFUSED
+      I18n.t("email_settings.connection_error")
+    when Net::OpenTimeout, Net::ReadTimeout
+      I18n.t("email_settings.timeout_error")
+    else
+      I18n.t("email_settings.unhandled_error", message: exception.message)
+    end
+  end
+
+  ##
+  # Attempts to authenticate and disconnect a POP3 session and if that raises
+  # an error then it is assumed the credentials or some other settings are wrong.
+  #
+  # @param debug [Boolean] - When set to true, any errors will be logged at a warning
+  #                          level before being re-raised.
+  def self.validate_pop3(
+    host:,
+    port:,
+    username:,
+    password:,
+    ssl: SiteSetting.pop3_polling_ssl,
+    openssl_verify: SiteSetting.pop3_polling_openssl_verify,
+    debug: false
+  )
+    begin
+      pop3 = Net::POP3.new(host, port)
+
+      # Note that we do not allow which verification mode to be specified
+      # like we do for SMTP, we just pick TLS1_2 if the SSL and openSSL verify
+      # options have been enabled.
+      if ssl
+        if openssl_verify
+          pop3.enable_ssl(max_version: OpenSSL::SSL::TLS1_2_VERSION)
+        else
+          pop3.enable_ssl(OpenSSL::SSL::VERIFY_NONE)
+        end
+      end
+
+      # This disconnects itself, unlike SMTP and IMAP.
+      pop3.auth_only(username, password)
+    rescue => err
+      log_and_raise(err, debug)
+    end
+  end
+
+  ##
+  # Attempts to start an SMTP session and if that raises an error then it is
+  # assumed the credentials or other settings are wrong.
+  #
+  # For Gmail, the port should be 587, enable_starttls_auto should be true,
+  # and enable_tls should be false.
+  #
+  # @param domain [String] - Used for HELO, will be the email sender's domain, so often
+  #                          will just be the host e.g. the domain for test@gmail.com is gmail.com.
+  #                          localhost can be used in development mode.
+  #                          See https://datatracker.ietf.org/doc/html/rfc788#section-4
+  # @param debug [Boolean] - When set to true, any errors will be logged at a warning
+  #                          level before being re-raised.
+  def self.validate_smtp(
+    host:,
+    port:,
+    domain:,
+    username:,
+    password:,
+    authentication: GlobalSetting.smtp_authentication,
+    enable_starttls_auto: GlobalSetting.smtp_enable_start_tls,
+    enable_tls: GlobalSetting.smtp_force_tls,
+    openssl_verify_mode: GlobalSetting.smtp_openssl_verify_mode,
+    debug: false
+  )
+    begin
+      if enable_tls && enable_starttls_auto
+        raise ArgumentError, "TLS and STARTTLS are mutually exclusive"
+      end
+
+      if ![:plain, :login, :cram_md5].include?(authentication.to_sym)
+        raise ArgumentError, "Invalid authentication method. Must be plain, login, or cram_md5."
+      end
+
+      smtp = Net::SMTP.new(host, port)
+
+      # These SSL options are cribbed from the Mail gem, which is used internally
+      # by ActionMailer. Unfortunately the mail gem hides this setup in private
+      # methods, e.g. https://github.com/mikel/mail/blob/master/lib/mail/network/delivery_methods/smtp.rb#L112-L147
+      #
+      # Relying on the GlobalSetting options is a good idea here.
+      #
+      # For specific use cases, options should be passed in from higher up. For example
+      # Gmail needs either port 465 and tls enabled, or port 587 and starttls_auto.
+      if openssl_verify_mode.kind_of?(String)
+        openssl_verify_mode = OpenSSL::SSL.const_get("VERIFY_#{openssl_verify_mode.upcase}")
+      end
+      ssl_context = Net::SMTP.default_ssl_context
+      ssl_context.verify_mode = openssl_verify_mode if openssl_verify_mode
+
+      smtp.enable_starttls_auto(ssl_context) if enable_starttls_auto
+      smtp.enable_tls(ssl_context) if enable_tls
+
+      smtp.start(domain, username, password, authentication.to_sym)
+      smtp.finish
+    rescue => err
+      log_and_raise(err, debug)
+    end
+  end
+
+  ##
+  # Attempts to login, logout, and disconnect an IMAP session and if that raises
+  # an error then it is assumed the credentials or some other settings are wrong.
+  #
+  # @param debug [Boolean] - When set to true, any errors will be logged at a warning
+  #                          level before being re-raised.
+  def self.validate_imap(
+    host:,
+    port:,
+    username:,
+    password:,
+    open_timeout: 10,
+    ssl: true,
+    debug: false
+  )
+    begin
+      imap = Net::IMAP.new(host, port: port, ssl: ssl, open_timeout: open_timeout)
+      imap.login(username, password)
+      imap.logout rescue nil
+      imap.disconnect
+    rescue => err
+      log_and_raise(err, debug)
+    end
+  end
+
+  def self.log_and_raise(err, debug)
+    if debug
+      Rails.logger.warn("[EmailSettingsValidator] Error encountered when validating email settings: #{err.message} #{err.backtrace.join("\n")}")
+    end
+    raise err
+  end
+end

config/locales/server.en.yml

@@ -986,6 +986,16 @@ en:
     no_drafts:
       self: "You have no drafts; begin composing a reply in any topic and it will be auto-saved as a new draft."
 
+  email_settings:
+    pop3_authentication_error: "There was an issue with the POP3 credentials provided, check the username and password and try again."
+    imap_authentication_error: "There was an issue with the IMAP credentials provided, check the username and password and try again."
+    imap_no_response_error: "An error occurred when communicating with the IMAP server. %{message}"
+    smtp_authentication_error: "There was an issue with the SMTP credentials provided, check the username and password and try again."
+    smtp_server_busy_error: "The SMTP server is currently busy, try again later."
+    smtp_unhandled_error: "There was an unhandled error when communicating with the SMTP server. %{message}"
+    connection_error: "There was an issue connecting with the server, check the server name and port and try again."
+    timeout_error: "Connection to the server timed out, check the server name and port and try again."
+    unhandled_error: "Unhandled error when testing email settings. %{message}"
   webauthn:
     validation:
       invalid_type_error: "The webauthn type provided was invalid. Valid types are webauthn.get and webauthn.create."

lib/tasks/emails.rake

@@ -80,6 +80,16 @@ task 'emails:test', [:email] => [:environment] do |_, args|
 
     puts "Testing sending to #{email} using #{smtp[:address]}:#{smtp[:port]}, username:#{smtp[:user_name]} with #{smtp[:authentication]} auth."
 
+    # TODO (martin, post-2.7 release) Change to use EmailSettingsValidator
+    # EmailSettingsValidator.validate_smtp(
+    #   host: smtp[:address],
+    #   port: smtp[:port],
+    #   domain: smtp[:domain] || 'localhost',
+    #   username: smtp[:user_name],
+    #   password: smtp[:password],
+    #   authentication: smtp[:authentication] || 'plain'
+    # )
+
     # We would like to do this, but Net::SMTP errors out using starttls
     #Net::SMTP.start(smtp[:address], smtp[:port]) do |s|
     #  s.starttls if !!smtp[:enable_starttls_auto] && s.capable_starttls?

lib/validators/pop3_polling_enabled_setting_validator.rb

@@ -33,6 +33,16 @@ class POP3PollingEnabledSettingValidator
   private
 
   def authentication_works?
+
+    # TODO (martin, post-2.7 release) Change to use EmailSettingsValidator
+    # EmailSettingsValidator.validate_pop3(
+    #   host: SiteSetting.pop3_polling_host,
+    #   port: SiteSetting.pop3_polling_port,
+    #   ssl: SiteSetting.pop3_polling_ssl,
+    #   username: SiteSetting.pop3_polling_username,
+    #   password: SiteSetting.pop3_polling_password,
+    #   openssl_verify: SiteSetting.pop3_polling_openssl_verify
+    # )
     @authentication_works ||= begin
       pop3 = Net::POP3.new(SiteSetting.pop3_polling_host, SiteSetting.pop3_polling_port)
       pop3.enable_ssl(OpenSSL::SSL::VERIFY_NONE) if SiteSetting.pop3_polling_ssl

spec/services/email_settings_validator_spec.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe EmailSettingsValidator do
+  let(:username) { "kwest@gmail.com" }
+  let(:password) { "mbdtf" }
+
+  describe "#friendly_exception_message" do
+    it "formats a Net::POPAuthenticationError" do
+      exception = Net::POPAuthenticationError.new("invalid credentials")
+      expect(subject.class.friendly_exception_message(exception)).to eq(
+        I18n.t("email_settings.pop3_authentication_error")
+      )
+    end
+
+    it "formats a Net::IMAP::NoResponseError for invalid credentials" do
+      exception = Net::IMAP::NoResponseError.new(stub(data: stub(text: "Invalid credentials")))
+      expect(subject.class.friendly_exception_message(exception)).to eq(
+        I18n.t("email_settings.imap_authentication_error")
+      )
+    end
+
+    it "formats a general Net::IMAP::NoResponseError" do
+      exception = Net::IMAP::NoResponseError.new(stub(data: stub(text: "NO bad problem (Failure)")))
+      expect(subject.class.friendly_exception_message(exception)).to eq(
+        I18n.t("email_settings.imap_no_response_error", message: "NO bad problem")
+      )
+    end
+
+    it "formats a Net::SMTPAuthenticationError" do
+      exception = Net::SMTPAuthenticationError.new("invalid credentials")
+      expect(subject.class.friendly_exception_message(exception)).to eq(
+        I18n.t("email_settings.smtp_authentication_error")
+      )
+    end
+
+    it "formats a Net::SMTPServerBusy" do
+      exception = Net::SMTPServerBusy.new("call me maybe later")
+      expect(subject.class.friendly_exception_message(exception)).to eq(
+        I18n.t("email_settings.smtp_server_busy_error")
+      )
+    end
+
+    it "formats a Net::SMTPSyntaxError, Net::SMTPFatalError, and Net::SMTPUnknownError" do
+      exception = Net::SMTPSyntaxError.new("bad syntax")
+      expect(subject.class.friendly_exception_message(exception)).to eq(
+        I18n.t("email_settings.smtp_unhandled_error", message: exception.message)
+      )
+      exception = Net::SMTPFatalError.new("fatal")
+      expect(subject.class.friendly_exception_message(exception)).to eq(
+        I18n.t("email_settings.smtp_unhandled_error", message: exception.message)
+      )
+      exception = Net::SMTPUnknownError.new("unknown")
+      expect(subject.class.friendly_exception_message(exception)).to eq(
+        I18n.t("email_settings.smtp_unhandled_error", message: exception.message)
+      )
+    end
+
+    it "formats a SocketError and Errno::ECONNREFUSED" do
+      exception = SocketError.new("bad socket")
+      expect(subject.class.friendly_exception_message(exception)).to eq(
+        I18n.t("email_settings.connection_error")
+      )
+      exception = Errno::ECONNREFUSED.new("no thanks")
+      expect(subject.class.friendly_exception_message(exception)).to eq(
+        I18n.t("email_settings.connection_error")
+      )
+    end
+
+    it "formats a Net::OpenTimeout and Net::ReadTimeout error" do
+      exception = Net::OpenTimeout.new("timed out")
+      expect(subject.class.friendly_exception_message(exception)).to eq(
+        I18n.t("email_settings.timeout_error")
+      )
+      exception = Net::ReadTimeout.new("timed out")
+      expect(subject.class.friendly_exception_message(exception)).to eq(
+        I18n.t("email_settings.timeout_error")
+      )
+    end
+
+    it "formats unhandled errors" do
+      exception = StandardError.new("unknown")
+      expect(subject.class.friendly_exception_message(exception)).to eq(
+        I18n.t("email_settings.unhandled_error", message: exception.message)
+      )
+    end
+  end
+
+  describe "#validate_imap" do
+    let(:host) { "imap.gmail.com" }
+    let(:port) { 993 }
+
+    let(:net_imap_stub) do
+      obj = mock()
+      obj.stubs(:login).returns(true)
+      obj
+    end
+
+    before do
+      Net::IMAP.stubs(:new).returns(net_imap_stub)
+    end
+
+    it "is valid if no error is raised" do
+      net_imap_stub.stubs(:logout).returns(true)
+      net_imap_stub.stubs(:disconnect).returns(true)
+      expect { subject.class.validate_imap(host: host, port: port, username: username, password: password) }.not_to raise_error
+    end
+
+    it "is invalid if an error is raised" do
+      net_imap_stub.stubs(:login).raises(Net::IMAP::NoResponseError, stub(data: stub(text: "no response")))
+      expect { subject.class.validate_imap(host: host, port: port, username: username, password: password, debug: true) }.to raise_error(Net::IMAP::NoResponseError)
+    end
+
+    it "logs a warning if debug: true passed in and still raises the error" do
+      net_imap_stub.stubs(:login).raises(Net::IMAP::NoResponseError, stub(data: stub(text: "no response")))
+      Rails.logger.expects(:warn).with(regexp_matches(/\[EmailSettingsValidator\] Error encountered/)).at_least_once
+      expect { subject.class.validate_imap(host: host, port: port, username: username, password: password, debug: true) }.to raise_error(Net::IMAP::NoResponseError)
+    end
+  end
+
+  describe "#validate_pop3" do
+    let(:host) { "pop.gmail.com" }
+    let(:port) { 995 }
+
+    let(:net_pop3_stub) do
+      obj = mock()
+      obj.stubs(:auth_only).returns(true)
+      obj.stubs(:enable_ssl).returns(true)
+      obj
+    end
+
+    before do
+      Net::POP3.stubs(:new).returns(net_pop3_stub)
+    end
+
+    it "is valid if no error is raised" do
+      expect { subject.class.validate_pop3(host: host, port: port, username: username, password: password) }.not_to raise_error
+    end
+
+    it "is invalid if an error is raised" do
+      net_pop3_stub.stubs(:auth_only).raises(Net::POPAuthenticationError, "invalid credentials")
+      expect { subject.class.validate_pop3(host: host, port: port, username: username, password: password, debug: true) }.to raise_error(Net::POPAuthenticationError)
+    end
+
+    it "logs a warning if debug: true passed in and still raises the error" do
+      Rails.logger.expects(:warn).with(regexp_matches(/\[EmailSettingsValidator\] Error encountered/)).at_least_once
+      net_pop3_stub.stubs(:auth_only).raises(Net::POPAuthenticationError, "invalid credentials")
+      expect { subject.class.validate_pop3(host: host, port: port, username: username, password: password, debug: true) }.to raise_error(Net::POPAuthenticationError)
+    end
+
+    it "uses the correct ssl verify params if those settings are enabled" do
+      SiteSetting.pop3_polling_ssl = true
+      SiteSetting.pop3_polling_openssl_verify = true
+      net_pop3_stub.expects(:enable_ssl).with(max_version: OpenSSL::SSL::TLS1_2_VERSION)
+      expect { subject.class.validate_pop3(host: host, port: port, username: username, password: password) }.not_to raise_error
+    end
+
+    it "uses the correct ssl verify params if openssl_verify is not enabled" do
+      SiteSetting.pop3_polling_ssl = true
+      SiteSetting.pop3_polling_openssl_verify = false
+      net_pop3_stub.expects(:enable_ssl).with(OpenSSL::SSL::VERIFY_NONE)
+      expect { subject.class.validate_pop3(host: host, port: port, username: username, password: password) }.not_to raise_error
+    end
+  end
+
+  describe "#validate_smtp" do
+    let(:host) { "smtp.gmail.com" }
+    let(:port) { 587 }
+    let(:domain) { "gmail.com" }
+
+    let(:net_smtp_stub) do
+      obj = mock()
+      obj.stubs(:start).returns(true)
+      obj.stubs(:finish).returns(true)
+      obj.stubs(:enable_tls).returns(true)
+      obj.stubs(:enable_starttls_auto).returns(true)
+      obj
+    end
+
+    before do
+      Net::SMTP.stubs(:new).returns(net_smtp_stub)
+    end
+
+    it "is valid if no error is raised" do
+      expect { subject.class.validate_smtp(host: host, port: port, username: username, password: password, domain: domain) }.not_to raise_error
+    end
+
+    it "is invalid if an error is raised" do
+      net_smtp_stub.stubs(:start).raises(Net::SMTPAuthenticationError, "invalid credentials")
+      expect { subject.class.validate_smtp(host: host, port: port, username: username, password: password, domain: domain) }.to raise_error(Net::SMTPAuthenticationError)
+    end
+
+    it "logs a warning if debug: true passed in and still raises the error" do
+      Rails.logger.expects(:warn).with(regexp_matches(/\[EmailSettingsValidator\] Error encountered/)).at_least_once
+      net_smtp_stub.stubs(:start).raises(Net::SMTPAuthenticationError, "invalid credentials")
+      expect { subject.class.validate_smtp(host: host, port: port, username: username, password: password, debug: true, domain: domain) }.to raise_error(Net::SMTPAuthenticationError)
+    end
+
+    it "uses the correct ssl verify params for enable_tls if those settings are enabled" do
+      net_smtp_stub.expects(:enable_tls)
+      expect { subject.class.validate_smtp(host: host, port: port, username: username, password: password, domain: domain, openssl_verify_mode: "peer", enable_tls: true, enable_starttls_auto: false) }.not_to raise_error
+    end
+
+    it "uses the correct ssl verify params for enable_starttls_auto if those settings are enabled" do
+      net_smtp_stub.expects(:enable_starttls_auto)
+      expect { subject.class.validate_smtp(host: host, port: port, username: username, password: password, domain: domain, openssl_verify_mode: "peer", enable_tls: false, enable_starttls_auto: true) }.not_to raise_error
+    end
+
+    it "raises an ArgumentError if both enable_tls is true and enable_starttls_auto is true" do
+      expect { subject.class.validate_smtp(host: host, port: port, username: username, password: password, domain: domain, enable_ssl: true, enable_starttls_auto: true) }.to raise_error(ArgumentError)
+    end
+
+    it "raises an ArgumentError if a bad authentication method is used" do
+      expect { subject.class.validate_smtp(host: host, port: port, username: username, password: password, domain: domain, authentication: :rubber_stamp) }.to raise_error(ArgumentError)
+    end
+  end
+end