EXPERIMENTAL: Allow logs to be shipped via different methods.

See https://github.com/dwbutler/logstash-logger#uri-configuration for
a list of available methods.

Gemfile

@@ -174,6 +174,7 @@ gem 'cppjieba_rb', require: false
 
 gem 'lograge', require: false
 gem 'logstash-event', require: false
+gem 'logstash-logger', require: false
 gem 'logster'
 
 gem 'sassc', require: false

Gemfile.lock

@@ -156,6 +156,8 @@ GEM
       railties (>= 4, < 5.2)
       request_store (~> 1.0)
     logstash-event (1.2.02)
+    logstash-logger (0.25.1)
+      logstash-event (~> 1.2)
     logster (1.2.8)
     loofah (2.1.1)
       crass (~> 1.0.2)
@@ -440,6 +442,7 @@ DEPENDENCIES
   listen
   lograge
   logstash-event
+  logstash-logger
   logster
   lru_redux
   mail

config/initializers/100-lograge.rb

@@ -8,8 +8,6 @@ if (Rails.env.production? && SiteSetting.logging_provider == 'lograge') || ENV["
   Rails.application.configure do
     config.lograge.enabled = true
 
-    logstash_formatter = ENV["LOGSTASH_FORMATTER"]
-
     config.lograge.custom_options = lambda do |event|
       exceptions = %w(controller action format id)
 
@@ -21,12 +19,17 @@ if (Rails.env.production? && SiteSetting.logging_provider == 'lograge') || ENV["
         database: RailsMultisite::ConnectionManagement.current_db,
       }
 
-      output[:time] = event.time unless logstash_formatter
       output
     end
 
-    if logstash_formatter
+    if ENV["LOGSTASH_URI"]
       config.lograge.formatter = Lograge::Formatters::Logstash.new
+
+      require 'discourse_logstash_logger'
+
+      config.lograge.logger = DiscourseLogstashLogger.logger(
+        uri: ENV['LOGSTASH_URI'], type: :rails
+      )
     end
   end
 end

config/unicorn.conf.rb

@@ -1,10 +1,8 @@
 # See http://unicorn.bogomips.org/Unicorn/Configurator.html
 
-if ENV["UNICORN_JSON_LOG_FORMAT"]
+if ENV["LOGSTASH_URI"]
-  unicorn_logger = Logger.new($stderr)
+  require_relative '../lib/discourse_logstash_logger'
-  require_relative '../lib/unicorn/unicorn_json_log_formatter'
+  logger DiscourseLogstashLogger.logger(uri: ENV['LOGSTASH_URI'], type: :unicorn)
-  unicorn_logger.formatter = UnicornJSONLogFormatter.new
-  logger unicorn_logger
 end
 
 # enable out of band gc out of the box, it is low risk and improves perf a lot

lib/discourse_logstash_logger.rb

@@ -0,0 +1,16 @@
+require 'logstash-logger'
+
+class DiscourseLogstashLogger
+  def self.logger(uri:, type:)
+    LogStashLogger.new(
+      uri: uri,
+      sync: true,
+      customize_event: ->(event) {
+        event['hostname'] = `hostname`.chomp
+        event['severity'] = Object.const_get("Logger::Severity::#{event['severity']}")
+        event['severity_name'] = event['severity']
+        event['type'] = type
+      },
+    )
+  end
+end

lib/unicorn/unicorn_json_log_formatter.rb

@@ -1,22 +0,0 @@
-require 'json'
-
-class UnicornJSONLogFormatter < Logger::Formatter
-  def call(severity, datetime, progname, message)
-    default = {
-      severity: Object.const_get("Logger::Severity::#{severity}"),
-      severity_name: severity,
-      datetime: DateTime.parse(datetime.to_s).to_s,
-      progname: progname || '',
-      pid: $$,
-    }
-
-    default[:message] =
-      if message.is_a?(Exception)
-        "#{message.message}: #{message.backtrace.join("\n")}"
-      else
-        message
-      end
-
-    "#{default.to_json}\n"
-  end
-end

spec/components/unicorn/unicorn_json_log_formatter_spec.rb

@@ -1,32 +0,0 @@
-require 'rails_helper'
-require 'unicorn/unicorn_json_log_formatter'
-
-RSpec.describe UnicornJSONLogFormatter do
-  context 'when message is an exception' do
-    it 'should include the backtrace' do
-      freeze_time do
-        begin
-          raise 'boom'
-        rescue => e
-          error = e
-
-          output = described_class.new.call(
-            'ERROR',
-            Time.zone.now,
-            '',
-            e
-          )
-        end
-
-        output = JSON.parse(output)
-
-        expect(output["severity"]).to eq(3)
-        expect(output["severity_name"]).to eq("ERROR")
-        expect(output["datetime"]).to be_present
-        expect(output["progname"]).to eq('')
-        expect(output["pid"]).to be_present
-        expect(output["message"]).to match(/boom:.*/)
-      end
-    end
-  end
-end