discourse/lib/demon/sidekiq.rb
Alan Guo Xiang Tan 23c38cbf11
DEV: Log Unicorn worker timeout backtraces to Rails.logger (#27257)
This commit introduces the following changes:

1. Introduce the `SignalTrapLogger` singleton which starts a single
   thread that polls a queue to log messages with the specified logger.
   This thread is necessary becasue most loggers cannot be used inside
   the `Signal.trap` context as they rely on mutexes which are not
   allowed within the context.

2. Moves the monkey patch in `freedom_patches/unicorn_http_server_patch.rb` to
   `config/unicorn.config.rb` which is already monkey patching
   `Unicorn::HttpServer`.

3. `Unicorn::HttpServer` will now automatically send a `USR2` signal to
   a unicorn worker 2 seconds before the worker is timed out by the
   Unicorn master.

4. When a Unicorn worker receives a `USR2` signal, it will now log only
   the main thread's backtraces to `Rails.logger`. Previously, it was
   `put`ing the backtraces to `STDOUT` which most people wouldn't read.
   Logging it via `Rails.logger` will make the backtraces easily
   accessible via `/logs`.
2024-06-03 12:51:12 +08:00

66 lines
1.7 KiB
Ruby

# frozen_string_literal: true
require "demon/base"
class Demon::Sidekiq < ::Demon::Base
def self.prefix
"sidekiq"
end
def self.after_fork(&blk)
blk ? (@blk = blk) : @blk
end
private
def suppress_stdout
false
end
def suppress_stderr
false
end
def log_in_trap(message, level: :info)
SignalTrapLogger.instance.log(@logger, message, level: level)
end
def after_fork
Demon::Sidekiq.after_fork&.call
SignalTrapLogger.instance.after_fork
log("Loading Sidekiq in process id #{Process.pid}")
require "sidekiq/cli"
cli = Sidekiq::CLI.instance
# Unicorn uses USR1 to indicate that log files have been rotated
Signal.trap("USR1") do
log_in_trap("Sidekiq reopening logs...")
Unicorn::Util.reopen_logs
log_in_trap("Sidekiq done reopening logs...")
end
options = ["-c", GlobalSetting.sidekiq_workers.to_s]
[["critical", 8], ["default", 4], ["low", 2], ["ultra_low", 1]].each do |queue_name, weight|
custom_queue_hostname = ENV["UNICORN_SIDEKIQ_#{queue_name.upcase}_QUEUE_HOSTNAME"]
if !custom_queue_hostname || custom_queue_hostname.split(",").include?(Discourse.os_hostname)
options << "-q"
options << "#{queue_name},#{weight}"
end
end
# Sidekiq not as high priority as web, in this environment it is forked so a web is very
# likely running
Discourse::Utils.execute_command("renice", "-n", "5", "-p", Process.pid.to_s)
cli.parse(options)
load Rails.root + "config/initializers/100-sidekiq.rb"
cli.run
rescue => e
log("Error encountered while starting Sidekiq: #{e.message}\n#{e.backtrace.join("\n")}")
exit 1
end
end