FIX: Improve failover for multisite clusters (#11150)

- Bump rails_failover for new per-backend callback feature
- If the master backend fails over, make all sites readonly. And vice-versa for fallback
- If a single backend fails over, make that individual site readonly. And vice-versa for fallback
- When a single backend fails, also check connection to the master backend

Gemfile.lock

@@ -275,7 +275,7 @@ GEM
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
-    rails_failover (0.5.9)
+    rails_failover (0.6.0)
       activerecord (~> 6.0)
       railties (~> 6.0)
     rails_multisite (2.5.0)

config/initializers/002-rails_failover.rb

@@ -36,19 +36,34 @@ if defined?(RailsFailover::ActiveRecord)
     end
   end
 
-  RailsFailover::ActiveRecord.on_failover do
-    if RailsMultisite::ConnectionManagement.current_db == RailsMultisite::ConnectionManagement::DEFAULT
+  RailsFailover::ActiveRecord.on_failover do |role|
+    if role == ActiveRecord::Base.writing_role # Multisite master
       RailsMultisite::ConnectionManagement.each_connection do
-        Sidekiq.pause!("pg_failover") if !Sidekiq.paused?
         Discourse.enable_readonly_mode(Discourse::PG_READONLY_MODE_KEY)
       end
+    else
+      ActiveRecord::Base.connected_to(role: role) do
+        Discourse.enable_readonly_mode(Discourse::PG_READONLY_MODE_KEY)
+      end
+
+      # Test connection to the master, and trigger master failover if needed
+      ActiveRecord::Base.connected_to(role: ActiveRecord::Base.writing_role) do
+        ActiveRecord::Base.connection.active?
+      rescue PG::ConnectionBad, PG::UnableToSend, PG::ServerError
+        RailsFailover::ActiveRecord.verify_primary(ActiveRecord::Base.writing_role)
+      end
     end
   end
 
-  RailsFailover::ActiveRecord.on_fallback do
+  RailsFailover::ActiveRecord.on_fallback do |role|
+    if role == ActiveRecord::Base.writing_role # Multisite master
       RailsMultisite::ConnectionManagement.each_connection do
         Discourse.disable_readonly_mode(Discourse::PG_READONLY_MODE_KEY)
-      Sidekiq.unpause! if Sidekiq.paused?
+      end
+    else
+      ActiveRecord::Base.connected_to(role: role) do
+        Discourse.disable_readonly_mode(Discourse::PG_READONLY_MODE_KEY)
+      end
     end
 
     if Rails.configuration.multisite

lib/discourse.rb

@@ -448,6 +448,10 @@ module Discourse
   ]
 
   def self.enable_readonly_mode(key = READONLY_MODE_KEY)
+    if key == PG_READONLY_MODE_KEY || key == PG_FORCE_READONLY_MODE_KEY
+      Sidekiq.pause!("pg_failover") if !Sidekiq.paused?
+    end
+
     if key == USER_READONLY_MODE_KEY || key == PG_FORCE_READONLY_MODE_KEY
       Discourse.redis.set(key, 1)
     else
@@ -497,6 +501,10 @@ module Discourse
   end
 
   def self.disable_readonly_mode(key = READONLY_MODE_KEY)
+    if key == PG_READONLY_MODE_KEY || key == PG_FORCE_READONLY_MODE_KEY
+      Sidekiq.unpause! if Sidekiq.paused?
+    end
+
     Discourse.redis.del(key)
     MessageBus.publish(readonly_channel, false)
     true
@@ -505,7 +513,6 @@ module Discourse
   def self.enable_pg_force_readonly_mode
     RailsMultisite::ConnectionManagement.each_connection do
       enable_readonly_mode(PG_FORCE_READONLY_MODE_KEY)
-      Sidekiq.pause!("pg_failover") if !Sidekiq.paused?
     end
 
     true
@@ -514,7 +521,6 @@ module Discourse
   def self.disable_pg_force_readonly_mode
     RailsMultisite::ConnectionManagement.each_connection do
       disable_readonly_mode(PG_FORCE_READONLY_MODE_KEY)
-      Sidekiq.unpause!
     end
 
     true