Use a different Redis key when PG failover sets site to readonly mode.

app/controllers/admin/backups_controller.rb

@@ -95,7 +95,14 @@ class Admin::BackupsController < Admin::AdminController
 
   def readonly
     enable = params.fetch(:enable).to_s == "true"
-    enable ? Discourse.enable_readonly_mode(user_enabled: true) : Discourse.disable_readonly_mode(user_enabled: true)
+    readonly_mode_key = Discourse::USER_READONLY_MODE_KEY
+
+    if enable
+      Discourse.enable_readonly_mode(readonly_mode_key)
+    else
+      Discourse.disable_readonly_mode(readonly_mode_key)
+    end
+
     render nothing: true
   end
 

lib/active_record/connection_adapters/postgresql_fallback_adapter.rb

@@ -52,7 +52,7 @@ class PostgreSQLFallbackHandler
             logger.warn "#{log_prefix}: Master server is active. Reconnecting..."
 
             self.master_up(key)
-            Discourse.disable_readonly_mode
+            Discourse.disable_readonly_mode(Discourse::PG_READONLY_MODE_KEY)
           end
         rescue => e
           logger.warn "#{log_prefix}: Connection to master PostgreSQL server failed with '#{e.message}'"
@@ -103,7 +103,7 @@ module ActiveRecord
         }))
 
         verify_replica(connection)
-        Discourse.enable_readonly_mode
+        Discourse.enable_readonly_mode(Discourse::PG_READONLY_MODE_KEY)
       else
         begin
           connection = postgresql_connection(config)

lib/discourse.rb

@@ -113,24 +113,6 @@ module Discourse
     end
   end
 
-  def self.last_read_only
-    @last_read_only ||= {}
-  end
-
-  def self.recently_readonly?
-    read_only = last_read_only[$redis.namespace]
-    return false unless read_only
-    read_only > 15.seconds.ago
-  end
-
-  def self.received_readonly!
-    last_read_only[$redis.namespace] = Time.zone.now
-  end
-
-  def self.clear_readonly!
-    last_read_only[$redis.namespace] = nil
-  end
-
   def self.disabled_plugin_names
     plugins.select { |p| !p.enabled? }.map(&:name)
   end
@@ -212,41 +194,67 @@ module Discourse
 
   READONLY_MODE_KEY_TTL ||= 60
   READONLY_MODE_KEY ||= 'readonly_mode'.freeze
+  PG_READONLY_MODE_KEY ||= 'readonly_mode:postgres'.freeze
   USER_READONLY_MODE_KEY ||= 'readonly_mode:user'.freeze
 
-  def self.enable_readonly_mode(user_enabled: false)
-    if user_enabled
-      $redis.set(USER_READONLY_MODE_KEY, 1)
+  READONLY_KEYS = [
+    READONLY_MODE_KEY,
+    PG_READONLY_MODE_KEY,
+    USER_READONLY_MODE_KEY
+  ]
+
+  def self.enable_readonly_mode(key = READONLY_MODE_KEY)
+    if key == USER_READONLY_MODE_KEY
+      $redis.set(key, 1)
     else
-      $redis.setex(READONLY_MODE_KEY, READONLY_MODE_KEY_TTL, 1)
-      keep_readonly_mode
+      $redis.setex(key, READONLY_MODE_KEY_TTL, 1)
+      keep_readonly_mode(key)
     end
 
     MessageBus.publish(readonly_channel, true)
     true
   end
 
-  def self.keep_readonly_mode
+  def self.keep_readonly_mode(key)
     # extend the expiry by 1 minute every 30 seconds
     unless Rails.env.test?
       Thread.new do
         while readonly_mode?
-          $redis.expire(READONLY_MODE_KEY, READONLY_MODE_KEY_TTL)
+          $redis.expire(key, READONLY_MODE_KEY_TTL)
           sleep 30.seconds
         end
       end
     end
   end
 
-  def self.disable_readonly_mode(user_enabled: false)
-    key = user_enabled ? USER_READONLY_MODE_KEY : READONLY_MODE_KEY
+  def self.disable_readonly_mode(key = READONLY_MODE_KEY)
     $redis.del(key)
     MessageBus.publish(readonly_channel, false)
     true
   end
 
   def self.readonly_mode?
-    recently_readonly? || !!$redis.get(READONLY_MODE_KEY) || !!$redis.get(USER_READONLY_MODE_KEY)
+    return true if recently_readonly?
+    READONLY_KEYS.each { |key| return true if !!$redis.get(key) }
+    false
+  end
+
+  def self.last_read_only
+    @last_read_only ||= {}
+  end
+
+  def self.recently_readonly?
+    read_only = last_read_only[$redis.namespace]
+    return false unless read_only
+    read_only > 15.seconds.ago
+  end
+
+  def self.received_readonly!
+    last_read_only[$redis.namespace] = Time.zone.now
+  end
+
+  def self.clear_readonly!
+    last_read_only[$redis.namespace] = nil
   end
 
   def self.request_refresh!

spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb

@@ -42,8 +42,13 @@ describe ActiveRecord::ConnectionHandling do
       end
 
       after do
-        with_multisite_db(multisite_db) { Discourse.disable_readonly_mode }
-        Discourse.disable_readonly_mode
+        pg_readonly_mode_key = Discourse::PG_READONLY_MODE_KEY
+
+        with_multisite_db(multisite_db) do
+          Discourse.disable_readonly_mode(pg_readonly_mode_key)
+        end
+
+        Discourse.disable_readonly_mode(pg_readonly_mode_key)
         ActiveRecord::Base.establish_connection(ActiveRecord::Base.configurations[Rails.env])
       end
 

spec/components/discourse_spec.rb

@@ -118,7 +118,7 @@ describe Discourse do
       context 'user enabled readonly mode' do
         it "adds a key in redis and publish a message through the message bus" do
           expect($redis.get(user_readonly_mode_key)).to eq(nil)
-          message = MessageBus.track_publish { Discourse.enable_readonly_mode(user_enabled: true) }.first
+          message = MessageBus.track_publish { Discourse.enable_readonly_mode(user_readonly_mode_key) }.first
           assert_readonly_mode(message, user_readonly_mode_key)
         end
       end
@@ -160,10 +160,10 @@ describe Discourse do
       end
 
       it "returns true when user enabled readonly mode key is present in redis" do
-        Discourse.enable_readonly_mode(user_enabled: true)
+        Discourse.enable_readonly_mode(user_readonly_mode_key)
         expect(Discourse.readonly_mode?).to eq(true)
 
-        Discourse.disable_readonly_mode(user_enabled: true)
+        Discourse.disable_readonly_mode(user_readonly_mode_key)
         expect(Discourse.readonly_mode?).to eq(false)
       end
     end