From cdd550e9471fc73b2ffc76fc5378da58b6897f5b Mon Sep 17 00:00:00 2001 From: Guo Xiang Tan Date: Wed, 11 Jan 2017 16:38:07 +0800 Subject: [PATCH] Use a different Redis key when PG failover sets site to readonly mode. --- app/controllers/admin/backups_controller.rb | 9 ++- .../postgresql_fallback_adapter.rb | 4 +- lib/discourse.rb | 64 +++++++++++-------- .../postgresql_fallback_adapter_spec.rb | 9 ++- spec/components/discourse_spec.rb | 6 +- 5 files changed, 56 insertions(+), 36 deletions(-) diff --git a/app/controllers/admin/backups_controller.rb b/app/controllers/admin/backups_controller.rb index 3c7855668c9..2c71943b488 100644 --- a/app/controllers/admin/backups_controller.rb +++ b/app/controllers/admin/backups_controller.rb @@ -95,7 +95,14 @@ class Admin::BackupsController < Admin::AdminController def readonly enable = params.fetch(:enable).to_s == "true" - enable ? Discourse.enable_readonly_mode(user_enabled: true) : Discourse.disable_readonly_mode(user_enabled: true) + readonly_mode_key = Discourse::USER_READONLY_MODE_KEY + + if enable + Discourse.enable_readonly_mode(readonly_mode_key) + else + Discourse.disable_readonly_mode(readonly_mode_key) + end + render nothing: true end diff --git a/lib/active_record/connection_adapters/postgresql_fallback_adapter.rb b/lib/active_record/connection_adapters/postgresql_fallback_adapter.rb index 2f2e75030cd..84500b50f77 100644 --- a/lib/active_record/connection_adapters/postgresql_fallback_adapter.rb +++ b/lib/active_record/connection_adapters/postgresql_fallback_adapter.rb @@ -52,7 +52,7 @@ class PostgreSQLFallbackHandler logger.warn "#{log_prefix}: Master server is active. Reconnecting..." self.master_up(key) - Discourse.disable_readonly_mode + Discourse.disable_readonly_mode(Discourse::PG_READONLY_MODE_KEY) end rescue => e logger.warn "#{log_prefix}: Connection to master PostgreSQL server failed with '#{e.message}'" @@ -103,7 +103,7 @@ module ActiveRecord })) verify_replica(connection) - Discourse.enable_readonly_mode + Discourse.enable_readonly_mode(Discourse::PG_READONLY_MODE_KEY) else begin connection = postgresql_connection(config) diff --git a/lib/discourse.rb b/lib/discourse.rb index 9f27827f4c5..9acae903e10 100644 --- a/lib/discourse.rb +++ b/lib/discourse.rb @@ -113,24 +113,6 @@ module Discourse end end - def self.last_read_only - @last_read_only ||= {} - end - - def self.recently_readonly? - read_only = last_read_only[$redis.namespace] - return false unless read_only - read_only > 15.seconds.ago - end - - def self.received_readonly! - last_read_only[$redis.namespace] = Time.zone.now - end - - def self.clear_readonly! - last_read_only[$redis.namespace] = nil - end - def self.disabled_plugin_names plugins.select { |p| !p.enabled? }.map(&:name) end @@ -212,41 +194,67 @@ module Discourse READONLY_MODE_KEY_TTL ||= 60 READONLY_MODE_KEY ||= 'readonly_mode'.freeze + PG_READONLY_MODE_KEY ||= 'readonly_mode:postgres'.freeze USER_READONLY_MODE_KEY ||= 'readonly_mode:user'.freeze - def self.enable_readonly_mode(user_enabled: false) - if user_enabled - $redis.set(USER_READONLY_MODE_KEY, 1) + READONLY_KEYS = [ + READONLY_MODE_KEY, + PG_READONLY_MODE_KEY, + USER_READONLY_MODE_KEY + ] + + def self.enable_readonly_mode(key = READONLY_MODE_KEY) + if key == USER_READONLY_MODE_KEY + $redis.set(key, 1) else - $redis.setex(READONLY_MODE_KEY, READONLY_MODE_KEY_TTL, 1) - keep_readonly_mode + $redis.setex(key, READONLY_MODE_KEY_TTL, 1) + keep_readonly_mode(key) end MessageBus.publish(readonly_channel, true) true end - def self.keep_readonly_mode + def self.keep_readonly_mode(key) # extend the expiry by 1 minute every 30 seconds unless Rails.env.test? Thread.new do while readonly_mode? - $redis.expire(READONLY_MODE_KEY, READONLY_MODE_KEY_TTL) + $redis.expire(key, READONLY_MODE_KEY_TTL) sleep 30.seconds end end end end - def self.disable_readonly_mode(user_enabled: false) - key = user_enabled ? USER_READONLY_MODE_KEY : READONLY_MODE_KEY + def self.disable_readonly_mode(key = READONLY_MODE_KEY) $redis.del(key) MessageBus.publish(readonly_channel, false) true end def self.readonly_mode? - recently_readonly? || !!$redis.get(READONLY_MODE_KEY) || !!$redis.get(USER_READONLY_MODE_KEY) + return true if recently_readonly? + READONLY_KEYS.each { |key| return true if !!$redis.get(key) } + false + end + + def self.last_read_only + @last_read_only ||= {} + end + + def self.recently_readonly? + read_only = last_read_only[$redis.namespace] + return false unless read_only + read_only > 15.seconds.ago + end + + def self.received_readonly! + last_read_only[$redis.namespace] = Time.zone.now + end + + def self.clear_readonly! + last_read_only[$redis.namespace] = nil end def self.request_refresh! diff --git a/spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb b/spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb index a9e61dbea15..74f48940754 100644 --- a/spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb +++ b/spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb @@ -42,8 +42,13 @@ describe ActiveRecord::ConnectionHandling do end after do - with_multisite_db(multisite_db) { Discourse.disable_readonly_mode } - Discourse.disable_readonly_mode + pg_readonly_mode_key = Discourse::PG_READONLY_MODE_KEY + + with_multisite_db(multisite_db) do + Discourse.disable_readonly_mode(pg_readonly_mode_key) + end + + Discourse.disable_readonly_mode(pg_readonly_mode_key) ActiveRecord::Base.establish_connection(ActiveRecord::Base.configurations[Rails.env]) end diff --git a/spec/components/discourse_spec.rb b/spec/components/discourse_spec.rb index 5d441eacc33..b1b21cfd5c9 100644 --- a/spec/components/discourse_spec.rb +++ b/spec/components/discourse_spec.rb @@ -118,7 +118,7 @@ describe Discourse do context 'user enabled readonly mode' do it "adds a key in redis and publish a message through the message bus" do expect($redis.get(user_readonly_mode_key)).to eq(nil) - message = MessageBus.track_publish { Discourse.enable_readonly_mode(user_enabled: true) }.first + message = MessageBus.track_publish { Discourse.enable_readonly_mode(user_readonly_mode_key) }.first assert_readonly_mode(message, user_readonly_mode_key) end end @@ -160,10 +160,10 @@ describe Discourse do end it "returns true when user enabled readonly mode key is present in redis" do - Discourse.enable_readonly_mode(user_enabled: true) + Discourse.enable_readonly_mode(user_readonly_mode_key) expect(Discourse.readonly_mode?).to eq(true) - Discourse.disable_readonly_mode(user_enabled: true) + Discourse.disable_readonly_mode(user_readonly_mode_key) expect(Discourse.readonly_mode?).to eq(false) end end