diff --git a/lib/active_record/connection_adapters/postgresql_fallback_adapter.rb b/lib/active_record/connection_adapters/postgresql_fallback_adapter.rb index d25cdded343..fca51bd2a63 100644 --- a/lib/active_record/connection_adapters/postgresql_fallback_adapter.rb +++ b/lib/active_record/connection_adapters/postgresql_fallback_adapter.rb @@ -9,10 +9,18 @@ class PostgreSQLFallbackHandler attr_reader :masters_down attr_accessor :initialized + DATABASE_DOWN_CHANNEL = '/global/database_down'.freeze + def initialize @masters_down = DistributedCache.new('masters_down', namespace: false) @mutex = Mutex.new @initialized = false + + MessageBus.subscribe(DATABASE_DOWN_CHANNEL) do |payload| + RailsMultisite::ConnectionManagement.with_connection(payload.data['db']) do + clear_connections + end + end end def verify_master @@ -38,10 +46,11 @@ class PostgreSQLFallbackHandler synchronize { @masters_down[namespace] } end - def master_down=(args) + def master_down synchronize do - @masters_down[namespace] = args - Sidekiq.pause! if args && !Sidekiq.paused? + @masters_down[namespace] = true + Sidekiq.pause! if !Sidekiq.paused? + MessageBus.publish(DATABASE_DOWN_CHANNEL, db: namespace) end end @@ -63,8 +72,7 @@ class PostgreSQLFallbackHandler if is_connection_active logger.warn "#{log_prefix}: Master server is active. Reconnecting..." - ActiveRecord::Base.clear_active_connections! - ActiveRecord::Base.clear_all_connections! + clear_connections self.master_up(key) disable_readonly_mode Sidekiq.unpause! @@ -82,6 +90,11 @@ class PostgreSQLFallbackHandler disable_readonly_mode end + def clear_connections + ActiveRecord::Base.clear_active_connections! + ActiveRecord::Base.clear_all_connections! + end + private def disable_readonly_mode @@ -130,12 +143,13 @@ module ActiveRecord connection = postgresql_connection(config) fallback_handler.initialized ||= true rescue PG::ConnectionBad => e - fallback_handler.master_down = true + fallback_handler.master_down fallback_handler.verify_master if !fallback_handler.initialized return postgresql_fallback_connection(config) else + fallback_handler.clear_connections raise e end end diff --git a/spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb b/spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb index 6f0b256c54a..c377a0d7981 100644 --- a/spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb +++ b/spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb @@ -30,9 +30,7 @@ describe ActiveRecord::ConnectionHandling do postgresql_fallback_handler.initialized = true ['default', multisite_db].each do |db| - with_multisite_db(db) do - postgresql_fallback_handler.master_down = false - end + postgresql_fallback_handler.master_up(db) end end @@ -75,10 +73,14 @@ describe ActiveRecord::ConnectionHandling do ).returns(@replica_connection) end - expect(postgresql_fallback_handler.master_down?).to eq(false) + expect(postgresql_fallback_handler.master_down?).to eq(nil) - expect { ActiveRecord::Base.postgresql_fallback_connection(config) } - .to raise_error(PG::ConnectionBad) + message = MessageBus.track_publish(PostgreSQLFallbackHandler::DATABASE_DOWN_CHANNEL) do + expect { ActiveRecord::Base.postgresql_fallback_connection(config) } + .to raise_error(PG::ConnectionBad) + end.first + + expect(message.data[:db]).to eq('default') expect { ActiveRecord::Base.postgresql_fallback_connection(config) } .to change { Discourse.readonly_mode? }.from(false).to(true) @@ -87,10 +89,14 @@ describe ActiveRecord::ConnectionHandling do expect(Sidekiq.paused?).to eq(true) with_multisite_db(multisite_db) do - expect(postgresql_fallback_handler.master_down?).to eq(false) + expect(postgresql_fallback_handler.master_down?).to eq(nil) - expect { ActiveRecord::Base.postgresql_fallback_connection(multisite_config) } - .to raise_error(PG::ConnectionBad) + message = MessageBus.track_publish(PostgreSQLFallbackHandler::DATABASE_DOWN_CHANNEL) do + expect { ActiveRecord::Base.postgresql_fallback_connection(multisite_config) } + .to raise_error(PG::ConnectionBad) + end.first + + expect(message.data[:db]).to eq(multisite_db) expect { ActiveRecord::Base.postgresql_fallback_connection(multisite_config) } .to change { Discourse.readonly_mode? }.from(false).to(true)