FIX: add safety so sidekiq can no longer be paused indefinitely

If the process pausing sidekiq dies sidekiq will come out of pause mode

lib/sidekiq/pausable.rb

@@ -1,25 +1,83 @@
-module Sidekiq
+require 'thread'
+
+class SidekiqPauser
+  def initialize
+    @mutex = Mutex.new
+    @done = ConditionVariable.new
+  end
+
+  def pause!
+    @mutex.synchronize do
+      @paused = true
+      @pause_thread ||= start_pause_thread
+      sleep 0.001 while !paused?
+    end
 
-  def self.pause!
-    Sidekiq.redis { |r| r.set(paused_key, 1) }
     true
   end
 
-  def self.paused?
+  def paused?
     Sidekiq.redis { |r| !!r.get(paused_key) }
   end
 
-  def self.unpause!
+  def unpause!
+    # concurrency is hard, perform signaling from a bg thread
+    # otherwise it acts weird
+    Thread.new do
+      @mutex.synchronize do
+        if @pause_thread
+          @paused = false
+          @done.signal
+        end
+      end
+    end.join
+
+    @mutex.synchronize do
+      @pause_thread.join if @pause_thread
+      @pause_thread = nil
+    end
+
     Sidekiq.redis { |r| r.del(paused_key) }
     true
   end
 
   private
 
-  def self.paused_key
-    "sidekiq_is_paused"
+  def start_pause_thread
+    Thread.new do
+      while @paused do
+        # TODO retries in case bad redis connectivity
+        Sidekiq.redis do |r|
+          r.setex paused_key, 60, "paused"
+        end
+
+        @mutex.synchronize do
+          return unless @paused
+          @done.wait(@mutex, 30)
+        end
+      end
+    end
   end
 
+  def paused_key
+    "sidekiq_is_paused_v2"
+  end
+
+end
+
+module Sidekiq
+  @pauser = SidekiqPauser.new
+  def self.pause!
+    @pauser.pause!
+  end
+
+  def self.paused?
+    @pauser.paused?
+  end
+
+  def self.unpause!
+    @pauser.unpause!
+  end
 end
 
 # server middleware that will reschedule work whenever Sidekiq is paused

spec/components/sidekiq/pausable_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+require_dependency 'sidekiq/pausable'
+
+describe Sidekiq do
+  it "can pause and unpause" do
+    Sidekiq.pause!
+    Sidekiq.paused?.should == true
+    Sidekiq.unpause!
+    Sidekiq.paused?.should == false
+  end
+end