diff --git a/app/models/notification.rb b/app/models/notification.rb
index d8ca8553df2..96e3e43f9e5 100644
--- a/app/models/notification.rb
+++ b/app/models/notification.rb
@@ -13,7 +13,10 @@ class Notification < ActiveRecord::Base
                             .where('topics.id IS NULL OR topics.deleted_at IS NULL') }
 
   after_commit :send_email
-  after_commit :refresh_notification_count
+  # This is super weird because the tests fail if we don't specify `on: :destroy`
+  # TODO: Revert back to default in Rails 5 
+  after_commit :refresh_notification_count, on: :destroy
+  after_commit :refresh_notification_count, on: [:create, :update]
 
   def self.ensure_consistency!
     Notification.exec_sql("
diff --git a/spec/models/notification_spec.rb b/spec/models/notification_spec.rb
index 02ee99917c7..fa45deb0e74 100644
--- a/spec/models/notification_spec.rb
+++ b/spec/models/notification_spec.rb
@@ -133,7 +133,7 @@ describe Notification do
 
       it 'updates the notification count on destroy' do
         Notification.any_instance.expects(:refresh_notification_count).returns(nil)
-        notification.destroy
+        notification.destroy!
       end
 
     end