PERF: stop rebuilding full directory item table on refresh

app/models/directory_item.rb

@@ -40,19 +40,39 @@ class DirectoryItem < ActiveRecord::Base
             end
 
     ActiveRecord::Base.transaction do
-      exec_sql "DELETE FROM directory_items WHERE period_type = :period_type", period_type: period_types[period_type]
+      exec_sql "DELETE FROM directory_items
-      exec_sql "INSERT INTO directory_items
+                USING directory_items di
-                  (period_type, user_id, likes_received, likes_given, topics_entered, days_visited, posts_read, topic_count, post_count)
+                LEFT JOIN users u ON u.id = user_id
-                  SELECT
+                WHERE di.id = directory_items.id AND
+                      u.id IS NULL AND
+                      di.period_type = :period_type", period_type: period_types[period_type]
+
+
+      exec_sql "INSERT INTO directory_items(period_type, user_id, likes_received, likes_given, topics_entered, days_visited, posts_read, topic_count, post_count)
+                SELECT
                     :period_type,
                     u.id,
-                    SUM(CASE WHEN ua.action_type = :was_liked_type THEN 1 ELSE 0 END),
+                    0,
-                    SUM(CASE WHEN ua.action_type = :like_type THEN 1 ELSE 0 END),
+                    0,
-                    COALESCE((SELECT COUNT(topic_id) FROM topic_views AS v WHERE v.user_id = u.id AND v.viewed_at >= :since), 0),
+                    0,
-                    COALESCE((SELECT COUNT(id) FROM user_visits AS uv WHERE uv.user_id = u.id AND uv.visited_at >= :since), 0),
+                    0,
-                    COALESCE((SELECT SUM(posts_read) FROM user_visits AS uv2 WHERE uv2.user_id = u.id AND uv2.visited_at >= :since), 0),
+                    0,
-                    SUM(CASE WHEN ua.action_type = :new_topic_type THEN 1 ELSE 0 END),
+                    0,
-                    SUM(CASE WHEN ua.action_type = :reply_type THEN 1 ELSE 0 END)
+                    0
+                FROM users u
+                LEFT JOIN directory_items di ON di.user_id = u.id AND di.period_type = :period_type
+                WHERE di.id IS NULL AND u.id > 0
+      ", period_type: period_types[period_type]
+
+      exec_sql "WITH x AS (SELECT
+                    u.id user_id,
+                    SUM(CASE WHEN ua.action_type = :was_liked_type THEN 1 ELSE 0 END) likes_received,
+                    SUM(CASE WHEN ua.action_type = :like_type THEN 1 ELSE 0 END) likes_given,
+                    COALESCE((SELECT COUNT(topic_id) FROM topic_views AS v WHERE v.user_id = u.id AND v.viewed_at >= :since), 0) topics_entered,
+                    COALESCE((SELECT COUNT(id) FROM user_visits AS uv WHERE uv.user_id = u.id AND uv.visited_at >= :since), 0) days_visited,
+                    COALESCE((SELECT SUM(posts_read) FROM user_visits AS uv2 WHERE uv2.user_id = u.id AND uv2.visited_at >= :since), 0) posts_read,
+                    SUM(CASE WHEN ua.action_type = :new_topic_type THEN 1 ELSE 0 END) topic_count,
+                    SUM(CASE WHEN ua.action_type = :reply_type THEN 1 ELSE 0 END) post_count
                   FROM users AS u
                   LEFT OUTER JOIN user_actions AS ua ON ua.user_id = u.id
                   LEFT OUTER JOIN topics AS t ON ua.target_topic_id = t.id AND t.archetype = 'regular'
@@ -67,7 +87,28 @@ class DirectoryItem < ActiveRecord::Base
                     AND (NOT (COALESCE(p.hidden, false)))
                     AND COALESCE(p.post_type, :regular_post_type) = :regular_post_type
                     AND u.id > 0
-                  GROUP BY u.id",
+                  GROUP BY u.id)
+      UPDATE directory_items di SET
+               likes_received = x.likes_received,
+               likes_given = x.likes_given,
+               topics_entered = x.likes_given,
+               days_visited = x.days_visited,
+               posts_read = x.posts_read,
+               topic_count = x.topic_count,
+               post_count = x.post_count
+      FROM x
+      WHERE
+        x.user_id = di.user_id AND
+        di.period_type = :period_type AND (
+        di.likes_received <> x.likes_received OR
+        di.likes_given <> x.likes_given OR
+        di.topics_entered <> x.likes_given OR
+        di.days_visited <> x.days_visited OR
+        di.posts_read <> x.posts_read OR
+        di.topic_count <> x.topic_count OR
+        di.post_count <> x.post_count )
+
+      ",
                   period_type: period_types[period_type],
                   since: since,
                   like_type: UserAction::LIKE,
@@ -76,6 +117,7 @@ class DirectoryItem < ActiveRecord::Base
                   reply_type: UserAction::REPLY,
                   regular_post_type: Post.types[:regular]
 
+
       if period_type == :all
         exec_sql <<SQL
         UPDATE user_stats s

db/migrate/20160302063432_rebuild_directory_item_with_index.rb

@@ -0,0 +1,12 @@
+class RebuildDirectoryItemWithIndex < ActiveRecord::Migration
+  def up
+    remove_index :directory_items, [:period_type]
+    execute "TRUNCATE TABLE directory_items RESTART IDENTITY"
+    add_index :directory_items, [:period_type, :user_id], unique: true
+  end
+
+  def down
+    remove_index :directory_items, [:period_type, :user_id]
+    add_index :directory_items, [:period_type]
+  end
+end

spec/models/directory_item_spec.rb

@@ -19,13 +19,17 @@ describe DirectoryItem do
   end
 
   context 'refresh' do
-    let!(:post) { Fabricate(:post) }
+    before do
+      ActiveRecord::Base.observers.enable :all
+    end
+
+    let!(:post) { create_post }
 
     it "creates the record for the user" do
       DirectoryItem.refresh!
       expect(DirectoryItem.where(period_type: DirectoryItem.period_types[:all])
                           .where(user_id: post.user.id)
-                          .exists?).to be_truthy
+                          .where(topic_count: 1).count).to eq(1)
     end
 
   end