From 7426764af456546a4b91f247b1c21e8e12de0cd5 Mon Sep 17 00:00:00 2001
From: Krzysztof Kotlarek <kotlarek.krzysztof@gmail.com>
Date: Fri, 12 Feb 2021 08:05:38 +1100
Subject: [PATCH] FIX: Optimize move to dismiss_new_topics migration (#12041)

This migration is quite heavy because of join to all potential topics which should be `dismissed` for each user. To make it a little bit more efficient I did two things:
- move conditions to join so it should use fewer rows
- do that in batches - 1000 users at the time
---
 ...10208022738_move_new_since_to_new_table.rb | 37 +++++++++++--------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/db/migrate/20210208022738_move_new_since_to_new_table.rb b/db/migrate/20210208022738_move_new_since_to_new_table.rb
index 6b975e464fa..109e4d6aa11 100644
--- a/db/migrate/20210208022738_move_new_since_to_new_table.rb
+++ b/db/migrate/20210208022738_move_new_since_to_new_table.rb
@@ -2,35 +2,40 @@
 
 class MoveNewSinceToNewTable < ActiveRecord::Migration[6.0]
   def up
-    sql = <<~SQL
+    User.find_in_batches do |users|
+      sql = <<~SQL
       INSERT INTO dismissed_topic_users (user_id, topic_id, created_at)
       SELECT users.id, topics.id, user_stats.new_since
       FROM user_stats
       JOIN users ON users.id = user_stats.user_id
       JOIN user_options ON user_options.user_id = users.id
       LEFT JOIN topics ON topics.created_at <= user_stats.new_since
+        AND topics.archetype <> :private_message
+        AND topics.created_at >= GREATEST(CASE
+                    WHEN COALESCE(user_options.new_topic_duration_minutes, :default_duration) = :always THEN users.created_at
+                    WHEN COALESCE(user_options.new_topic_duration_minutes, :default_duration) = :last_visit THEN COALESCE(users.previous_visit_at,users.created_at)
+                    ELSE (:now::timestamp - INTERVAL '1 MINUTE' * COALESCE(user_options.new_topic_duration_minutes, :default_duration))
+                 END, :min_date)
       LEFT JOIN topic_users ON topics.id = topic_users.topic_id AND users.id = topic_users.user_id
       LEFT JOIN dismissed_topic_users ON dismissed_topic_users.topic_id = topics.id AND users.id = dismissed_topic_users.user_id
       WHERE user_stats.new_since IS NOT NULL
       AND topic_users.id IS NULL
+      AND topics.id IS NOT NULL
       AND dismissed_topic_users.id IS NULL
-      AND topics.archetype <> :private_message
-      AND topics.created_at >= GREATEST(CASE
-                  WHEN COALESCE(user_options.new_topic_duration_minutes, :default_duration) = :always THEN users.created_at
-                  WHEN COALESCE(user_options.new_topic_duration_minutes, :default_duration) = :last_visit THEN COALESCE(users.previous_visit_at,users.created_at)
-                  ELSE (:now::timestamp - INTERVAL '1 MINUTE' * COALESCE(user_options.new_topic_duration_minutes, :default_duration))
-               END, :min_date)
+      AND users.id IN (:user_ids)
       ORDER BY topics.created_at DESC
       LIMIT :max_new_topics
-    SQL
-    DB.exec(sql,
-            now: DateTime.now,
-            last_visit: User::NewTopicDuration::LAST_VISIT,
-            always: User::NewTopicDuration::ALWAYS,
-            default_duration: SiteSetting.default_other_new_topic_duration_minutes,
-            min_date: Time.at(SiteSetting.min_new_topics_time).to_datetime,
-            private_message: Archetype.private_message,
-            max_new_topics: SiteSetting.max_new_topics)
+      SQL
+      DB.exec(sql,
+              now: DateTime.now,
+              last_visit: User::NewTopicDuration::LAST_VISIT,
+              always: User::NewTopicDuration::ALWAYS,
+              default_duration: SiteSetting.default_other_new_topic_duration_minutes,
+              min_date: Time.at(SiteSetting.min_new_topics_time).to_datetime,
+              private_message: Archetype.private_message,
+              user_ids: users.map(&:id),
+              max_new_topics: SiteSetting.max_new_topics)
+    end
   end
 
   def down