diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 511d6a54815..910120219b8 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -26,10 +26,13 @@ jobs:
       fail-fast: false
 
       matrix:
-        build_type: [backend, frontend]
+        build_type: [backend, frontend, annotations]
         target: [core, plugins]
         postgres: ["13"]
         redis: ["6.x"]
+        exclude:
+          - build_type: annotations
+            target: plugins
 
     services:
       postgres:
@@ -131,3 +134,20 @@ jobs:
         if: matrix.build_type == 'frontend' && matrix.target == 'plugins'
         run: bin/rake plugin:qunit['*','1200000']
         timeout-minutes: 30
+
+      - name: Check Annotations
+        if: matrix.build_type == 'annotations'
+        run: |
+          bin/rake annotate:ensure_all_indexes
+          bin/annotate --models --model-dir app/models
+
+          if [ ! -z "$(git status --porcelain app/models/)" ]; then
+            echo "Core annotations are not up to date. To resolve, run:"
+            echo "  bin/rake annotate:clean"
+            echo
+            echo "Or manually apply the diff printed below:"
+            echo "---------------------------------------------"
+            git -c color.ui=always diff app/models/
+            exit 1
+          fi
+        timeout-minutes: 30
diff --git a/Gemfile b/Gemfile
index c2680dc0b06..80d264469e9 100644
--- a/Gemfile
+++ b/Gemfile
@@ -165,6 +165,8 @@ group :test, :development do
   gem 'parallel_tests'
 
   gem 'rswag-specs'
+
+  gem 'annotate'
 end
 
 group :development do
@@ -173,7 +175,6 @@ group :development do
   gem 'better_errors', platform: :mri, require: !!ENV['BETTER_ERRORS']
   gem 'binding_of_caller'
   gem 'yaml-lint'
-  gem 'annotate'
   gem 'discourse_dev_assets'
   gem 'faker', "~> 2.16"
 end
diff --git a/lib/tasks/annotate.rake b/lib/tasks/annotate.rake
new file mode 100644
index 00000000000..b7ceb0f1b9b
--- /dev/null
+++ b/lib/tasks/annotate.rake
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+desc "ensure the asynchronously-created post_search_data index is present"
+task "annotate" => :environment do |task, args|
+  raise if !system "bin/annotate --models"
+  STDERR.puts "Annotate executed successfully"
+
+  non_core_plugins = Dir["plugins/*"].filter do |plugin_path|
+    `git check-ignore #{plugin_path}`.present?
+  end
+  if non_core_plugins.length > 0
+    STDERR.puts "Warning: you have non-core plugins installed which may affect the annotations"
+    STDERR.puts "For core annotations, consider running `bin/rake annotate:clean`"
+  end
+end
+
+desc "ensure the asynchronously-created post_search_data index is present"
+task "annotate:ensure_all_indexes" => :environment do |task, args|
+  # One of the indexes on post_search_data is created by a sidekiq job
+  # We need to do some acrobatics to create it on-demand
+  SeedData::Topics.with_default_locale.create(include_welcome_topics: true)
+  SiteSetting.search_enable_recent_regular_posts_offset_size = 1
+  Jobs::CreateRecentPostSearchIndexes.new.execute([])
+end
+
+desc "regenerate core model annotations using a temporary database"
+task "annotate:clean" => :environment do |task, args|
+  db = TemporaryDb.new
+  db.start
+  db.with_env do
+    raise if !system "RAILS_ENV=test LOAD_PLUGINS=0 bin/rake db:migrate"
+    raise if !system "RAILS_ENV=test LOAD_PLUGINS=0 bin/rake annotate:ensure_all_indexes"
+    raise if !system "RAILS_ENV=test LOAD_PLUGINS=0 bin/annotate --models --model-dir app/models"
+  end
+  STDERR.puts "Annotate executed successfully"
+ensure
+  db&.stop
+  db&.remove
+end
diff --git a/lib/temporary_db.rb b/lib/temporary_db.rb
index 97300a1dbb2..3b7a42e9b5e 100644
--- a/lib/temporary_db.rb
+++ b/lib/temporary_db.rb
@@ -99,6 +99,28 @@ class TemporaryDb
     `#{pg_ctl_path} -D '#{PG_TEMP_PATH}' stop`
   end
 
+  def with_env(&block)
+    old_host = ENV["PGHOST"]
+    old_user = ENV["PGUSER"]
+    old_port = ENV["PGPORT"]
+    old_dev_db = ENV["DISCOURSE_DEV_DB"]
+    old_rails_db = ENV["RAILS_DB"]
+
+    ENV["PGHOST"] = "localhost"
+    ENV["PGUSER"] = "discourse"
+    ENV["PGPORT"] = pg_port.to_s
+    ENV["DISCOURSE_DEV_DB"] = "discourse"
+    ENV["RAILS_DB"] = "discourse"
+
+    yield
+  ensure
+    ENV["PGHOST"] = old_host
+    ENV["PGUSER"] = old_user
+    ENV["PGPORT"] = old_port
+    ENV["DISCOURSE_DEV_DB"] = old_dev_db
+    ENV["RAILS_DB"] = old_rails_db
+  end
+
   def remove
     raise "Error: the database must be stopped before it can be removed" if @started
     FileUtils.rm_rf PG_TEMP_PATH