FEATURE: Add WebHookEventsDailyAggregate (#27542)

* FEATURE: Add WebHookEventsDailyAggregate

Add WebHookEventsDailyAggregate model to store daily aggregates of web hook events.
Add AggregateWebHooksEvents job to aggregate web hook events daily.
Add spec for WebHookEventsDailyAggregate model.

* DEV: Update annotations for web_hook_events_daily_aggregate.rb

* DEV: Update app/jobs/scheduled/aggregate_web_hooks_events.rb

Co-authored-by: Martin Brennan <martin@discourse.org>

* DEV: Address review feedback

Solves:
- https://github.com/discourse/discourse/pull/27542#discussion_r1646961101
- https://github.com/discourse/discourse/pull/27542#discussion_r1646958890
- https://github.com/discourse/discourse/pull/27542#discussion_r1646976808
- https://github.com/discourse/discourse/pull/27542#discussion_r1646979846
- https://github.com/discourse/discourse/pull/27542#discussion_r1646981036

* A11Y: Add translation to retain_web_hook_events_aggregate_days key

* FEATURE: Purge old web hook events daily aggregate

Solves: https://github.com/discourse/discourse/pull/27542#discussion_r1646961101

* DEV:  Update tests for web_hook_events_daily_aggregate

Update WebHookEventsDailyAggregate to not use save! at the end
Solves: https://github.com/discourse/discourse/pull/27542#discussion_r1646984601

* PERF: Change job query to use WebHook table instead of WebHookEvent table

* DEV: Update tests to use `fab!`

* DEV: Address code review feedback.

Add idempotency to job
Add has_many to WebHook

* DEV: add test case for job and change job query

* DEV: Change AggregateWebHooksEvents job test name

---------

Co-authored-by: Martin Brennan <martin@discourse.org>

app/jobs/scheduled/aggregate_web_hooks_events.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Jobs
+  class AggregateWebHooksEvents < ::Jobs::Scheduled
+    every 1.day
+
+    def execute(args = {})
+      date = args[:date].present? ? args[:date] : Time.zone.now.to_date
+      WebHook
+        .joins(
+          "LEFT JOIN web_hook_events_daily_aggregates ON web_hooks.id = web_hook_events_daily_aggregates.web_hook_id AND web_hook_events_daily_aggregates.date = '#{date}'",
+        )
+        .where(active: true)
+        .where(web_hook_events_daily_aggregates: { id: nil })
+        .distinct
+        .each do |web_hook|
+          WebHookEventsDailyAggregate.create!(web_hook_id: web_hook.id, date: date)
+        end
+    end
+  end
+end

app/jobs/scheduled/purge_old_web_hook_events_daily_aggregate.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Jobs
+  class PurgeOldWebHookEventsDailyAggregate < ::Jobs::Scheduled
+    every 1.day
+
+    def execute(_)
+      WebHookEventsDailyAggregate.purge_old
+    end
+  end
+end

app/models/web_hook.rb

@@ -7,6 +7,7 @@ class WebHook < ActiveRecord::Base
   has_and_belongs_to_many :tags
 
   has_many :web_hook_events, dependent: :destroy
+  has_many :web_hook_events_daily_aggregates, dependent: :destroy
 
   default_scope { order("id ASC") }
 

app/models/web_hook_events_daily_aggregate.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+class WebHookEventsDailyAggregate < ActiveRecord::Base
+  belongs_to :web_hook
+
+  default_scope { order("created_at DESC") }
+  before_create :aggregate!
+
+  def self.purge_old
+    where("created_at < ?", SiteSetting.retain_web_hook_events_aggregate_days.days.ago).delete_all
+  end
+
+  def self.by_day(start_date, end_date, web_hook_id = nil)
+    result = where("date >= ? AND date <= ?", start_date.to_date, end_date.to_date)
+    result = result.where(web_hook_id: web_hook_id) if web_hook_id
+    result
+  end
+
+  def aggregate!
+    events =
+      WebHookEvent.where(
+        "created_at >= ? AND created_at < ? AND web_hook_id = ?",
+        self.date,
+        self.date + 1.day,
+        self.web_hook_id,
+      )
+
+    self.mean_duration = events.sum(:duration) / events.count
+
+    self.successful_event_count = events.where("status >= 200 AND status <= 299").count
+    self.failed_event_count = events.where("status < 200 OR status > 299").count
+  end
+end
+
+# == Schema Information
+#
+# Table name: web_hook_events_daily_aggregates
+#
+#  id                     :bigint           not null, primary key
+#  web_hook_id            :bigint           not null
+#  date                   :date
+#  successful_event_count :integer
+#  failed_event_count     :integer
+#  mean_duration          :integer          default(0)
+#  created_at             :datetime         not null
+#  updated_at             :datetime         not null
+#
+# Indexes
+#
+#  index_web_hook_events_daily_aggregates_on_web_hook_id  (web_hook_id)
+#

config/locales/server.en.yml

@@ -2564,6 +2564,7 @@ en:
     default_sidebar_switch_panel_position: "Position of button on sidebar to switch to chat"
 
     retain_web_hook_events_period_days: "Number of days to retain web hook event records."
+    retain_web_hook_events_aggregate_days: "Number of days to retain web hook event aggregate records."
     retry_web_hook_events: "Automatically retry failed web hook events for 4 times. Time gaps between the retries are 1, 5, 25 and 125 minutes."
     revoke_api_keys_unused_days: "Number of days since an API key was last used before it is automatically revoked (0 for never)"
     revoke_api_keys_maxlife_days: "Number of days before an API key is automatically revoked (0 for never)"

config/site_settings.yml

@@ -3047,6 +3047,9 @@ api:
   retain_web_hook_events_period_days:
     default: 30
     max: 36500
+  retain_web_hook_events_aggregate_days:
+    default: 720
+    max: 36500
   retry_web_hook_events:
     default: false
   api_key_last_used_epoch:

db/migrate/20240619123052_create_web_hook_events_daily_aggregates.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CreateWebHookEventsDailyAggregates < ActiveRecord::Migration[7.0]
+  def change
+    create_table :web_hook_events_daily_aggregates do |t|
+      t.belongs_to :web_hook, null: false, index: true
+      t.date :date
+      t.integer :successful_event_count
+      t.integer :failed_event_count
+      t.integer :mean_duration, default: 0
+
+      t.timestamps
+    end
+  end
+end

spec/models/web_hook_events_daily_aggregate_spec.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+RSpec.describe WebHookEventsDailyAggregate do
+  fab!(:web_hook)
+  fab!(:event) do
+    Fabricate(
+      :web_hook_event,
+      status: 200,
+      web_hook: web_hook,
+      created_at: 1.days.ago,
+      duration: 280,
+    )
+  end
+  fab!(:event_today) { Fabricate(:web_hook_event, status: 200, web_hook: web_hook, duration: 300) }
+
+  fab!(:failed_event) do
+    Fabricate(
+      :web_hook_event,
+      status: 400,
+      created_at: 1.days.ago,
+      web_hook: web_hook,
+      duration: 200,
+    )
+  end
+
+  fab!(:failed_event2) do
+    Fabricate(
+      :web_hook_event,
+      status: 400,
+      web_hook: web_hook,
+      created_at: 1.days.ago,
+      duration: 200,
+    )
+  end
+  fab!(:failed_event_today) do
+    Fabricate(:web_hook_event, status: 400, web_hook: web_hook, duration: 200)
+  end
+  describe ".purge_old" do
+    before { SiteSetting.retain_web_hook_events_aggregate_days = 1 }
+
+    it "should be able to purge old web hook event aggregates" do
+      web_hook = Fabricate(:web_hook)
+      WebHookEvent.create!(status: 200, web_hook: web_hook, created_at: 1.days.ago, duration: 180)
+      WebHookEvent.create!(status: 200, web_hook: web_hook, created_at: 2.days.ago, duration: 180)
+
+      yesterday_aggregate =
+        WebHookEventsDailyAggregate.create!(web_hook_id: web_hook.id, date: 1.days.ago)
+
+      WebHookEventsDailyAggregate.create!(
+        web_hook_id: web_hook.id,
+        date: 2.days.ago,
+        created_at: 2.days.ago,
+      )
+
+      expect { described_class.purge_old }.to change { WebHookEventsDailyAggregate.count }.by(-1)
+
+      expect(WebHookEventsDailyAggregate.find(yesterday_aggregate.id)).to eq(yesterday_aggregate)
+    end
+  end
+
+  describe "aggregation works" do
+    it "should be able to aggregate web hook events" do
+      yesterday_aggregate =
+        WebHookEventsDailyAggregate.create!(web_hook_id: web_hook.id, date: 1.days.ago)
+      yesterday_events = [event, failed_event, failed_event2]
+
+      expect(WebHookEventsDailyAggregate.count).to eq(1)
+      expect(yesterday_aggregate.web_hook_id).to eq(web_hook.id)
+      expect(yesterday_aggregate.date).to eq(1.days.ago.to_date)
+
+      expect(yesterday_aggregate.mean_duration).to eq(
+        yesterday_events.sum(&:duration) / yesterday_events.count,
+      )
+      expect(yesterday_aggregate.successful_event_count).to eq(1)
+      expect(yesterday_aggregate.failed_event_count).to eq(2)
+    end
+
+    it "should be able to filter by day" do
+      WebHookEventsDailyAggregate.create!(web_hook_id: web_hook.id, date: 1.days.ago)
+      WebHookEventsDailyAggregate.create!(web_hook_id: web_hook.id, date: 0.days.ago)
+      yesterday_events = [event, failed_event, failed_event2]
+      today_events = [event_today, failed_event_today]
+
+      yesterday_aggregate = WebHookEventsDailyAggregate.by_day(1.days.ago, 1.days.ago)
+      expect(yesterday_aggregate.count).to eq(1)
+      expect(yesterday_aggregate.first.date).to eq(1.days.ago.to_date)
+
+      expect(WebHookEventsDailyAggregate.count).to eq(2)
+
+      today_and_yesterday_aggregate = WebHookEventsDailyAggregate.by_day(1.days.ago, 0.days.ago)
+
+      expect(today_and_yesterday_aggregate.count).to eq(2)
+      expect(today_and_yesterday_aggregate.map(&:date)).to eq(
+        [0.days.ago.to_date, 1.days.ago.to_date],
+      )
+      expect(today_and_yesterday_aggregate.map(&:mean_duration)).to eq(
+        [
+          today_events.sum(&:duration) / today_events.count,
+          yesterday_events.sum(&:duration) / yesterday_events.count,
+        ],
+      )
+    end
+
+    it "should not create a new WebHookEventsDailyAggregate row if AggregateWebHooksEvents runs twice" do
+      expect { Jobs::AggregateWebHooksEvents.new.execute(date: 1.days.ago) }.to change {
+        WebHookEventsDailyAggregate.count
+      }.by(1)
+
+      expect { Jobs::AggregateWebHooksEvents.new.execute(date: 1.days.ago) }.not_to change {
+        WebHookEventsDailyAggregate.count
+      }
+    end
+  end
+end