diff --git a/app/jobs/scheduled/aggregate_web_hooks_events.rb b/app/jobs/scheduled/aggregate_web_hooks_events.rb new file mode 100644 index 00000000000..856cbde059f --- /dev/null +++ b/app/jobs/scheduled/aggregate_web_hooks_events.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Jobs + class AggregateWebHooksEvents < ::Jobs::Scheduled + every 1.day + + def execute(args = {}) + date = args[:date].present? ? args[:date] : Time.zone.now.to_date + WebHook + .joins( + "LEFT JOIN web_hook_events_daily_aggregates ON web_hooks.id = web_hook_events_daily_aggregates.web_hook_id AND web_hook_events_daily_aggregates.date = '#{date}'", + ) + .where(active: true) + .where(web_hook_events_daily_aggregates: { id: nil }) + .distinct + .each do |web_hook| + WebHookEventsDailyAggregate.create!(web_hook_id: web_hook.id, date: date) + end + end + end +end diff --git a/app/jobs/scheduled/purge_old_web_hook_events_daily_aggregate.rb b/app/jobs/scheduled/purge_old_web_hook_events_daily_aggregate.rb new file mode 100644 index 00000000000..f42f66693dc --- /dev/null +++ b/app/jobs/scheduled/purge_old_web_hook_events_daily_aggregate.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Jobs + class PurgeOldWebHookEventsDailyAggregate < ::Jobs::Scheduled + every 1.day + + def execute(_) + WebHookEventsDailyAggregate.purge_old + end + end +end diff --git a/app/models/web_hook.rb b/app/models/web_hook.rb index 2381e1ef0ac..c9a08428229 100644 --- a/app/models/web_hook.rb +++ b/app/models/web_hook.rb @@ -7,6 +7,7 @@ class WebHook < ActiveRecord::Base has_and_belongs_to_many :tags has_many :web_hook_events, dependent: :destroy + has_many :web_hook_events_daily_aggregates, dependent: :destroy default_scope { order("id ASC") } diff --git a/app/models/web_hook_events_daily_aggregate.rb b/app/models/web_hook_events_daily_aggregate.rb new file mode 100644 index 00000000000..67b2f4a7026 --- /dev/null +++ b/app/models/web_hook_events_daily_aggregate.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +class WebHookEventsDailyAggregate < ActiveRecord::Base + belongs_to :web_hook + + default_scope { order("created_at DESC") } + before_create :aggregate! + + def self.purge_old + where("created_at < ?", SiteSetting.retain_web_hook_events_aggregate_days.days.ago).delete_all + end + + def self.by_day(start_date, end_date, web_hook_id = nil) + result = where("date >= ? AND date <= ?", start_date.to_date, end_date.to_date) + result = result.where(web_hook_id: web_hook_id) if web_hook_id + result + end + + def aggregate! + events = + WebHookEvent.where( + "created_at >= ? AND created_at < ? AND web_hook_id = ?", + self.date, + self.date + 1.day, + self.web_hook_id, + ) + + self.mean_duration = events.sum(:duration) / events.count + + self.successful_event_count = events.where("status >= 200 AND status <= 299").count + self.failed_event_count = events.where("status < 200 OR status > 299").count + end +end + +# == Schema Information +# +# Table name: web_hook_events_daily_aggregates +# +# id :bigint not null, primary key +# web_hook_id :bigint not null +# date :date +# successful_event_count :integer +# failed_event_count :integer +# mean_duration :integer default(0) +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# index_web_hook_events_daily_aggregates_on_web_hook_id (web_hook_id) +# diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index 1a9389e8bb7..ccfb29d6040 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -2564,6 +2564,7 @@ en: default_sidebar_switch_panel_position: "Position of button on sidebar to switch to chat" retain_web_hook_events_period_days: "Number of days to retain web hook event records." + retain_web_hook_events_aggregate_days: "Number of days to retain web hook event aggregate records." retry_web_hook_events: "Automatically retry failed web hook events for 4 times. Time gaps between the retries are 1, 5, 25 and 125 minutes." revoke_api_keys_unused_days: "Number of days since an API key was last used before it is automatically revoked (0 for never)" revoke_api_keys_maxlife_days: "Number of days before an API key is automatically revoked (0 for never)" diff --git a/config/site_settings.yml b/config/site_settings.yml index 104276c3065..0ddfcbdce58 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -3047,6 +3047,9 @@ api: retain_web_hook_events_period_days: default: 30 max: 36500 + retain_web_hook_events_aggregate_days: + default: 720 + max: 36500 retry_web_hook_events: default: false api_key_last_used_epoch: diff --git a/db/migrate/20240619123052_create_web_hook_events_daily_aggregates.rb b/db/migrate/20240619123052_create_web_hook_events_daily_aggregates.rb new file mode 100644 index 00000000000..d7bab02f191 --- /dev/null +++ b/db/migrate/20240619123052_create_web_hook_events_daily_aggregates.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class CreateWebHookEventsDailyAggregates < ActiveRecord::Migration[7.0] + def change + create_table :web_hook_events_daily_aggregates do |t| + t.belongs_to :web_hook, null: false, index: true + t.date :date + t.integer :successful_event_count + t.integer :failed_event_count + t.integer :mean_duration, default: 0 + + t.timestamps + end + end +end diff --git a/spec/models/web_hook_events_daily_aggregate_spec.rb b/spec/models/web_hook_events_daily_aggregate_spec.rb new file mode 100644 index 00000000000..1f01f641c5b --- /dev/null +++ b/spec/models/web_hook_events_daily_aggregate_spec.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +RSpec.describe WebHookEventsDailyAggregate do + fab!(:web_hook) + fab!(:event) do + Fabricate( + :web_hook_event, + status: 200, + web_hook: web_hook, + created_at: 1.days.ago, + duration: 280, + ) + end + fab!(:event_today) { Fabricate(:web_hook_event, status: 200, web_hook: web_hook, duration: 300) } + + fab!(:failed_event) do + Fabricate( + :web_hook_event, + status: 400, + created_at: 1.days.ago, + web_hook: web_hook, + duration: 200, + ) + end + + fab!(:failed_event2) do + Fabricate( + :web_hook_event, + status: 400, + web_hook: web_hook, + created_at: 1.days.ago, + duration: 200, + ) + end + fab!(:failed_event_today) do + Fabricate(:web_hook_event, status: 400, web_hook: web_hook, duration: 200) + end + describe ".purge_old" do + before { SiteSetting.retain_web_hook_events_aggregate_days = 1 } + + it "should be able to purge old web hook event aggregates" do + web_hook = Fabricate(:web_hook) + WebHookEvent.create!(status: 200, web_hook: web_hook, created_at: 1.days.ago, duration: 180) + WebHookEvent.create!(status: 200, web_hook: web_hook, created_at: 2.days.ago, duration: 180) + + yesterday_aggregate = + WebHookEventsDailyAggregate.create!(web_hook_id: web_hook.id, date: 1.days.ago) + + WebHookEventsDailyAggregate.create!( + web_hook_id: web_hook.id, + date: 2.days.ago, + created_at: 2.days.ago, + ) + + expect { described_class.purge_old }.to change { WebHookEventsDailyAggregate.count }.by(-1) + + expect(WebHookEventsDailyAggregate.find(yesterday_aggregate.id)).to eq(yesterday_aggregate) + end + end + + describe "aggregation works" do + it "should be able to aggregate web hook events" do + yesterday_aggregate = + WebHookEventsDailyAggregate.create!(web_hook_id: web_hook.id, date: 1.days.ago) + yesterday_events = [event, failed_event, failed_event2] + + expect(WebHookEventsDailyAggregate.count).to eq(1) + expect(yesterday_aggregate.web_hook_id).to eq(web_hook.id) + expect(yesterday_aggregate.date).to eq(1.days.ago.to_date) + + expect(yesterday_aggregate.mean_duration).to eq( + yesterday_events.sum(&:duration) / yesterday_events.count, + ) + expect(yesterday_aggregate.successful_event_count).to eq(1) + expect(yesterday_aggregate.failed_event_count).to eq(2) + end + + it "should be able to filter by day" do + WebHookEventsDailyAggregate.create!(web_hook_id: web_hook.id, date: 1.days.ago) + WebHookEventsDailyAggregate.create!(web_hook_id: web_hook.id, date: 0.days.ago) + yesterday_events = [event, failed_event, failed_event2] + today_events = [event_today, failed_event_today] + + yesterday_aggregate = WebHookEventsDailyAggregate.by_day(1.days.ago, 1.days.ago) + expect(yesterday_aggregate.count).to eq(1) + expect(yesterday_aggregate.first.date).to eq(1.days.ago.to_date) + + expect(WebHookEventsDailyAggregate.count).to eq(2) + + today_and_yesterday_aggregate = WebHookEventsDailyAggregate.by_day(1.days.ago, 0.days.ago) + + expect(today_and_yesterday_aggregate.count).to eq(2) + expect(today_and_yesterday_aggregate.map(&:date)).to eq( + [0.days.ago.to_date, 1.days.ago.to_date], + ) + expect(today_and_yesterday_aggregate.map(&:mean_duration)).to eq( + [ + today_events.sum(&:duration) / today_events.count, + yesterday_events.sum(&:duration) / yesterday_events.count, + ], + ) + end + + it "should not create a new WebHookEventsDailyAggregate row if AggregateWebHooksEvents runs twice" do + expect { Jobs::AggregateWebHooksEvents.new.execute(date: 1.days.ago) }.to change { + WebHookEventsDailyAggregate.count + }.by(1) + + expect { Jobs::AggregateWebHooksEvents.new.execute(date: 1.days.ago) }.not_to change { + WebHookEventsDailyAggregate.count + } + end + end +end