mirror of
https://github.com/discourse/discourse.git
synced 2024-11-26 06:43:44 +08:00
4580844c56
Followup 94fe31e5b3,
change the color of the "Known Crawler" bar on the
new "Consolidated Pageviews with Browser Detection (Experimental)"
report to be purple, like it was on the original
"Consolidated Pageviews" report to allow for easier
visual comparison.
Also removes the report colors to named keys in a hash
for easier reference than having to look up the
index of the array all the time.
452 lines
13 KiB
Ruby
452 lines
13 KiB
Ruby
# frozen_string_literal: true
|
||
|
||
class Report
|
||
# Change this line each time report format change
|
||
# and you want to ensure cache is reset
|
||
SCHEMA_VERSION = 4
|
||
|
||
FILTERS = %i[
|
||
name
|
||
start_date
|
||
end_date
|
||
category
|
||
group
|
||
trust_level
|
||
file_extension
|
||
include_subcategories
|
||
]
|
||
|
||
include Reports::PostEdits
|
||
include Reports::TopTrafficSources
|
||
include Reports::TopicsWithNoResponse
|
||
include Reports::DauByMau
|
||
include Reports::FlagsStatus
|
||
include Reports::Emails
|
||
include Reports::Likes
|
||
include Reports::SystemPrivateMessages
|
||
include Reports::UsersByType
|
||
include Reports::StorageStats
|
||
include Reports::NotifyModeratorsPrivateMessages
|
||
include Reports::SuspiciousLogins
|
||
include Reports::TopReferredTopics
|
||
include Reports::Signups
|
||
include Reports::NotifyUserPrivateMessages
|
||
include Reports::NewContributors
|
||
include Reports::TrendingSearch
|
||
include Reports::UserToUserPrivateMessages
|
||
include Reports::Flags
|
||
include Reports::Topics
|
||
include Reports::Posts
|
||
include Reports::Bookmarks
|
||
include Reports::StaffLogins
|
||
include Reports::DailyEngagedUsers
|
||
include Reports::UserToUserPrivateMessagesWithReplies
|
||
include Reports::MobileVisits
|
||
include Reports::TopReferrers
|
||
include Reports::WebCrawlers
|
||
include Reports::ModeratorsActivity
|
||
include Reports::TopIgnoredUsers
|
||
include Reports::UserFlaggingRatio
|
||
include Reports::TrustLevelGrowth
|
||
include Reports::ConsolidatedPageViews
|
||
include Reports::ConsolidatedPageViewsBrowserDetection
|
||
include Reports::ConsolidatedApiRequests
|
||
include Reports::Visits
|
||
include Reports::TimeToFirstResponse
|
||
include Reports::UsersByTrustLevel
|
||
include Reports::ModeratorWarningPrivateMessages
|
||
include Reports::ProfileViews
|
||
include Reports::TopUploads
|
||
include Reports::TopUsersByLikesReceived
|
||
include Reports::TopUsersByLikesReceivedFromInferiorTrustLevel
|
||
include Reports::TopUsersByLikesReceivedFromAVarietyOfPeople
|
||
|
||
attr_accessor :type,
|
||
:data,
|
||
:total,
|
||
:prev30Days,
|
||
:start_date,
|
||
:end_date,
|
||
:labels,
|
||
:prev_period,
|
||
:facets,
|
||
:limit,
|
||
:average,
|
||
:percent,
|
||
:higher_is_better,
|
||
:icon,
|
||
:modes,
|
||
:prev_data,
|
||
:dates_filtering,
|
||
:error,
|
||
:primary_color,
|
||
:secondary_color,
|
||
:filters,
|
||
:available_filters
|
||
|
||
def self.default_days
|
||
30
|
||
end
|
||
|
||
def self.default_labels
|
||
[
|
||
{ type: :date, property: :x, title: I18n.t("reports.default.labels.day") },
|
||
{ type: :number, property: :y, title: I18n.t("reports.default.labels.count") },
|
||
]
|
||
end
|
||
|
||
def initialize(type)
|
||
@type = type
|
||
@start_date ||= Report.default_days.days.ago.utc.beginning_of_day
|
||
@end_date ||= Time.now.utc.end_of_day
|
||
@prev_end_date = @start_date
|
||
@average = false
|
||
@percent = false
|
||
@higher_is_better = true
|
||
@modes = %i[table chart]
|
||
@prev_data = nil
|
||
@dates_filtering = true
|
||
@available_filters = {}
|
||
@filters = {}
|
||
|
||
tertiary = ColorScheme.hex_for_name("tertiary") || "0088cc"
|
||
@primary_color = rgba_color(tertiary)
|
||
@secondary_color = rgba_color(tertiary, 0.1)
|
||
end
|
||
|
||
def self.cache_key(report)
|
||
[
|
||
"reports",
|
||
report.type,
|
||
report.start_date.to_date.strftime("%Y%m%d"),
|
||
report.end_date.to_date.strftime("%Y%m%d"),
|
||
report.facets,
|
||
report.limit,
|
||
report.filters.blank? ? nil : MultiJson.dump(report.filters),
|
||
SCHEMA_VERSION,
|
||
].compact.map(&:to_s).join(":")
|
||
end
|
||
|
||
def add_filter(name, options = {})
|
||
available_filters[name] = options
|
||
end
|
||
|
||
def remove_filter(name)
|
||
available_filters.delete(name)
|
||
end
|
||
|
||
def add_category_filter
|
||
category_id = filters[:category].to_i if filters[:category].present?
|
||
add_filter("category", type: "category", default: category_id)
|
||
return if category_id.blank?
|
||
|
||
include_subcategories = filters[:include_subcategories]
|
||
include_subcategories = !!ActiveRecord::Type::Boolean.new.cast(include_subcategories)
|
||
add_filter("include_subcategories", type: "bool", default: include_subcategories)
|
||
|
||
[category_id, include_subcategories]
|
||
end
|
||
|
||
def self.clear_cache(type = nil)
|
||
pattern = type ? "reports:#{type}:*" : "reports:*"
|
||
|
||
Discourse.cache.keys(pattern).each { |key| Discourse.cache.redis.del(key) }
|
||
end
|
||
|
||
def self.wrap_slow_query(timeout = 20_000)
|
||
ActiveRecord::Base.connection.transaction do
|
||
# Allows only read only transactions
|
||
DB.exec "SET TRANSACTION READ ONLY"
|
||
# Set a statement timeout so we can't tie up the server
|
||
DB.exec "SET LOCAL statement_timeout = #{timeout}"
|
||
yield
|
||
end
|
||
end
|
||
|
||
def prev_start_date
|
||
self.start_date - (self.end_date - self.start_date)
|
||
end
|
||
|
||
def prev_end_date
|
||
self.start_date
|
||
end
|
||
|
||
def as_json(options = nil)
|
||
description = I18n.t("reports.#{type}.description", default: "")
|
||
description_link = I18n.t("reports.#{type}.description_link", default: "")
|
||
|
||
{
|
||
type: type,
|
||
title: I18n.t("reports.#{type}.title", default: nil),
|
||
xaxis: I18n.t("reports.#{type}.xaxis", default: nil),
|
||
yaxis: I18n.t("reports.#{type}.yaxis", default: nil),
|
||
description: description.presence ? description : nil,
|
||
description_link: description_link.presence ? description_link : nil,
|
||
data: data,
|
||
start_date: start_date&.iso8601,
|
||
end_date: end_date&.iso8601,
|
||
prev_data: self.prev_data,
|
||
prev_start_date: prev_start_date&.iso8601,
|
||
prev_end_date: prev_end_date&.iso8601,
|
||
prev30Days: self.prev30Days,
|
||
dates_filtering: self.dates_filtering,
|
||
report_key: Report.cache_key(self),
|
||
primary_color: self.primary_color,
|
||
secondary_color: self.secondary_color,
|
||
available_filters: self.available_filters.map { |k, v| { id: k }.merge(v) },
|
||
labels: labels || Report.default_labels,
|
||
average: self.average,
|
||
percent: self.percent,
|
||
higher_is_better: self.higher_is_better,
|
||
modes: self.modes,
|
||
}.tap do |json|
|
||
json[:icon] = self.icon if self.icon
|
||
json[:error] = self.error if self.error
|
||
json[:total] = self.total if self.total
|
||
json[:prev_period] = self.prev_period if self.prev_period
|
||
json[:prev30Days] = self.prev30Days if self.prev30Days
|
||
json[:limit] = self.limit if self.limit
|
||
|
||
if type == "page_view_crawler_reqs"
|
||
json[:related_report] = Report.find(
|
||
"web_crawlers",
|
||
start_date: start_date,
|
||
end_date: end_date,
|
||
)&.as_json
|
||
end
|
||
end
|
||
end
|
||
|
||
def Report.add_report(name, &block)
|
||
singleton_class.instance_eval { define_method("report_#{name}", &block) }
|
||
end
|
||
|
||
def self._get(type, opts = nil)
|
||
opts ||= {}
|
||
|
||
# Load the report
|
||
report = Report.new(type)
|
||
report.start_date = opts[:start_date] if opts[:start_date]
|
||
report.end_date = opts[:end_date] if opts[:end_date]
|
||
report.facets = opts[:facets] || %i[total prev30Days]
|
||
report.limit = opts[:limit] if opts[:limit]
|
||
report.average = opts[:average] if opts[:average]
|
||
report.percent = opts[:percent] if opts[:percent]
|
||
report.filters = opts[:filters] if opts[:filters]
|
||
report.labels = Report.default_labels
|
||
|
||
report
|
||
end
|
||
|
||
def self.find_cached(type, opts = nil)
|
||
report = _get(type, opts)
|
||
Discourse.cache.read(cache_key(report))
|
||
end
|
||
|
||
def self.cache(report)
|
||
duration = report.error == :exception ? 1.minute : 35.minutes
|
||
Discourse.cache.write(cache_key(report), report.as_json, expires_in: duration)
|
||
end
|
||
|
||
def self.find(type, opts = nil)
|
||
opts ||= {}
|
||
|
||
begin
|
||
report = _get(type, opts)
|
||
report_method = :"report_#{type}"
|
||
|
||
begin
|
||
wrap_slow_query do
|
||
if respond_to?(report_method)
|
||
public_send(report_method, report)
|
||
elsif type =~ /_reqs\z/
|
||
req_report(report, type.split(/_reqs\z/)[0].to_sym)
|
||
else
|
||
return nil
|
||
end
|
||
end
|
||
rescue ActiveRecord::QueryCanceled, PG::QueryCanceled => e
|
||
report.error = :timeout
|
||
end
|
||
rescue Exception => e
|
||
# In test mode, don't swallow exceptions by default to help debug errors.
|
||
raise if Rails.env.test? && !opts[:wrap_exceptions_in_test]
|
||
|
||
# ensures that if anything unexpected prevents us from
|
||
# creating a report object we fail elegantly and log an error
|
||
if !report
|
||
Rails.logger.error("Couldn’t create report `#{type}`: <#{e.class} #{e.message}>")
|
||
return nil
|
||
end
|
||
|
||
report.error = :exception
|
||
|
||
# given reports can be added by plugins we don’t want dashboard failures
|
||
# on report computation, however we do want to log which report is provoking
|
||
# an error
|
||
Rails.logger.error(
|
||
"Error while computing report `#{report.type}`: #{e.message}\n#{e.backtrace.join("\n")}",
|
||
)
|
||
end
|
||
|
||
report
|
||
end
|
||
|
||
def self.req_report(report, filter = nil)
|
||
data =
|
||
if filter == :page_view_total
|
||
ApplicationRequest.where(
|
||
req_type: [
|
||
ApplicationRequest
|
||
.req_types
|
||
.reject { |k, v| k =~ /mobile/ }
|
||
.map { |k, v| v if k =~ /page_view/ }
|
||
.compact,
|
||
].flatten,
|
||
)
|
||
else
|
||
ApplicationRequest.where(req_type: ApplicationRequest.req_types[filter])
|
||
end
|
||
|
||
report.icon = "file" if filter == :page_view_total
|
||
|
||
report.data = []
|
||
data
|
||
.where("date >= ? AND date <= ?", report.start_date, report.end_date)
|
||
.order(date: :asc)
|
||
.group(:date)
|
||
.sum(:count)
|
||
.each { |date, count| report.data << { x: date, y: count } }
|
||
|
||
report.total = data.sum(:count)
|
||
|
||
report.prev30Days =
|
||
data.where("date >= ? AND date < ?", (report.start_date - 31.days), report.start_date).sum(
|
||
:count,
|
||
)
|
||
end
|
||
|
||
def self.report_about(report, subject_class, report_method = :count_per_day)
|
||
basic_report_about report, subject_class, report_method, report.start_date, report.end_date
|
||
add_counts report, subject_class
|
||
end
|
||
|
||
def self.basic_report_about(report, subject_class, report_method, *args)
|
||
report.data = []
|
||
|
||
subject_class
|
||
.public_send(report_method, *args)
|
||
.each { |date, count| report.data << { x: date, y: count } }
|
||
end
|
||
|
||
def self.add_prev_data(report, subject_class, report_method, *args)
|
||
if report.modes.include?(:chart) && report.facets.include?(:prev_period)
|
||
prev_data = subject_class.public_send(report_method, *args)
|
||
report.prev_data = prev_data.map { |k, v| { x: k, y: v } }
|
||
end
|
||
end
|
||
|
||
def self.add_counts(report, subject_class, query_column = "created_at")
|
||
if report.facets.include?(:prev_period)
|
||
prev_data =
|
||
subject_class.where(
|
||
"#{query_column} >= ? and #{query_column} < ?",
|
||
report.prev_start_date,
|
||
report.prev_end_date,
|
||
)
|
||
|
||
report.prev_period = prev_data.count
|
||
end
|
||
|
||
report.total = subject_class.count if report.facets.include?(:total)
|
||
|
||
if report.facets.include?(:prev30Days)
|
||
report.prev30Days =
|
||
subject_class.where(
|
||
"#{query_column} >= ? and #{query_column} < ?",
|
||
report.start_date - 30.days,
|
||
report.start_date,
|
||
).count
|
||
end
|
||
end
|
||
|
||
def self.post_action_report(report, post_action_type)
|
||
category_id, include_subcategories = report.add_category_filter
|
||
|
||
report.data = []
|
||
PostAction
|
||
.count_per_day_for_type(
|
||
post_action_type,
|
||
category_id: category_id,
|
||
include_subcategories: include_subcategories,
|
||
start_date: report.start_date,
|
||
end_date: report.end_date,
|
||
)
|
||
.each { |date, count| report.data << { x: date, y: count } }
|
||
|
||
countable = PostAction.unscoped.where(post_action_type_id: post_action_type)
|
||
if category_id
|
||
if include_subcategories
|
||
countable =
|
||
countable.joins(post: :topic).where(
|
||
"topics.category_id IN (?)",
|
||
Category.subcategory_ids(category_id),
|
||
)
|
||
else
|
||
countable = countable.joins(post: :topic).where("topics.category_id = ?", category_id)
|
||
end
|
||
end
|
||
|
||
add_counts report, countable, "post_actions.created_at"
|
||
end
|
||
|
||
def self.private_messages_report(report, topic_subtype)
|
||
report.icon = "envelope"
|
||
subject = Topic.where("topics.user_id > 0")
|
||
basic_report_about report,
|
||
subject,
|
||
:private_message_topics_count_per_day,
|
||
report.start_date,
|
||
report.end_date,
|
||
topic_subtype
|
||
subject = Topic.private_messages.where("topics.user_id > 0").with_subtype(topic_subtype)
|
||
add_counts report, subject, "topics.created_at"
|
||
end
|
||
|
||
def rgba_color(hex, opacity = 1)
|
||
rgbs = hex_to_rgbs(adjust_hex(hex))
|
||
"rgba(#{rgbs.join(",")},#{opacity})"
|
||
end
|
||
|
||
def colors
|
||
{
|
||
turquoise: "#1EB8D1",
|
||
lime: "#9BC53D",
|
||
purple: "#721D8D",
|
||
magenta: "#E84A5F",
|
||
brown: "#8A6916",
|
||
}
|
||
end
|
||
|
||
private
|
||
|
||
def adjust_hex(hex)
|
||
hex = hex.gsub("#", "")
|
||
|
||
if hex.size == 3
|
||
chars = hex.scan(/\w/)
|
||
hex = chars.zip(chars).flatten.join
|
||
end
|
||
|
||
hex = hex.ljust(6, hex.last) if hex.size < 3
|
||
|
||
hex
|
||
end
|
||
|
||
def hex_to_rgbs(hex_color)
|
||
hex_color = hex_color.gsub("#", "")
|
||
rgbs = hex_color.scan(/../)
|
||
rgbs.map! { |color| color.hex }.map! { |rgb| rgb.to_i }
|
||
end
|
||
end
|