FEATURE: makes reports loadable in bulk (#6309)

This commit is contained in:
Joffrey JAFFEUX 2018-08-24 15:28:01 +02:00 committed by GitHub
parent 52a2a1f0d8
commit 82dcc5cbfa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 388 additions and 342 deletions

View File

@ -1,7 +1,7 @@
import ReportLoader from "discourse/lib/reports-loader";
import Category from "discourse/models/category";
import { exportEntity } from "discourse/lib/export-csv";
import { outputExportResult } from "discourse/lib/export-result";
import { ajax } from "discourse/lib/ajax";
import { SCHEMA_VERSION, default as Report } from "admin/models/report";
import computed from "ember-addons/ember-computed-decorators";
import {
@ -95,7 +95,7 @@ export default Ember.Component.extend({
this.get("currentMode")
);
} else if (this.get("dataSourceName")) {
this._fetchReport().finally(() => this._computeReport());
this._fetchReport();
}
},
@ -306,29 +306,31 @@ export default Ember.Component.extend({
this.setProperties({ isLoading: true, rateLimitationString: null });
let payload = this._buildPayload(["prev_period"]);
Ember.run.next(() => {
let payload = this._buildPayload(["prev_period"]);
return ajax(this.get("dataSource"), payload)
.then(response => {
if (response && response.report) {
this._reports.push(this._loadReport(response.report));
} else {
console.log("failed loading", this.get("dataSource"));
const callback = response => {
if (!this.element || this.isDestroying || this.isDestroyed) {
return;
}
})
.catch(data => {
if (data.jqXHR && data.jqXHR.status === 429) {
this.set("isLoading", false);
if (response === 429) {
this.set(
"rateLimitationString",
I18n.t("admin.dashboard.too_many_requests")
);
} else if (response === 500) {
this.set("model.error", "exception");
} else if (response) {
this._reports.push(this._loadReport(response));
this._computeReport();
}
})
.finally(() => {
if (this.element && !this.isDestroying && !this.isDestroyed) {
this.set("isLoading", false);
}
});
};
ReportLoader.enqueue(this.get("dataSourceName"), payload.data, callback);
});
},
_buildPayload(facets) {

View File

@ -0,0 +1,87 @@
import { ajax } from "discourse/lib/ajax";
const { debounce } = Ember.run;
let _queue = [];
let _processing = 0;
// max number of reports which will be requested in one bulk request
const MAX_JOB_SIZE = 5;
// max number of concurrent bulk requests
const MAX_CONCURRENCY = 3;
// max number of jobs stored, first entered jobs will be evicted first
const MAX_QUEUE_SIZE = 20;
const BULK_REPORTS_ENDPOINT = "/admin/reports/bulk";
const DEBOUNCING_DELAY = 50;
export default {
enqueue(type, params, callback) {
// makes sures the queue is not filling indefinitely
if (_queue.length >= MAX_QUEUE_SIZE) {
const removedJobs = _queue.splice(0, 1)[0];
removedJobs.forEach(job => {
// this is technically not a 429, but it's the result
// of client doing too many requests so we want the same
// behavior
job.runnable()(429);
});
}
_queue.push({ runnable: () => callback, type, params });
debounce(this, this._processQueue, DEBOUNCING_DELAY);
},
_processQueue() {
if (_queue.length === 0) return;
if (_processing >= MAX_CONCURRENCY) return;
_processing++;
const jobs = _queue.splice(0, MAX_JOB_SIZE);
// if queue has still jobs after splice, we request a future processing
if (_queue.length > 0) {
debounce(this, this._processQueue, DEBOUNCING_DELAY);
}
let reports = {};
jobs.forEach(job => {
reports[job.type] = job.params;
});
ajax(BULK_REPORTS_ENDPOINT, { data: { reports } })
.then(response => {
jobs.forEach(job => {
const report = response.reports.findBy("type", job.type);
job.runnable()(report);
});
})
.catch(data => {
jobs.forEach(job => {
if (data.jqXHR && data.jqXHR.status === 429) {
job.runnable()(429);
} else if (data.jqXHR && data.jqXHR.status === 500) {
job.runnable()(500);
} else {
job.runnable()();
}
});
})
.finally(() => {
_processing--;
// when a request is done we want to start processing queue
// without waiting for debouncing
debounce(this, this._processQueue, DEBOUNCING_DELAY, true);
});
},
_reset() {
_queue = [];
_processing = 0;
}
};

View File

@ -18,44 +18,41 @@ class Admin::ReportsController < Admin::AdminController
render_json_dump(reports: reports.sort_by { |report| report[:title] })
end
def bulk
reports = []
hijack do
params[:reports].each do |report_type, report_params|
args = parse_params(report_params)
report = nil
if (report_params[:cache])
report = Report.find_cached(report_type, args)
end
if report
reports << report
else
report = Report.find(report_type, args)
if (report_params[:cache]) && report
Report.cache(report, 35.minutes)
end
reports << report if report
end
end
render_json_dump(reports: reports)
end
end
def show
report_type = params[:type]
raise Discourse::NotFound unless report_type =~ /^[a-z0-9\_]+$/
start_date = (params[:start_date].present? ? Time.parse(params[:start_date]).to_date : 1.days.ago).beginning_of_day
end_date = (params[:end_date].present? ? Time.parse(params[:end_date]).to_date : start_date + 30.days).end_of_day
if params.has_key?(:category_id) && params[:category_id].to_i > 0
category_id = params[:category_id].to_i
else
category_id = nil
end
if params.has_key?(:group_id) && params[:group_id].to_i > 0
group_id = params[:group_id].to_i
else
group_id = nil
end
facets = nil
if Array === params[:facets]
facets = params[:facets].map { |s| s.to_s.to_sym }
end
limit = nil
if params.has_key?(:limit) && params[:limit].to_i > 0
limit = params[:limit].to_i
end
args = {
start_date: start_date,
end_date: end_date,
category_id: category_id,
group_id: group_id,
facets: facets,
limit: limit
}
args = parse_params(params)
report = nil
if (params[:cache])
@ -77,7 +74,43 @@ class Admin::ReportsController < Admin::AdminController
render_json_dump(report: report)
end
end
private
def parse_params(report_params)
start_date = (report_params[:start_date].present? ? Time.parse(report_params[:start_date]).to_date : 1.days.ago).beginning_of_day
end_date = (report_params[:end_date].present? ? Time.parse(report_params[:end_date]).to_date : start_date + 30.days).end_of_day
if report_params.has_key?(:category_id) && report_params[:category_id].to_i > 0
category_id = report_params[:category_id].to_i
else
category_id = nil
end
if report_params.has_key?(:group_id) && report_params[:group_id].to_i > 0
group_id = report_params[:group_id].to_i
else
group_id = nil
end
facets = nil
if Array === report_params[:facets]
facets = report_params[:facets].map { |s| s.to_s.to_sym }
end
limit = nil
if report_params.has_key?(:limit) && report_params[:limit].to_i > 0
limit = report_params[:limit].to_i
end
{
start_date: start_date,
end_date: end_date,
category_id: category_id,
group_id: group_id,
facets: facets,
limit: limit
}
end
end

View File

@ -76,6 +76,7 @@ Discourse::Application.routes.draw do
end
get "reports" => "reports#index"
get "reports/bulk" => "reports#bulk"
get "reports/:type" => "reports#show"
resources :groups, constraints: AdminConstraint.new do

View File

@ -13,6 +13,40 @@ describe Admin::ReportsController do
sign_in(admin)
end
describe '#bulk' do
context "valid params" do
it "renders the reports as JSON" do
Fabricate(:topic)
get "/admin/reports/bulk.json", params: {
reports: {
topics: { limit: 10 },
likes: { limit: 10 }
}
}
expect(response.status).to eq(200)
expect(JSON.parse(response.body)["reports"].count).to eq(2)
end
end
context "invalid params" do
context "inexisting report" do
it "returns only existing reports" do
get "/admin/reports/bulk.json", params: {
reports: {
topics: { limit: 10 },
xxx: { limit: 10 }
}
}
expect(response.status).to eq(200)
expect(JSON.parse(response.body)["reports"].count).to eq(1)
expect(JSON.parse(response.body)["reports"][0]["type"]).to eq("topics")
end
end
end
end
describe '#show' do
context "invalid id form" do
let(:invalid_id) { "!!&asdfasdf" }

View File

@ -138,7 +138,7 @@ componentTest("rate limited", {
};
// prettier-ignore
server.get("/admin/reports/signups_rate_limited", () => { //eslint-disable-line
server.get("/admin/reports/bulk", () => { //eslint-disable-line
return response({"errors":["You’ve performed this action too many times. Please wait 10 seconds before trying again."],"error_type":"rate_limit","extras":{"wait_seconds":10}});
});
},

View File

@ -1,7 +0,0 @@
export default {
"/admin/reports/daily_engaged_users": {
report: {
report_key: "daily_engaged_users"
}
}
};

View File

@ -1,7 +0,0 @@
export default {
"/admin/reports/dau_by_mau": {
report: {
report_key: "dau_by_mau"
}
}
};

View File

@ -1,7 +0,0 @@
export default {
"/admin/reports/flags": {
report: {
report_key: "flags"
}
}
};

View File

@ -1,7 +0,0 @@
export default {
"/admin/reports/likes": {
report: {
report_key: "likes"
}
}
};

View File

@ -1,7 +0,0 @@
export default {
"/admin/reports/new_contributors": {
report: {
report_key: "new_contributors"
}
}
};

View File

@ -1,90 +0,0 @@
const startDate = moment()
.locale("en")
.utc()
.startOf("day")
.subtract(1, "month");
const endDate = moment()
.locale("en")
.utc()
.endOf("day");
const data = [
851,
3805,
2437,
3768,
4476,
3021,
1285,
1120,
3932,
2777,
3298,
3198,
3601,
1249,
1046,
3212,
3358,
3306,
2618,
2679,
910,
875,
3877,
2342,
2305,
3534,
3713,
1133,
1350,
4048,
2523,
1062
];
export default {
"/admin/reports/page_view_total_reqs": {
report: {
type: "page_view_total_reqs",
title: "Pageviews",
xaxis: "Day",
yaxis: "Total Pageviews",
description: null,
data: [...data].map((d, i) => {
return {
x: moment(startDate)
.add(i, "days")
.format("YYYY-MM-DD"),
y: d
};
}),
start_date: startDate.toISOString(),
end_date: endDate.toISOString(),
prev_data: null,
prev_start_date: "2018-06-20T00:00:00Z",
prev_end_date: "2018-07-23T00:00:00Z",
category_id: null,
group_id: null,
prev30Days: 58110,
dates_filtering: true,
report_key: `reports:page_view_total_reqs:${startDate.format(
"YYYYMMDD"
)}:${endDate.format("YYYYMMDD")}:[:prev_period]:2`,
labels: [
{ type: "date", property: "x", title: "Day" },
{ type: "number", property: "y", title: "Count" }
],
processing: false,
average: false,
percent: false,
higher_is_better: true,
category_filtering: false,
group_filtering: false,
modes: ["table", "chart"],
icon: "file",
total: 921672
}
}
};

View File

@ -1,7 +0,0 @@
export default {
"/admin/reports/posts": {
report: {
report_key: "posts"
}
}
};

View File

@ -0,0 +1,178 @@
let signups = {
type: "signups",
title: "Signups",
xaxis: "Day",
yaxis: "Number of signups",
description: "New account registrations for this period",
data: [
{ x: "2018-06-16", y: 12 },
{ x: "2018-06-17", y: 16 },
{ x: "2018-06-18", y: 42 },
{ x: "2018-06-19", y: 38 },
{ x: "2018-06-20", y: 41 },
{ x: "2018-06-21", y: 32 },
{ x: "2018-06-22", y: 23 },
{ x: "2018-06-23", y: 23 },
{ x: "2018-06-24", y: 17 },
{ x: "2018-06-25", y: 27 },
{ x: "2018-06-26", y: 32 },
{ x: "2018-06-27", y: 7 }
],
start_date: "2018-06-16T00:00:00Z",
end_date: "2018-07-16T23:59:59Z",
prev_data: [
{ x: "2018-05-17", y: 32 },
{ x: "2018-05-18", y: 30 },
{ x: "2018-05-19", y: 12 },
{ x: "2018-05-20", y: 23 },
{ x: "2018-05-21", y: 50 },
{ x: "2018-05-22", y: 39 },
{ x: "2018-05-23", y: 51 },
{ x: "2018-05-24", y: 48 },
{ x: "2018-05-25", y: 37 },
{ x: "2018-05-26", y: 17 },
{ x: "2018-05-27", y: 6 },
{ x: "2018-05-28", y: 20 },
{ x: "2018-05-29", y: 37 },
{ x: "2018-05-30", y: 37 },
{ x: "2018-05-31", y: 37 },
{ x: "2018-06-01", y: 38 },
{ x: "2018-06-02", y: 23 },
{ x: "2018-06-03", y: 18 },
{ x: "2018-06-04", y: 39 },
{ x: "2018-06-05", y: 26 },
{ x: "2018-06-06", y: 39 },
{ x: "2018-06-07", y: 52 },
{ x: "2018-06-08", y: 35 },
{ x: "2018-06-09", y: 19 },
{ x: "2018-06-10", y: 15 },
{ x: "2018-06-11", y: 31 },
{ x: "2018-06-12", y: 38 },
{ x: "2018-06-13", y: 30 },
{ x: "2018-06-14", y: 45 },
{ x: "2018-06-15", y: 37 },
{ x: "2018-06-16", y: 12 }
],
prev_start_date: "2018-05-17T00:00:00Z",
prev_end_date: "2018-06-17T00:00:00Z",
category_id: null,
group_id: null,
prev30Days: null,
dates_filtering: true,
report_key: "reports:signups::20180616:20180716::[:prev_period]:",
labels: [
{ type: "date", properties: ["x"], title: "Day" },
{ type: "number", properties: ["y"], title: "Count" }
],
processing: false,
average: false,
percent: false,
higher_is_better: true,
category_filtering: false,
group_filtering: true,
modes: ["table", "chart"],
prev_period: 961
};
let signups_fixture = JSON.parse(JSON.stringify(signups));
signups_fixture.type = "signups_exception";
signups_fixture.error = "exception";
const signups_exception = signups_fixture;
signups_fixture = JSON.parse(JSON.stringify(signups));
signups_fixture.type = "signups_timeout";
signups_fixture.error = "timeout";
const signups_timeout = signups_fixture;
const startDate = moment()
.locale("en")
.utc()
.startOf("day")
.subtract(1, "month");
const endDate = moment()
.locale("en")
.utc()
.endOf("day");
const data = [
851,
3805,
2437,
3768,
4476,
3021,
1285,
1120,
3932,
2777,
3298,
3198,
3601,
1249,
1046,
3212,
3358,
3306,
2618,
2679,
910,
875,
3877,
2342,
2305,
3534,
3713,
1133,
1350,
4048,
2523,
1062
];
const page_view_total_reqs = {
type: "page_view_total_reqs",
title: "Pageviews",
xaxis: "Day",
yaxis: "Total Pageviews",
description: null,
data: [...data].map((d, i) => {
return {
x: moment(startDate)
.add(i, "days")
.format("YYYY-MM-DD"),
y: d
};
}),
start_date: startDate.toISOString(),
end_date: endDate.toISOString(),
prev_data: null,
prev_start_date: "2018-06-20T00:00:00Z",
prev_end_date: "2018-07-23T00:00:00Z",
category_id: null,
group_id: null,
prev30Days: 58110,
dates_filtering: true,
report_key: `reports:page_view_total_reqs:${startDate.format(
"YYYYMMDD"
)}:${endDate.format("YYYYMMDD")}:[:prev_period]:2`,
labels: [
{ type: "date", property: "x", title: "Day" },
{ type: "number", property: "y", title: "Count" }
],
processing: false,
average: false,
percent: false,
higher_is_better: true,
category_filtering: false,
group_filtering: false,
modes: ["table", "chart"],
icon: "file",
total: 921672
};
export default {
"/admin/reports/bulk": {
reports: [signups, signups_exception, signups_timeout, page_view_total_reqs]
}
};

View File

@ -1,79 +0,0 @@
export default {
"/admin/reports/signups": {
report: {
type: "signups",
title: "Signups",
xaxis: "Day",
yaxis: "Number of signups",
description: "New account registrations for this period",
data: [
{ x: "2018-06-16", y: 12 },
{ x: "2018-06-17", y: 16 },
{ x: "2018-06-18", y: 42 },
{ x: "2018-06-19", y: 38 },
{ x: "2018-06-20", y: 41 },
{ x: "2018-06-21", y: 32 },
{ x: "2018-06-22", y: 23 },
{ x: "2018-06-23", y: 23 },
{ x: "2018-06-24", y: 17 },
{ x: "2018-06-25", y: 27 },
{ x: "2018-06-26", y: 32 },
{ x: "2018-06-27", y: 7 }
],
start_date: "2018-06-16T00:00:00Z",
end_date: "2018-07-16T23:59:59Z",
prev_data: [
{ x: "2018-05-17", y: 32 },
{ x: "2018-05-18", y: 30 },
{ x: "2018-05-19", y: 12 },
{ x: "2018-05-20", y: 23 },
{ x: "2018-05-21", y: 50 },
{ x: "2018-05-22", y: 39 },
{ x: "2018-05-23", y: 51 },
{ x: "2018-05-24", y: 48 },
{ x: "2018-05-25", y: 37 },
{ x: "2018-05-26", y: 17 },
{ x: "2018-05-27", y: 6 },
{ x: "2018-05-28", y: 20 },
{ x: "2018-05-29", y: 37 },
{ x: "2018-05-30", y: 37 },
{ x: "2018-05-31", y: 37 },
{ x: "2018-06-01", y: 38 },
{ x: "2018-06-02", y: 23 },
{ x: "2018-06-03", y: 18 },
{ x: "2018-06-04", y: 39 },
{ x: "2018-06-05", y: 26 },
{ x: "2018-06-06", y: 39 },
{ x: "2018-06-07", y: 52 },
{ x: "2018-06-08", y: 35 },
{ x: "2018-06-09", y: 19 },
{ x: "2018-06-10", y: 15 },
{ x: "2018-06-11", y: 31 },
{ x: "2018-06-12", y: 38 },
{ x: "2018-06-13", y: 30 },
{ x: "2018-06-14", y: 45 },
{ x: "2018-06-15", y: 37 },
{ x: "2018-06-16", y: 12 }
],
prev_start_date: "2018-05-17T00:00:00Z",
prev_end_date: "2018-06-17T00:00:00Z",
category_id: null,
group_id: null,
prev30Days: null,
dates_filtering: true,
report_key: "reports:signups::20180616:20180716::[:prev_period]:",
labels: [
{ type: "date", properties: ["x"], title: "Day" },
{ type: "number", properties: ["y"], title: "Count" }
],
processing: false,
average: false,
percent: false,
higher_is_better: true,
category_filtering: false,
group_filtering: true,
modes: ["table", "chart"],
prev_period: 961
}
}
};

View File

@ -1,11 +0,0 @@
import signups from "fixtures/signups";
const signupsExceptionKey = "/admin/reports/signups_exception";
const signupsKey = "/admin/reports/signups";
let fixture = {};
fixture[signupsExceptionKey] = JSON.parse(JSON.stringify(signups[signupsKey]));
fixture[signupsExceptionKey].report.error = "exception";
export default fixture;

View File

@ -1,11 +0,0 @@
import signups from "fixtures/signups";
const signupsTimeoutKey = "/admin/reports/signups_timeout";
const signupsKey = "/admin/reports/signups";
let fixture = {};
fixture[signupsTimeoutKey] = JSON.parse(JSON.stringify(signups[signupsKey]));
fixture[signupsTimeoutKey].report.error = "timeout";
export default fixture;

View File

@ -1,7 +0,0 @@
export default {
"/admin/reports/time_to_first_response": {
report: {
report_key: "time_to_first_response"
}
}
};

View File

@ -1,7 +0,0 @@
export default {
"/admin/reports/top_referred_topics": {
report: {
report_key: "top_referred_topics"
}
}
};

View File

@ -1,7 +0,0 @@
export default {
"/admin/reports/topics": {
report: {
report_key: "topics"
}
}
};

View File

@ -1,7 +0,0 @@
export default {
"/admin/reports/trending_search": {
report: {
report_key: "trending_search"
}
}
};

View File

@ -1,7 +0,0 @@
export default {
"/admin/reports/user_to_user_private_messages_with_replies": {
report: {
report_key: "user_to_user_private_messages_with_replies"
}
}
};

View File

@ -1,7 +0,0 @@
export default {
"/admin/reports/users_by_trust_level": {
report: {
report_key: "users_by_trust_level"
}
}
};

View File

@ -1,7 +0,0 @@
export default {
"/admin/reports/users_by_type": {
report: {
report_key: "users_by_type"
}
}
};

View File

@ -1,7 +0,0 @@
export default {
"/admin/reports/visits": {
report: {
report_key: "posts"
}
}
};