mirror of
https://github.com/discourse/discourse.git
synced 2024-12-03 00:35:18 +08:00
2f2da72747
Our 'page_view_crawler' / 'page_view_anon' metrics are based purely on the User Agent sent by clients. This means that 'badly behaved' bots which are imitating real user agents are counted towards 'anon' page views. This commit introduces a new method of tracking visitors. When an initial HTML request is made, we assume it is a 'non-browser' request (i.e. a bot). Then, once the JS application has booted, we notify the server to count it as a 'browser' request. This reliance on a JavaScript-capable browser matches up more closely to dedicated analytics systems like Google Analytics. Existing data collection and graphs are unchanged. Data collected via the new technique is available in a new 'experimental' report.
820 lines
26 KiB
Ruby
820 lines
26 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
RSpec.describe Middleware::RequestTracker do
|
|
def env(opts = {})
|
|
path = opts.delete(:path) || "/path?bla=1"
|
|
create_request_env(path: path).merge(
|
|
"HTTP_HOST" => "http://test.com",
|
|
"HTTP_USER_AGENT" =>
|
|
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
|
|
"REQUEST_METHOD" => "GET",
|
|
"HTTP_ACCEPT" =>
|
|
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
|
|
"rack.input" => StringIO.new,
|
|
).merge(opts)
|
|
end
|
|
|
|
before do
|
|
ApplicationRequest.enable
|
|
CachedCounting.reset
|
|
CachedCounting.enable
|
|
end
|
|
|
|
after do
|
|
CachedCounting.reset
|
|
ApplicationRequest.disable
|
|
CachedCounting.disable
|
|
end
|
|
|
|
describe "full request" do
|
|
it "can handle rogue user agents" do
|
|
agent = (+"Evil Googlebot String \xc3\x28").force_encoding("Windows-1252")
|
|
|
|
middleware =
|
|
Middleware::RequestTracker.new(->(env) { ["200", { "Content-Type" => "text/html" }, [""]] })
|
|
middleware.call(env("HTTP_USER_AGENT" => agent))
|
|
|
|
CachedCounting.flush
|
|
|
|
expect(WebCrawlerRequest.where(user_agent: agent.encode("utf-8")).count).to eq(1)
|
|
end
|
|
end
|
|
|
|
describe "log_request" do
|
|
before do
|
|
freeze_time
|
|
ApplicationRequest.clear_cache!
|
|
end
|
|
|
|
def log_tracked_view(val)
|
|
data =
|
|
Middleware::RequestTracker.get_data(
|
|
env("HTTP_DISCOURSE_TRACK_VIEW" => val),
|
|
["200", { "Content-Type" => "text/html" }],
|
|
0.2,
|
|
)
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
end
|
|
|
|
it "can exclude/include based on custom header" do
|
|
log_tracked_view("true")
|
|
log_tracked_view("1")
|
|
log_tracked_view("false")
|
|
log_tracked_view("0")
|
|
|
|
CachedCounting.flush
|
|
|
|
expect(ApplicationRequest.page_view_anon.first.count).to eq(2)
|
|
expect(ApplicationRequest.page_view_anon_browser.first.count).to eq(2)
|
|
end
|
|
|
|
it "can log requests correctly" do
|
|
data =
|
|
Middleware::RequestTracker.get_data(
|
|
env("HTTP_USER_AGENT" => "AdsBot-Google (+http://www.google.com/adsbot.html)"),
|
|
["200", { "Content-Type" => "text/html" }],
|
|
0.1,
|
|
)
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
|
|
data =
|
|
Middleware::RequestTracker.get_data(
|
|
env("HTTP_DISCOURSE_TRACK_VIEW" => "1"),
|
|
["200", {}],
|
|
0.1,
|
|
)
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
|
|
data =
|
|
Middleware::RequestTracker.get_data(
|
|
env(
|
|
"HTTP_USER_AGENT" =>
|
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4",
|
|
),
|
|
["200", { "Content-Type" => "text/html" }],
|
|
0.1,
|
|
)
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
|
|
# /srv/status is never a tracked view because content-type is text/plain
|
|
data =
|
|
Middleware::RequestTracker.get_data(
|
|
env("HTTP_USER_AGENT" => "kube-probe/1.18", "REQUEST_URI" => "/srv/status?shutdown_ok=1"),
|
|
["200", { "Content-Type" => "text/plain" }],
|
|
0.1,
|
|
)
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
|
|
CachedCounting.flush
|
|
|
|
expect(ApplicationRequest.http_total.first.count).to eq(4)
|
|
expect(ApplicationRequest.http_2xx.first.count).to eq(4)
|
|
|
|
expect(ApplicationRequest.page_view_anon.first.count).to eq(2)
|
|
expect(ApplicationRequest.page_view_crawler.first.count).to eq(1)
|
|
expect(ApplicationRequest.page_view_anon_mobile.first.count).to eq(1)
|
|
|
|
expect(ApplicationRequest.page_view_crawler.first.count).to eq(1)
|
|
|
|
expect(ApplicationRequest.page_view_anon_browser.first.count).to eq(1)
|
|
end
|
|
|
|
it "logs deferred pageviews correctly" do
|
|
data =
|
|
Middleware::RequestTracker.get_data(
|
|
env(:path => "/message-bus/abcde/poll", "HTTP_DISCOURSE_DEFERRED_TRACK_VIEW" => "1"),
|
|
["200", { "Content-Type" => "text/html" }],
|
|
0.1,
|
|
)
|
|
Middleware::RequestTracker.log_request(data)
|
|
|
|
expect(data[:deferred_track]).to eq(true)
|
|
CachedCounting.flush
|
|
|
|
expect(ApplicationRequest.page_view_anon_browser.first.count).to eq(1)
|
|
end
|
|
|
|
it "logs API requests correctly" do
|
|
data =
|
|
Middleware::RequestTracker.get_data(
|
|
env("_DISCOURSE_API" => "1"),
|
|
["200", { "Content-Type" => "text/json" }],
|
|
0.1,
|
|
)
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
|
|
data =
|
|
Middleware::RequestTracker.get_data(
|
|
env("_DISCOURSE_API" => "1"),
|
|
["404", { "Content-Type" => "text/json" }],
|
|
0.1,
|
|
)
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
|
|
data =
|
|
Middleware::RequestTracker.get_data(env("_DISCOURSE_USER_API" => "1"), ["200", {}], 0.1)
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
CachedCounting.flush
|
|
|
|
expect(ApplicationRequest.http_total.first.count).to eq(3)
|
|
expect(ApplicationRequest.http_2xx.first.count).to eq(2)
|
|
|
|
expect(ApplicationRequest.api.first.count).to eq(2)
|
|
expect(ApplicationRequest.user_api.first.count).to eq(1)
|
|
end
|
|
|
|
it "can log Discourse user agent requests correctly" do
|
|
# log discourse api agents as crawlers for page view stats...
|
|
data =
|
|
Middleware::RequestTracker.get_data(
|
|
env("HTTP_USER_AGENT" => "DiscourseAPI Ruby Gem 0.19.0"),
|
|
["200", { "Content-Type" => "text/html" }],
|
|
0.1,
|
|
)
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
|
|
CachedCounting.flush
|
|
CachedCounting.reset
|
|
|
|
expect(ApplicationRequest.page_view_crawler.first.count).to eq(1)
|
|
|
|
# ...but count our mobile app user agents as regular visits
|
|
data =
|
|
Middleware::RequestTracker.get_data(
|
|
env("HTTP_USER_AGENT" => "Mozilla/5.0 AppleWebKit/605.1.15 Mobile/15E148 DiscourseHub)"),
|
|
["200", { "Content-Type" => "text/html" }],
|
|
0.1,
|
|
)
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
|
|
CachedCounting.flush
|
|
|
|
expect(ApplicationRequest.page_view_crawler.first.count).to eq(1)
|
|
expect(ApplicationRequest.page_view_anon.first.count).to eq(1)
|
|
end
|
|
|
|
context "when ignoring anonymous page views" do
|
|
let(:anon_data) do
|
|
Middleware::RequestTracker.get_data(
|
|
env(
|
|
"HTTP_USER_AGENT" =>
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36",
|
|
),
|
|
["200", { "Content-Type" => "text/html" }],
|
|
0.1,
|
|
)
|
|
end
|
|
|
|
let(:logged_in_data) do
|
|
user = Fabricate(:user, active: true)
|
|
token = UserAuthToken.generate!(user_id: user.id)
|
|
cookie =
|
|
create_auth_cookie(
|
|
token: token.unhashed_auth_token,
|
|
user_id: user.id,
|
|
trust_level: user.trust_level,
|
|
issued_at: 5.minutes.ago,
|
|
)
|
|
Middleware::RequestTracker.get_data(
|
|
env(
|
|
"HTTP_USER_AGENT" =>
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36",
|
|
"HTTP_COOKIE" => "_t=#{cookie};",
|
|
),
|
|
["200", { "Content-Type" => "text/html" }],
|
|
0.1,
|
|
)
|
|
end
|
|
|
|
it "does not ignore anonymous requests for public sites" do
|
|
SiteSetting.login_required = false
|
|
|
|
Middleware::RequestTracker.log_request(anon_data)
|
|
Middleware::RequestTracker.log_request(logged_in_data)
|
|
|
|
CachedCounting.flush
|
|
|
|
expect(ApplicationRequest.http_total.first.count).to eq(2)
|
|
expect(ApplicationRequest.http_2xx.first.count).to eq(2)
|
|
|
|
expect(ApplicationRequest.page_view_logged_in.first.count).to eq(1)
|
|
expect(ApplicationRequest.page_view_anon.first.count).to eq(1)
|
|
end
|
|
|
|
it "ignores anonymous requests for private sites" do
|
|
SiteSetting.login_required = true
|
|
|
|
Middleware::RequestTracker.log_request(anon_data)
|
|
Middleware::RequestTracker.log_request(logged_in_data)
|
|
|
|
CachedCounting.flush
|
|
|
|
expect(ApplicationRequest.http_total.first.count).to eq(2)
|
|
expect(ApplicationRequest.http_2xx.first.count).to eq(2)
|
|
|
|
expect(ApplicationRequest.page_view_logged_in.first.count).to eq(1)
|
|
expect(ApplicationRequest.page_view_anon.first).to eq(nil)
|
|
end
|
|
end
|
|
end
|
|
|
|
describe "rate limiting" do
|
|
before do
|
|
RateLimiter.enable
|
|
RateLimiter.clear_all_global!
|
|
|
|
@orig_logger = Rails.logger
|
|
Rails.logger = @fake_logger = FakeLogger.new
|
|
|
|
# rate limiter tests depend on checks for retry-after
|
|
# they can be sensitive to clock skew during test runs
|
|
freeze_time_safe
|
|
end
|
|
|
|
use_redis_snapshotting
|
|
|
|
after { Rails.logger = @orig_logger }
|
|
|
|
let :middleware do
|
|
app = lambda { |env| [200, {}, ["OK"]] }
|
|
|
|
Middleware::RequestTracker.new(app)
|
|
end
|
|
|
|
it "does nothing if configured to do nothing" do
|
|
global_setting :max_reqs_per_ip_mode, "none"
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
|
|
|
status, _ = middleware.call(env)
|
|
status, _ = middleware.call(env)
|
|
|
|
expect(status).to eq(200)
|
|
end
|
|
|
|
it "blocks private IPs if not skipped" do
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
|
global_setting :max_reqs_per_ip_mode, "warn+block"
|
|
global_setting :max_reqs_rate_limit_on_private, true
|
|
|
|
addresses = %w[
|
|
127.1.2.3
|
|
127.0.0.2
|
|
192.168.1.2
|
|
10.0.1.2
|
|
172.16.9.8
|
|
172.19.1.2
|
|
172.20.9.8
|
|
172.29.1.2
|
|
172.30.9.8
|
|
172.31.1.2
|
|
]
|
|
warn_count = 1
|
|
addresses.each do |addr|
|
|
env1 = env("REMOTE_ADDR" => addr)
|
|
|
|
status, _ = middleware.call(env1)
|
|
status, _ = middleware.call(env1)
|
|
|
|
expect(@fake_logger.warnings.count { |w| w.include?("Global rate limit exceeded") }).to eq(
|
|
warn_count,
|
|
)
|
|
expect(status).to eq(429)
|
|
warn_count += 1
|
|
end
|
|
end
|
|
|
|
it "blocks if the ip isn't static skipped" do
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
|
global_setting :max_reqs_per_ip_mode, "block"
|
|
|
|
env1 = env("REMOTE_ADDR" => "1.1.1.1")
|
|
status, _ = middleware.call(env1)
|
|
status, _ = middleware.call(env1)
|
|
expect(status).to eq(429)
|
|
end
|
|
|
|
it "doesn't block if rate limiter is enabled but IP is on the static exception list" do
|
|
stub_const(
|
|
Middleware::RequestTracker,
|
|
"STATIC_IP_SKIPPER",
|
|
"177.33.14.73 191.209.88.192/30".split.map { |ip| IPAddr.new(ip) },
|
|
) do
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
|
global_setting :max_reqs_per_ip_mode, "block"
|
|
|
|
env1 = env("REMOTE_ADDR" => "177.33.14.73")
|
|
env2 = env("REMOTE_ADDR" => "191.209.88.194")
|
|
|
|
status, _ = middleware.call(env1)
|
|
expect(status).to eq(200)
|
|
|
|
status, _ = middleware.call(env1)
|
|
expect(status).to eq(200)
|
|
|
|
status, _ = middleware.call(env2)
|
|
expect(status).to eq(200)
|
|
|
|
status, _ = middleware.call(env2)
|
|
expect(status).to eq(200)
|
|
end
|
|
end
|
|
|
|
describe "register_ip_skipper" do
|
|
before do
|
|
Middleware::RequestTracker.register_ip_skipper { |ip| ip == "1.1.1.2" }
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
|
global_setting :max_reqs_per_ip_mode, "block"
|
|
end
|
|
|
|
after { Middleware::RequestTracker.unregister_ip_skipper }
|
|
|
|
it "won't block if the ip is skipped" do
|
|
env1 = env("REMOTE_ADDR" => "1.1.1.2")
|
|
status, _ = middleware.call(env1)
|
|
status, _ = middleware.call(env1)
|
|
expect(status).to eq(200)
|
|
end
|
|
|
|
it "blocks if the ip isn't skipped" do
|
|
env1 = env("REMOTE_ADDR" => "1.1.1.1")
|
|
status, _ = middleware.call(env1)
|
|
status, _ = middleware.call(env1)
|
|
expect(status).to eq(429)
|
|
end
|
|
end
|
|
|
|
it "does nothing for private IPs if skipped" do
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
|
global_setting :max_reqs_per_ip_mode, "warn+block"
|
|
global_setting :max_reqs_rate_limit_on_private, false
|
|
|
|
addresses = %w[
|
|
127.1.2.3
|
|
127.0.3.1
|
|
192.168.1.2
|
|
10.0.1.2
|
|
172.16.9.8
|
|
172.19.1.2
|
|
172.20.9.8
|
|
172.29.1.2
|
|
172.30.9.8
|
|
172.31.1.2
|
|
]
|
|
addresses.each do |addr|
|
|
env1 = env("REMOTE_ADDR" => addr)
|
|
|
|
status, _ = middleware.call(env1)
|
|
status, _ = middleware.call(env1)
|
|
|
|
expect(@fake_logger.warnings.count { |w| w.include?("Global rate limit exceeded") }).to eq(
|
|
0,
|
|
)
|
|
expect(status).to eq(200)
|
|
end
|
|
end
|
|
|
|
it "does warn if rate limiter is enabled via warn+block" do
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
|
global_setting :max_reqs_per_ip_mode, "warn+block"
|
|
|
|
status, _ = middleware.call(env)
|
|
status, headers = middleware.call(env)
|
|
|
|
expect(@fake_logger.warnings.count { |w| w.include?("Global rate limit exceeded") }).to eq(1)
|
|
expect(status).to eq(429)
|
|
expect(headers["Retry-After"]).to eq("10")
|
|
end
|
|
|
|
it "does warn if rate limiter is enabled" do
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
|
global_setting :max_reqs_per_ip_mode, "warn"
|
|
|
|
status, _ = middleware.call(env)
|
|
status, _ = middleware.call(env)
|
|
|
|
expect(@fake_logger.warnings.count { |w| w.include?("Global rate limit exceeded") }).to eq(1)
|
|
expect(status).to eq(200)
|
|
end
|
|
|
|
it "allows assets for more requests" do
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
|
global_setting :max_reqs_per_ip_mode, "block"
|
|
global_setting :max_asset_reqs_per_ip_per_10_seconds, 3
|
|
|
|
env1 = env("REMOTE_ADDR" => "1.1.1.1", "DISCOURSE_IS_ASSET_PATH" => 1)
|
|
|
|
status, _ = middleware.call(env1)
|
|
expect(status).to eq(200)
|
|
status, _ = middleware.call(env1)
|
|
expect(status).to eq(200)
|
|
status, _ = middleware.call(env1)
|
|
expect(status).to eq(200)
|
|
status, headers = middleware.call(env1)
|
|
expect(status).to eq(429)
|
|
expect(headers["Retry-After"]).to eq("10")
|
|
|
|
env2 = env("REMOTE_ADDR" => "1.1.1.1")
|
|
|
|
status, headers = middleware.call(env2)
|
|
expect(status).to eq(429)
|
|
expect(headers["Retry-After"]).to eq("10")
|
|
end
|
|
|
|
it "does block if rate limiter is enabled" do
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
|
global_setting :max_reqs_per_ip_mode, "block"
|
|
|
|
env1 = env("REMOTE_ADDR" => "1.1.1.1")
|
|
env2 = env("REMOTE_ADDR" => "1.1.1.2")
|
|
|
|
status, _ = middleware.call(env1)
|
|
expect(status).to eq(200)
|
|
|
|
status, headers = middleware.call(env1)
|
|
expect(status).to eq(429)
|
|
expect(headers["Retry-After"]).to eq("10")
|
|
|
|
status, _ = middleware.call(env2)
|
|
expect(status).to eq(200)
|
|
end
|
|
|
|
describe "diagnostic information" do
|
|
it "is included when the requests-per-10-seconds limit is reached" do
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
|
called = 0
|
|
app =
|
|
lambda do |_|
|
|
called += 1
|
|
[200, {}, ["OK"]]
|
|
end
|
|
env = env("REMOTE_ADDR" => "1.1.1.1")
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
status, = middleware.call(env)
|
|
expect(status).to eq(200)
|
|
expect(called).to eq(1)
|
|
|
|
env = env("REMOTE_ADDR" => "1.1.1.1")
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
status, headers, response = middleware.call(env)
|
|
expect(status).to eq(429)
|
|
expect(called).to eq(1)
|
|
expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("ip_10_secs_limit")
|
|
expect(response.first).to include("too many requests from this IP address")
|
|
expect(response.first).to include("Error code: ip_10_secs_limit.")
|
|
end
|
|
|
|
it "is included when the requests-per-minute limit is reached" do
|
|
global_setting :max_reqs_per_ip_per_minute, 1
|
|
called = 0
|
|
app =
|
|
lambda do |_|
|
|
called += 1
|
|
[200, {}, ["OK"]]
|
|
end
|
|
env = env("REMOTE_ADDR" => "1.1.1.1")
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
status, = middleware.call(env)
|
|
expect(status).to eq(200)
|
|
expect(called).to eq(1)
|
|
|
|
env = env("REMOTE_ADDR" => "1.1.1.1")
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
status, headers, response = middleware.call(env)
|
|
expect(status).to eq(429)
|
|
expect(called).to eq(1)
|
|
expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("ip_60_secs_limit")
|
|
expect(response.first).to include("too many requests from this IP address")
|
|
expect(response.first).to include("Error code: ip_60_secs_limit.")
|
|
end
|
|
|
|
it "is included when the assets-requests-per-10-seconds limit is reached" do
|
|
global_setting :max_asset_reqs_per_ip_per_10_seconds, 1
|
|
called = 0
|
|
app =
|
|
lambda do |env|
|
|
called += 1
|
|
env["DISCOURSE_IS_ASSET_PATH"] = true
|
|
[200, {}, ["OK"]]
|
|
end
|
|
env = env("REMOTE_ADDR" => "1.1.1.1")
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
status, = middleware.call(env)
|
|
expect(status).to eq(200)
|
|
expect(called).to eq(1)
|
|
|
|
env = env("REMOTE_ADDR" => "1.1.1.1")
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
status, headers, response = middleware.call(env)
|
|
expect(status).to eq(429)
|
|
expect(called).to eq(1)
|
|
expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("ip_assets_10_secs_limit")
|
|
expect(response.first).to include("too many requests from this IP address")
|
|
expect(response.first).to include("Error code: ip_assets_10_secs_limit.")
|
|
end
|
|
end
|
|
|
|
it "users with high enough trust level are not rate limited per ip" do
|
|
global_setting :max_reqs_per_ip_per_minute, 1
|
|
global_setting :skip_per_ip_rate_limit_trust_level, 3
|
|
|
|
envs =
|
|
3.times.map do |n|
|
|
user = Fabricate(:user, trust_level: 3)
|
|
token = UserAuthToken.generate!(user_id: user.id)
|
|
cookie =
|
|
create_auth_cookie(
|
|
token: token.unhashed_auth_token,
|
|
user_id: user.id,
|
|
trust_level: user.trust_level,
|
|
issued_at: 5.minutes.ago,
|
|
)
|
|
env("HTTP_COOKIE" => "_t=#{cookie}", "REMOTE_ADDR" => "1.1.1.1")
|
|
end
|
|
|
|
called = 0
|
|
app =
|
|
lambda do |env|
|
|
called += 1
|
|
[200, {}, ["OK"]]
|
|
end
|
|
envs.each do |env|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
status, = middleware.call(env)
|
|
expect(status).to eq(200)
|
|
end
|
|
expect(called).to eq(3)
|
|
|
|
envs.each do |env|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
status, headers, response = middleware.call(env)
|
|
expect(status).to eq(429)
|
|
expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("id_60_secs_limit")
|
|
expect(response.first).to include("too many requests from this user")
|
|
expect(response.first).to include("Error code: id_60_secs_limit.")
|
|
end
|
|
expect(called).to eq(3)
|
|
end
|
|
|
|
it "falls back to IP rate limiting if the cookie is too old" do
|
|
unfreeze_time
|
|
global_setting :max_reqs_per_ip_per_minute, 1
|
|
global_setting :skip_per_ip_rate_limit_trust_level, 3
|
|
user = Fabricate(:user, trust_level: 3)
|
|
token = UserAuthToken.generate!(user_id: user.id)
|
|
cookie =
|
|
create_auth_cookie(
|
|
token: token.unhashed_auth_token,
|
|
user_id: user.id,
|
|
trust_level: user.trust_level,
|
|
issued_at: 5.minutes.ago,
|
|
)
|
|
env = env("HTTP_COOKIE" => "_t=#{cookie}", "REMOTE_ADDR" => "1.1.1.1")
|
|
|
|
called = 0
|
|
app =
|
|
lambda do |_|
|
|
called += 1
|
|
[200, {}, ["OK"]]
|
|
end
|
|
freeze_time(12.minutes.from_now) do
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
status, = middleware.call(env)
|
|
expect(status).to eq(200)
|
|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
status, headers, response = middleware.call(env)
|
|
expect(status).to eq(429)
|
|
expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("ip_60_secs_limit")
|
|
expect(response.first).to include("too many requests from this IP address")
|
|
expect(response.first).to include("Error code: ip_60_secs_limit.")
|
|
end
|
|
end
|
|
|
|
it "falls back to IP rate limiting if the cookie is tampered with" do
|
|
unfreeze_time
|
|
global_setting :max_reqs_per_ip_per_minute, 1
|
|
global_setting :skip_per_ip_rate_limit_trust_level, 3
|
|
user = Fabricate(:user, trust_level: 3)
|
|
token = UserAuthToken.generate!(user_id: user.id)
|
|
cookie =
|
|
create_auth_cookie(
|
|
token: token.unhashed_auth_token,
|
|
user_id: user.id,
|
|
trust_level: user.trust_level,
|
|
issued_at: Time.zone.now,
|
|
)
|
|
cookie = swap_2_different_characters(cookie)
|
|
env = env("HTTP_COOKIE" => "_t=#{cookie}", "REMOTE_ADDR" => "1.1.1.1")
|
|
|
|
called = 0
|
|
app =
|
|
lambda do |_|
|
|
called += 1
|
|
[200, {}, ["OK"]]
|
|
end
|
|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
status, = middleware.call(env)
|
|
expect(status).to eq(200)
|
|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
status, headers, response = middleware.call(env)
|
|
expect(status).to eq(429)
|
|
expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("ip_60_secs_limit")
|
|
expect(response.first).to include("too many requests from this IP address")
|
|
expect(response.first).to include("Error code: ip_60_secs_limit.")
|
|
end
|
|
end
|
|
|
|
describe "callbacks" do
|
|
def app(result, sql_calls: 0, redis_calls: 0)
|
|
lambda do |env|
|
|
sql_calls.times { User.where(id: -100).pluck(:id) }
|
|
redis_calls.times { Discourse.redis.get("x") }
|
|
yield if block_given?
|
|
result
|
|
end
|
|
end
|
|
|
|
let(:logger) do
|
|
->(env, data) do
|
|
@env = env
|
|
@data = data
|
|
end
|
|
end
|
|
|
|
before { Middleware::RequestTracker.register_detailed_request_logger(logger) }
|
|
|
|
after { Middleware::RequestTracker.unregister_detailed_request_logger(logger) }
|
|
|
|
it "can report data from anon cache" do
|
|
Middleware::AnonymousCache.enable_anon_cache
|
|
|
|
cache = Middleware::AnonymousCache.new(app([200, {}, ["i am a thing"]]))
|
|
tracker = Middleware::RequestTracker.new(cache)
|
|
|
|
uri = "/path?#{SecureRandom.hex}"
|
|
|
|
request_params = { "a" => "b", "action" => "bob", "controller" => "jane" }
|
|
|
|
tracker.call(
|
|
env(
|
|
"REQUEST_URI" => uri,
|
|
"ANON_CACHE_DURATION" => 60,
|
|
"action_dispatch.request.parameters" => request_params,
|
|
),
|
|
)
|
|
expect(@data[:cache]).to eq("skip")
|
|
|
|
tracker.call(
|
|
env(
|
|
"REQUEST_URI" => uri,
|
|
"ANON_CACHE_DURATION" => 60,
|
|
"action_dispatch.request.parameters" => request_params,
|
|
),
|
|
)
|
|
expect(@data[:cache]).to eq("store")
|
|
|
|
tracker.call(env("REQUEST_URI" => uri, "ANON_CACHE_DURATION" => 60))
|
|
expect(@data[:cache]).to eq("true")
|
|
|
|
# not allowlisted
|
|
request_params.delete("a")
|
|
|
|
expect(@env["action_dispatch.request.parameters"]).to eq(request_params)
|
|
end
|
|
|
|
it "can correctly log detailed data" do
|
|
global_setting :enable_performance_http_headers, true
|
|
|
|
# ensure pg is warmed up with the select 1 query
|
|
User.where(id: -100).pluck(:id)
|
|
|
|
freeze_time
|
|
start = Time.now.to_f
|
|
|
|
freeze_time 1.minute.from_now
|
|
|
|
tracker = Middleware::RequestTracker.new(app([200, {}, []], sql_calls: 2, redis_calls: 2))
|
|
_, headers, _ = tracker.call(env("HTTP_X_REQUEST_START" => "t=#{start}"))
|
|
|
|
expect(@data[:queue_seconds]).to eq(60)
|
|
|
|
timing = @data[:timing]
|
|
expect(timing[:total_duration]).to be > 0
|
|
|
|
expect(timing[:sql][:duration]).to be > 0
|
|
expect(timing[:sql][:calls]).to eq 2
|
|
|
|
expect(timing[:redis][:duration]).to be > 0
|
|
expect(timing[:redis][:calls]).to eq 2
|
|
|
|
expect(headers["X-Queue-Time"]).to eq("60.000000")
|
|
|
|
expect(headers["X-Redis-Calls"]).to eq("2")
|
|
expect(headers["X-Redis-Time"].to_f).to be > 0
|
|
|
|
expect(headers["X-Sql-Calls"]).to eq("2")
|
|
expect(headers["X-Sql-Time"].to_f).to be > 0
|
|
|
|
expect(headers["X-Runtime"].to_f).to be > 0
|
|
end
|
|
|
|
it "correctly logs GC stats when `instrument_gc_stat_per_request` site setting has been enabled" do
|
|
tracker =
|
|
Middleware::RequestTracker.new(
|
|
app([200, {}, []]) do
|
|
GC.start(full_mark: true) # Major GC
|
|
GC.start(full_mark: false) # Minor GC
|
|
end,
|
|
)
|
|
|
|
tracker.call(env)
|
|
|
|
expect(@data[:timing][:gc]).to eq(nil)
|
|
|
|
SiteSetting.instrument_gc_stat_per_request = true
|
|
|
|
tracker =
|
|
Middleware::RequestTracker.new(
|
|
app([200, {}, []]) do
|
|
GC.start(full_mark: true) # Major GC
|
|
GC.start(full_mark: false) # Minor GC
|
|
end,
|
|
)
|
|
|
|
tracker.call(env)
|
|
|
|
expect(@data[:timing][:gc][:time]).to be > 0.0
|
|
expect(@data[:timing][:gc][:major_count]).to eq(1)
|
|
expect(@data[:timing][:gc][:minor_count]).to eq(1)
|
|
end
|
|
|
|
it "can correctly log messagebus request types" do
|
|
tracker = Middleware::RequestTracker.new(app([200, {}, []]))
|
|
|
|
tracker.call(env(path: "/message-bus/abcde/poll"))
|
|
expect(@data[:is_background]).to eq(true)
|
|
expect(@data[:background_type]).to eq("message-bus")
|
|
|
|
tracker.call(env(path: "/message-bus/abcde/poll?dlp=t"))
|
|
expect(@data[:is_background]).to eq(true)
|
|
expect(@data[:background_type]).to eq("message-bus-dlp")
|
|
|
|
tracker.call(env("HTTP_DONT_CHUNK" => "True", :path => "/message-bus/abcde/poll"))
|
|
expect(@data[:is_background]).to eq(true)
|
|
expect(@data[:background_type]).to eq("message-bus-dontchunk")
|
|
end
|
|
end
|
|
end
|