2019-04-30 08:27:42 +08:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2022-07-28 10:27:38 +08:00
|
|
|
RSpec.describe Middleware::RequestTracker do
|
2015-02-05 13:08:52 +08:00
|
|
|
def env(opts = {})
|
2023-01-06 19:26:18 +08:00
|
|
|
path = opts.delete(:path) || "/path?bla=1"
|
|
|
|
create_request_env(path: path).merge(
|
2015-02-05 13:08:52 +08:00
|
|
|
"HTTP_HOST" => "http://test.com",
|
2023-01-09 19:18:21 +08:00
|
|
|
"HTTP_USER_AGENT" =>
|
|
|
|
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
|
2015-02-05 13:08:52 +08:00
|
|
|
"REQUEST_METHOD" => "GET",
|
2023-01-09 19:18:21 +08:00
|
|
|
"HTTP_ACCEPT" =>
|
|
|
|
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
|
|
|
|
"rack.input" => StringIO.new,
|
FEATURE: Apply rate limits per user instead of IP for trusted users (#14706)
Currently, Discourse rate limits all incoming requests by the IP address they
originate from regardless of the user making the request. This can be
frustrating if there are multiple users using Discourse simultaneously while
sharing the same IP address (e.g. employees in an office).
This commit implements a new feature to make Discourse apply rate limits by
user id rather than IP address for users at or higher than the configured trust
level (1 is the default).
For example, let's say a Discourse instance is configured to allow 200 requests
per minute per IP address, and we have 10 users at trust level 4 using
Discourse simultaneously from the same IP address. Before this feature, the 10
users could only make a total of 200 requests per minute before they got rate
limited. But with the new feature, each user is allowed to make 200 requests
per minute because the rate limits are applied on user id rather than the IP
address.
The minimum trust level for applying user-id-based rate limits can be
configured by the `skip_per_ip_rate_limit_trust_level` global setting. The
default is 1, but it can be changed by either adding the
`DISCOURSE_SKIP_PER_IP_RATE_LIMIT_TRUST_LEVEL` environment variable with the
desired value to your `app.yml`, or changing the setting's value in the
`discourse.conf` file.
Requests made with API keys are still rate limited by IP address and the
relevant global settings that control API keys rate limits.
Before this commit, Discourse's auth cookie (`_t`) was simply a 32 characters
string that Discourse used to lookup the current user from the database and the
cookie contained no additional information about the user. However, we had to
change the cookie content in this commit so we could identify the user from the
cookie without making a database query before the rate limits logic and avoid
introducing a bottleneck on busy sites.
Besides the 32 characters auth token, the cookie now includes the user id,
trust level and the cookie's generation date, and we encrypt/sign the cookie to
prevent tampering.
Internal ticket number: t54739.
2021-11-18 04:27:30 +08:00
|
|
|
).merge(opts)
|
2015-02-05 13:08:52 +08:00
|
|
|
end
|
|
|
|
|
2020-05-18 17:22:39 +08:00
|
|
|
before do
|
|
|
|
ApplicationRequest.enable
|
2022-02-23 00:45:25 +08:00
|
|
|
CachedCounting.reset
|
|
|
|
CachedCounting.enable
|
2020-05-18 17:22:39 +08:00
|
|
|
end
|
|
|
|
|
|
|
|
after do
|
|
|
|
ApplicationRequest.disable
|
2022-02-23 00:45:25 +08:00
|
|
|
CachedCounting.disable
|
2020-05-18 17:22:39 +08:00
|
|
|
end
|
|
|
|
|
2022-07-28 00:14:14 +08:00
|
|
|
describe "full request" do
|
2019-12-09 14:43:51 +08:00
|
|
|
it "can handle rogue user agents" do
|
|
|
|
agent = (+"Evil Googlebot String \xc3\x28").force_encoding("Windows-1252")
|
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
middleware =
|
|
|
|
Middleware::RequestTracker.new(->(env) { ["200", { "Content-Type" => "text/html" }, [""]] })
|
2019-12-09 14:43:51 +08:00
|
|
|
middleware.call(env("HTTP_USER_AGENT" => agent))
|
|
|
|
|
2022-02-23 00:45:25 +08:00
|
|
|
CachedCounting.flush
|
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
expect(WebCrawlerRequest.where(user_agent: agent.encode("utf-8")).count).to eq(1)
|
2019-12-09 14:43:51 +08:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2022-07-28 00:14:14 +08:00
|
|
|
describe "log_request" do
|
2015-02-26 08:40:57 +08:00
|
|
|
before do
|
2022-05-05 09:53:54 +08:00
|
|
|
freeze_time
|
2015-02-05 13:08:52 +08:00
|
|
|
ApplicationRequest.clear_cache!
|
2015-02-26 08:40:57 +08:00
|
|
|
end
|
|
|
|
|
|
|
|
def log_tracked_view(val)
|
2023-01-09 19:18:21 +08:00
|
|
|
data =
|
|
|
|
Middleware::RequestTracker.get_data(
|
|
|
|
env("HTTP_DISCOURSE_TRACK_VIEW" => val),
|
|
|
|
["200", { "Content-Type" => "text/html" }],
|
|
|
|
0.2,
|
|
|
|
)
|
2015-02-26 08:40:57 +08:00
|
|
|
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
|
|
end
|
|
|
|
|
|
|
|
it "can exclude/include based on custom header" do
|
|
|
|
log_tracked_view("true")
|
|
|
|
log_tracked_view("1")
|
|
|
|
log_tracked_view("false")
|
|
|
|
log_tracked_view("0")
|
2022-02-23 00:45:25 +08:00
|
|
|
|
|
|
|
CachedCounting.flush
|
2015-02-26 08:40:57 +08:00
|
|
|
|
2015-04-25 23:18:35 +08:00
|
|
|
expect(ApplicationRequest.page_view_anon.first.count).to eq(2)
|
2015-02-26 08:40:57 +08:00
|
|
|
end
|
|
|
|
|
|
|
|
it "can log requests correctly" do
|
2023-01-09 19:18:21 +08:00
|
|
|
data =
|
|
|
|
Middleware::RequestTracker.get_data(
|
|
|
|
env("HTTP_USER_AGENT" => "AdsBot-Google (+http://www.google.com/adsbot.html)"),
|
|
|
|
["200", { "Content-Type" => "text/html" }],
|
|
|
|
0.1,
|
|
|
|
)
|
2015-02-10 14:03:33 +08:00
|
|
|
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
data =
|
|
|
|
Middleware::RequestTracker.get_data(
|
|
|
|
env("HTTP_DISCOURSE_TRACK_VIEW" => "1"),
|
|
|
|
["200", {}],
|
|
|
|
0.1,
|
|
|
|
)
|
2015-02-10 14:03:33 +08:00
|
|
|
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
2015-02-05 13:08:52 +08:00
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
data =
|
|
|
|
Middleware::RequestTracker.get_data(
|
|
|
|
env(
|
|
|
|
"HTTP_USER_AGENT" =>
|
|
|
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4",
|
|
|
|
),
|
|
|
|
["200", { "Content-Type" => "text/html" }],
|
|
|
|
0.1,
|
|
|
|
)
|
2015-07-04 05:02:57 +08:00
|
|
|
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
|
|
|
2020-03-24 13:28:07 +08:00
|
|
|
# /srv/status is never a tracked view because content-type is text/plain
|
2023-01-09 19:18:21 +08:00
|
|
|
data =
|
|
|
|
Middleware::RequestTracker.get_data(
|
|
|
|
env("HTTP_USER_AGENT" => "kube-probe/1.18", "REQUEST_URI" => "/srv/status?shutdown_ok=1"),
|
|
|
|
["200", { "Content-Type" => "text/plain" }],
|
|
|
|
0.1,
|
|
|
|
)
|
2020-03-24 13:28:07 +08:00
|
|
|
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
|
|
|
2022-02-23 00:45:25 +08:00
|
|
|
CachedCounting.flush
|
2015-02-05 13:08:52 +08:00
|
|
|
|
2020-03-24 13:28:07 +08:00
|
|
|
expect(ApplicationRequest.http_total.first.count).to eq(4)
|
|
|
|
expect(ApplicationRequest.http_2xx.first.count).to eq(4)
|
2015-02-05 13:08:52 +08:00
|
|
|
|
2015-07-04 05:02:57 +08:00
|
|
|
expect(ApplicationRequest.page_view_anon.first.count).to eq(2)
|
2015-04-25 23:18:35 +08:00
|
|
|
expect(ApplicationRequest.page_view_crawler.first.count).to eq(1)
|
2015-07-04 05:02:57 +08:00
|
|
|
expect(ApplicationRequest.page_view_anon_mobile.first.count).to eq(1)
|
2019-05-08 22:38:55 +08:00
|
|
|
|
2019-11-04 22:16:50 +08:00
|
|
|
expect(ApplicationRequest.page_view_crawler.first.count).to eq(1)
|
|
|
|
end
|
|
|
|
|
2022-11-29 19:07:42 +08:00
|
|
|
it "logs API requests correctly" do
|
2023-01-09 19:18:21 +08:00
|
|
|
data =
|
|
|
|
Middleware::RequestTracker.get_data(
|
|
|
|
env("_DISCOURSE_API" => "1"),
|
|
|
|
["200", { "Content-Type" => "text/json" }],
|
|
|
|
0.1,
|
|
|
|
)
|
2022-11-29 19:07:42 +08:00
|
|
|
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
data =
|
|
|
|
Middleware::RequestTracker.get_data(
|
|
|
|
env("_DISCOURSE_API" => "1"),
|
|
|
|
["404", { "Content-Type" => "text/json" }],
|
|
|
|
0.1,
|
|
|
|
)
|
2022-11-29 19:07:42 +08:00
|
|
|
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
data =
|
|
|
|
Middleware::RequestTracker.get_data(env("_DISCOURSE_USER_API" => "1"), ["200", {}], 0.1)
|
2022-11-29 19:07:42 +08:00
|
|
|
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
|
|
|
CachedCounting.flush
|
|
|
|
|
|
|
|
expect(ApplicationRequest.http_total.first.count).to eq(3)
|
|
|
|
expect(ApplicationRequest.http_2xx.first.count).to eq(2)
|
|
|
|
|
|
|
|
expect(ApplicationRequest.api.first.count).to eq(2)
|
|
|
|
expect(ApplicationRequest.user_api.first.count).to eq(1)
|
|
|
|
end
|
|
|
|
|
2019-11-04 22:16:50 +08:00
|
|
|
it "can log Discourse user agent requests correctly" do
|
|
|
|
# log discourse api agents as crawlers for page view stats...
|
2023-01-09 19:18:21 +08:00
|
|
|
data =
|
|
|
|
Middleware::RequestTracker.get_data(
|
|
|
|
env("HTTP_USER_AGENT" => "DiscourseAPI Ruby Gem 0.19.0"),
|
|
|
|
["200", { "Content-Type" => "text/html" }],
|
|
|
|
0.1,
|
|
|
|
)
|
2019-05-08 22:38:55 +08:00
|
|
|
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
2022-02-23 00:45:25 +08:00
|
|
|
|
|
|
|
CachedCounting.flush
|
|
|
|
CachedCounting.reset
|
|
|
|
|
2019-11-04 22:16:50 +08:00
|
|
|
expect(ApplicationRequest.page_view_crawler.first.count).to eq(1)
|
2019-05-08 22:38:55 +08:00
|
|
|
|
2019-11-04 22:16:50 +08:00
|
|
|
# ...but count our mobile app user agents as regular visits
|
2023-01-09 19:18:21 +08:00
|
|
|
data =
|
|
|
|
Middleware::RequestTracker.get_data(
|
|
|
|
env("HTTP_USER_AGENT" => "Mozilla/5.0 AppleWebKit/605.1.15 Mobile/15E148 DiscourseHub)"),
|
|
|
|
["200", { "Content-Type" => "text/html" }],
|
|
|
|
0.1,
|
|
|
|
)
|
2019-11-04 22:16:50 +08:00
|
|
|
|
|
|
|
Middleware::RequestTracker.log_request(data)
|
2022-02-23 00:45:25 +08:00
|
|
|
|
|
|
|
CachedCounting.flush
|
2017-10-18 09:10:12 +08:00
|
|
|
|
2019-11-04 22:16:50 +08:00
|
|
|
expect(ApplicationRequest.page_view_crawler.first.count).to eq(1)
|
|
|
|
expect(ApplicationRequest.page_view_anon.first.count).to eq(1)
|
|
|
|
end
|
2021-04-26 19:19:47 +08:00
|
|
|
|
2022-07-28 00:14:14 +08:00
|
|
|
context "when ignoring anonymous page views" do
|
2021-04-26 19:19:47 +08:00
|
|
|
let(:anon_data) do
|
2023-01-09 19:18:21 +08:00
|
|
|
Middleware::RequestTracker.get_data(
|
|
|
|
env(
|
|
|
|
"HTTP_USER_AGENT" =>
|
|
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36",
|
|
|
|
),
|
|
|
|
["200", { "Content-Type" => "text/html" }],
|
|
|
|
0.1,
|
|
|
|
)
|
2021-04-26 19:19:47 +08:00
|
|
|
end
|
|
|
|
|
|
|
|
let(:logged_in_data) do
|
|
|
|
user = Fabricate(:user, active: true)
|
|
|
|
token = UserAuthToken.generate!(user_id: user.id)
|
2023-01-09 19:18:21 +08:00
|
|
|
cookie =
|
|
|
|
create_auth_cookie(
|
|
|
|
token: token.unhashed_auth_token,
|
|
|
|
user_id: user.id,
|
|
|
|
trust_level: user.trust_level,
|
|
|
|
issued_at: 5.minutes.ago,
|
|
|
|
)
|
|
|
|
Middleware::RequestTracker.get_data(
|
|
|
|
env(
|
|
|
|
"HTTP_USER_AGENT" =>
|
|
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36",
|
|
|
|
"HTTP_COOKIE" => "_t=#{cookie};",
|
|
|
|
),
|
|
|
|
["200", { "Content-Type" => "text/html" }],
|
|
|
|
0.1,
|
FEATURE: Apply rate limits per user instead of IP for trusted users (#14706)
Currently, Discourse rate limits all incoming requests by the IP address they
originate from regardless of the user making the request. This can be
frustrating if there are multiple users using Discourse simultaneously while
sharing the same IP address (e.g. employees in an office).
This commit implements a new feature to make Discourse apply rate limits by
user id rather than IP address for users at or higher than the configured trust
level (1 is the default).
For example, let's say a Discourse instance is configured to allow 200 requests
per minute per IP address, and we have 10 users at trust level 4 using
Discourse simultaneously from the same IP address. Before this feature, the 10
users could only make a total of 200 requests per minute before they got rate
limited. But with the new feature, each user is allowed to make 200 requests
per minute because the rate limits are applied on user id rather than the IP
address.
The minimum trust level for applying user-id-based rate limits can be
configured by the `skip_per_ip_rate_limit_trust_level` global setting. The
default is 1, but it can be changed by either adding the
`DISCOURSE_SKIP_PER_IP_RATE_LIMIT_TRUST_LEVEL` environment variable with the
desired value to your `app.yml`, or changing the setting's value in the
`discourse.conf` file.
Requests made with API keys are still rate limited by IP address and the
relevant global settings that control API keys rate limits.
Before this commit, Discourse's auth cookie (`_t`) was simply a 32 characters
string that Discourse used to lookup the current user from the database and the
cookie contained no additional information about the user. However, we had to
change the cookie content in this commit so we could identify the user from the
cookie without making a database query before the rate limits logic and avoid
introducing a bottleneck on busy sites.
Besides the 32 characters auth token, the cookie now includes the user id,
trust level and the cookie's generation date, and we encrypt/sign the cookie to
prevent tampering.
Internal ticket number: t54739.
2021-11-18 04:27:30 +08:00
|
|
|
)
|
2021-04-26 19:19:47 +08:00
|
|
|
end
|
|
|
|
|
|
|
|
it "does not ignore anonymous requests for public sites" do
|
|
|
|
SiteSetting.login_required = false
|
|
|
|
|
|
|
|
Middleware::RequestTracker.log_request(anon_data)
|
|
|
|
Middleware::RequestTracker.log_request(logged_in_data)
|
|
|
|
|
2022-02-23 00:45:25 +08:00
|
|
|
CachedCounting.flush
|
2021-04-26 19:19:47 +08:00
|
|
|
|
|
|
|
expect(ApplicationRequest.http_total.first.count).to eq(2)
|
|
|
|
expect(ApplicationRequest.http_2xx.first.count).to eq(2)
|
|
|
|
|
|
|
|
expect(ApplicationRequest.page_view_logged_in.first.count).to eq(1)
|
|
|
|
expect(ApplicationRequest.page_view_anon.first.count).to eq(1)
|
|
|
|
end
|
|
|
|
|
|
|
|
it "ignores anonymous requests for private sites" do
|
|
|
|
SiteSetting.login_required = true
|
|
|
|
|
|
|
|
Middleware::RequestTracker.log_request(anon_data)
|
|
|
|
Middleware::RequestTracker.log_request(logged_in_data)
|
|
|
|
|
2022-02-23 00:45:25 +08:00
|
|
|
CachedCounting.flush
|
2021-04-26 19:19:47 +08:00
|
|
|
|
|
|
|
expect(ApplicationRequest.http_total.first.count).to eq(2)
|
|
|
|
expect(ApplicationRequest.http_2xx.first.count).to eq(2)
|
|
|
|
|
|
|
|
expect(ApplicationRequest.page_view_logged_in.first.count).to eq(1)
|
|
|
|
expect(ApplicationRequest.page_view_anon.first).to eq(nil)
|
|
|
|
end
|
|
|
|
end
|
2017-10-18 09:10:12 +08:00
|
|
|
end
|
2017-12-11 14:21:00 +08:00
|
|
|
|
2022-07-28 00:14:14 +08:00
|
|
|
describe "rate limiting" do
|
2017-12-11 14:21:00 +08:00
|
|
|
before do
|
|
|
|
RateLimiter.enable
|
|
|
|
RateLimiter.clear_all_global!
|
FEATURE: Apply rate limits per user instead of IP for trusted users (#14706)
Currently, Discourse rate limits all incoming requests by the IP address they
originate from regardless of the user making the request. This can be
frustrating if there are multiple users using Discourse simultaneously while
sharing the same IP address (e.g. employees in an office).
This commit implements a new feature to make Discourse apply rate limits by
user id rather than IP address for users at or higher than the configured trust
level (1 is the default).
For example, let's say a Discourse instance is configured to allow 200 requests
per minute per IP address, and we have 10 users at trust level 4 using
Discourse simultaneously from the same IP address. Before this feature, the 10
users could only make a total of 200 requests per minute before they got rate
limited. But with the new feature, each user is allowed to make 200 requests
per minute because the rate limits are applied on user id rather than the IP
address.
The minimum trust level for applying user-id-based rate limits can be
configured by the `skip_per_ip_rate_limit_trust_level` global setting. The
default is 1, but it can be changed by either adding the
`DISCOURSE_SKIP_PER_IP_RATE_LIMIT_TRUST_LEVEL` environment variable with the
desired value to your `app.yml`, or changing the setting's value in the
`discourse.conf` file.
Requests made with API keys are still rate limited by IP address and the
relevant global settings that control API keys rate limits.
Before this commit, Discourse's auth cookie (`_t`) was simply a 32 characters
string that Discourse used to lookup the current user from the database and the
cookie contained no additional information about the user. However, we had to
change the cookie content in this commit so we could identify the user from the
cookie without making a database query before the rate limits logic and avoid
introducing a bottleneck on busy sites.
Besides the 32 characters auth token, the cookie now includes the user id,
trust level and the cookie's generation date, and we encrypt/sign the cookie to
prevent tampering.
Internal ticket number: t54739.
2021-11-18 04:27:30 +08:00
|
|
|
RateLimiter.clear_all!
|
2017-12-11 14:21:00 +08:00
|
|
|
|
2022-05-05 09:53:54 +08:00
|
|
|
@orig_logger = Rails.logger
|
|
|
|
Rails.logger = @fake_logger = FakeLogger.new
|
2021-03-11 07:47:23 +08:00
|
|
|
|
|
|
|
# rate limiter tests depend on checks for retry-after
|
|
|
|
# they can be sensitive to clock skew during test runs
|
2023-01-09 19:18:21 +08:00
|
|
|
freeze_time DateTime.parse("2021-01-01 01:00")
|
2017-12-11 14:21:00 +08:00
|
|
|
end
|
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
after { Rails.logger = @orig_logger }
|
2017-12-11 14:21:00 +08:00
|
|
|
|
|
|
|
let :middleware do
|
2023-01-09 19:18:21 +08:00
|
|
|
app = lambda { |env| [200, {}, ["OK"]] }
|
2017-12-11 14:21:00 +08:00
|
|
|
|
|
|
|
Middleware::RequestTracker.new(app)
|
|
|
|
end
|
|
|
|
|
2019-11-18 13:05:58 +08:00
|
|
|
it "does nothing if configured to do nothing" do
|
|
|
|
global_setting :max_reqs_per_ip_mode, "none"
|
2018-01-22 10:18:30 +08:00
|
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
2017-12-11 14:21:00 +08:00
|
|
|
|
|
|
|
status, _ = middleware.call(env)
|
|
|
|
status, _ = middleware.call(env)
|
|
|
|
|
|
|
|
expect(status).to eq(200)
|
|
|
|
end
|
|
|
|
|
2018-01-08 05:39:17 +08:00
|
|
|
it "blocks private IPs if not skipped" do
|
2018-01-22 10:18:30 +08:00
|
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
2023-01-09 19:18:21 +08:00
|
|
|
global_setting :max_reqs_per_ip_mode, "warn+block"
|
2018-01-22 10:18:30 +08:00
|
|
|
global_setting :max_reqs_rate_limit_on_private, true
|
2018-01-08 05:39:17 +08:00
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
addresses = %w[
|
|
|
|
127.1.2.3
|
|
|
|
127.0.0.2
|
|
|
|
192.168.1.2
|
|
|
|
10.0.1.2
|
|
|
|
172.16.9.8
|
|
|
|
172.19.1.2
|
|
|
|
172.20.9.8
|
|
|
|
172.29.1.2
|
|
|
|
172.30.9.8
|
|
|
|
172.31.1.2
|
|
|
|
]
|
2021-03-22 11:56:32 +08:00
|
|
|
warn_count = 1
|
|
|
|
addresses.each do |addr|
|
|
|
|
env1 = env("REMOTE_ADDR" => addr)
|
2018-01-08 05:39:17 +08:00
|
|
|
|
2021-03-22 11:56:32 +08:00
|
|
|
status, _ = middleware.call(env1)
|
|
|
|
status, _ = middleware.call(env1)
|
2018-01-08 05:39:17 +08:00
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
expect(
|
|
|
|
@fake_logger.warnings.count { |w| w.include?("Global IP rate limit exceeded") },
|
|
|
|
).to eq(warn_count)
|
2021-03-22 11:56:32 +08:00
|
|
|
expect(status).to eq(429)
|
|
|
|
warn_count += 1
|
|
|
|
end
|
2018-01-08 05:39:17 +08:00
|
|
|
end
|
2021-08-13 23:00:23 +08:00
|
|
|
|
|
|
|
it "blocks if the ip isn't static skipped" do
|
|
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
2023-01-09 19:18:21 +08:00
|
|
|
global_setting :max_reqs_per_ip_mode, "block"
|
2021-08-13 23:00:23 +08:00
|
|
|
|
|
|
|
env1 = env("REMOTE_ADDR" => "1.1.1.1")
|
|
|
|
status, _ = middleware.call(env1)
|
|
|
|
status, _ = middleware.call(env1)
|
|
|
|
expect(status).to eq(429)
|
|
|
|
end
|
|
|
|
|
|
|
|
it "doesn't block if rate limiter is enabled but IP is on the static exception list" do
|
2023-01-09 19:18:21 +08:00
|
|
|
stub_const(
|
|
|
|
Middleware::RequestTracker,
|
|
|
|
"STATIC_IP_SKIPPER",
|
|
|
|
"177.33.14.73 191.209.88.192/30"&.split&.map { |ip| IPAddr.new(ip) },
|
|
|
|
) do
|
2021-08-13 23:00:23 +08:00
|
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
2023-01-09 19:18:21 +08:00
|
|
|
global_setting :max_reqs_per_ip_mode, "block"
|
2021-08-13 23:00:23 +08:00
|
|
|
|
|
|
|
env1 = env("REMOTE_ADDR" => "177.33.14.73")
|
|
|
|
env2 = env("REMOTE_ADDR" => "191.209.88.194")
|
|
|
|
|
|
|
|
status, _ = middleware.call(env1)
|
|
|
|
expect(status).to eq(200)
|
|
|
|
|
|
|
|
status, _ = middleware.call(env1)
|
|
|
|
expect(status).to eq(200)
|
|
|
|
|
|
|
|
status, _ = middleware.call(env2)
|
|
|
|
expect(status).to eq(200)
|
|
|
|
|
|
|
|
status, _ = middleware.call(env2)
|
|
|
|
expect(status).to eq(200)
|
|
|
|
end
|
|
|
|
end
|
2018-01-08 05:39:17 +08:00
|
|
|
|
2018-02-06 06:45:25 +08:00
|
|
|
describe "register_ip_skipper" do
|
|
|
|
before do
|
2023-01-09 19:18:21 +08:00
|
|
|
Middleware::RequestTracker.register_ip_skipper { |ip| ip == "1.1.1.2" }
|
2018-02-06 06:45:25 +08:00
|
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
2023-01-09 19:18:21 +08:00
|
|
|
global_setting :max_reqs_per_ip_mode, "block"
|
2018-02-06 06:45:25 +08:00
|
|
|
end
|
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
after { Middleware::RequestTracker.unregister_ip_skipper }
|
2018-02-06 07:38:15 +08:00
|
|
|
|
2018-02-06 06:45:25 +08:00
|
|
|
it "won't block if the ip is skipped" do
|
|
|
|
env1 = env("REMOTE_ADDR" => "1.1.1.2")
|
|
|
|
status, _ = middleware.call(env1)
|
|
|
|
status, _ = middleware.call(env1)
|
|
|
|
expect(status).to eq(200)
|
|
|
|
end
|
|
|
|
|
|
|
|
it "blocks if the ip isn't skipped" do
|
|
|
|
env1 = env("REMOTE_ADDR" => "1.1.1.1")
|
|
|
|
status, _ = middleware.call(env1)
|
|
|
|
status, _ = middleware.call(env1)
|
|
|
|
expect(status).to eq(429)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-01-08 05:39:17 +08:00
|
|
|
it "does nothing for private IPs if skipped" do
|
2018-01-22 10:18:30 +08:00
|
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
2023-01-09 19:18:21 +08:00
|
|
|
global_setting :max_reqs_per_ip_mode, "warn+block"
|
2018-01-22 10:18:30 +08:00
|
|
|
global_setting :max_reqs_rate_limit_on_private, false
|
2018-01-08 05:39:17 +08:00
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
addresses = %w[
|
|
|
|
127.1.2.3
|
|
|
|
127.0.3.1
|
|
|
|
192.168.1.2
|
|
|
|
10.0.1.2
|
|
|
|
172.16.9.8
|
|
|
|
172.19.1.2
|
|
|
|
172.20.9.8
|
|
|
|
172.29.1.2
|
|
|
|
172.30.9.8
|
|
|
|
172.31.1.2
|
|
|
|
]
|
2021-03-22 11:56:32 +08:00
|
|
|
addresses.each do |addr|
|
|
|
|
env1 = env("REMOTE_ADDR" => addr)
|
2018-01-08 05:39:17 +08:00
|
|
|
|
2021-03-22 11:56:32 +08:00
|
|
|
status, _ = middleware.call(env1)
|
|
|
|
status, _ = middleware.call(env1)
|
2018-01-08 05:39:17 +08:00
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
expect(
|
|
|
|
@fake_logger.warnings.count { |w| w.include?("Global IP rate limit exceeded") },
|
|
|
|
).to eq(0)
|
2021-03-22 11:56:32 +08:00
|
|
|
expect(status).to eq(200)
|
|
|
|
end
|
2018-01-08 05:39:17 +08:00
|
|
|
end
|
|
|
|
|
|
|
|
it "does warn if rate limiter is enabled via warn+block" do
|
2018-01-22 10:18:30 +08:00
|
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
2023-01-09 19:18:21 +08:00
|
|
|
global_setting :max_reqs_per_ip_mode, "warn+block"
|
2018-01-08 05:39:17 +08:00
|
|
|
|
|
|
|
status, _ = middleware.call(env)
|
2021-01-19 17:35:46 +08:00
|
|
|
status, headers = middleware.call(env)
|
2018-01-08 05:39:17 +08:00
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
expect(@fake_logger.warnings.count { |w| w.include?("Global IP rate limit exceeded") }).to eq(
|
|
|
|
1,
|
|
|
|
)
|
2018-01-08 05:39:17 +08:00
|
|
|
expect(status).to eq(429)
|
2021-03-24 03:32:36 +08:00
|
|
|
expect(headers["Retry-After"]).to eq("10")
|
2018-01-08 05:39:17 +08:00
|
|
|
end
|
|
|
|
|
2017-12-11 14:21:00 +08:00
|
|
|
it "does warn if rate limiter is enabled" do
|
2018-01-22 10:18:30 +08:00
|
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
2023-01-09 19:18:21 +08:00
|
|
|
global_setting :max_reqs_per_ip_mode, "warn"
|
2017-12-11 14:21:00 +08:00
|
|
|
|
|
|
|
status, _ = middleware.call(env)
|
|
|
|
status, _ = middleware.call(env)
|
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
expect(@fake_logger.warnings.count { |w| w.include?("Global IP rate limit exceeded") }).to eq(
|
|
|
|
1,
|
|
|
|
)
|
2017-12-11 14:21:00 +08:00
|
|
|
expect(status).to eq(200)
|
|
|
|
end
|
|
|
|
|
2018-03-06 12:20:39 +08:00
|
|
|
it "allows assets for more requests" do
|
|
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
2023-01-09 19:18:21 +08:00
|
|
|
global_setting :max_reqs_per_ip_mode, "block"
|
2018-03-06 12:20:39 +08:00
|
|
|
global_setting :max_asset_reqs_per_ip_per_10_seconds, 3
|
|
|
|
|
|
|
|
env1 = env("REMOTE_ADDR" => "1.1.1.1", "DISCOURSE_IS_ASSET_PATH" => 1)
|
|
|
|
|
|
|
|
status, _ = middleware.call(env1)
|
|
|
|
expect(status).to eq(200)
|
|
|
|
status, _ = middleware.call(env1)
|
|
|
|
expect(status).to eq(200)
|
|
|
|
status, _ = middleware.call(env1)
|
|
|
|
expect(status).to eq(200)
|
2021-01-19 17:35:46 +08:00
|
|
|
status, headers = middleware.call(env1)
|
2018-03-06 12:20:39 +08:00
|
|
|
expect(status).to eq(429)
|
2021-03-24 03:32:36 +08:00
|
|
|
expect(headers["Retry-After"]).to eq("10")
|
2018-03-06 12:20:39 +08:00
|
|
|
|
|
|
|
env2 = env("REMOTE_ADDR" => "1.1.1.1")
|
|
|
|
|
2021-01-19 17:35:46 +08:00
|
|
|
status, headers = middleware.call(env2)
|
2018-03-06 12:20:39 +08:00
|
|
|
expect(status).to eq(429)
|
2021-03-24 03:32:36 +08:00
|
|
|
expect(headers["Retry-After"]).to eq("10")
|
2018-03-06 12:20:39 +08:00
|
|
|
end
|
|
|
|
|
2017-12-11 14:21:00 +08:00
|
|
|
it "does block if rate limiter is enabled" do
|
2018-01-22 10:18:30 +08:00
|
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
2023-01-09 19:18:21 +08:00
|
|
|
global_setting :max_reqs_per_ip_mode, "block"
|
2017-12-11 14:21:00 +08:00
|
|
|
|
|
|
|
env1 = env("REMOTE_ADDR" => "1.1.1.1")
|
|
|
|
env2 = env("REMOTE_ADDR" => "1.1.1.2")
|
|
|
|
|
|
|
|
status, _ = middleware.call(env1)
|
2018-03-06 12:20:39 +08:00
|
|
|
expect(status).to eq(200)
|
2017-12-11 14:21:00 +08:00
|
|
|
|
2021-01-19 17:35:46 +08:00
|
|
|
status, headers = middleware.call(env1)
|
2017-12-11 14:21:00 +08:00
|
|
|
expect(status).to eq(429)
|
2021-03-24 03:32:36 +08:00
|
|
|
expect(headers["Retry-After"]).to eq("10")
|
2017-12-11 14:21:00 +08:00
|
|
|
|
|
|
|
status, _ = middleware.call(env2)
|
|
|
|
expect(status).to eq(200)
|
|
|
|
end
|
FEATURE: Apply rate limits per user instead of IP for trusted users (#14706)
Currently, Discourse rate limits all incoming requests by the IP address they
originate from regardless of the user making the request. This can be
frustrating if there are multiple users using Discourse simultaneously while
sharing the same IP address (e.g. employees in an office).
This commit implements a new feature to make Discourse apply rate limits by
user id rather than IP address for users at or higher than the configured trust
level (1 is the default).
For example, let's say a Discourse instance is configured to allow 200 requests
per minute per IP address, and we have 10 users at trust level 4 using
Discourse simultaneously from the same IP address. Before this feature, the 10
users could only make a total of 200 requests per minute before they got rate
limited. But with the new feature, each user is allowed to make 200 requests
per minute because the rate limits are applied on user id rather than the IP
address.
The minimum trust level for applying user-id-based rate limits can be
configured by the `skip_per_ip_rate_limit_trust_level` global setting. The
default is 1, but it can be changed by either adding the
`DISCOURSE_SKIP_PER_IP_RATE_LIMIT_TRUST_LEVEL` environment variable with the
desired value to your `app.yml`, or changing the setting's value in the
`discourse.conf` file.
Requests made with API keys are still rate limited by IP address and the
relevant global settings that control API keys rate limits.
Before this commit, Discourse's auth cookie (`_t`) was simply a 32 characters
string that Discourse used to lookup the current user from the database and the
cookie contained no additional information about the user. However, we had to
change the cookie content in this commit so we could identify the user from the
cookie without making a database query before the rate limits logic and avoid
introducing a bottleneck on busy sites.
Besides the 32 characters auth token, the cookie now includes the user id,
trust level and the cookie's generation date, and we encrypt/sign the cookie to
prevent tampering.
Internal ticket number: t54739.
2021-11-18 04:27:30 +08:00
|
|
|
|
|
|
|
describe "diagnostic information" do
|
|
|
|
it "is included when the requests-per-10-seconds limit is reached" do
|
|
|
|
global_setting :max_reqs_per_ip_per_10_seconds, 1
|
|
|
|
called = 0
|
2023-01-09 19:18:21 +08:00
|
|
|
app =
|
|
|
|
lambda do |_|
|
|
|
|
called += 1
|
|
|
|
[200, {}, ["OK"]]
|
|
|
|
end
|
FEATURE: Apply rate limits per user instead of IP for trusted users (#14706)
Currently, Discourse rate limits all incoming requests by the IP address they
originate from regardless of the user making the request. This can be
frustrating if there are multiple users using Discourse simultaneously while
sharing the same IP address (e.g. employees in an office).
This commit implements a new feature to make Discourse apply rate limits by
user id rather than IP address for users at or higher than the configured trust
level (1 is the default).
For example, let's say a Discourse instance is configured to allow 200 requests
per minute per IP address, and we have 10 users at trust level 4 using
Discourse simultaneously from the same IP address. Before this feature, the 10
users could only make a total of 200 requests per minute before they got rate
limited. But with the new feature, each user is allowed to make 200 requests
per minute because the rate limits are applied on user id rather than the IP
address.
The minimum trust level for applying user-id-based rate limits can be
configured by the `skip_per_ip_rate_limit_trust_level` global setting. The
default is 1, but it can be changed by either adding the
`DISCOURSE_SKIP_PER_IP_RATE_LIMIT_TRUST_LEVEL` environment variable with the
desired value to your `app.yml`, or changing the setting's value in the
`discourse.conf` file.
Requests made with API keys are still rate limited by IP address and the
relevant global settings that control API keys rate limits.
Before this commit, Discourse's auth cookie (`_t`) was simply a 32 characters
string that Discourse used to lookup the current user from the database and the
cookie contained no additional information about the user. However, we had to
change the cookie content in this commit so we could identify the user from the
cookie without making a database query before the rate limits logic and avoid
introducing a bottleneck on busy sites.
Besides the 32 characters auth token, the cookie now includes the user id,
trust level and the cookie's generation date, and we encrypt/sign the cookie to
prevent tampering.
Internal ticket number: t54739.
2021-11-18 04:27:30 +08:00
|
|
|
env = env("REMOTE_ADDR" => "1.1.1.1")
|
|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
|
|
status, = middleware.call(env)
|
|
|
|
expect(status).to eq(200)
|
|
|
|
expect(called).to eq(1)
|
|
|
|
|
|
|
|
env = env("REMOTE_ADDR" => "1.1.1.1")
|
|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
|
|
status, headers, response = middleware.call(env)
|
|
|
|
expect(status).to eq(429)
|
|
|
|
expect(called).to eq(1)
|
|
|
|
expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("ip_10_secs_limit")
|
|
|
|
expect(response.first).to include("Error code: ip_10_secs_limit.")
|
|
|
|
end
|
|
|
|
|
|
|
|
it "is included when the requests-per-minute limit is reached" do
|
|
|
|
global_setting :max_reqs_per_ip_per_minute, 1
|
|
|
|
called = 0
|
2023-01-09 19:18:21 +08:00
|
|
|
app =
|
|
|
|
lambda do |_|
|
|
|
|
called += 1
|
|
|
|
[200, {}, ["OK"]]
|
|
|
|
end
|
FEATURE: Apply rate limits per user instead of IP for trusted users (#14706)
Currently, Discourse rate limits all incoming requests by the IP address they
originate from regardless of the user making the request. This can be
frustrating if there are multiple users using Discourse simultaneously while
sharing the same IP address (e.g. employees in an office).
This commit implements a new feature to make Discourse apply rate limits by
user id rather than IP address for users at or higher than the configured trust
level (1 is the default).
For example, let's say a Discourse instance is configured to allow 200 requests
per minute per IP address, and we have 10 users at trust level 4 using
Discourse simultaneously from the same IP address. Before this feature, the 10
users could only make a total of 200 requests per minute before they got rate
limited. But with the new feature, each user is allowed to make 200 requests
per minute because the rate limits are applied on user id rather than the IP
address.
The minimum trust level for applying user-id-based rate limits can be
configured by the `skip_per_ip_rate_limit_trust_level` global setting. The
default is 1, but it can be changed by either adding the
`DISCOURSE_SKIP_PER_IP_RATE_LIMIT_TRUST_LEVEL` environment variable with the
desired value to your `app.yml`, or changing the setting's value in the
`discourse.conf` file.
Requests made with API keys are still rate limited by IP address and the
relevant global settings that control API keys rate limits.
Before this commit, Discourse's auth cookie (`_t`) was simply a 32 characters
string that Discourse used to lookup the current user from the database and the
cookie contained no additional information about the user. However, we had to
change the cookie content in this commit so we could identify the user from the
cookie without making a database query before the rate limits logic and avoid
introducing a bottleneck on busy sites.
Besides the 32 characters auth token, the cookie now includes the user id,
trust level and the cookie's generation date, and we encrypt/sign the cookie to
prevent tampering.
Internal ticket number: t54739.
2021-11-18 04:27:30 +08:00
|
|
|
env = env("REMOTE_ADDR" => "1.1.1.1")
|
|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
|
|
status, = middleware.call(env)
|
|
|
|
expect(status).to eq(200)
|
|
|
|
expect(called).to eq(1)
|
|
|
|
|
|
|
|
env = env("REMOTE_ADDR" => "1.1.1.1")
|
|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
|
|
status, headers, response = middleware.call(env)
|
|
|
|
expect(status).to eq(429)
|
|
|
|
expect(called).to eq(1)
|
|
|
|
expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("ip_60_secs_limit")
|
|
|
|
expect(response.first).to include("Error code: ip_60_secs_limit.")
|
|
|
|
end
|
|
|
|
|
|
|
|
it "is included when the assets-requests-per-10-seconds limit is reached" do
|
|
|
|
global_setting :max_asset_reqs_per_ip_per_10_seconds, 1
|
|
|
|
called = 0
|
2023-01-09 19:18:21 +08:00
|
|
|
app =
|
|
|
|
lambda do |env|
|
|
|
|
called += 1
|
|
|
|
env["DISCOURSE_IS_ASSET_PATH"] = true
|
|
|
|
[200, {}, ["OK"]]
|
|
|
|
end
|
FEATURE: Apply rate limits per user instead of IP for trusted users (#14706)
Currently, Discourse rate limits all incoming requests by the IP address they
originate from regardless of the user making the request. This can be
frustrating if there are multiple users using Discourse simultaneously while
sharing the same IP address (e.g. employees in an office).
This commit implements a new feature to make Discourse apply rate limits by
user id rather than IP address for users at or higher than the configured trust
level (1 is the default).
For example, let's say a Discourse instance is configured to allow 200 requests
per minute per IP address, and we have 10 users at trust level 4 using
Discourse simultaneously from the same IP address. Before this feature, the 10
users could only make a total of 200 requests per minute before they got rate
limited. But with the new feature, each user is allowed to make 200 requests
per minute because the rate limits are applied on user id rather than the IP
address.
The minimum trust level for applying user-id-based rate limits can be
configured by the `skip_per_ip_rate_limit_trust_level` global setting. The
default is 1, but it can be changed by either adding the
`DISCOURSE_SKIP_PER_IP_RATE_LIMIT_TRUST_LEVEL` environment variable with the
desired value to your `app.yml`, or changing the setting's value in the
`discourse.conf` file.
Requests made with API keys are still rate limited by IP address and the
relevant global settings that control API keys rate limits.
Before this commit, Discourse's auth cookie (`_t`) was simply a 32 characters
string that Discourse used to lookup the current user from the database and the
cookie contained no additional information about the user. However, we had to
change the cookie content in this commit so we could identify the user from the
cookie without making a database query before the rate limits logic and avoid
introducing a bottleneck on busy sites.
Besides the 32 characters auth token, the cookie now includes the user id,
trust level and the cookie's generation date, and we encrypt/sign the cookie to
prevent tampering.
Internal ticket number: t54739.
2021-11-18 04:27:30 +08:00
|
|
|
env = env("REMOTE_ADDR" => "1.1.1.1")
|
|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
|
|
status, = middleware.call(env)
|
|
|
|
expect(status).to eq(200)
|
|
|
|
expect(called).to eq(1)
|
|
|
|
|
|
|
|
env = env("REMOTE_ADDR" => "1.1.1.1")
|
|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
|
|
status, headers, response = middleware.call(env)
|
|
|
|
expect(status).to eq(429)
|
|
|
|
expect(called).to eq(1)
|
|
|
|
expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("ip_assets_10_secs_limit")
|
|
|
|
expect(response.first).to include("Error code: ip_assets_10_secs_limit.")
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
it "users with high enough trust level are not rate limited per ip" do
|
|
|
|
global_setting :max_reqs_per_ip_per_minute, 1
|
|
|
|
global_setting :skip_per_ip_rate_limit_trust_level, 3
|
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
envs =
|
|
|
|
3.times.map do |n|
|
|
|
|
user = Fabricate(:user, trust_level: 3)
|
|
|
|
token = UserAuthToken.generate!(user_id: user.id)
|
|
|
|
cookie =
|
|
|
|
create_auth_cookie(
|
|
|
|
token: token.unhashed_auth_token,
|
|
|
|
user_id: user.id,
|
|
|
|
trust_level: user.trust_level,
|
|
|
|
issued_at: 5.minutes.ago,
|
|
|
|
)
|
|
|
|
env("HTTP_COOKIE" => "_t=#{cookie}", "REMOTE_ADDR" => "1.1.1.1")
|
|
|
|
end
|
FEATURE: Apply rate limits per user instead of IP for trusted users (#14706)
Currently, Discourse rate limits all incoming requests by the IP address they
originate from regardless of the user making the request. This can be
frustrating if there are multiple users using Discourse simultaneously while
sharing the same IP address (e.g. employees in an office).
This commit implements a new feature to make Discourse apply rate limits by
user id rather than IP address for users at or higher than the configured trust
level (1 is the default).
For example, let's say a Discourse instance is configured to allow 200 requests
per minute per IP address, and we have 10 users at trust level 4 using
Discourse simultaneously from the same IP address. Before this feature, the 10
users could only make a total of 200 requests per minute before they got rate
limited. But with the new feature, each user is allowed to make 200 requests
per minute because the rate limits are applied on user id rather than the IP
address.
The minimum trust level for applying user-id-based rate limits can be
configured by the `skip_per_ip_rate_limit_trust_level` global setting. The
default is 1, but it can be changed by either adding the
`DISCOURSE_SKIP_PER_IP_RATE_LIMIT_TRUST_LEVEL` environment variable with the
desired value to your `app.yml`, or changing the setting's value in the
`discourse.conf` file.
Requests made with API keys are still rate limited by IP address and the
relevant global settings that control API keys rate limits.
Before this commit, Discourse's auth cookie (`_t`) was simply a 32 characters
string that Discourse used to lookup the current user from the database and the
cookie contained no additional information about the user. However, we had to
change the cookie content in this commit so we could identify the user from the
cookie without making a database query before the rate limits logic and avoid
introducing a bottleneck on busy sites.
Besides the 32 characters auth token, the cookie now includes the user id,
trust level and the cookie's generation date, and we encrypt/sign the cookie to
prevent tampering.
Internal ticket number: t54739.
2021-11-18 04:27:30 +08:00
|
|
|
|
|
|
|
called = 0
|
2023-01-09 19:18:21 +08:00
|
|
|
app =
|
|
|
|
lambda do |env|
|
|
|
|
called += 1
|
|
|
|
[200, {}, ["OK"]]
|
|
|
|
end
|
FEATURE: Apply rate limits per user instead of IP for trusted users (#14706)
Currently, Discourse rate limits all incoming requests by the IP address they
originate from regardless of the user making the request. This can be
frustrating if there are multiple users using Discourse simultaneously while
sharing the same IP address (e.g. employees in an office).
This commit implements a new feature to make Discourse apply rate limits by
user id rather than IP address for users at or higher than the configured trust
level (1 is the default).
For example, let's say a Discourse instance is configured to allow 200 requests
per minute per IP address, and we have 10 users at trust level 4 using
Discourse simultaneously from the same IP address. Before this feature, the 10
users could only make a total of 200 requests per minute before they got rate
limited. But with the new feature, each user is allowed to make 200 requests
per minute because the rate limits are applied on user id rather than the IP
address.
The minimum trust level for applying user-id-based rate limits can be
configured by the `skip_per_ip_rate_limit_trust_level` global setting. The
default is 1, but it can be changed by either adding the
`DISCOURSE_SKIP_PER_IP_RATE_LIMIT_TRUST_LEVEL` environment variable with the
desired value to your `app.yml`, or changing the setting's value in the
`discourse.conf` file.
Requests made with API keys are still rate limited by IP address and the
relevant global settings that control API keys rate limits.
Before this commit, Discourse's auth cookie (`_t`) was simply a 32 characters
string that Discourse used to lookup the current user from the database and the
cookie contained no additional information about the user. However, we had to
change the cookie content in this commit so we could identify the user from the
cookie without making a database query before the rate limits logic and avoid
introducing a bottleneck on busy sites.
Besides the 32 characters auth token, the cookie now includes the user id,
trust level and the cookie's generation date, and we encrypt/sign the cookie to
prevent tampering.
Internal ticket number: t54739.
2021-11-18 04:27:30 +08:00
|
|
|
envs.each do |env|
|
|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
|
|
status, = middleware.call(env)
|
|
|
|
expect(status).to eq(200)
|
|
|
|
end
|
|
|
|
expect(called).to eq(3)
|
|
|
|
|
|
|
|
envs.each do |env|
|
|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
|
|
status, headers, response = middleware.call(env)
|
|
|
|
expect(status).to eq(429)
|
|
|
|
expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("id_60_secs_limit")
|
|
|
|
expect(response.first).to include("Error code: id_60_secs_limit.")
|
|
|
|
end
|
|
|
|
expect(called).to eq(3)
|
|
|
|
end
|
|
|
|
|
|
|
|
it "falls back to IP rate limiting if the cookie is too old" do
|
|
|
|
unfreeze_time
|
|
|
|
global_setting :max_reqs_per_ip_per_minute, 1
|
|
|
|
global_setting :skip_per_ip_rate_limit_trust_level, 3
|
|
|
|
user = Fabricate(:user, trust_level: 3)
|
|
|
|
token = UserAuthToken.generate!(user_id: user.id)
|
2023-01-09 19:18:21 +08:00
|
|
|
cookie =
|
|
|
|
create_auth_cookie(
|
|
|
|
token: token.unhashed_auth_token,
|
|
|
|
user_id: user.id,
|
|
|
|
trust_level: user.trust_level,
|
|
|
|
issued_at: 5.minutes.ago,
|
|
|
|
)
|
FEATURE: Apply rate limits per user instead of IP for trusted users (#14706)
Currently, Discourse rate limits all incoming requests by the IP address they
originate from regardless of the user making the request. This can be
frustrating if there are multiple users using Discourse simultaneously while
sharing the same IP address (e.g. employees in an office).
This commit implements a new feature to make Discourse apply rate limits by
user id rather than IP address for users at or higher than the configured trust
level (1 is the default).
For example, let's say a Discourse instance is configured to allow 200 requests
per minute per IP address, and we have 10 users at trust level 4 using
Discourse simultaneously from the same IP address. Before this feature, the 10
users could only make a total of 200 requests per minute before they got rate
limited. But with the new feature, each user is allowed to make 200 requests
per minute because the rate limits are applied on user id rather than the IP
address.
The minimum trust level for applying user-id-based rate limits can be
configured by the `skip_per_ip_rate_limit_trust_level` global setting. The
default is 1, but it can be changed by either adding the
`DISCOURSE_SKIP_PER_IP_RATE_LIMIT_TRUST_LEVEL` environment variable with the
desired value to your `app.yml`, or changing the setting's value in the
`discourse.conf` file.
Requests made with API keys are still rate limited by IP address and the
relevant global settings that control API keys rate limits.
Before this commit, Discourse's auth cookie (`_t`) was simply a 32 characters
string that Discourse used to lookup the current user from the database and the
cookie contained no additional information about the user. However, we had to
change the cookie content in this commit so we could identify the user from the
cookie without making a database query before the rate limits logic and avoid
introducing a bottleneck on busy sites.
Besides the 32 characters auth token, the cookie now includes the user id,
trust level and the cookie's generation date, and we encrypt/sign the cookie to
prevent tampering.
Internal ticket number: t54739.
2021-11-18 04:27:30 +08:00
|
|
|
env = env("HTTP_COOKIE" => "_t=#{cookie}", "REMOTE_ADDR" => "1.1.1.1")
|
|
|
|
|
|
|
|
called = 0
|
2023-01-09 19:18:21 +08:00
|
|
|
app =
|
|
|
|
lambda do |_|
|
|
|
|
called += 1
|
|
|
|
[200, {}, ["OK"]]
|
|
|
|
end
|
FEATURE: Apply rate limits per user instead of IP for trusted users (#14706)
Currently, Discourse rate limits all incoming requests by the IP address they
originate from regardless of the user making the request. This can be
frustrating if there are multiple users using Discourse simultaneously while
sharing the same IP address (e.g. employees in an office).
This commit implements a new feature to make Discourse apply rate limits by
user id rather than IP address for users at or higher than the configured trust
level (1 is the default).
For example, let's say a Discourse instance is configured to allow 200 requests
per minute per IP address, and we have 10 users at trust level 4 using
Discourse simultaneously from the same IP address. Before this feature, the 10
users could only make a total of 200 requests per minute before they got rate
limited. But with the new feature, each user is allowed to make 200 requests
per minute because the rate limits are applied on user id rather than the IP
address.
The minimum trust level for applying user-id-based rate limits can be
configured by the `skip_per_ip_rate_limit_trust_level` global setting. The
default is 1, but it can be changed by either adding the
`DISCOURSE_SKIP_PER_IP_RATE_LIMIT_TRUST_LEVEL` environment variable with the
desired value to your `app.yml`, or changing the setting's value in the
`discourse.conf` file.
Requests made with API keys are still rate limited by IP address and the
relevant global settings that control API keys rate limits.
Before this commit, Discourse's auth cookie (`_t`) was simply a 32 characters
string that Discourse used to lookup the current user from the database and the
cookie contained no additional information about the user. However, we had to
change the cookie content in this commit so we could identify the user from the
cookie without making a database query before the rate limits logic and avoid
introducing a bottleneck on busy sites.
Besides the 32 characters auth token, the cookie now includes the user id,
trust level and the cookie's generation date, and we encrypt/sign the cookie to
prevent tampering.
Internal ticket number: t54739.
2021-11-18 04:27:30 +08:00
|
|
|
freeze_time(12.minutes.from_now) do
|
|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
|
|
status, = middleware.call(env)
|
|
|
|
expect(status).to eq(200)
|
|
|
|
|
|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
|
|
status, headers, response = middleware.call(env)
|
|
|
|
expect(status).to eq(429)
|
|
|
|
expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("ip_60_secs_limit")
|
|
|
|
expect(response.first).to include("Error code: ip_60_secs_limit.")
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
it "falls back to IP rate limiting if the cookie is tampered with" do
|
|
|
|
unfreeze_time
|
|
|
|
global_setting :max_reqs_per_ip_per_minute, 1
|
|
|
|
global_setting :skip_per_ip_rate_limit_trust_level, 3
|
|
|
|
user = Fabricate(:user, trust_level: 3)
|
|
|
|
token = UserAuthToken.generate!(user_id: user.id)
|
2023-01-09 19:18:21 +08:00
|
|
|
cookie =
|
|
|
|
create_auth_cookie(
|
|
|
|
token: token.unhashed_auth_token,
|
|
|
|
user_id: user.id,
|
|
|
|
trust_level: user.trust_level,
|
|
|
|
issued_at: Time.zone.now,
|
|
|
|
)
|
FEATURE: Apply rate limits per user instead of IP for trusted users (#14706)
Currently, Discourse rate limits all incoming requests by the IP address they
originate from regardless of the user making the request. This can be
frustrating if there are multiple users using Discourse simultaneously while
sharing the same IP address (e.g. employees in an office).
This commit implements a new feature to make Discourse apply rate limits by
user id rather than IP address for users at or higher than the configured trust
level (1 is the default).
For example, let's say a Discourse instance is configured to allow 200 requests
per minute per IP address, and we have 10 users at trust level 4 using
Discourse simultaneously from the same IP address. Before this feature, the 10
users could only make a total of 200 requests per minute before they got rate
limited. But with the new feature, each user is allowed to make 200 requests
per minute because the rate limits are applied on user id rather than the IP
address.
The minimum trust level for applying user-id-based rate limits can be
configured by the `skip_per_ip_rate_limit_trust_level` global setting. The
default is 1, but it can be changed by either adding the
`DISCOURSE_SKIP_PER_IP_RATE_LIMIT_TRUST_LEVEL` environment variable with the
desired value to your `app.yml`, or changing the setting's value in the
`discourse.conf` file.
Requests made with API keys are still rate limited by IP address and the
relevant global settings that control API keys rate limits.
Before this commit, Discourse's auth cookie (`_t`) was simply a 32 characters
string that Discourse used to lookup the current user from the database and the
cookie contained no additional information about the user. However, we had to
change the cookie content in this commit so we could identify the user from the
cookie without making a database query before the rate limits logic and avoid
introducing a bottleneck on busy sites.
Besides the 32 characters auth token, the cookie now includes the user id,
trust level and the cookie's generation date, and we encrypt/sign the cookie to
prevent tampering.
Internal ticket number: t54739.
2021-11-18 04:27:30 +08:00
|
|
|
cookie = swap_2_different_characters(cookie)
|
|
|
|
env = env("HTTP_COOKIE" => "_t=#{cookie}", "REMOTE_ADDR" => "1.1.1.1")
|
|
|
|
|
|
|
|
called = 0
|
2023-01-09 19:18:21 +08:00
|
|
|
app =
|
|
|
|
lambda do |_|
|
|
|
|
called += 1
|
|
|
|
[200, {}, ["OK"]]
|
|
|
|
end
|
FEATURE: Apply rate limits per user instead of IP for trusted users (#14706)
Currently, Discourse rate limits all incoming requests by the IP address they
originate from regardless of the user making the request. This can be
frustrating if there are multiple users using Discourse simultaneously while
sharing the same IP address (e.g. employees in an office).
This commit implements a new feature to make Discourse apply rate limits by
user id rather than IP address for users at or higher than the configured trust
level (1 is the default).
For example, let's say a Discourse instance is configured to allow 200 requests
per minute per IP address, and we have 10 users at trust level 4 using
Discourse simultaneously from the same IP address. Before this feature, the 10
users could only make a total of 200 requests per minute before they got rate
limited. But with the new feature, each user is allowed to make 200 requests
per minute because the rate limits are applied on user id rather than the IP
address.
The minimum trust level for applying user-id-based rate limits can be
configured by the `skip_per_ip_rate_limit_trust_level` global setting. The
default is 1, but it can be changed by either adding the
`DISCOURSE_SKIP_PER_IP_RATE_LIMIT_TRUST_LEVEL` environment variable with the
desired value to your `app.yml`, or changing the setting's value in the
`discourse.conf` file.
Requests made with API keys are still rate limited by IP address and the
relevant global settings that control API keys rate limits.
Before this commit, Discourse's auth cookie (`_t`) was simply a 32 characters
string that Discourse used to lookup the current user from the database and the
cookie contained no additional information about the user. However, we had to
change the cookie content in this commit so we could identify the user from the
cookie without making a database query before the rate limits logic and avoid
introducing a bottleneck on busy sites.
Besides the 32 characters auth token, the cookie now includes the user id,
trust level and the cookie's generation date, and we encrypt/sign the cookie to
prevent tampering.
Internal ticket number: t54739.
2021-11-18 04:27:30 +08:00
|
|
|
|
|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
|
|
status, = middleware.call(env)
|
|
|
|
expect(status).to eq(200)
|
|
|
|
|
|
|
|
middleware = Middleware::RequestTracker.new(app)
|
|
|
|
status, headers, response = middleware.call(env)
|
|
|
|
expect(status).to eq(429)
|
|
|
|
expect(headers["Discourse-Rate-Limit-Error-Code"]).to eq("ip_60_secs_limit")
|
|
|
|
expect(response.first).to include("Error code: ip_60_secs_limit.")
|
|
|
|
end
|
2017-12-11 14:21:00 +08:00
|
|
|
end
|
2017-10-18 09:10:12 +08:00
|
|
|
|
2022-07-28 00:14:14 +08:00
|
|
|
describe "callbacks" do
|
2017-10-18 09:10:12 +08:00
|
|
|
def app(result, sql_calls: 0, redis_calls: 0)
|
|
|
|
lambda do |env|
|
2023-01-09 19:18:21 +08:00
|
|
|
sql_calls.times { User.where(id: -100).pluck(:id) }
|
|
|
|
redis_calls.times { Discourse.redis.get("x") }
|
2017-10-18 09:10:12 +08:00
|
|
|
result
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2022-05-05 09:53:54 +08:00
|
|
|
let(:logger) do
|
2023-01-09 19:18:21 +08:00
|
|
|
->(env, data) {
|
2017-10-18 09:10:12 +08:00
|
|
|
@env = env
|
|
|
|
@data = data
|
2023-01-09 19:18:21 +08:00
|
|
|
}
|
2017-10-18 09:10:12 +08:00
|
|
|
end
|
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
before { Middleware::RequestTracker.register_detailed_request_logger(logger) }
|
2017-10-18 09:10:12 +08:00
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
after { Middleware::RequestTracker.unregister_detailed_request_logger(logger) }
|
2017-10-18 09:10:12 +08:00
|
|
|
|
2019-09-02 16:45:35 +08:00
|
|
|
it "can report data from anon cache" do
|
|
|
|
cache = Middleware::AnonymousCache.new(app([200, {}, ["i am a thing"]]))
|
|
|
|
tracker = Middleware::RequestTracker.new(cache)
|
|
|
|
|
|
|
|
uri = "/path?#{SecureRandom.hex}"
|
2019-09-03 08:51:49 +08:00
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
request_params = { "a" => "b", "action" => "bob", "controller" => "jane" }
|
2019-09-03 08:51:49 +08:00
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
tracker.call(
|
|
|
|
env(
|
|
|
|
"REQUEST_URI" => uri,
|
|
|
|
"ANON_CACHE_DURATION" => 60,
|
|
|
|
"action_dispatch.request.parameters" => request_params,
|
|
|
|
),
|
|
|
|
)
|
2019-09-04 15:18:32 +08:00
|
|
|
expect(@data[:cache]).to eq("skip")
|
2019-09-02 16:45:35 +08:00
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
tracker.call(
|
|
|
|
env(
|
|
|
|
"REQUEST_URI" => uri,
|
|
|
|
"ANON_CACHE_DURATION" => 60,
|
|
|
|
"action_dispatch.request.parameters" => request_params,
|
|
|
|
),
|
|
|
|
)
|
2019-09-02 16:45:35 +08:00
|
|
|
expect(@data[:cache]).to eq("store")
|
|
|
|
|
|
|
|
tracker.call(env("REQUEST_URI" => uri, "ANON_CACHE_DURATION" => 60))
|
|
|
|
expect(@data[:cache]).to eq("true")
|
2019-09-03 08:51:49 +08:00
|
|
|
|
2020-07-27 08:23:54 +08:00
|
|
|
# not allowlisted
|
2019-09-03 08:51:49 +08:00
|
|
|
request_params.delete("a")
|
|
|
|
|
|
|
|
expect(@env["action_dispatch.request.parameters"]).to eq(request_params)
|
2019-09-02 16:45:35 +08:00
|
|
|
end
|
|
|
|
|
2017-10-18 09:10:12 +08:00
|
|
|
it "can correctly log detailed data" do
|
2019-06-05 14:08:11 +08:00
|
|
|
global_setting :enable_performance_http_headers, true
|
|
|
|
|
2018-03-27 14:57:19 +08:00
|
|
|
# ensure pg is warmed up with the select 1 query
|
|
|
|
User.where(id: -100).pluck(:id)
|
|
|
|
|
2018-04-17 16:05:51 +08:00
|
|
|
freeze_time
|
|
|
|
start = Time.now.to_f
|
|
|
|
|
|
|
|
freeze_time 1.minute.from_now
|
|
|
|
|
2017-10-18 09:10:12 +08:00
|
|
|
tracker = Middleware::RequestTracker.new(app([200, {}, []], sql_calls: 2, redis_calls: 2))
|
2019-06-05 14:08:11 +08:00
|
|
|
_, headers, _ = tracker.call(env("HTTP_X_REQUEST_START" => "t=#{start}"))
|
2018-04-17 16:05:51 +08:00
|
|
|
|
|
|
|
expect(@data[:queue_seconds]).to eq(60)
|
2017-10-18 09:10:12 +08:00
|
|
|
|
|
|
|
timing = @data[:timing]
|
|
|
|
expect(timing[:total_duration]).to be > 0
|
|
|
|
|
|
|
|
expect(timing[:sql][:duration]).to be > 0
|
|
|
|
expect(timing[:sql][:calls]).to eq 2
|
|
|
|
|
|
|
|
expect(timing[:redis][:duration]).to be > 0
|
|
|
|
expect(timing[:redis][:calls]).to eq 2
|
2019-06-05 14:08:11 +08:00
|
|
|
|
|
|
|
expect(headers["X-Queue-Time"]).to eq("60.000000")
|
|
|
|
|
|
|
|
expect(headers["X-Redis-Calls"]).to eq("2")
|
|
|
|
expect(headers["X-Redis-Time"].to_f).to be > 0
|
|
|
|
|
|
|
|
expect(headers["X-Sql-Calls"]).to eq("2")
|
|
|
|
expect(headers["X-Sql-Time"].to_f).to be > 0
|
|
|
|
|
|
|
|
expect(headers["X-Runtime"].to_f).to be > 0
|
2017-10-18 09:10:12 +08:00
|
|
|
end
|
2023-01-06 19:26:18 +08:00
|
|
|
|
|
|
|
it "can correctly log messagebus request types" do
|
|
|
|
tracker = Middleware::RequestTracker.new(app([200, {}, []]))
|
|
|
|
|
|
|
|
tracker.call(env(path: "/message-bus/abcde/poll"))
|
|
|
|
expect(@data[:is_background]).to eq(true)
|
|
|
|
expect(@data[:background_type]).to eq("message-bus")
|
|
|
|
|
|
|
|
tracker.call(env(path: "/message-bus/abcde/poll?dlp=t"))
|
|
|
|
expect(@data[:is_background]).to eq(true)
|
|
|
|
expect(@data[:background_type]).to eq("message-bus-dlp")
|
|
|
|
|
2023-01-09 19:18:21 +08:00
|
|
|
tracker.call(env("HTTP_DONT_CHUNK" => "True", :path => "/message-bus/abcde/poll"))
|
2023-01-06 19:26:18 +08:00
|
|
|
expect(@data[:is_background]).to eq(true)
|
|
|
|
expect(@data[:background_type]).to eq("message-bus-dontchunk")
|
|
|
|
end
|
2015-02-05 13:08:52 +08:00
|
|
|
end
|
|
|
|
end
|