2019-04-30 08:27:42 +08:00
|
|
|
|
# frozen_string_literal: true
|
|
|
|
|
|
|
2017-05-23 00:23:04 +08:00
|
|
|
|
require "final_destination"
|
|
|
|
|
|
|
2022-07-28 10:27:38 +08:00
|
|
|
|
RSpec.describe FinalDestination do
|
2017-05-23 00:23:04 +08:00
|
|
|
|
let(:opts) do
|
2017-06-27 03:38:23 +08:00
|
|
|
|
{
|
|
|
|
|
|
ignore_redirects: ["https://ignore-me.com"],
|
2021-05-14 03:48:35 +08:00
|
|
|
|
force_get_hosts: %w[https://force.get.com https://*.ihaveawildcard.com/],
|
2018-12-19 14:27:07 +08:00
|
|
|
|
preserve_fragment_url_hosts: ["https://eviltrout.com"],
|
2017-05-23 00:23:04 +08:00
|
|
|
|
}
|
|
|
|
|
|
end
|
|
|
|
|
|
|
2017-05-26 15:19:09 +08:00
|
|
|
|
let(:doc_response) { { status: 200, headers: { "Content-Type" => "text/html" } } }
|
2017-05-23 00:23:04 +08:00
|
|
|
|
|
2021-07-31 01:36:30 +08:00
|
|
|
|
let(:image_response) { { status: 200, headers: { "Content-Type" => "image/jpeg" } } }
|
|
|
|
|
|
|
2022-02-14 09:11:09 +08:00
|
|
|
|
let(:body_response) { { status: 200, body: "<body>test</body>" } }
|
|
|
|
|
|
|
2022-11-02 00:33:17 +08:00
|
|
|
|
def fd_stub_request(method, url)
|
|
|
|
|
|
uri = URI.parse(url)
|
|
|
|
|
|
|
|
|
|
|
|
host = uri.hostname
|
|
|
|
|
|
ip = "1.2.3.4"
|
|
|
|
|
|
|
|
|
|
|
|
# In Excon we pass the IP in the URL, so we need to stub
|
|
|
|
|
|
# that version as well
|
|
|
|
|
|
uri.hostname = "HOSTNAME_PLACEHOLDER"
|
|
|
|
|
|
matcher =
|
|
|
|
|
|
Regexp.escape(uri.to_s).sub(
|
|
|
|
|
|
"HOSTNAME_PLACEHOLDER",
|
|
|
|
|
|
"(#{Regexp.escape(host)}|#{Regexp.escape(ip)})",
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
stub_request(method, /\A#{matcher}\z/).with(headers: { "Host" => host })
|
|
|
|
|
|
end
|
|
|
|
|
|
|
2021-10-01 23:48:21 +08:00
|
|
|
|
def canonical_follow(from, dest)
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:get, from).to_return(
|
2021-10-01 23:48:21 +08:00
|
|
|
|
status: 200,
|
|
|
|
|
|
body: "<head><link rel=\"canonical\" href=\"#{dest}\"></head>",
|
|
|
|
|
|
)
|
|
|
|
|
|
end
|
|
|
|
|
|
|
2017-05-23 00:23:04 +08:00
|
|
|
|
def redirect_response(from, dest)
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, from).to_return(status: 302, headers: { "Location" => dest })
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
2017-05-23 23:51:23 +08:00
|
|
|
|
def fd(url)
|
|
|
|
|
|
FinalDestination.new(url, opts)
|
|
|
|
|
|
end
|
|
|
|
|
|
|
2018-08-26 22:31:02 +08:00
|
|
|
|
it "correctly parses ignored hostnames" do
|
|
|
|
|
|
fd =
|
|
|
|
|
|
FinalDestination.new(
|
|
|
|
|
|
"https://meta.discourse.org",
|
|
|
|
|
|
ignore_redirects: %w[http://google.com youtube.com https://meta.discourse.org ://bing.com],
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
expect(fd.ignored).to eq(%w[test.localhost google.com meta.discourse.org])
|
|
|
|
|
|
end
|
|
|
|
|
|
|
2017-05-23 00:23:04 +08:00
|
|
|
|
describe ".resolve" do
|
|
|
|
|
|
it "has a ready status code before anything happens" do
|
2017-05-23 23:51:23 +08:00
|
|
|
|
expect(fd("https://eviltrout.com").status).to eq(:ready)
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
2017-09-27 14:52:49 +08:00
|
|
|
|
it "returns nil for an invalid url" do
|
2017-05-23 23:51:23 +08:00
|
|
|
|
expect(fd(nil).resolve).to be_nil
|
|
|
|
|
|
expect(fd("asdf").resolve).to be_nil
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
2022-11-02 00:33:17 +08:00
|
|
|
|
it "returns nil for unresolvable url" do
|
|
|
|
|
|
FinalDestination::SSRFDetector.stubs(:lookup_ips).raises(SocketError)
|
|
|
|
|
|
expect(fd("https://example.com").resolve).to eq(nil)
|
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "returns nil for url timeout" do
|
|
|
|
|
|
FinalDestination::SSRFDetector.stubs(:lookup_ips).raises(Timeout::Error)
|
|
|
|
|
|
expect(fd("https://example.com").resolve).to eq(nil)
|
|
|
|
|
|
end
|
|
|
|
|
|
|
2017-09-27 14:52:49 +08:00
|
|
|
|
it "returns nil when read timeouts" do
|
|
|
|
|
|
Excon.expects(:public_send).raises(Excon::Errors::Timeout)
|
|
|
|
|
|
|
|
|
|
|
|
expect(fd("https://discourse.org").resolve).to eq(nil)
|
|
|
|
|
|
end
|
|
|
|
|
|
|
2017-05-23 00:23:04 +08:00
|
|
|
|
context "without redirects" do
|
2022-11-02 00:33:17 +08:00
|
|
|
|
before { fd_stub_request(:head, "https://eviltrout.com/").to_return(doc_response) }
|
2017-05-23 00:23:04 +08:00
|
|
|
|
|
|
|
|
|
|
it "returns the final url" do
|
2017-05-23 23:51:23 +08:00
|
|
|
|
final = FinalDestination.new("https://eviltrout.com", opts)
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("https://eviltrout.com")
|
|
|
|
|
|
expect(final.redirected?).to eq(false)
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
2017-05-24 01:31:20 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
2017-06-27 03:38:23 +08:00
|
|
|
|
it "ignores redirects" do
|
|
|
|
|
|
final = FinalDestination.new("https://ignore-me.com/some-url", opts)
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("https://ignore-me.com/some-url")
|
|
|
|
|
|
expect(final.redirected?).to eq(false)
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
|
|
|
|
|
end
|
|
|
|
|
|
|
2022-07-28 00:14:14 +08:00
|
|
|
|
context "with underscores in URLs" do
|
2022-11-02 00:33:17 +08:00
|
|
|
|
before { fd_stub_request(:head, "https://some_thing.example.com").to_return(doc_response) }
|
2017-05-24 01:31:20 +08:00
|
|
|
|
|
|
|
|
|
|
it "doesn't raise errors with underscores in urls" do
|
|
|
|
|
|
final = FinalDestination.new("https://some_thing.example.com", opts)
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("https://some_thing.example.com")
|
|
|
|
|
|
expect(final.redirected?).to eq(false)
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
|
|
|
|
|
end
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
context "with a couple of redirects" do
|
|
|
|
|
|
before do
|
2017-05-26 13:04:25 +08:00
|
|
|
|
redirect_response("https://eviltrout.com", "https://codinghorror.com/blog")
|
|
|
|
|
|
redirect_response("https://codinghorror.com/blog", "https://discourse.org")
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, "https://discourse.org").to_return(doc_response)
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "returns the final url" do
|
2017-05-23 23:51:23 +08:00
|
|
|
|
final = FinalDestination.new("https://eviltrout.com", opts)
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("https://discourse.org")
|
|
|
|
|
|
expect(final.redirected?).to eq(true)
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
context "with too many redirects" do
|
|
|
|
|
|
before do
|
2017-05-26 13:04:25 +08:00
|
|
|
|
redirect_response("https://eviltrout.com", "https://codinghorror.com/blog")
|
|
|
|
|
|
redirect_response("https://codinghorror.com/blog", "https://discourse.org")
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, "https://discourse.org").to_return(doc_response)
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "returns the final url" do
|
2017-05-23 23:51:23 +08:00
|
|
|
|
final = FinalDestination.new("https://eviltrout.com", opts.merge(max_redirects: 1))
|
|
|
|
|
|
expect(final.resolve).to be_nil
|
|
|
|
|
|
expect(final.redirected?).to eq(true)
|
|
|
|
|
|
expect(final.status).to eq(:too_many_redirects)
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
context "with a redirect to an internal IP" do
|
|
|
|
|
|
before do
|
2017-05-26 13:04:25 +08:00
|
|
|
|
redirect_response("https://eviltrout.com", "https://private-host.com")
|
2022-11-02 00:33:17 +08:00
|
|
|
|
FinalDestination::SSRFDetector
|
|
|
|
|
|
.stubs(:lookup_and_filter_ips)
|
|
|
|
|
|
.with("eviltrout.com")
|
|
|
|
|
|
.returns(["1.2.3.4"])
|
|
|
|
|
|
FinalDestination::SSRFDetector
|
|
|
|
|
|
.stubs(:lookup_and_filter_ips)
|
|
|
|
|
|
.with("private-host.com")
|
|
|
|
|
|
.raises(FinalDestination::SSRFDetector::DisallowedIpError)
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "returns the final url" do
|
2017-05-23 23:51:23 +08:00
|
|
|
|
final = FinalDestination.new("https://eviltrout.com", opts)
|
|
|
|
|
|
expect(final.resolve).to be_nil
|
|
|
|
|
|
expect(final.redirected?).to eq(true)
|
|
|
|
|
|
expect(final.status).to eq(:invalid_address)
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
end
|
2017-06-07 01:53:49 +08:00
|
|
|
|
|
2019-08-07 18:56:03 +08:00
|
|
|
|
context "with a redirect to login path" do
|
|
|
|
|
|
before { redirect_response("https://eviltrout.com/t/xyz/1", "https://eviltrout.com/login") }
|
|
|
|
|
|
|
|
|
|
|
|
it "does not follow redirect" do
|
|
|
|
|
|
final = FinalDestination.new("https://eviltrout.com/t/xyz/1", opts)
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("https://eviltrout.com/t/xyz/1")
|
|
|
|
|
|
expect(final.redirected?).to eq(false)
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
|
|
|
|
|
end
|
|
|
|
|
|
end
|
|
|
|
|
|
|
2022-02-14 09:11:09 +08:00
|
|
|
|
it "raises error when response is too big" do
|
|
|
|
|
|
stub_const(described_class, "MAX_REQUEST_SIZE_BYTES", 1) do
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:get, "https://codinghorror.com/blog").to_return(body_response)
|
2022-02-14 09:11:09 +08:00
|
|
|
|
final =
|
|
|
|
|
|
FinalDestination.new("https://codinghorror.com/blog", opts.merge(follow_canonical: true))
|
|
|
|
|
|
expect { final.resolve }.to raise_error(
|
|
|
|
|
|
Excon::Errors::ExpectationFailed,
|
|
|
|
|
|
"response size too big: https://codinghorror.com/blog",
|
|
|
|
|
|
)
|
|
|
|
|
|
end
|
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "raises error when response is too slow" do
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:get, "https://codinghorror.com/blog").to_return(
|
|
|
|
|
|
lambda do |request|
|
|
|
|
|
|
freeze_time(11.seconds.from_now)
|
|
|
|
|
|
body_response
|
2023-01-09 19:18:21 +08:00
|
|
|
|
end,
|
2022-11-02 00:33:17 +08:00
|
|
|
|
)
|
2022-02-14 09:11:09 +08:00
|
|
|
|
final =
|
|
|
|
|
|
FinalDestination.new("https://codinghorror.com/blog", opts.merge(follow_canonical: true))
|
|
|
|
|
|
expect { final.resolve }.to raise_error(
|
|
|
|
|
|
Excon::Errors::ExpectationFailed,
|
|
|
|
|
|
"connect timeout reached: https://codinghorror.com/blog",
|
|
|
|
|
|
)
|
|
|
|
|
|
end
|
|
|
|
|
|
|
2022-07-28 00:14:14 +08:00
|
|
|
|
context "when following canonical links" do
|
2021-10-01 23:48:21 +08:00
|
|
|
|
it "resolves the canonical link as the final destination" do
|
|
|
|
|
|
canonical_follow("https://eviltrout.com", "https://codinghorror.com/blog")
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, "https://codinghorror.com/blog").to_return(doc_response)
|
2021-10-01 23:48:21 +08:00
|
|
|
|
|
|
|
|
|
|
final = FinalDestination.new("https://eviltrout.com", opts.merge(follow_canonical: true))
|
|
|
|
|
|
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("https://codinghorror.com/blog")
|
|
|
|
|
|
expect(final.redirected?).to eq(false)
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
|
|
|
|
|
end
|
|
|
|
|
|
|
2021-11-06 01:20:14 +08:00
|
|
|
|
it "resolves the canonical link when the URL is relative" do
|
|
|
|
|
|
host = "https://codinghorror.com"
|
|
|
|
|
|
|
|
|
|
|
|
canonical_follow("#{host}/blog", "/blog/canonical")
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, "#{host}/blog/canonical").to_return(doc_response)
|
2021-11-06 01:20:14 +08:00
|
|
|
|
|
|
|
|
|
|
final = FinalDestination.new("#{host}/blog", opts.merge(follow_canonical: true))
|
|
|
|
|
|
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("#{host}/blog/canonical")
|
|
|
|
|
|
expect(final.redirected?).to eq(false)
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "resolves the canonical link when the URL is relative and does not start with the / symbol" do
|
|
|
|
|
|
host = "https://codinghorror.com"
|
|
|
|
|
|
canonical_follow("#{host}/blog", "blog/canonical")
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, "#{host}/blog/canonical").to_return(doc_response)
|
2021-11-06 01:20:14 +08:00
|
|
|
|
|
|
|
|
|
|
final = FinalDestination.new("#{host}/blog", opts.merge(follow_canonical: true))
|
|
|
|
|
|
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("#{host}/blog/canonical")
|
|
|
|
|
|
expect(final.redirected?).to eq(false)
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
|
|
|
|
|
end
|
|
|
|
|
|
|
2021-10-01 23:48:21 +08:00
|
|
|
|
it "does not follow the canonical link if it's the same as the current URL" do
|
|
|
|
|
|
canonical_follow("https://eviltrout.com", "https://eviltrout.com")
|
|
|
|
|
|
|
|
|
|
|
|
final = FinalDestination.new("https://eviltrout.com", opts.merge(follow_canonical: true))
|
|
|
|
|
|
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("https://eviltrout.com")
|
|
|
|
|
|
expect(final.redirected?).to eq(false)
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "does not follow the canonical link if it's invalid" do
|
|
|
|
|
|
canonical_follow("https://eviltrout.com", "")
|
|
|
|
|
|
|
|
|
|
|
|
final = FinalDestination.new("https://eviltrout.com", opts.merge(follow_canonical: true))
|
|
|
|
|
|
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("https://eviltrout.com")
|
|
|
|
|
|
expect(final.redirected?).to eq(false)
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
|
|
|
|
|
end
|
|
|
|
|
|
end
|
|
|
|
|
|
|
2022-07-28 00:14:14 +08:00
|
|
|
|
context "when forcing GET" do
|
2017-08-08 17:44:27 +08:00
|
|
|
|
it "will do a GET when forced" do
|
2022-11-02 00:33:17 +08:00
|
|
|
|
url = "https://force.get.com/posts?page=4"
|
|
|
|
|
|
get_stub = fd_stub_request(:get, url)
|
|
|
|
|
|
head_stub = fd_stub_request(:head, url)
|
|
|
|
|
|
|
|
|
|
|
|
final = FinalDestination.new(url, opts)
|
|
|
|
|
|
expect(final.resolve.to_s).to eq(url)
|
2017-08-08 17:44:27 +08:00
|
|
|
|
expect(final.status).to eq(:resolved)
|
2022-11-02 00:33:17 +08:00
|
|
|
|
expect(get_stub).to have_been_requested
|
|
|
|
|
|
expect(head_stub).to_not have_been_requested
|
2017-08-08 17:44:27 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "will do a HEAD if not forced" do
|
2022-11-02 00:33:17 +08:00
|
|
|
|
url = "https://eviltrout.com/posts?page=2"
|
|
|
|
|
|
get_stub = fd_stub_request(:get, url)
|
|
|
|
|
|
head_stub = fd_stub_request(:head, url)
|
|
|
|
|
|
|
|
|
|
|
|
final = FinalDestination.new(url, opts)
|
|
|
|
|
|
expect(final.resolve.to_s).to eq(url)
|
2017-08-08 17:44:27 +08:00
|
|
|
|
expect(final.status).to eq(:resolved)
|
2022-11-02 00:33:17 +08:00
|
|
|
|
expect(get_stub).to_not have_been_requested
|
|
|
|
|
|
expect(head_stub).to have_been_requested
|
2017-08-08 17:44:27 +08:00
|
|
|
|
end
|
2021-05-14 03:48:35 +08:00
|
|
|
|
|
|
|
|
|
|
it "will do a GET when forced on a wildcard subdomain" do
|
2022-11-02 00:33:17 +08:00
|
|
|
|
url = "https://any-subdomain.ihaveawildcard.com/some/other/content"
|
|
|
|
|
|
get_stub = fd_stub_request(:get, url)
|
|
|
|
|
|
head_stub = fd_stub_request(:head, url)
|
|
|
|
|
|
|
|
|
|
|
|
final = FinalDestination.new(url, opts)
|
|
|
|
|
|
expect(final.resolve.to_s).to eq(url)
|
2021-05-14 03:48:35 +08:00
|
|
|
|
expect(final.status).to eq(:resolved)
|
2022-11-02 00:33:17 +08:00
|
|
|
|
expect(get_stub).to have_been_requested
|
|
|
|
|
|
expect(head_stub).to_not have_been_requested
|
2021-05-14 03:48:35 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "will do a HEAD if on a subdomain of a forced get domain without a wildcard" do
|
2022-11-02 00:33:17 +08:00
|
|
|
|
url = "https://particularly.eviltrout.com/has/a/secret/plan"
|
|
|
|
|
|
get_stub = fd_stub_request(:get, url)
|
|
|
|
|
|
head_stub = fd_stub_request(:head, url)
|
|
|
|
|
|
|
|
|
|
|
|
final = FinalDestination.new(url, opts)
|
|
|
|
|
|
expect(final.resolve.to_s).to eq(url)
|
2021-05-14 03:48:35 +08:00
|
|
|
|
expect(final.status).to eq(:resolved)
|
2022-11-02 00:33:17 +08:00
|
|
|
|
expect(get_stub).to_not have_been_requested
|
|
|
|
|
|
expect(head_stub).to have_been_requested
|
2021-05-14 03:48:35 +08:00
|
|
|
|
end
|
2017-08-08 17:44:27 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
2022-07-28 00:14:14 +08:00
|
|
|
|
context "when HEAD not supported" do
|
2017-06-07 01:53:49 +08:00
|
|
|
|
before do
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:get, "https://eviltrout.com").to_return(
|
2017-06-07 01:53:49 +08:00
|
|
|
|
status: 301,
|
|
|
|
|
|
headers: {
|
|
|
|
|
|
"Location" => "https://discourse.org",
|
|
|
|
|
|
"Set-Cookie" => "evil=trout",
|
|
|
|
|
|
},
|
|
|
|
|
|
)
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, "https://discourse.org")
|
2017-06-07 01:53:49 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
context "when the status code is 405" do
|
2022-11-02 00:33:17 +08:00
|
|
|
|
before { fd_stub_request(:head, "https://eviltrout.com").to_return(status: 405) }
|
2017-06-07 01:53:49 +08:00
|
|
|
|
|
|
|
|
|
|
it "will try a GET" do
|
|
|
|
|
|
final = FinalDestination.new("https://eviltrout.com", opts)
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("https://discourse.org")
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
|
|
|
|
|
expect(final.cookie).to eq("evil=trout")
|
|
|
|
|
|
end
|
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
context "when the status code is 501" do
|
2022-11-02 00:33:17 +08:00
|
|
|
|
before { fd_stub_request(:head, "https://eviltrout.com").to_return(status: 501) }
|
2017-06-07 01:53:49 +08:00
|
|
|
|
|
|
|
|
|
|
it "will try a GET" do
|
|
|
|
|
|
final = FinalDestination.new("https://eviltrout.com", opts)
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("https://discourse.org")
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
|
|
|
|
|
expect(final.cookie).to eq("evil=trout")
|
|
|
|
|
|
end
|
|
|
|
|
|
end
|
2018-02-18 23:08:07 +08:00
|
|
|
|
|
|
|
|
|
|
it "correctly extracts cookies during GET" do
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, "https://eviltrout.com").to_return(status: 405)
|
2018-02-18 23:08:07 +08:00
|
|
|
|
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:get, "https://eviltrout.com").to_return(
|
2018-02-18 23:08:07 +08:00
|
|
|
|
status: 302,
|
|
|
|
|
|
body: "",
|
|
|
|
|
|
headers: {
|
|
|
|
|
|
"Location" => "https://eviltrout.com",
|
|
|
|
|
|
"Set-Cookie" => [
|
|
|
|
|
|
"foo=219ffwef9w0f; expires=Mon, 19-Feb-2018 10:44:24 GMT; path=/; domain=eviltrout.com",
|
|
|
|
|
|
"bar=1",
|
|
|
|
|
|
"baz=2; expires=Tue, 19-Feb-2019 10:14:24 GMT; path=/; domain=eviltrout.com",
|
2023-01-09 19:18:21 +08:00
|
|
|
|
],
|
2018-02-18 23:08:07 +08:00
|
|
|
|
},
|
|
|
|
|
|
)
|
|
|
|
|
|
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, "https://eviltrout.com").with(
|
2018-02-18 23:08:07 +08:00
|
|
|
|
headers: {
|
|
|
|
|
|
"Cookie" => "bar=1; baz=2; foo=219ffwef9w0f",
|
|
|
|
|
|
},
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
final = FinalDestination.new("https://eviltrout.com", opts)
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("https://eviltrout.com")
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
|
|
|
|
|
expect(final.cookie).to eq("bar=1; baz=2; foo=219ffwef9w0f")
|
|
|
|
|
|
end
|
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "should use the correct format for cookies when there is only one cookie" do
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, "https://eviltrout.com").to_return(
|
2018-02-24 19:35:57 +08:00
|
|
|
|
status: 302,
|
|
|
|
|
|
headers: {
|
2018-02-18 23:08:07 +08:00
|
|
|
|
"Location" => "https://eviltrout.com",
|
|
|
|
|
|
"Set-Cookie" =>
|
|
|
|
|
|
"foo=219ffwef9w0f; expires=Mon, 19-Feb-2018 10:44:24 GMT; path=/; domain=eviltrout.com",
|
|
|
|
|
|
},
|
|
|
|
|
|
)
|
|
|
|
|
|
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, "https://eviltrout.com").with(
|
2018-02-18 23:08:07 +08:00
|
|
|
|
headers: {
|
|
|
|
|
|
"Cookie" => "foo=219ffwef9w0f",
|
|
|
|
|
|
},
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
final = FinalDestination.new("https://eviltrout.com", opts)
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("https://eviltrout.com")
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
|
|
|
|
|
expect(final.cookie).to eq("foo=219ffwef9w0f")
|
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "should use the correct format for cookies when there are multiple cookies" do
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, "https://eviltrout.com").to_return(
|
2018-02-24 19:35:57 +08:00
|
|
|
|
status: 302,
|
|
|
|
|
|
headers: {
|
2018-02-18 23:08:07 +08:00
|
|
|
|
"Location" => "https://eviltrout.com",
|
|
|
|
|
|
"Set-Cookie" => [
|
|
|
|
|
|
"foo=219ffwef9w0f; expires=Mon, 19-Feb-2018 10:44:24 GMT; path=/; domain=eviltrout.com",
|
|
|
|
|
|
"bar=1",
|
|
|
|
|
|
"baz=2; expires=Tue, 19-Feb-2019 10:14:24 GMT; path=/; domain=eviltrout.com",
|
2023-01-09 19:18:21 +08:00
|
|
|
|
],
|
2018-02-18 23:08:07 +08:00
|
|
|
|
},
|
|
|
|
|
|
)
|
2023-01-09 19:18:21 +08:00
|
|
|
|
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, "https://eviltrout.com").with(
|
2018-02-18 23:08:07 +08:00
|
|
|
|
headers: {
|
|
|
|
|
|
"Cookie" => "bar=1; baz=2; foo=219ffwef9w0f",
|
|
|
|
|
|
},
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
final = FinalDestination.new("https://eviltrout.com", opts)
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("https://eviltrout.com")
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
|
|
|
|
|
expect(final.cookie).to eq("bar=1; baz=2; foo=219ffwef9w0f")
|
2017-06-07 01:53:49 +08:00
|
|
|
|
end
|
2018-12-19 14:27:07 +08:00
|
|
|
|
|
|
|
|
|
|
it "persists fragment url" do
|
|
|
|
|
|
origin_url = "https://eviltrout.com/origin/lib/code/foobar.rb"
|
|
|
|
|
|
upstream_url = "https://eviltrout.com/upstream/lib/code/foobar.rb"
|
|
|
|
|
|
|
|
|
|
|
|
redirect_response(origin_url, upstream_url)
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, upstream_url).to_return(doc_response)
|
2018-12-19 14:27:07 +08:00
|
|
|
|
|
|
|
|
|
|
final = FinalDestination.new("#{origin_url}#L154-L205", opts)
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("#{upstream_url}#L154-L205")
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
|
|
|
|
|
end
|
2021-07-31 01:36:30 +08:00
|
|
|
|
|
2022-07-28 00:14:14 +08:00
|
|
|
|
context "with content_type" do
|
2021-07-31 01:36:30 +08:00
|
|
|
|
before do
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, "https://eviltrout.com/this/is/an/image").to_return(image_response)
|
2021-07-31 01:36:30 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "returns a content_type" do
|
|
|
|
|
|
final = FinalDestination.new("https://eviltrout.com/this/is/an/image", opts)
|
|
|
|
|
|
expect(final.resolve.to_s).to eq("https://eviltrout.com/this/is/an/image")
|
|
|
|
|
|
expect(final.content_type).to eq("image/jpeg")
|
|
|
|
|
|
expect(final.status).to eq(:resolved)
|
|
|
|
|
|
end
|
|
|
|
|
|
end
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
2022-07-21 21:58:17 +08:00
|
|
|
|
describe "#get" do
|
|
|
|
|
|
let(:fd) { FinalDestination.new("http://wikipedia.com", opts.merge(verbose: true)) }
|
2018-01-29 12:36:52 +08:00
|
|
|
|
|
2022-07-27 18:09:47 +08:00
|
|
|
|
before { described_class.clear_https_cache!("wikipedia.com") }
|
|
|
|
|
|
|
2022-07-21 21:58:17 +08:00
|
|
|
|
context "when there is a redirect" do
|
|
|
|
|
|
before do
|
|
|
|
|
|
stub_request(:get, "http://wikipedia.com/").to_return(
|
|
|
|
|
|
status: 302,
|
|
|
|
|
|
body: "",
|
|
|
|
|
|
headers: {
|
|
|
|
|
|
"location" => "https://wikipedia.com/",
|
|
|
|
|
|
},
|
|
|
|
|
|
)
|
|
|
|
|
|
# webmock does not do chunks
|
|
|
|
|
|
stub_request(:get, "https://wikipedia.com/").to_return(
|
|
|
|
|
|
status: 200,
|
|
|
|
|
|
body: "<html><head>",
|
|
|
|
|
|
headers: {
|
2023-01-09 19:18:21 +08:00
|
|
|
|
},
|
2022-07-21 21:58:17 +08:00
|
|
|
|
)
|
|
|
|
|
|
end
|
|
|
|
|
|
|
2022-07-25 18:38:54 +08:00
|
|
|
|
after { WebMock.reset! }
|
|
|
|
|
|
|
2022-07-21 21:58:17 +08:00
|
|
|
|
it "correctly streams" do
|
|
|
|
|
|
chunk = nil
|
|
|
|
|
|
result =
|
|
|
|
|
|
fd.get do |resp, c|
|
|
|
|
|
|
chunk = c
|
|
|
|
|
|
throw :done
|
|
|
|
|
|
end
|
2018-01-29 12:36:52 +08:00
|
|
|
|
|
2022-07-21 21:58:17 +08:00
|
|
|
|
expect(result).to eq("https://wikipedia.com/")
|
|
|
|
|
|
expect(chunk).to eq("<html><head>")
|
|
|
|
|
|
end
|
|
|
|
|
|
end
|
2018-01-29 12:36:52 +08:00
|
|
|
|
|
2022-07-21 21:58:17 +08:00
|
|
|
|
context "when there is a timeout" do
|
|
|
|
|
|
subject(:get) { fd.get {} }
|
2018-01-29 12:36:52 +08:00
|
|
|
|
|
2022-07-21 21:58:17 +08:00
|
|
|
|
before { fd.stubs(:safe_session).raises(Timeout::Error) }
|
2018-01-29 12:36:52 +08:00
|
|
|
|
|
2022-07-21 21:58:17 +08:00
|
|
|
|
it "logs the exception" do
|
2022-07-27 18:09:47 +08:00
|
|
|
|
Rails
|
|
|
|
|
|
.logger
|
|
|
|
|
|
.expects(:warn)
|
|
|
|
|
|
.with(regexp_matches(/FinalDestination could not resolve URL \(timeout\)/))
|
2022-07-21 21:58:17 +08:00
|
|
|
|
get
|
2018-01-29 12:36:52 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
2022-07-21 21:58:17 +08:00
|
|
|
|
it "returns nothing" do
|
|
|
|
|
|
expect(get).to be_blank
|
|
|
|
|
|
end
|
2018-01-29 12:36:52 +08:00
|
|
|
|
end
|
2022-08-09 18:13:56 +08:00
|
|
|
|
|
|
|
|
|
|
context "when there is an SSL error" do
|
|
|
|
|
|
subject(:get) { fd.get {} }
|
|
|
|
|
|
|
|
|
|
|
|
before { fd.stubs(:safe_session).raises(OpenSSL::SSL::SSLError) }
|
|
|
|
|
|
|
|
|
|
|
|
it "logs the exception" do
|
|
|
|
|
|
Rails.logger.expects(:warn).with(regexp_matches(/an error with ssl occurred/i))
|
|
|
|
|
|
get
|
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "returns nothing" do
|
|
|
|
|
|
expect(get).to be_blank
|
|
|
|
|
|
end
|
|
|
|
|
|
end
|
2018-01-29 12:36:52 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
2017-05-23 00:23:04 +08:00
|
|
|
|
describe ".validate_url_format" do
|
|
|
|
|
|
it "supports http urls" do
|
2017-05-23 23:51:23 +08:00
|
|
|
|
expect(fd("http://eviltrout.com").validate_uri_format).to eq(true)
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "supports https urls" do
|
2017-05-23 23:51:23 +08:00
|
|
|
|
expect(fd("https://eviltrout.com").validate_uri_format).to eq(true)
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "doesn't support ftp urls" do
|
2017-05-23 23:51:23 +08:00
|
|
|
|
expect(fd("ftp://eviltrout.com").validate_uri_format).to eq(false)
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
2017-05-24 01:07:18 +08:00
|
|
|
|
it "doesn't support IP urls" do
|
|
|
|
|
|
expect(fd("http://104.25.152.10").validate_uri_format).to eq(false)
|
|
|
|
|
|
expect(fd("https://[2001:abc:de:01:0:3f0:6a65:c2bf]").validate_uri_format).to eq(false)
|
|
|
|
|
|
end
|
|
|
|
|
|
|
2017-05-23 00:23:04 +08:00
|
|
|
|
it "returns false for schemeless URL" do
|
2017-05-23 23:51:23 +08:00
|
|
|
|
expect(fd("eviltrout.com").validate_uri_format).to eq(false)
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "returns false for nil URL" do
|
2017-05-23 23:51:23 +08:00
|
|
|
|
expect(fd(nil).validate_uri_format).to eq(false)
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
2023-08-23 09:18:33 +08:00
|
|
|
|
it "returns false for invalid https ports" do
|
2017-05-23 23:51:23 +08:00
|
|
|
|
expect(fd("https://eviltrout.com:8000").validate_uri_format).to eq(false)
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
2023-08-23 09:18:33 +08:00
|
|
|
|
it "returns true for valid http and https ports" do
|
2017-05-23 23:51:23 +08:00
|
|
|
|
expect(fd("http://eviltrout.com:80").validate_uri_format).to eq(true)
|
|
|
|
|
|
expect(fd("https://eviltrout.com:443").validate_uri_format).to eq(true)
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
2023-08-23 09:18:33 +08:00
|
|
|
|
|
|
|
|
|
|
it "returns false for invalid http port" do
|
|
|
|
|
|
expect(fd("http://eviltrout.com:21").validate_uri_format).to eq(false)
|
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
context "when s3_endpoint defined" do
|
|
|
|
|
|
before { SiteSetting.s3_endpoint = "http://minio.local:9000" }
|
|
|
|
|
|
|
|
|
|
|
|
it "returns false if the host is not in allowed_internal_hosts" do
|
|
|
|
|
|
expect(fd("http://discoursetest.minio.local:9000").validate_uri_format).to eq(false)
|
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
it "returns true if the host is in allowed_internal_hosts" do
|
|
|
|
|
|
SiteSetting.allowed_internal_hosts = %w[minio.local discoursetest.minio.local].join("|")
|
|
|
|
|
|
expect(fd("http://discoursetest.minio.local:9000").validate_uri_format).to eq(true)
|
|
|
|
|
|
end
|
|
|
|
|
|
end
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
|
2017-10-17 13:22:38 +08:00
|
|
|
|
describe "https cache" do
|
|
|
|
|
|
it "will cache https lookups" do
|
|
|
|
|
|
FinalDestination.clear_https_cache!("wikipedia.com")
|
|
|
|
|
|
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, "http://wikipedia.com/image.png").to_return(
|
2017-10-17 13:22:38 +08:00
|
|
|
|
status: 302,
|
|
|
|
|
|
body: "",
|
|
|
|
|
|
headers: {
|
|
|
|
|
|
location: "https://wikipedia.com/image.png",
|
|
|
|
|
|
},
|
|
|
|
|
|
)
|
2018-02-24 19:35:57 +08:00
|
|
|
|
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, "https://wikipedia.com/image.png")
|
2017-10-17 13:22:38 +08:00
|
|
|
|
|
|
|
|
|
|
fd("http://wikipedia.com/image.png").resolve
|
|
|
|
|
|
|
2022-11-02 00:33:17 +08:00
|
|
|
|
fd_stub_request(:head, "https://wikipedia.com/image2.png")
|
2017-10-17 13:22:38 +08:00
|
|
|
|
|
|
|
|
|
|
fd("http://wikipedia.com/image2.png").resolve
|
|
|
|
|
|
end
|
|
|
|
|
|
end
|
|
|
|
|
|
|
2022-08-09 18:28:29 +08:00
|
|
|
|
describe "#normalized_url" do
|
|
|
|
|
|
it "correctly normalizes url" do
|
2017-07-30 00:42:04 +08:00
|
|
|
|
fragment_url =
|
|
|
|
|
|
"https://eviltrout.com/2016/02/25/fixing-android-performance.html#discourse-comments"
|
|
|
|
|
|
|
2022-08-09 18:28:29 +08:00
|
|
|
|
expect(fd(fragment_url).normalized_url.to_s).to eq(fragment_url)
|
2017-09-26 18:34:54 +08:00
|
|
|
|
|
2022-08-09 18:28:29 +08:00
|
|
|
|
expect(fd("https://eviltrout.com?s=180&d=mm&r=g").normalized_url.to_s).to eq(
|
2020-12-04 06:16:01 +08:00
|
|
|
|
"https://eviltrout.com?s=180&d=mm&%23038;r=g",
|
|
|
|
|
|
)
|
2017-09-26 18:34:54 +08:00
|
|
|
|
|
2022-08-09 18:28:29 +08:00
|
|
|
|
expect(fd("http://example.com/?a=\11\15").normalized_url.to_s).to eq(
|
|
|
|
|
|
"http://example.com/?a=%09%0D",
|
|
|
|
|
|
)
|
2017-09-26 18:34:54 +08:00
|
|
|
|
|
2022-08-09 18:28:29 +08:00
|
|
|
|
expect(
|
|
|
|
|
|
fd("https://ru.wikipedia.org/wiki/%D0%A1%D0%B2%D0%BE%D0%B1%D0%BE").normalized_url.to_s,
|
2017-09-26 18:34:54 +08:00
|
|
|
|
).to eq("https://ru.wikipedia.org/wiki/%D0%A1%D0%B2%D0%BE%D0%B1%D0%BE")
|
|
|
|
|
|
|
2022-08-09 18:28:29 +08:00
|
|
|
|
expect(fd("https://ru.wikipedia.org/wiki/Свобо").normalized_url.to_s).to eq(
|
2017-09-26 18:34:54 +08:00
|
|
|
|
"https://ru.wikipedia.org/wiki/%D0%A1%D0%B2%D0%BE%D0%B1%D0%BE",
|
|
|
|
|
|
)
|
2017-07-30 00:42:04 +08:00
|
|
|
|
end
|
|
|
|
|
|
end
|
2017-05-23 00:23:04 +08:00
|
|
|
|
end
|
