diff --git a/lib/final_destination.rb b/lib/final_destination.rb index 9bc5d4040ee..9ec7c6876d1 100644 --- a/lib/final_destination.rb +++ b/lib/final_destination.rb @@ -145,6 +145,8 @@ class FinalDestination return if @stop_at_blocked_pages && blocked_domain?(@uri) result, headers_subset = safe_get(@uri, &blk) + return nil if !result + cookie = headers_subset.set_cookie location = headers_subset.location diff --git a/lib/retrieve_title.rb b/lib/retrieve_title.rb index c91ea639a65..7a9a37294ff 100644 --- a/lib/retrieve_title.rb +++ b/lib/retrieve_title.rb @@ -62,6 +62,8 @@ module RetrieveTitle fd.get do |_response, chunk, uri| unless Net::HTTPRedirection === _response + throw :done if uri.blank? + if current current << chunk else diff --git a/spec/lib/retrieve_title_spec.rb b/spec/lib/retrieve_title_spec.rb index 1b554123083..91852ca6c90 100644 --- a/spec/lib/retrieve_title_spec.rb +++ b/spec/lib/retrieve_title_spec.rb @@ -136,6 +136,12 @@ describe RetrieveTitle do expect(RetrieveTitle.crawl("https://cat.com/meow/no-onebox")).to be_blank end + + it "doesn't return a title if response is unsuccessful" do + stub_request(:get, "https://example.com").to_return(status: 404, body: "") + + expect(RetrieveTitle.crawl("https://example.com")).to eq(nil) + end end context 'fetch_title' do