FIX: Don't crawl non-http/s links

This commit is contained in:
Robin Ward 2017-05-19 16:57:19 -04:00
parent 558a274187
commit ea9f93dcc5

View File

@ -27,6 +27,8 @@ module Jobs
uri = URI(url)
return if uri.blank? || uri.host.blank?
return unless ['https', 'http'].include?(uri.scheme)
headers = CrawlTopicLink.request_headers(uri)
head = Excon.head(url, read_timeout: 20, headers: headers)