FEATURE: Whitelist hosts for internal crawling

config/locales/server.en.yml

@@ -1034,6 +1034,7 @@ en:
     cors_origins: "Allowed origins for cross-origin requests (CORS). Each origin must include http:// or https://. The DISCOURSE_ENABLE_CORS env variable must be set to true to enable CORS."
     use_admin_ip_whitelist: "Admins can only log in if they are at an IP address defined in the Screened IPs list (Admin > Logs > Screened Ips)."
     blacklist_ip_blocks: "A list of private IP blocks that should never be crawled by Discourse"
+    whitelist_internal_hosts: "A list of internal hosts that discourse can safely crawl for oneboxing and other purposes"
     top_menu: "Determine which items appear in the homepage navigation, and in what order. Example latest|new|unread|categories|top|read|posted|bookmarks"
     post_menu: "Determine which items appear on the post menu, and in what order. Example like|edit|flag|delete|share|bookmark|reply"
     post_menu_hidden_items: "The menu items to hide by default in the post menu unless an expansion ellipsis is clicked on."

config/site_settings.yml

@@ -897,6 +897,9 @@ security:
     default: ''
     type: list
     shadowed_by_global: true
+  whitelist_internal_hosts:
+    default: ''
+    type: list
 
 onebox:
   enable_flash_video_onebox: false

lib/final_destination.rb

@@ -143,6 +143,12 @@ class FinalDestination
       hostname_matches?(GlobalSetting.try(:cdn_url)) ||
       hostname_matches?(Discourse.base_url_no_prefix)
 
+    if SiteSetting.whitelist_internal_hosts.present?
+      SiteSetting.whitelist_internal_hosts.split('|').each do |h|
+        return true if @uri.hostname.downcase == h.downcase
+      end
+    end
+
     address_s = @opts[:lookup_ip].call(@uri.hostname)
     return false unless address_s
 

spec/components/final_destination_spec.rb

@@ -237,7 +237,7 @@ describe FinalDestination do
       expect(fd("https://[2001:470:1:3a8::251]").is_dest_valid?).to eq(true)
     end
 
-    it "returns true for private ipv6" do
+    it "returns false for private ipv6" do
       expect(fd("https://[fdd7:b450:d4d1:6b44::1]").is_dest_valid?).to eq(false)
     end
 
@@ -255,6 +255,11 @@ describe FinalDestination do
       GlobalSetting.stubs(:cdn_url).returns("https://cdn.example.com/discourse")
       expect(fd("https://cdn.example.com/some/asset").is_dest_valid?).to eq(true)
     end
+
+    it 'supports whitelisting via a site setting' do
+      SiteSetting.whitelist_internal_hosts = 'private-host.com'
+      expect(fd("https://private-host.com/some/url").is_dest_valid?).to eq(true)
+    end
   end
 
 end