FEATURE: Add site setting for specific hosts using custom user agent when oneboxing

Followup to #00c406

config/locales/server.en.yml

@@ -386,13 +386,13 @@ en:
       time_must_be_provided: "time must be provided for all reminders except '%{reminder_type}'"
 
     reminders:
-        at_desktop: "Next time I'm at my desktop"
-        later_today: "Later today <br/>{{date}}"
-        next_business_day: "Next business day <br/>{{date}}"
-        tomorrow: "Tomorrow <br/>{{date}}"
-        next_week: "Next week <br/>{{date}}"
-        next_month: "Next month <br/>{{date}}"
-        custom: "Custom date and time"
+      at_desktop: "Next time I'm at my desktop"
+      later_today: "Later today <br/>{{date}}"
+      next_business_day: "Next business day <br/>{{date}}"
+      tomorrow: "Tomorrow <br/>{{date}}"
+      next_week: "Next week <br/>{{date}}"
+      next_month: "Next month <br/>{{date}}"
+      custom: "Custom date and time"
 
   groups:
     success:
@@ -1451,6 +1451,7 @@ en:
     onebox_domains_blacklist: "A list of domains that will never be oneboxed."
     inline_onebox_domains_whitelist: "A list of domains that will be oneboxed in miniature form if linked without a title"
     enable_inline_onebox_on_all_domains: "Ignore inline_onebox_domain_whitelist site setting and allow inline onebox on all domains."
+    force_custom_user_agent_hosts: "Hosts for which to use the custom onebox user agent on all requests. (Especially useful for hosts that limit access by user agent)."
     max_oneboxes_per_post: "Maximum number of oneboxes in a post."
 
     logo: "The logo image at the top left of your site. Use a wide rectangular image with a height of 120 and an aspect ratio greater than 3:1. If left blank, the site title text will be shown."

config/site_settings.yml

@@ -1436,7 +1436,9 @@ onebox:
     list_type: compact
   enable_inline_onebox_on_all_domains:
     default: false
-
+  force_custom_user_agent_hosts:
+    default: "http://codepen.io"
+    type: list
 spam:
   add_rel_nofollow_to_user_content: true
   hide_post_sensitivity:

lib/oneboxer.rb

@@ -31,7 +31,7 @@ module Oneboxer
   end
 
   def self.force_custom_user_agent_hosts
-    @force_custom_user_agent_hosts ||= ['http://codepen.io']
+    @force_custom_user_agent_hosts ||= SiteSetting.force_custom_user_agent_hosts.split('|')
   end
 
   def self.allowed_post_types

spec/components/oneboxer_spec.rb

@@ -3,6 +3,9 @@
 require 'rails_helper'
 
 describe Oneboxer do
+  before do
+    SiteSetting.force_custom_user_agent_hosts = "http://codepen.io|https://video.discourse.org/"
+  end
 
   it "returns blank string for an invalid onebox" do
     stub_request(:head, "http://boom.com")
@@ -169,4 +172,15 @@ describe Oneboxer do
 
     expect(Oneboxer.external_onebox(url)[:onebox]).to be_present
   end
+
+  it "uses the Onebox custom user agent on specified hosts" do
+    url = 'https://video.discourse.org/presentation.mp4'
+
+    stub_request(:head, url).to_return(status: 403, body: "", headers: {})
+    stub_request(:get, url).to_return(status: 403, body: "", headers: {})
+    stub_request(:head, url).with(headers: { "User-Agent" => Onebox.options.user_agent }).to_return(status: 200, body: "", headers: {})
+    stub_request(:get, url).with(headers: { "User-Agent" => Onebox.options.user_agent }).to_return(status: 200, body: "", headers: {})
+
+    expect(Oneboxer.preview(url, invalidate_oneboxes: true)).to be_present
+  end
 end