FEATURE: Allow oneboxing private GitHub URLs (#27705)

This commit adds the ability to onebox private GitHub
commits, pull requests, issues, blobs, and actions using
a new `github_onebox_access_token` site setting. The token
must be set up in correctly to have access to the repos needed.

To do this successfully with the Oneboxer, we need to skip
redirects on the github.com host, otherwise we get a 404
on the URL before it is translated into a GitHub API URL
and has the appropriate headers added.

config/locales/server.en.yml

@@ -1757,6 +1757,7 @@ en:
     force_custom_user_agent_hosts: "Hosts for which to use the custom onebox user agent on all requests. (Especially useful for hosts that limit access by user agent)."
     max_oneboxes_per_post: "Set the maximum number of oneboxes that can be included in a single post. Oneboxes provide a preview of linked content within the post."
     facebook_app_access_token: "A token generated from your Facebook app ID and secret. Used to generate Instagram oneboxes."
+    github_onebox_access_token: "A GitHub access token which is used to generate GitHub oneboxes for private repos, commits, pull requests, issues, and file contents. Without this, only public GitHub URLs will be oneboxed."
 
     logo: "The logo image at the top left of your site. Use a wide rectangular image with a height of 120 and an aspect ratio greater than 3:1. If left blank, the site title text will be shown."
     logo_small: "The small logo image at the top left of your site, seen when scrolling down. Use a square 120 × 120 image. If left blank, a home glyph will be shown."

config/site_settings.yml

@@ -2125,6 +2125,9 @@ onebox:
   cache_onebox_user_agent:
     default: ""
     hidden: true
+  github_onebox_access_token:
+    default: ""
+    secret: true
 spam:
   add_rel_nofollow_to_user_content: true
   hide_post_sensitivity:

lib/onebox/engine/github_actions_onebox.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "../mixins/github_body"
+require_relative "../mixins/github_auth_header"
 
 module Onebox
   module Engine
@@ -8,6 +9,7 @@ module Onebox
       include Engine
       include LayoutSupport
       include JSON
+      include Onebox::Mixins::GithubAuthHeader
 
       matches_regexp(
         %r{^https?://(?:www\.)?(?:(?:\w)+\.)?github\.com/(?<org>.+)/(?<repo>.+)/(actions/runs/[[:digit:]]+|pull/[[:digit:]]*/checks\?check_run_id=[[:digit:]]+)},
@@ -63,20 +65,22 @@ module Onebox
       end
 
       def data
+        result = raw(github_auth_header).clone
+
         status = "unknown"
-        if raw["status"] == "completed"
+        if result["status"] == "completed"
-          if raw["conclusion"] == "success"
+          if result["conclusion"] == "success"
             status = "success"
-          elsif raw["conclusion"] == "failure"
+          elsif result["conclusion"] == "failure"
             status = "failure"
           end
-        elsif raw["status"] == "in_progress"
+        elsif result["status"] == "in_progress"
           status = "pending"
         end
 
         title =
           if type == :actions_run
-            raw["head_commit"]["message"].lines.first
+            result["head_commit"]["message"].lines.first
           elsif type == :pr_run
             pr_url =
               "https://api.github.com/repos/#{match[:org]}/#{match[:repo]}/pulls/#{match[:pr_id]}"
@@ -86,8 +90,8 @@ module Onebox
         {
           :link => @url,
           :title => title,
-          :name => raw["name"],
+          :name => result["name"],
-          :run_number => raw["run_number"],
+          :run_number => result["run_number"],
           status => true,
         }
       end

lib/onebox/engine/github_blob_onebox.rb

@@ -1,10 +1,13 @@
 # frozen_string_literal: true
 
 require_relative "../mixins/git_blob_onebox"
+require_relative "../mixins/github_auth_header"
 
 module Onebox
   module Engine
     class GithubBlobOnebox
+      include Onebox::Mixins::GithubAuthHeader
+
       def self.git_regexp
         %r{^https?://(www\.)?github\.com.*/blob/}
       end
@@ -35,6 +38,10 @@ module Onebox
       def title
         Sanitize.fragment(Onebox::Helpers.uri_unencode(link).sub(%r{^https?\://github\.com/}, ""))
       end
+
+      def auth_headers
+        github_auth_header
+      end
     end
   end
 end

lib/onebox/engine/github_commit_onebox.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "../mixins/github_body"
+require_relative "../mixins/github_auth_header"
 
 module Onebox
   module Engine
@@ -9,6 +10,7 @@ module Onebox
       include LayoutSupport
       include JSON
       include Onebox::Mixins::GithubBody
+      include Onebox::Mixins::GithubAuthHeader
 
       matches_regexp(%r{^https?://(?:www\.)?(?:(?:\w)+\.)?(github)\.com(?:/)?(?:.)*/commit/})
       always_https
@@ -33,7 +35,7 @@ module Onebox
       end
 
       def data
-        result = raw.clone
+        result = raw(github_auth_header).clone
 
         lines = result["commit"]["message"].split("\n")
         result["title"] = lines.first

lib/onebox/engine/github_issue_onebox.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "../mixins/github_body"
+require_relative "../mixins/github_auth_header"
 
 module Onebox
   module Engine
@@ -10,6 +11,7 @@ module Onebox
       include LayoutSupport
       include JSON
       include Onebox::Mixins::GithubBody
+      include Onebox::Mixins::GithubAuthHeader
 
       matches_regexp(
         %r{^https?://(?:www\.)?(?:(?:\w)+\.)?github\.com/(?<org>.+)/(?<repo>.+)/issues/([[:digit:]]+)},
@@ -35,31 +37,32 @@ module Onebox
       end
 
       def data
-        created_at = Time.parse(raw["created_at"])
+        result = raw(github_auth_header).clone
-        closed_at = Time.parse(raw["closed_at"]) if raw["closed_at"]
+        created_at = Time.parse(result["created_at"])
-        body, excerpt = compute_body(raw["body"])
+        closed_at = Time.parse(result["closed_at"]) if result["closed_at"]
+        body, excerpt = compute_body(result["body"])
         ulink = URI(link)
 
         labels =
-          raw["labels"].map do |l|
+          result["labels"].map do |l|
             { name: Emoji.codes_to_img(Onebox::Helpers.sanitize(l["name"])) }
           end
 
         {
           link: @url,
-          title: raw["title"],
+          title: result["title"],
           body: body,
           excerpt: excerpt,
           labels: labels,
-          user: raw["user"],
+          user: result["user"],
           created_at: created_at.strftime("%I:%M%p - %d %b %y %Z"),
           created_at_date: created_at.strftime("%F"),
           created_at_time: created_at.strftime("%T"),
           closed_at: closed_at&.strftime("%I:%M%p - %d %b %y %Z"),
           closed_at_date: closed_at&.strftime("%F"),
           closed_at_time: closed_at&.strftime("%T"),
-          closed_by: raw["closed_by"],
+          closed_by: result["closed_by"],
-          avatar: "https://avatars1.githubusercontent.com/u/#{raw["user"]["id"]}?v=2&s=96",
+          avatar: "https://avatars1.githubusercontent.com/u/#{result["user"]["id"]}?v=2&s=96",
           domain: "#{ulink.host}/#{ulink.path.split("/")[1]}/#{ulink.path.split("/")[2]}",
           i18n: i18n,
         }

lib/onebox/engine/github_pull_request_onebox.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "../mixins/github_body"
+require_relative "../mixins/github_auth_header"
 
 module Onebox
   module Engine
@@ -9,6 +10,7 @@ module Onebox
       include LayoutSupport
       include JSON
       include Onebox::Mixins::GithubBody
+      include Onebox::Mixins::GithubAuthHeader
 
       GITHUB_COMMENT_REGEX = /(<!--.*?-->\r\n)/
 
@@ -27,7 +29,7 @@ module Onebox
       end
 
       def data
-        result = raw.clone
+        result = raw(github_auth_header).clone
         result["link"] = link
 
         created_at = Time.parse(result["created_at"])

lib/onebox/engine/gitlab_blob_onebox.rb

@@ -33,6 +33,10 @@ module Onebox
       def title
         Sanitize.fragment(Onebox::Helpers.uri_unencode(link).sub(%r{^https?\://gitlab\.com/}, ""))
       end
+
+      def auth_headers
+        {}
+      end
     end
   end
 end

lib/onebox/engine/json.rb

@@ -5,8 +5,12 @@ module Onebox
     module JSON
       private
 
-      def raw
+      def raw(http_headers = {})
-        @raw ||= ::MultiJson.load(URI.parse(url).open(read_timeout: timeout))
+        @raw ||= ::MultiJson.load(URI.parse(url).open(options.merge(http_headers)))
+      end
+
+      def options
+        { read_timeout: timeout }
       end
     end
   end

lib/onebox/mixins/git_blob_onebox.rb

@@ -170,7 +170,11 @@ module Onebox
               @model_file = @lang.dup
               @raw = "https://render.githubusercontent.com/view/solid?url=" + self.raw_template(m)
             else
-              contents = URI.parse(self.raw_template(m)).open(read_timeout: timeout).read
+              contents =
+                URI
+                  .parse(self.raw_template(m))
+                  .open({ read_timeout: timeout }.merge(self.auth_headers))
+                  .read
 
               if contents.encoding == Encoding::BINARY || contents.bytes.include?(0)
                 @raw = nil

lib/onebox/mixins/github_auth_header.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Onebox
+  module Mixins
+    module GithubAuthHeader
+      def github_auth_header
+        return {} if SiteSetting.github_onebox_access_token.blank?
+        { "Authorization" => "Bearer #{SiteSetting.github_onebox_access_token}" }
+      end
+    end
+  end
+end

lib/oneboxer.rb

@@ -681,6 +681,14 @@ module Oneboxer
 
     uri = URI(url)
 
+    # For private GitHub repos, we get a 404 when trying to use
+    # FinalDestination to request the final URL because no auth headers
+    # are sent. In this case we can ignore redirects and go straight to
+    # using Onebox.preview
+    if SiteSetting.github_onebox_access_token.present? && uri.hostname == "github.com"
+      fd_options[:ignore_redirects] << "https://github.com"
+    end
+
     strategy = Oneboxer.ordered_strategies(uri.hostname).shift if strategy.blank?
 
     if strategy && Oneboxer.strategies[strategy][:force_get_host]

spec/lib/onebox/engine/github_actions_onebox_spec.rb

@@ -4,16 +4,18 @@ RSpec.describe Onebox::Engine::GithubActionsOnebox do
   describe "PR check run" do
     before do
       @link = "https://github.com/discourse/discourse/pull/13128/checks?check_run_id=2660861130"
+      @pr_run_uri = "https://api.github.com/repos/discourse/discourse/pulls/13128"
+      @run_uri = "https://api.github.com/repos/discourse/discourse/check-runs/2660861130"
 
-      stub_request(:get, "https://api.github.com/repos/discourse/discourse/pulls/13128").to_return(
+      stub_request(:get, @pr_run_uri).to_return(
         status: 200,
         body: onebox_response("githubactions_pr"),
       )
 
-      stub_request(
+      stub_request(:get, @run_uri).to_return(
-        :get,
+        status: 200,
-        "https://api.github.com/repos/discourse/discourse/check-runs/2660861130",
+        body: onebox_response("githubactions_pr_run"),
-      ).to_return(status: 200, body: onebox_response("githubactions_pr_run"))
+      )
     end
 
     include_context "with engines"
@@ -28,16 +30,30 @@ RSpec.describe Onebox::Engine::GithubActionsOnebox do
         expect(html).to include("simplify post and topic deletion language")
       end
     end
+
+    context "when github_onebox_access_token is configured" do
+      before { SiteSetting.github_onebox_access_token = "1234" }
+
+      it "sends it as part of the request" do
+        html
+        expect(WebMock).to have_requested(:get, @run_uri).with(
+          headers: {
+            "Authorization" => "Bearer #{SiteSetting.github_onebox_access_token}",
+          },
+        )
+      end
+    end
   end
 
   describe "GitHub Actions run" do
     before do
       @link = "https://github.com/discourse/discourse/actions/runs/873214216"
+      @pr_run_uri = "https://api.github.com/repos/discourse/discourse/actions/runs/873214216"
 
-      stub_request(
+      stub_request(:get, @pr_run_uri).to_return(
-        :get,
+        status: 200,
-        "https://api.github.com/repos/discourse/discourse/actions/runs/873214216",
+        body: onebox_response("githubactions_actions_run"),
-      ).to_return(status: 200, body: onebox_response("githubactions_actions_run"))
+      )
     end
 
     include_context "with engines"
@@ -56,5 +72,18 @@ RSpec.describe Onebox::Engine::GithubActionsOnebox do
         expect(html).to include("Linting")
       end
     end
+
+    context "when github_onebox_access_token is configured" do
+      before { SiteSetting.github_onebox_access_token = "1234" }
+
+      it "sends it as part of the request" do
+        html
+        expect(WebMock).to have_requested(:get, @pr_run_uri).with(
+          headers: {
+            "Authorization" => "Bearer #{SiteSetting.github_onebox_access_token}",
+          },
+        )
+      end
+    end
   end
 end

spec/lib/onebox/engine/github_blob_onebox_spec.rb

@@ -5,10 +5,12 @@ RSpec.describe Onebox::Engine::GithubBlobOnebox do
     @link =
       "https://github.com/discourse/onebox/blob/master/lib/onebox/engine/github_blob_onebox.rb"
     @uri = URI.parse(@link)
-    stub_request(
+    @raw_uri =
-      :get,
+      "https://raw.githubusercontent.com/discourse/onebox/master/lib/onebox/engine/github_blob_onebox.rb"
-      "https://raw.githubusercontent.com/discourse/onebox/master/lib/onebox/engine/github_blob_onebox.rb",
+    stub_request(:get, @raw_uri).to_return(
-    ).to_return(status: 200, body: onebox_response(described_class.onebox_name))
+      status: 200,
+      body: onebox_response(described_class.onebox_name),
+    )
   end
 
   include_context "with engines"
@@ -38,5 +40,18 @@ RSpec.describe Onebox::Engine::GithubBlobOnebox do
       expect(html).not_to include("/Pages")
       expect(html).to include("This file is binary.")
     end
+
+    context "when github_onebox_access_token is configured" do
+      before { SiteSetting.github_onebox_access_token = "1234" }
+
+      it "sends it as part of the request" do
+        html
+        expect(WebMock).to have_requested(:get, @raw_uri).with(
+          headers: {
+            "Authorization" => "Bearer #{SiteSetting.github_onebox_access_token}",
+          },
+        )
+      end
+    end
   end
 end

spec/lib/onebox/engine/github_commit_onebox_spec.rb

@@ -51,6 +51,19 @@ RSpec.describe Onebox::Engine::GithubCommitOnebox do
         expect(html).to include("2 deletions")
       end
     end
+
+    context "when github_onebox_access_token is configured" do
+      before { SiteSetting.github_onebox_access_token = "1234" }
+
+      it "sends it as part of the request" do
+        html
+        expect(WebMock).to have_requested(:get, @uri).with(
+          headers: {
+            "Authorization" => "Bearer #{SiteSetting.github_onebox_access_token}",
+          },
+        )
+      end
+    end
   end
 
   describe "PR with commit URL" do
@@ -58,12 +71,9 @@ RSpec.describe Onebox::Engine::GithubCommitOnebox do
       @link =
         "https://github.com/discourse/discourse/pull/4662/commit/803d023e2307309f8b776ab3b8b7e38ba91c0919"
       @uri =
-        "https://api.github.com/repos/discourse/discourse/commit/803d023e2307309f8b776ab3b8b7e38ba91c0919"
+        "https://api.github.com/repos/discourse/discourse/commits/803d023e2307309f8b776ab3b8b7e38ba91c0919"
 
-      stub_request(
+      stub_request(:get, @uri).to_return(status: 200, body: onebox_response("githubcommit"))
-        :get,
-        "https://api.github.com/repos/discourse/discourse/commits/803d023e2307309f8b776ab3b8b7e38ba91c0919",
-      ).to_return(status: 200, body: onebox_response("githubcommit"))
     end
 
     include_context "with engines"
@@ -107,5 +117,18 @@ RSpec.describe Onebox::Engine::GithubCommitOnebox do
         expect(html).to include("2 deletions")
       end
     end
+
+    context "when github_onebox_access_token is configured" do
+      before { SiteSetting.github_onebox_access_token = "1234" }
+
+      it "sends it as part of the request" do
+        html
+        expect(WebMock).to have_requested(:get, @uri).with(
+          headers: {
+            "Authorization" => "Bearer #{SiteSetting.github_onebox_access_token}",
+          },
+        )
+      end
+    end
   end
 end

spec/lib/onebox/engine/github_issue_onebox_spec.rb

@@ -3,8 +3,9 @@
 RSpec.describe Onebox::Engine::GithubIssueOnebox do
   before do
     @link = "https://github.com/discourse/discourse/issues/1"
+    @issue_uri = "https://api.github.com/repos/discourse/discourse/issues/1"
 
-    stub_request(:get, "https://api.github.com/repos/discourse/discourse/issues/1").to_return(
+    stub_request(:get, @issue_uri).to_return(
       status: 200,
       body: onebox_response("github_issue_onebox"),
     )
@@ -20,5 +21,18 @@ RSpec.describe Onebox::Engine::GithubIssueOnebox do
 
       expect(html).to include(sanitized_label)
     end
+
+    context "when github_onebox_access_token is configured" do
+      before { SiteSetting.github_onebox_access_token = "1234" }
+
+      it "sends it as part of the request" do
+        html
+        expect(WebMock).to have_requested(:get, @issue_uri).with(
+          headers: {
+            "Authorization" => "Bearer #{SiteSetting.github_onebox_access_token}",
+          },
+        )
+      end
+    end
   end
 end

spec/lib/onebox/engine/github_pull_request_onebox_spec.rb

@@ -90,4 +90,17 @@ RSpec.describe Onebox::Engine::GithubPullRequestOnebox do
       expect(html).to include("You've signed the CLA")
     end
   end
+
+  context "when github_onebox_access_token is configured" do
+    before { SiteSetting.github_onebox_access_token = "1234" }
+
+    it "sends it as part of the request" do
+      html
+      expect(WebMock).to have_requested(:get, @uri).with(
+        headers: {
+          "Authorization" => "Bearer #{SiteSetting.github_onebox_access_token}",
+        },
+      )
+    end
+  end
 end