diff --git a/lib/onebox/engine/twitter_status_onebox.rb b/lib/onebox/engine/twitter_status_onebox.rb index c409e20071e..0c63b46a175 100644 --- a/lib/onebox/engine/twitter_status_onebox.rb +++ b/lib/onebox/engine/twitter_status_onebox.rb @@ -6,17 +6,13 @@ module Onebox include Engine include LayoutSupport include HTML + include ActionView::Helpers::NumberHelper matches_regexp( %r{^https?://(mobile\.|www\.)?twitter\.com/.+?/status(es)?/\d+(/(video|photo)/\d?+)?+(/?\?.*)?/?$}, ) always_https - def self.===(other) - client = Onebox.options.twitter_client - client && !client.twitter_credentials_missing? && super - end - def http_params { "User-Agent" => "DiscourseBot/1.0" } end @@ -27,10 +23,46 @@ module Onebox private + def get_twitter_data + response = + begin + Onebox::Helpers.fetch_response(url, headers: http_params) + rescue StandardError + return nil + end + html = Nokogiri.HTML(response) + twitter_data = {} + html + .css("meta") + .each do |m| + if m.attribute("property") && m.attribute("property").to_s.match(/^og:/i) + m_content = m.attribute("content").to_s.strip + m_property = m.attribute("property").to_s.gsub("og:", "").gsub(":", "_") + twitter_data[m_property.to_sym] = m_content + end + end + twitter_data + end + def match @match ||= @url.match(%r{twitter\.com/.+?/status(es)?/(?\d+)}) end + def twitter_data + @twitter_data ||= get_twitter_data + end + + def guess_tweet_index + usernames = meta_tags_data("additionalName").compact + usernames.each_with_index do |username, index| + return index if twitter_data[:url].to_s.include?(username) + end + end + + def tweet_index + @tweet_index ||= guess_tweet_index + end + def client Onebox.options.twitter_client end @@ -39,66 +71,139 @@ module Onebox client && !client.twitter_credentials_missing? end - def raw - @raw ||= client.status(match[:id]).to_hash if twitter_api_credentials_present? + def symbolize_keys(obj) + case obj + when Array + obj.map { |item| symbolize_keys(item) } + when Hash + obj.each_with_object({}) do |(key, value), result| + result[key.to_sym] = symbolize_keys(value) + end + else + obj + end end - def access(*keys) - keys.reduce(raw) do |memo, key| - next unless memo - memo[key] || memo[key.to_s] + def raw + if twitter_api_credentials_present? + @raw ||= symbolize_keys(client.status(match[:id])) + else + super end end def tweet - client.prettify_tweet(raw)&.strip + if twitter_api_credentials_present? + client.prettify_tweet(raw)&.strip + else + twitter_data[:description].gsub(/“(.+?)”/im) { $1 } if twitter_data[:description] + end end def timestamp - date = DateTime.strptime(access(:created_at), "%a %b %d %H:%M:%S %z %Y") - user_offset = access(:user, :utc_offset).to_i - offset = (user_offset >= 0 ? "+" : "-") + Time.at(user_offset.abs).gmtime.strftime("%H%M") - date.new_offset(offset).strftime("%-l:%M %p - %-d %b %Y") + if twitter_api_credentials_present? && (created_at = raw.dig(:data, :created_at)) + date = DateTime.strptime(created_at, "%Y-%m-%dT%H:%M:%S.%L%z") + date.strftime("%-l:%M %p - %-d %b %Y") + end end def title - access(:user, :name) + if twitter_api_credentials_present? + raw.dig(:includes, :users)&.first&.dig(:name) + else + meta_tags_data("givenName")[tweet_index] + end end def screen_name - access(:user, :screen_name) + if twitter_api_credentials_present? + raw.dig(:includes, :users)&.first&.dig(:username) + else + meta_tags_data("additionalName")[tweet_index] + end end def avatar - access(:user, :profile_image_url_https).sub("normal", "400x400") + if twitter_api_credentials_present? + raw.dig(:includes, :users)&.first&.dig(:profile_image_url) + end end def likes - prettify_number(access(:favorite_count).to_i) + if twitter_api_credentials_present? + prettify_number(raw.dig(:data, :public_metrics, :like_count).to_i) + end end def retweets - prettify_number(access(:retweet_count).to_i) + if twitter_api_credentials_present? + prettify_number(raw.dig(:data, :public_metrics, :retweet_count).to_i) + end end def quoted_full_name - access(:quoted_status, :user, :name) + if twitter_api_credentials_present? && quoted_tweet_author.present? + quoted_tweet_author[:name] + end end def quoted_screen_name - access(:quoted_status, :user, :screen_name) + if twitter_api_credentials_present? && quoted_tweet_author.present? + quoted_tweet_author[:username] + end end - def quoted_tweet - access(:quoted_status, :full_text) + def quoted_text + quoted_tweet[:text] if twitter_api_credentials_present? && quoted_tweet.present? end def quoted_link - "https://twitter.com/#{quoted_screen_name}/status/#{access(:quoted_status, :id)}" + if twitter_api_credentials_present? + "https://twitter.com/#{quoted_screen_name}/status/#{quoted_status_id}" + end + end + + def quoted_status_id + raw.dig(:data, :referenced_tweets)&.find { |ref| ref[:type] == "quoted" }&.dig(:id) + end + + def quoted_tweet + raw.dig(:includes, :tweets)&.find { |tweet| tweet[:id] == quoted_status_id } + end + + def quoted_tweet_author + raw.dig(:includes, :users)&.find { |user| user[:id] == quoted_tweet&.dig(:author_id) } end def prettify_number(count) - count > 0 ? client.prettify_number(count) : nil + if count > 0 + number_to_human( + count, + format: "%n%u", + precision: 2, + units: { + thousand: "K", + million: "M", + billion: "B", + }, + ) + end + end + + def attr_at_css(css_property, attribute_name) + raw.at_css(css_property)&.attr(attribute_name) + end + + def meta_tags_data(attribute_name) + data = [] + raw + .css("meta") + .each do |m| + if m.attribute("itemprop") && m.attribute("itemprop").to_s.strip == attribute_name + data.push(m.attribute("content").to_s.strip) + end + end + data end def data @@ -111,7 +216,7 @@ module Onebox avatar: avatar, likes: likes, retweets: retweets, - quoted_tweet: quoted_tweet, + quoted_text: quoted_text, quoted_full_name: quoted_full_name, quoted_screen_name: quoted_screen_name, quoted_link: quoted_link, diff --git a/lib/onebox/templates/twitterstatus.mustache b/lib/onebox/templates/twitterstatus.mustache index 13fe2cfafd9..4c94a40d081 100644 --- a/lib/onebox/templates/twitterstatus.mustache +++ b/lib/onebox/templates/twitterstatus.mustache @@ -4,15 +4,15 @@
{{{tweet}}} - {{#quoted_tweet}} + {{#quoted_text}}

{{quoted_full_name}} @{{quoted_screen_name}}

 -
{{quoted_tweet}}
+
{{quoted_text}}
- {{/quoted_tweet}} + {{/quoted_text}}
diff --git a/lib/twitter_api.rb b/lib/twitter_api.rb index c7f5126e1e9..51330ccf4aa 100644 --- a/lib/twitter_api.rb +++ b/lib/twitter_api.rb @@ -3,17 +3,21 @@ # lightweight Twitter api calls class TwitterApi class << self - include ActionView::Helpers::NumberHelper - BASE_URL = "https://api.twitter.com" + URL_PARAMS = %w[ + tweet.fields=id,author_id,text,created_at,entities,referenced_tweets,public_metrics + user.fields=id,name,username,profile_image_url + media.fields=type,height,width,variants,preview_image_url,url + expansions=attachments.media_keys,referenced_tweets.id.author_id + ] def prettify_tweet(tweet) - text = tweet["full_text"].dup - if (entities = tweet["entities"]) && (urls = entities["urls"]) + text = tweet[:data][:text].dup.to_s + if (entities = tweet[:data][:entities]) && (urls = entities[:urls]) urls.each do |url| text.gsub!( - url["url"], - "#{url["display_url"]}", + url[:url], + "#{url[:display_url]}", ) end end @@ -22,25 +26,23 @@ class TwitterApi result = Rinku.auto_link(text, :all, 'target="_blank"').to_s - if tweet["extended_entities"] && media = tweet["extended_entities"]["media"] + if tweet[:includes] && media = tweet[:includes][:media] media.each do |m| - if m["type"] == "photo" - if large = m["sizes"]["large"] - result << "
" - end - elsif m["type"] == "video" || m["type"] == "animated_gif" + if m[:type] == "photo" + result << "
" + elsif m[:type] == "video" || m[:type] == "animated_gif" video_to_display = - m["video_info"]["variants"] - .select { |v| v["content_type"] == "video/mp4" } - .sort { |v| v["bitrate"] } + m[:variants] + .select { |v| v[:content_type] == "video/mp4" } + .sort { |v| v[:bit_rate] } .last # choose highest bitrate - if video_to_display && url = video_to_display["url"] - width = m["sizes"]["large"]["w"] - height = m["sizes"]["large"]["h"] + if video_to_display && url = video_to_display[:url] + width = m[:width] + height = m[:height] attributes = - if m["type"] == "animated_gif" + if m[:type] == "animated_gif" %w[playsinline loop muted autoplay disableRemotePlayback disablePictureInPicture] else %w[controls playsinline] @@ -52,7 +54,7 @@ class TwitterApi
@@ -66,19 +68,6 @@ class TwitterApi result end - def prettify_number(count) - number_to_human( - count, - format: "%n%u", - precision: 2, - units: { - thousand: "K", - million: "M", - billion: "B", - }, - ) - end - def tweet_for(id) JSON.parse(twitter_get(tweet_uri_for(id))) end @@ -111,7 +100,7 @@ class TwitterApi end def tweet_uri_for(id) - URI.parse "#{BASE_URL}/1.1/statuses/show.json?id=#{id}&tweet_mode=extended" + URI.parse "#{BASE_URL}/2/tweets/#{id}?#{URL_PARAMS.join("&")}" end def twitter_get(uri) diff --git a/spec/fixtures/onebox/twitterstatus.response b/spec/fixtures/onebox/twitterstatus.response index 8bb442d5d8e..1761117c994 100644 --- a/spec/fixtures/onebox/twitterstatus.response +++ b/spec/fixtures/onebox/twitterstatus.response @@ -1,2814 +1,1867 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Vyki Englert on Twitter: "I'm a sucker for pledges. @Peers Pledge #sharingeconomy http://t.co/T4Sc47KAzh" - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-
-
- - -
-
- - - - Previous - - - -
-
- - - - Next - - - -
-
-
-
- - - -
- -
- - - - - - - - - - - - - - - - - - - - -
-
-
- - -
-
    -
  • -

    Add a location to your Tweets

    -

    - When you tweet with a location, Twitter stores that location. - You can switch location on/off before each Tweet and always have the option to delete your location history. - Learn more -

    -
    - - -
    -
    • -
-
- -
- -
-
-
- - -
-
-
- - -
-
-
    -
    -
    - -
    - - -
    -
    -
    -
    - - -
    -

    Profile summary

    -
    - -
    - -
    - -
    - -
    -
    -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -