# frozen_string_literal: true require "active_support/test_case" require "fileutils" require "json" require "nokogiri" require "open-uri" require "file_helper" EMOJI_GROUPS_PATH ||= "lib/emoji/groups.json" EMOJI_DB_PATH ||= "lib/emoji/db.json" EMOJI_IMAGES_PATH ||= "public/images/emoji" EMOJI_ORDERING_URL ||= "http://www.unicode.org/emoji/charts/emoji-ordering.html" # emoji aliases are actually created as images # eg: "right_anger_bubble" => [ "anger_right" ] # your app will physically have right_anger_bubble.png and anger_right.png EMOJI_ALIASES ||= { "right_anger_bubble" => ["anger_right"], "ballot_box" => ["ballot_box_with_ballot"], "basketball_man" => %w[basketball_player person_with_ball], "beach_umbrella" => %w[umbrella_on_ground beach beach_with_umbrella], "parasol_on_ground" => ["umbrella_on_ground"], "bellhop_bell" => ["bellhop"], "biohazard" => ["biohazard_sign"], "bow_and_arrow" => ["archery"], "spiral_calendar" => %w[calendar_spiral spiral_calendar_pad], "card_file_box" => ["card_box"], "champagne" => ["bottle_with_popping_cork"], "cheese" => ["cheese_wedge"], "city_sunset" => ["city_dusk"], "couch_and_lamp" => ["couch"], "crayon" => ["lower_left_crayon"], "cricket_bat_and_ball" => ["cricket_bat_ball"], "latin_cross" => ["cross"], "dagger" => ["dagger_knife"], "desktop_computer" => ["desktop"], "card_index_dividers" => ["dividers"], "dove" => ["dove_of_peace"], "footprints" => ["feet"], "fire" => ["flame"], "black_flag" => %w[flag_black waving_black_flag], "cn" => ["flag_cn"], "de" => ["flag_de"], "es" => ["flag_es"], "fr" => ["flag_fr"], "uk" => %w[gb flag_gb], "it" => ["flag_it"], "jp" => ["flag_jp"], "kr" => ["flag_kr"], "ru" => ["flag_ru"], "us" => ["flag_us"], "white_flag" => %w[flag_white waving_white_flag], "plate_with_cutlery" => %w[fork_knife_plate fork_and_knife_with_plate], "framed_picture" => %w[frame_photo frame_with_picture], "hammer_and_pick" => ["hammer_pick"], "heavy_heart_exclamation" => %w[heart_exclamation heavy_heart_exclamation_mark_ornament], "houses" => %w[homes house_buildings], "hotdog" => ["hot_dog"], "derelict_house" => %w[house_abandoned derelict_house_building], "desert_island" => ["island"], "old_key" => ["key2"], "laughing" => ["satisfied"], "business_suit_levitating" => %w[levitate man_in_business_suit_levitating], "weight_lifting_man" => %w[lifter weight_lifter], "medal_sports" => %w[medal sports_medal], "metal" => ["sign_of_the_horns"], "fu" => %w[middle_finger reversed_hand_with_middle_finger_extended], "motorcycle" => ["racing_motorcycle"], "mountain_snow" => ["snow_capped_mountain"], "newspaper_roll" => %w[newspaper2 rolled_up_newspaper], "spiral_notepad" => %w[notepad_spiral spiral_note_pad], "oil_drum" => ["oil"], "older_woman" => ["grandma"], "paintbrush" => ["lower_left_paintbrush"], "paperclips" => ["linked_paperclips"], "pause_button" => ["double_vertical_bar"], "peace_symbol" => ["peace"], "fountain_pen" => %w[pen_fountain lower_left_fountain_pen], "ping_pong" => ["table_tennis"], "place_of_worship" => ["worship_symbol"], "poop" => %w[shit hankey poo], "radioactive" => ["radioactive_sign"], "railway_track" => ["railroad_track"], "robot" => ["robot_face"], "skull" => ["skeleton"], "skull_and_crossbones" => ["skull_crossbones"], "speaking_head" => ["speaking_head_in_silhouette"], "male_detective" => %w[spy sleuth_or_spy], "thinking" => ["thinking_face"], "-1" => ["thumbsdown"], "+1" => ["thumbsup"], "cloud_with_lightning_and_rain" => %w[thunder_cloud_rain thunder_cloud_and_rain], "tickets" => ["admission_tickets"], "next_track_button" => %w[track_next next_track], "previous_track_button" => %w[track_previous previous_track], "unicorn" => ["unicorn_face"], "funeral_urn" => ["urn"], "sun_behind_large_cloud" => %w[white_sun_cloud white_sun_behind_cloud], "sun_behind_rain_cloud" => %w[white_sun_rain_cloud white_sun_behind_cloud_with_rain], "partly_sunny" => %w[white_sun_small_cloud white_sun_with_small_cloud], "open_umbrella" => ["umbrella2"], "hammer_and_wrench" => ["tools"], "face_with_thermometer" => ["thermometer_face"], "timer_clock" => ["timer"], "keycap_ten" => ["ten"], "memo" => ["pencil"], "rescue_worker_helmet" => %w[helmet_with_cross helmet_with_white_cross], "slightly_smiling_face" => %w[slightly_smiling slight_smile], "construction_worker_man" => ["construction_worker"], "upside_down_face" => ["upside_down"], "money_mouth_face" => ["money_mouth"], "nerd_face" => ["nerd"], "hugs" => %w[hugging hugging_face], "roll_eyes" => %w[rolling_eyes face_with_rolling_eyes], "slightly_frowning_face" => %w[frowning slight_frown], "frowning_face" => %w[frowning2 white_frowning_face], "zipper_mouth_face" => ["zipper_mouth"], "face_with_head_bandage" => ["head_bandage"], "raised_hand_with_fingers_splayed" => ["hand_splayed"], "raised_hand" => ["hand"], "vulcan_salute" => %w[vulcan raised_hand_with_part_between_middle_and_ring_fingers], "policeman" => ["cop"], "running_man" => ["runner"], "walking_man" => ["walking"], "bowing_man" => ["bow"], "no_good_woman" => ["no_good"], "raising_hand_woman" => ["raising_hand"], "pouting_woman" => ["person_with_pouting_face"], "frowning_woman" => ["person_frowning"], "haircut_woman" => ["haircut"], "massage_woman" => ["massage"], "tshirt" => ["shirt"], "biking_man" => ["bicyclist"], "mountain_biking_man" => ["mountain_bicyclist"], "passenger_ship" => ["cruise_ship"], "motor_boat" => %w[motorboat boat], "flight_arrival" => ["airplane_arriving"], "flight_departure" => ["airplane_departure"], "small_airplane" => ["airplane_small"], "racing_car" => ["race_car"], "family_man_woman_boy_boy" => ["family_man_woman_boys"], "family_man_woman_girl_girl" => ["family_man_woman_girls"], "family_woman_woman_boy" => ["family_women_boy"], "family_woman_woman_girl" => ["family_women_girl"], "family_woman_woman_girl_boy" => ["family_women_girl_boy"], "family_woman_woman_boy_boy" => ["family_women_boys"], "family_woman_woman_girl_girl" => ["family_women_girls"], "family_man_man_boy" => ["family_men_boy"], "family_man_man_girl" => ["family_men_girl"], "family_man_man_girl_boy" => ["family_men_girl_boy"], "family_man_man_boy_boy" => ["family_men_boys"], "family_man_man_girl_girl" => ["family_men_girls"], "cloud_with_lightning" => ["cloud_lightning"], "tornado" => %w[cloud_tornado cloud_with_tornado], "cloud_with_rain" => ["cloud_rain"], "cloud_with_snow" => ["cloud_snow"], "asterisk" => ["keycap_star"], "studio_microphone" => ["microphone2"], "medal_military" => ["military_medal"], "couple_with_heart_woman_woman" => ["female_couple_with_heart"], "couple_with_heart_man_man" => ["male_couple_with_heart"], "couplekiss_woman_woman" => ["female_couplekiss"], "couplekiss_man_man" => ["male_couplekiss"], "honeybee" => ["bee"], "lion" => ["lion_face"], "artificial_satellite" => ["satellite_orbital"], "computer_mouse" => %w[mouse_three_button three_button_mouse], "hocho" => ["knife"], "swimming_man" => ["swimmer"], "wind_face" => ["wind_blowing_face"], "golfing_man" => ["golfer"], "facepunch" => ["punch"], "building_construction" => ["construction_site"], "family_man_woman_girl_boy" => ["family"], "ice_hockey" => ["hockey"], "snowman_with_snow" => ["snowman2"], "play_or_pause_button" => ["play_pause"], "film_projector" => ["projector"], "shopping" => ["shopping_bags"], "open_book" => ["book"], "national_park" => ["park"], "world_map" => ["map"], "pen" => %w[pen_ballpoint lower_left_ballpoint_pen], "email" => %w[envelope e-mail], "phone" => ["telephone"], "atom_symbol" => ["atom"], "mantelpiece_clock" => ["clock"], "camera_flash" => ["camera_with_flash"], "film_strip" => ["film_frames"], "balance_scale" => ["scales"], "surfing_man" => ["surfer"], "couplekiss_man_woman" => ["couplekiss"], "couple_with_heart_woman_man" => ["couple_with_heart"], "clamp" => ["compression"], "dancing_women" => ["dancers"], "blonde_man" => ["person_with_blond_hair"], "sleeping_bed" => ["sleeping_accommodation"], "om" => ["om_symbol"], "tipping_hand_woman" => ["information_desk_person"], "rowing_man" => ["rowboat"], "new_moon" => ["moon"], "oncoming_automobile" => %w[car automobile], "fleur_de_lis" => ["fleur-de-lis"], "face_vomiting" => ["puke"], "smile" => ["grinning_face_with_smiling_eyes"], "frowning_with_open_mouth" => ["frowning_face_with_open_mouth"], } EMOJI_GROUPS ||= [ { "name" => "smileys_&_emotion", "tabicon" => "grinning" }, { "name" => "people_&_body", "tabicon" => "wave" }, { "name" => "animals_&_nature", "tabicon" => "evergreen_tree" }, { "name" => "food_&_drink", "tabicon" => "hamburger" }, { "name" => "travel_&_places", "tabicon" => "airplane" }, { "name" => "activities", "tabicon" => "soccer" }, { "name" => "objects", "tabicon" => "eyeglasses" }, { "name" => "symbols", "tabicon" => "white_check_mark" }, { "name" => "flags", "tabicon" => "checkered_flag" }, ] FITZPATRICK_SCALE ||= %w[1f3fb 1f3fc 1f3fd 1f3fe 1f3ff] DEFAULT_SET ||= "twitter" # Replace the platform by another when downloading the image (accepts names or categories) EMOJI_IMAGES_PATCH ||= { "apple" => { "snowboarder" => "twitter", }, "windows" => { "country-flag" => "twitter", }, } EMOJI_SETS ||= { "apple" => "apple", "google" => "google", "google_blob" => "google_classic", "facebook" => "facebook_messenger", "twitter" => "twitter", "windows" => "win10", } EMOJI_DB_REPO ||= "git@github.com:xfalcox/emoji-db.git" EMOJI_DB_REPO_PATH ||= File.join("tmp", "emoji-db") GENERATED_PATH ||= File.join(EMOJI_DB_REPO_PATH, "generated") def search_aliases(emojis) # Format is search pattern => associated emojis # eg: "cry" => [ "sob" ] # for a "cry" query should return: cry and sob @aliases ||= begin aliases = { "sad" => %w[frowning_face slightly_frowning_face sob crying_cat_face cry], "cry" => ["sob"], } emojis.each do |_, config| next if config["search_aliases"].blank? config["search_aliases"].each do |name| aliases[name] ||= [] aliases[name] << config["name"] end end aliases.map { |_, names| names.uniq! } aliases end end desc "update emoji images" task "emoji:update" do abort("This task can't be run on production.") if Rails.env.production? copy_emoji_db json_db = File.read(File.join(GENERATED_PATH, "db.json")) db = JSON.parse(json_db) write_db_json(db["emojis"], db["translations"], search_aliases(db["emojis"])) fix_incomplete_sets(db["emojis"]) write_aliases groups = generate_emoji_groups(db["emojis"], db["sections"]) write_js_groups(db["emojis"], groups) optimize_images(Dir.glob(File.join(Rails.root, EMOJI_IMAGES_PATH, "/**/*.png"))) TestEmojiUpdate.run_and_summarize FileUtils.rm_rf(EMOJI_DB_REPO_PATH) end desc "test the emoji generation script" task "emoji:test" do ENV["EMOJI_TEST"] = "1" Rake::Task["emoji:update"].invoke end def optimize_images(images) images.each do |filename| FileHelper.image_optim(allow_pngquant: true, strip_image_metadata: true).optimize_image!( filename, ) end end def copy_emoji_db `rm -rf tmp/emoji-db && git clone -b unicodeorg-as-source-of-truth --depth 1 #{EMOJI_DB_REPO} tmp/emoji-db` path = "#{EMOJI_IMAGES_PATH}/**/*" confirm_overwrite(path) puts "Cleaning emoji folder..." emoji_assets = Dir.glob(path) emoji_assets.delete_if { |x| x == "#{EMOJI_IMAGES_PATH}/emoji_one" } FileUtils.rm_rf(emoji_assets) EMOJI_SETS.each do |set_name, set_destination| origin = File.join(GENERATED_PATH, set_name) destination = File.join(EMOJI_IMAGES_PATH, set_destination) FileUtils.mv(origin, destination) end end def fix_incomplete_sets(emojis) emojis.each do |code, config| EMOJI_SETS.each do |set_name, set_destination| patch_set = EMOJI_SETS[EMOJI_IMAGES_PATCH.dig(set_name, config["name"])] || EMOJI_SETS[EMOJI_IMAGES_PATCH.dig(set_name, config["category"])] if patch_set || !File.exist?(File.join(EMOJI_IMAGES_PATH, set_destination, "#{config["name"]}.png")) origin = File.join(EMOJI_IMAGES_PATH, patch_set || EMOJI_SETS[DEFAULT_SET], config["name"]) FileUtils.cp( "#{origin}.png", File.join(EMOJI_IMAGES_PATH, set_destination, "#{config["name"]}.png"), ) if File.directory?(origin) FileUtils.cp_r(origin, File.join(EMOJI_IMAGES_PATH, set_destination, config["name"])) end end end end end def generate_emoji_groups(keywords, sections) puts "Generating groups..." list = URI.parse(EMOJI_ORDERING_URL).read doc = Nokogiri.HTML5(list) table = doc.css("table")[0] EMOJI_GROUPS.map do |group| group["icons"] ||= [] sub_sections = sections[group["name"]]["sub_sections"] sub_sections.each do |section| title_section = table.css("tr th a[@name='#{section}']") emoji_list_section = title_section.first.parent.parent.next_element emoji_list_section .css("a.plain img") .each do |link| emoji_code = link .attr("title") .scan(/U\+(.{4,5})\b/) .flatten .map { |code| code.downcase.strip } .join("_") emoji_char = code_to_emoji(emoji_code) if emoji = keywords[emoji_char] group["icons"] << { name: emoji["name"], diversity: emoji["fitzpatrick_scale"] } end end end group.delete("sections") group end end def write_aliases EMOJI_ALIASES.each do |original, aliases| aliases.each do |emoji_alias| EMOJI_SETS.each do |set_name, set_destination| origin_file = File.join(EMOJI_IMAGES_PATH, set_destination, "#{original}.png") origin_dir = File.join(EMOJI_IMAGES_PATH, set_destination, original) FileUtils.cp( origin_file, File.join(EMOJI_IMAGES_PATH, set_destination, "#{emoji_alias}.png"), ) if File.directory?(origin_dir) FileUtils.cp_r(origin_dir, File.join(EMOJI_IMAGES_PATH, set_destination, emoji_alias)) end end end end end def write_db_json(emojis, translations, search_aliases) puts "Writing #{EMOJI_DB_PATH}..." confirm_overwrite(EMOJI_DB_PATH) FileUtils.mkdir_p(File.expand_path("..", EMOJI_DB_PATH)) # skin tones variations of emojis shouldn’t appear in autocomplete emojis_without_tones = emojis .select do |char, config| !FITZPATRICK_SCALE.any? do |scale| codepoints_to_code(char.codepoints, config["fitzpatrick_scale"])[scale] end end .map do |char, config| { "code" => codepoints_to_code(char.codepoints, config["fitzpatrick_scale"]).tr("_", "-"), "name" => config["name"], } end emoji_with_tones = emojis .select { |code, config| config["fitzpatrick_scale"] } .map { |code, config| config["name"] } db = { "emojis" => emojis_without_tones, "tonableEmojis" => emoji_with_tones, "aliases" => EMOJI_ALIASES, "searchAliases" => search_aliases, "translations" => translations, } File.write(EMOJI_DB_PATH, JSON.pretty_generate(db)) end def write_js_groups(emojis, groups) puts "Writing #{EMOJI_GROUPS_PATH}..." confirm_overwrite(EMOJI_GROUPS_PATH) template = JSON.pretty_generate(groups) FileUtils.mkdir_p(File.expand_path("..", EMOJI_GROUPS_PATH)) File.write(EMOJI_GROUPS_PATH, template) end def code_to_emoji(code) code.split("_").map { |e| e.to_i(16) }.pack "U*" end def codepoints_to_code(codepoints, fitzpatrick_scale) codepoints = codepoints.map { |c| c.to_s(16).rjust(4, "0") }.join("_").downcase codepoints.gsub!(/_fe0f\z/, "") if !fitzpatrick_scale codepoints end def confirm_overwrite(path) return if ENV["EMOJI_TEST"] STDOUT.puts( "[!] You are about to overwrite #{path}, are you sure? [CTRL+c] to cancel, [ENTER] to continue", ) STDIN.gets.chomp end class TestEmojiUpdate < Minitest::Test def self.run_and_summarize puts "Runnings tests..." reporter = Minitest::SummaryReporter.new TestEmojiUpdate.run(reporter) puts reporter end def image_path(style, name) File.join("public", "images", "emoji", style, "#{name}.png") end def test_code_to_emoji assert_equal "😎", code_to_emoji("1f60e") end def test_codepoints_to_code assert_equal "1f6b5_200d_2640", codepoints_to_code([128_693, 8205, 9792, 65_039], false) end def test_codepoints_to_code_with_scale assert_equal "1f6b5_200d_2640_fe0f", codepoints_to_code([128_693, 8205, 9792, 65_039], true) end def test_groups_js_es6_creation assert File.exist?(EMOJI_GROUPS_PATH) assert File.size?(EMOJI_GROUPS_PATH) end def test_db_json_creation assert File.exist?(EMOJI_DB_PATH) assert File.size?(EMOJI_DB_PATH) end def test_alias_creation original_image = image_path("apple", "right_anger_bubble") alias_image = image_path("apple", "anger_right") assert_equal File.size(original_image), File.size(alias_image) end def test_cell_index_patch original_image = image_path("apple", "snowboarder") alias_image = image_path("twitter", "snowboarder") assert_equal File.size(original_image), File.size(alias_image) end def test_scales original_image = image_path("apple", "blonde_woman") assert File.exist?(original_image) assert File.size?(original_image) (2..6).each do |scale| image = image_path("apple", "blonde_woman/#{scale}") assert File.exist?(image) assert File.size?(image) end end def test_default_set original_image = image_path("twitter", "snowboarder") alias_image = image_path("apple", "snowboarder") assert_equal File.size(original_image), File.size(alias_image) original_image = image_path("twitter", "macau") alias_image = image_path("win10", "macau") assert_equal File.size(original_image), File.size(alias_image) end end