2014-02-21 05:07:02 +08:00
module CrawlerDetection
2015-01-06 17:05:45 +08:00
# added 'ia_archiver' based on https://meta.discourse.org/t/unable-to-archive-discourse-pages-with-the-internet-archive/21232
# added 'Wayback Save Page' based on https://meta.discourse.org/t/unable-to-archive-discourse-with-the-internet-archive-save-page-now-button/22875
2015-05-02 18:18:58 +08:00
# added 'Swiftbot' based on https://meta.discourse.org/t/how-to-add-html-markup-or-meta-tags-for-external-search-engine/28220
2015-02-14 22:24:51 +08:00
2014-02-21 05:07:02 +08:00
def self . crawler? ( user_agent )
2016-07-27 01:21:13 +08:00
! / Googlebot|Mediapartners|AdsBot|curl|Twitterbot|facebookexternalhit|bingbot|Baiduspider|ia_archiver|Wayback Save Page|360Spider|Swiftbot|YandexBot / . match ( user_agent ) . nil?
2014-02-15 06:10:08 +08:00
end
end
