2014-02-21 05:07:02 +08:00
|
|
|
module CrawlerDetection
|
2015-01-06 17:05:45 +08:00
|
|
|
# added 'ia_archiver' based on https://meta.discourse.org/t/unable-to-archive-discourse-pages-with-the-internet-archive/21232
|
|
|
|
|
# added 'Wayback Save Page' based on https://meta.discourse.org/t/unable-to-archive-discourse-with-the-internet-archive-save-page-now-button/22875
|
2014-02-21 05:07:02 +08:00
|
|
|
def self.crawler?(user_agent)
|
2015-01-06 17:05:45 +08:00
|
|
|
!/Googlebot|Mediapartners|AdsBot|curl|Twitterbot|facebookexternalhit|bingbot|Baiduspider|ia_archiver|Wayback Save Page/.match(user_agent).nil?
|
2014-02-15 06:10:08 +08:00
|
|
|
end
|
|
|
|
|
end
|
