From da72ecc3fcc32118d18a84aa2239e84f8b00f792 Mon Sep 17 00:00:00 2001 From: Gordon Mohr Date: Sun, 29 Oct 2017 02:30:29 -0700 Subject: [PATCH] add alt 'archive.org_bot' user-agent (#5273) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add `archive.org_bot' – another user-agent used by Internet Archive when crawling for Wayback Machine --- config/site_settings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/site_settings.yml b/config/site_settings.yml index 8fbb3f3fa8d..1878e2ce41a 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -929,7 +929,7 @@ security: allow_index_in_robots_txt: true allow_moderators_to_create_categories: false crawler_user_agents: - default: 'Googlebot|Mediapartners|AdsBot|curl|HTTrack|Twitterbot|facebookexternalhit|bingbot|Baiduspider|ia_archiver|Wayback Save Page|360Spider|Swiftbot|YandexBot' + default: 'Googlebot|Mediapartners|AdsBot|curl|HTTrack|Twitterbot|facebookexternalhit|bingbot|Baiduspider|ia_archiver|archive.org_bot|Wayback Save Page|360Spider|Swiftbot|YandexBot' type: list cors_origins: default: ''