From c587df7e2a03c2962f4c8cefd6f03969bb87d525 Mon Sep 17 00:00:00 2001 From: Sam Saffron Date: Tue, 30 Jul 2019 11:33:38 +1000 Subject: [PATCH] Revert "FEATURE: add Noindex to robots.txt for disallowed routes" This reverts commit d84256a876a9fa4fc7bcb4b8ac8c5865f8c10701. This is not supported by Google and causes robots.txt to be flagged as invalid Removing Noindex --- app/views/robots_txt/index.erb | 1 - spec/requests/robots_txt_controller_spec.rb | 1 - 2 files changed, 2 deletions(-) diff --git a/app/views/robots_txt/index.erb b/app/views/robots_txt/index.erb index d86f72edd79..71ca94baa7b 100644 --- a/app/views/robots_txt/index.erb +++ b/app/views/robots_txt/index.erb @@ -10,7 +10,6 @@ Crawl-delay: <%= agent[:delay] %> <%- end -%> <% agent[:disallow].each do |path| %> Disallow: <%= path %> -Noindex: <%= path %> <% end %> diff --git a/spec/requests/robots_txt_controller_spec.rb b/spec/requests/robots_txt_controller_spec.rb index d18d3d4967e..b7e283e59f2 100644 --- a/spec/requests/robots_txt_controller_spec.rb +++ b/spec/requests/robots_txt_controller_spec.rb @@ -20,7 +20,6 @@ RSpec.describe RobotsTxtController do Discourse.stubs(:base_uri).returns('/forum') get '/robots.txt' expect(response.body).to include("\nDisallow: /forum/admin") - expect(response.body).to include("\nNoindex: /forum/admin") end end