From 709bed5d1cb12694c32e71618b8224aadd9981d2 Mon Sep 17 00:00:00 2001 From: Tobias Eigen Date: Tue, 13 Feb 2024 20:09:24 -0800 Subject: [PATCH] removed broken link and comments from no_index.erb (#25648) We intend to move guidance about how to configure robots.txt to the admin interface and a documentation topic on meta. --- app/views/robots_txt/no_index.erb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app/views/robots_txt/no_index.erb b/app/views/robots_txt/no_index.erb index 88d46ae9edb..44f3503f95d 100644 --- a/app/views/robots_txt/no_index.erb +++ b/app/views/robots_txt/no_index.erb @@ -1,9 +1,3 @@ -# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file -# - -# Googlebot must be allowed to index so it can remove items from the index -# we return the X-Robots-Tag with noindex, nofollow which will ensure -# indexing is minimized and nothing shows up in Google search results User-agent: googlebot Allow: <%= Discourse.base_path + "/" %> Disallow: <%= Discourse.base_path + "/uploads/*" %>