mirror of
https://github.com/discourse/discourse.git
synced 2025-02-25 16:01:36 +08:00
FIX: Remove paths from robots.txt in favor of noindex header
Google no longer supports the use of robots.txt to block indexing. See https://support.google.com/webmasters/answer/6062608 and https://support.google.com/webmasters/answer/93710 Previous commits have added the `noindex` header to appropriate pages, now we need to remove the paths from robots.txt so the pages can be crawled. Follow up to: 13f229808a22db9e1032832a313ab701b66614c8 b6765aac4b532c026418a7ffd9effd0741ab8a37 676be3a853454a33cf627c3d570feb37d3bb0bfd 07b728c5e557c9aae91c51f3eaac5c32d479f2a2 c94e6a9a66757ea48d99e3ee8d880523871cb6f4
This commit is contained in:
parent
01b6349a67
commit
b52143feff
@ -10,14 +10,6 @@ class RobotsTxtController < ApplicationController
|
|||||||
DISALLOWED_PATHS ||= %w{
|
DISALLOWED_PATHS ||= %w{
|
||||||
/auth/
|
/auth/
|
||||||
/assets/browser-update*.js
|
/assets/browser-update*.js
|
||||||
/users/
|
|
||||||
/u/
|
|
||||||
/my/
|
|
||||||
/badges/
|
|
||||||
/search
|
|
||||||
/search/
|
|
||||||
/tags
|
|
||||||
/tags/
|
|
||||||
/email/
|
/email/
|
||||||
/session
|
/session
|
||||||
/session/
|
/session/
|
||||||
@ -27,11 +19,6 @@ class RobotsTxtController < ApplicationController
|
|||||||
/user-api-key/
|
/user-api-key/
|
||||||
/*?api_key*
|
/*?api_key*
|
||||||
/*?*api_key*
|
/*?*api_key*
|
||||||
/groups
|
|
||||||
/groups/
|
|
||||||
/t/*/*.rss
|
|
||||||
/tags/*.rss
|
|
||||||
/c/*.rss
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def index
|
def index
|
||||||
|
Loading…
x
Reference in New Issue
Block a user