added support for disabling indexing by google using SiteSetting.allow_index_in_robots_txt = false

app/controllers/robots_txt_controller.rb

@@ -0,0 +1,14 @@
+class RobotsTxtController < ApplicationController
+  layout false
+  skip_before_filter :check_xhr
+
+  def index
+    path = if SiteSetting.allow_index_in_robots_txt && !SiteSetting.restrict_access
+      :index
+    else
+      :no_index
+    end
+    
+    render path, content_type: 'text/plain'
+  end
+end

app/models/site_setting.rb

@@ -102,6 +102,8 @@ class SiteSetting < ActiveRecord::Base
   # we need to think of a way to force users to enter certain settings, this is a minimal config thing
   setting(:notification_email, 'info@discourse.org')
 
+  setting(:allow_index_in_robots_txt, true)
+
   setting(:send_welcome_message, true)
 
   setting(:twitter_consumer_key, '')

app/views/robots_txt/index.erb

@@ -1,5 +1,2 @@
 # See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
 #
-# To ban all spiders from the entire site uncomment the next two lines:
-# User-Agent: *
-# Disallow: /

app/views/robots_txt/no_index.erb

@@ -0,0 +1,6 @@
+# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
+#
+User-Agent: *
+Disallow: /
+
+

config/locales/en.yml

@@ -257,6 +257,9 @@ en:
     posts_per_page: "How many posts are returned on a topic page"
     system_username: "Username that sends system messages"
     send_welcome_message: "Do new users get a welcome private message?"
+    
+    allow_index_in_robots_txt: "Site should be indexed by search engines (update robots.txt)"
+
     port: "If you'd like to specify a port in the URL. Useful in development mode. Leave blank for none."
     force_hostname: "If you'd like to specify a hostname in the URL. Useful in development mode. Leave blank for none."
 

config/routes.rb

@@ -207,6 +207,9 @@ Discourse::Application.routes.draw do
   post 'draft' => 'draft#update'
   delete 'draft' => 'draft#destroy'
 
+
+  get 'robots.txt' => 'robots_txt#index'
+
   # You can have the root of your site routed with "root"
   # just remember to delete public/index.html.
   root :to => 'list#index'

spec/controllers/robots_txt_controller_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe RobotsTxtController do
+
+  context '.index' do
+    it "returns noindex when indexing is disallowed" do
+      SiteSetting.stubs(:allow_index_in_robots_txt).returns(true)
+      get :index
+      response.should render_template :index
+    end
+
+    it "returns index when indexing is allowed" do
+      SiteSetting.stubs(:allow_index_in_robots_txt).returns(false)
+      get :index
+      response.should render_template :no_index
+    end    
+    
+    it "serves it regardless if a site is in private mode" do 
+      SiteSetting.stubs(:allow_index_in_robots_txt).returns(true)
+      SiteSetting.stubs(:restrict_access).returns(true)
+      get :index
+      response.should render_template :no_index
+    end
+
+  end
+end