From 507e5a0ebc0c12fe65eea4199ae5237733c4e987 Mon Sep 17 00:00:00 2001
From: Jake Yesbeck <jake@apartmentlist.com>
Date: Mon, 4 May 2015 23:11:41 -0700
Subject: [PATCH] Robots controller and custom robots.txt files

New route to /robots.txt and some specific logic to switch
on if the subdomain is staging or not.

Staging will not allow any crawlers at all, production does not
have any current limitations
---
 app/controllers/robots_controller.rb       | 17 ++++++++
 config/robots.staging.txt                  |  5 +++
 {public => config}/robots.txt              |  0
 config/routes.rb                           |  1 +
 spec/controllers/robots_controller_spec.rb | 47 ++++++++++++++++++++++
 5 files changed, 70 insertions(+)
 create mode 100644 app/controllers/robots_controller.rb
 create mode 100644 config/robots.staging.txt
 rename {public => config}/robots.txt (100%)
 create mode 100644 spec/controllers/robots_controller_spec.rb

diff --git a/app/controllers/robots_controller.rb b/app/controllers/robots_controller.rb
new file mode 100644
index 000000000..2b6542ac2
--- /dev/null
+++ b/app/controllers/robots_controller.rb
@@ -0,0 +1,17 @@
+class RobotsController < ApplicationController
+  def robots
+    subdomain = request.subdomain
+
+    filename = if subdomain.present? && subdomain != 'www'
+      "robots.#{ subdomain }.txt"
+    else
+      "robots.txt"
+    end
+
+    robots_text = File.read(Rails.root.join('config', filename))
+
+    respond_to do |format|
+      format.text { render text: robots_text, layout: false }
+    end
+  end
+end
diff --git a/config/robots.staging.txt b/config/robots.staging.txt
new file mode 100644
index 000000000..8e1d42063
--- /dev/null
+++ b/config/robots.staging.txt
@@ -0,0 +1,5 @@
+# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
+#
+# To ban all spiders from the entire site uncomment the next two lines:
+User-Agent: *
+Disallow: /
diff --git a/public/robots.txt b/config/robots.txt
similarity index 100%
rename from public/robots.txt
rename to config/robots.txt
diff --git a/config/routes.rb b/config/routes.rb
index 542cf8c51..b40793264 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -1,5 +1,6 @@
 Growstuff::Application.routes.draw do
 
+  get '/robots.txt' => 'robots#robots'
 
   resources :plant_parts
 
diff --git a/spec/controllers/robots_controller_spec.rb b/spec/controllers/robots_controller_spec.rb
new file mode 100644
index 000000000..ee2b73196
--- /dev/null
+++ b/spec/controllers/robots_controller_spec.rb
@@ -0,0 +1,47 @@
+require 'rails_helper'
+require 'pry'
+
+describe RobotsController do 
+  describe '#robots' do 
+
+    before do
+      @request.host = "#{ subdomain }.localhost.com" 
+    end
+
+    context 'subdomain is staging' do
+      let(:subdomain) { 'staging' }
+
+      it 'loads the staging robots.txt file' do
+        expect(File).to receive(:read).with(Rails.root.join('config', 'robots.staging.txt'))
+
+        get :robots, format: :text
+
+        expect(response).to be_success
+      end 
+    end
+
+    context 'subdomain is www' do
+      let(:subdomain) { 'www' }
+
+      it 'loads the production robots.txt file' do 
+        expect(File).to receive(:read).with(Rails.root.join('config', 'robots.txt'))
+        
+        get :robots, format: :text
+
+        expect(response).to be_success
+      end 
+    end
+
+    context 'subdomain is not present' do 
+      let(:subdomain) { '' }
+
+      it 'loads the production robots.txt file' do 
+        expect(File).to receive(:read).with(Rails.root.join('config', 'robots.txt'))
+        
+        get :robots, format: :text
+
+        expect(response).to be_success
+      end
+    end
+  end
+end