Robots controller and custom robots.txt files

New route to /robots.txt and some specific logic to switch
on if the subdomain is staging or not.

Staging will not allow any crawlers at all, production does not
have any current limitations

app/controllers/robots_controller.rb

@@ -0,0 +1,17 @@
+class RobotsController < ApplicationController
+  def robots
+    subdomain = request.subdomain
+
+    filename = if subdomain.present? && subdomain != 'www'
+      "robots.#{ subdomain }.txt"
+    else
+      "robots.txt"
+    end
+
+    robots_text = File.read(Rails.root.join('config', filename))
+
+    respond_to do |format|
+      format.text { render text: robots_text, layout: false }
+    end
+  end
+end

config/robots.staging.txt

@@ -0,0 +1,5 @@
+# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
+#
+# To ban all spiders from the entire site uncomment the next two lines:
+User-Agent: *
+Disallow: /

config/routes.rb

@@ -1,5 +1,6 @@
 Growstuff::Application.routes.draw do
 
+  get '/robots.txt' => 'robots#robots'
 
   resources :plant_parts
 

spec/controllers/robots_controller_spec.rb

@@ -0,0 +1,47 @@
+require 'rails_helper'
+require 'pry'
+
+describe RobotsController do 
+  describe '#robots' do 
+
+    before do
+      @request.host = "#{ subdomain }.localhost.com" 
+    end
+
+    context 'subdomain is staging' do
+      let(:subdomain) { 'staging' }
+
+      it 'loads the staging robots.txt file' do
+        expect(File).to receive(:read).with(Rails.root.join('config', 'robots.staging.txt'))
+
+        get :robots, format: :text
+
+        expect(response).to be_success
+      end 
+    end
+
+    context 'subdomain is www' do
+      let(:subdomain) { 'www' }
+
+      it 'loads the production robots.txt file' do 
+        expect(File).to receive(:read).with(Rails.root.join('config', 'robots.txt'))
+        
+        get :robots, format: :text
+
+        expect(response).to be_success
+      end 
+    end
+
+    context 'subdomain is not present' do 
+      let(:subdomain) { '' }
+
+      it 'loads the production robots.txt file' do 
+        expect(File).to receive(:read).with(Rails.root.join('config', 'robots.txt'))
+        
+        get :robots, format: :text
+
+        expect(response).to be_success
+      end
+    end
+  end
+end