merge from upstream

.travis.yml

@@ -14,6 +14,8 @@ before_script:
 script:
 - bundle exec rake db:migrate --trace
 - bundle exec rspec spec/
+services:
+  - elasticsearch
 before_deploy:
   - bundle exec script/heroku_maintenance.rb on
 deploy:
@@ -32,3 +34,4 @@ deploy:
     - restart
 after_deploy:
   - bundle exec script/heroku_maintenance.rb off
+

Gemfile

@@ -31,6 +31,7 @@ gem 'figaro'                       # for handling config via ENV variables
 gem 'cancancan', '~> 1.9'          # for checking member privileges
 gem 'gibbon'                       # for Mailchimp newsletter subscriptions
 gem 'csv_shaper'                   # CSV export
+gem 'ruby-units'                   # for unit conversion
 
 gem 'comfortable_mexican_sofa', '~> 1.12.0' # content management system
 
@@ -78,6 +79,12 @@ gem 'omniauth'
 gem 'omniauth-twitter'
 gem 'omniauth-flickr', '>= 0.0.15'
 
+# client for Elasticsearch. Elasticsearch is a flexible
+# and powerful, distributed, real-time search and analytics engine. 
+# An example of the use in the project is fuzzy crop search.
+gem "elasticsearch-model"
+gem "elasticsearch-rails"
+
 gem 'rake', '>= 10.0.0'
 
 group :production, :staging do
@@ -85,6 +92,7 @@ group :production, :staging do
   gem 'dalli'
   gem 'memcachier'
   gem 'rails_12factor' # supresses heroku plugin injection
+  gem 'bonsai-elasticsearch-rails'  # Integration with Bonsa-Elasticsearch on heroku
 end
 
 group :development do

Gemfile.lock

@@ -61,6 +61,7 @@ GEM
     binding_of_caller (0.7.2)
       debug_inspector (>= 0.0.1)
     bluecloth (2.2.0)
+    bonsai-elasticsearch-rails (0.0.4)
     bootstrap-datepicker-rails (1.3.0.2)
       railties (>= 3.0)
     bootstrap-sass (3.3.3)
@@ -139,6 +140,19 @@ GEM
       json
       thread
       thread_safe
+    elasticsearch (1.0.6)
+      elasticsearch-api (= 1.0.6)
+      elasticsearch-transport (= 1.0.6)
+    elasticsearch-api (1.0.6)
+      multi_json
+    elasticsearch-model (0.1.6)
+      activesupport (> 3)
+      elasticsearch (> 0.4)
+      hashie
+    elasticsearch-rails (0.1.6)
+    elasticsearch-transport (1.0.6)
+      faraday
+      multi_json
     erubis (2.7.0)
     excon (0.43.0)
     execjs (2.2.2)
@@ -147,6 +161,8 @@ GEM
     factory_girl_rails (4.5.0)
       factory_girl (~> 4.5.0)
       railties (>= 3.0.0)
+    faraday (0.9.1)
+      multipart-post (>= 1.2, < 3)
     figaro (1.0.0)
       thor (~> 0.14)
     flickraw (0.9.8)
@@ -226,6 +242,7 @@ GEM
     minitest (5.5.1)
     multi_json (1.10.1)
     multi_xml (0.5.5)
+    multipart-post (2.0.0)
     netrc (0.10.0)
     newrelic_rpm (3.9.8.273)
     nokogiri (1.6.5)
@@ -316,6 +333,7 @@ GEM
       rspec-mocks (~> 3.1.0)
       rspec-support (~> 3.1.0)
     rspec-support (3.1.2)
+    ruby-units (1.4.5)
     ruby_parser (3.1.3)
       sexp_processor (~> 4.1)
     sass (3.2.19)
@@ -382,6 +400,7 @@ DEPENDENCIES
   better_errors
   binding_of_caller
   bluecloth
+  bonsai-elasticsearch-rails
   bootstrap-datepicker-rails
   bundler (>= 1.1.5)
   byebug
@@ -395,6 +414,8 @@ DEPENDENCIES
   dalli
   database_cleaner (~> 1.3.0)
   devise (~> 3.4.1)
+  elasticsearch-model
+  elasticsearch-rails
   factory_girl_rails (~> 4.5.0)
   figaro
   flickraw
@@ -430,6 +451,7 @@ DEPENDENCIES
   rake (>= 10.0.0)
   rspec-activemodel-mocks
   rspec-rails (~> 3.1.0)
+  ruby-units
   sass-rails (~> 4.0.4)
   therubyracer (~> 0.12)
   uglifier (~> 2.5.3)

app/assets/stylesheets/overrides.css.less

@@ -1,5 +1,3 @@
-
-
 @import "twitter/bootstrap/bootstrap";
 @import "custom_bootstrap/variables";
 // this padding needs to be done before the responsive stuff is imported
@@ -285,6 +283,12 @@ html, body {
   text-decoration: underline;
 }
 
+.alert {
+  a {
+    font-weight: 800;
+  }
+}
+
 // Overrides applying only to mobile view. This must be at the end of the overrides file.
 
 @media only screen and (max-width: 767px) {
@@ -302,3 +306,13 @@ html, body {
     display: block;
   }
 }
+
+/* override "info" alert boxes to be green, not blue, on Growstuff */
+@state-info-text: darken(@green, 10%);
+@state-info-bg: lighten(@green, 50%);
+
+/* and set "success" to be the same, as it was just very slightly
+ * different because the default bootstrap green is slightly different
+ * from ours */
+@state-success-text: darken(@green, 10%);
+@state-success-bg: lighten(@green, 50%);

app/controllers/crops_controller.rb

@@ -49,18 +49,16 @@ class CropsController < ApplicationController
 
   # GET /crops/search
   def search
-    @search = params[:search]
-    @exact_match = Crop.find_by_name(params[:search])
-
-    @partial_matches = Crop.search(params[:search])
-    # exclude exact match from partial match list
-    @partial_matches = @partial_matches.reject{ |r| @exact_match && r.eql?(@exact_match) }
-
-    @fuzzy = Crop.search(params[:term])
+    @all_matches = Crop.search(params[:search])
+    exact_match = Crop.find_by_name(params[:search])
+    if exact_match
+      @all_matches.delete(exact_match)
+      @all_matches.unshift(exact_match)
+    end
 
     respond_to do |format|
       format.html
-      format.json { render :json => @fuzzy }
+      format.json { render :json => Crop.search(params[:term]) }
     end
   end
 

app/controllers/harvests_controller.rb

@@ -107,6 +107,6 @@ class HarvestsController < ApplicationController
 
   def harvest_params
     params.require(:harvest).permit(:crop_id, :harvested_at, :description, :owner_id,
-    :quantity, :unit, :weight_quantity, :weight_unit, :plant_part_id, :slug)
+    :quantity, :unit, :weight_quantity, :weight_unit, :plant_part_id, :slug, :si_weight)
   end
 end

app/models/alternate_name.rb

@@ -1,4 +1,5 @@
 class AlternateName < ActiveRecord::Base
+  after_commit { |an| an.crop.__elasticsearch__.index_document if an.crop }
   belongs_to :crop
   belongs_to :creator, :class_name => 'Member'
 end

app/models/crop.rb

@@ -2,12 +2,12 @@ class Crop < ActiveRecord::Base
   extend FriendlyId
   friendly_id :name, use: [:slugged, :finders]
 
-  has_many :scientific_names
+  has_many :scientific_names, after_add: :update_index, after_remove: :update_index
   accepts_nested_attributes_for :scientific_names,
     :allow_destroy => true,
     :reject_if     => :all_blank
 
-  has_many :alternate_names
+  has_many :alternate_names, after_add: :update_index, after_remove: :update_index
   has_many :plantings
   has_many :photos, :through => :plantings
   has_many :seeds
@@ -31,6 +31,63 @@ class Crop < ActiveRecord::Base
       :with => /\Ahttps?:\/\/en\.wikipedia\.org\/wiki/,
       :message => 'is not a valid English Wikipedia URL'
     }
+  
+  ####################################
+  # Elastic search configuration
+  include Elasticsearch::Model
+  include Elasticsearch::Model::Callbacks
+  # In order to avoid clashing between different environments,
+  # use Rails.env as a part of index name (eg. development_growstuff)
+  index_name [Rails.env, "growstuff"].join('_')
+  settings index: { number_of_shards: 1 },
+    analysis: {
+      tokenizer: {
+        gs_edgeNGram_tokenizer: {
+          type: "edgeNGram",  # edgeNGram: NGram match from the start of a token
+          min_gram: 3,
+          max_gram: 10,
+          # token_chars: Elasticsearch will split on characters
+          # that don’t belong to any of these classes
+          token_chars: [ "letter", "digit" ] 
+        }
+      },
+      analyzer: {
+        gs_edgeNGram_analyzer: {
+          tokenizer: "gs_edgeNGram_tokenizer",
+          filter: ["lowercase"]
+        }
+      },
+    } do
+    mappings dynamic: 'false' do
+      indexes :id, type: 'long'
+      indexes :name, type: 'string', analyzer: 'gs_edgeNGram_analyzer'
+      indexes :scientific_names do
+        indexes :scientific_name,
+          type: 'string',
+          analyzer: 'gs_edgeNGram_analyzer',
+          # Disabling field-length norm (norm). If the norm option is turned on(by default), 
+          # higher weigh would be given for shorter fields, which in our case is irrelevant.
+          norms: { enabled: false }
+      end
+      indexes :alternate_names do
+        indexes :name, type: 'string', analyzer: 'gs_edgeNGram_analyzer'
+      end
+    end
+  end
+
+  def as_indexed_json(options={})
+    self.as_json(
+      only: [:id, :name],
+      include: {
+        scientific_names: { only: :scientific_name },
+        alternate_names: { only: :name }
+             })
+  end
+
+  def update_index(name_obj)
+    __elasticsearch__.index_document
+  end
+  ####################################
 
   def to_s
     return name
@@ -201,10 +258,24 @@ class Crop < ActiveRecord::Base
   end
 
   # Crop.search(string)
-  # searches for crops whose names match the string given
-  # just uses SQL LIKE for now, but can be made fancier later
   def self.search(query)
-    where("name ILIKE ?", "%#{query}%")
+    if ENV['GROWSTUFF_ELASTICSEARCH'] == "true"
+      search_str = query.nil? ? "" : query.downcase
+      response = __elasticsearch__.search( {
+          # Finds documents which match any field, but uses the _score from 
+          # the best field insead of adding up _score from each field.
+          query: {
+            multi_match: {
+              query: "#{search_str}",
+              fields: ["name", "scientific_names.scientific_name", "alternate_names.name"]
+            }
+          },
+          size: 50
+        }
+      )
+      return response.records.to_a
+    else
+      where("name ILIKE ?", "%#{query}%") 
+    end
   end
-
 end

app/models/harvest.rb

@@ -59,6 +59,17 @@ class Harvest < ActiveRecord::Base
 
   after_validation :cleanup_quantities
 
+  before_save :set_si_weight
+
+  # we're storing the harvest weight in kilograms in the db too 
+  # to make data manipulation easier
+  def set_si_weight
+    if self.weight_unit != nil
+      weight_string = "#{self.weight_quantity} #{self.weight_unit}"
+      self.si_weight = Unit(weight_string).convert_to("kg").to_s("%0.3f").delete(" kg").to_f
+    end
+  end
+
   def cleanup_quantities
     if quantity == 0
       self.quantity = nil

app/models/scientific_name.rb

@@ -1,4 +1,5 @@
 class ScientificName < ActiveRecord::Base
+  after_commit { |sn| sn.crop.__elasticsearch__.index_document if sn.crop }
   belongs_to :crop
   belongs_to :creator, :class_name => 'Member'
 end

app/views/crops/search.html.haml

@@ -1,6 +1,6 @@
 - content_for :title, "Crops matching #{@search}"
 
-- unless (@exact_match || @partial_matches.length > 0)
+- if @all_matches.empty?
   %h2 No results found
   %p
     Sorry, we couldn't find any crops that matched your search for "#{@search}".
@@ -9,18 +9,10 @@
     = link_to "browsing our crop database", crops_path
     instead.
 
-- if @exact_match
-  %div#exact_match
-    %h2 Exact match
-
+- else 
+  %div#all_matches
     .row
-      .col-md-2.six-across
-        = render :partial => "thumbnail", :locals => { :crop => @exact_match }
-
-- if ! @partial_matches.empty?
-  %div#partial_matches
-    %h2 Partial matches
-    .row
-      - @partial_matches.each do |c|
+      - @all_matches.each do |c|
         .col-md-2.six-across
           = render :partial => "thumbnail", :locals => { :crop => c }
+

app/views/harvests/index.csv.shaper

@@ -10,6 +10,7 @@ csv.headers :id,
   :unit,
   :weight_quantity,
   :weight_unit,
+  :si_weight,
   :date_harvested,
   :description,
   :date_added,
@@ -31,7 +32,7 @@ csv.headers :id,
     csv.cell :plant_part_id, h.plant_part ? h.plant_part.id : ''
     csv.cell :plant_part_name, h.plant_part ? h.plant_part.to_s : ''
 
-    csv.cells :quantity, :unit, :weight_quantity, :weight_unit
+    csv.cells :quantity, :unit, :weight_quantity, :weight_unit, :si_weight
 
     csv.cell :date_harvested, h.created_at.to_s(:db)
 

app/views/posts/show.html.haml

@@ -1,5 +1,19 @@
 = content_for :title, @post.subject
 
+- unless current_member
+  .alert.alert-info
+    = link_to @post.author.login_name, member_path(@post.author)
+    is using
+    = link_to ENV["GROWSTUFF_SITE_NAME"], root_path
+    to discuss #{ @post.subject } with a community of food gardeners worldwide.
+    We have advice on growing
+    = succeed "," do
+      = link_to 'hundreds of different crops', crops_url
+    and a community from all around the world.
+
+    = render :partial => "shared/signin_signup", 
+      :locals => { :to => "or to start using #{ENV["GROWSTUFF_SITE_NAME"]} to track what you're planting and harvesting" }
+
 = render :partial => "single", :locals => { :post => @post, :subject => false, :hide_comments => true }
 
 - if can? :edit, @post or can? :destroy, @post

config/application.yml.example

@@ -62,6 +62,13 @@ GROWSTUFF_PAYPAL_USERNAME: "dummy"
 GROWSTUFF_PAYPAL_PASSWORD: "dummy"
 GROWSTUFF_PAYPAL_SIGNATURE: "dummy"
 
+# Elasticsearch is used for flexible search and it requires another component
+# to be installed. To make it easy for people who don't need to test this feature
+# it's been turned off for test and development environment as a default.
+# If you want to test this functionality, install elasticsearch and
+# set this flag to "true".
+GROWSTUFF_ELASTICSEARCH: "false"
+
 ##############################################################################
 # Other environments
 # You can override the above for staging, production, etc.
@@ -78,6 +85,8 @@ test:
 
 staging:
   GROWSTUFF_SITE_NAME: Growstuff (staging)
+  GROWSTUFF_ELASTICSEARCH: "true"
 
 production:
   GROWSTUFF_SITE_NAME: Growstuff
+  GROWSTUFF_ELASTICSEARCH: "true"

db/migrate/20150129034206_add_si_weight_to_harvest.rb

@@ -0,0 +1,5 @@
+class AddSiWeightToHarvest < ActiveRecord::Migration
+  def change
+    add_column :harvests, :si_weight, :float
+  end
+end

lib/tasks/growstuff.rake

@@ -314,6 +314,19 @@ namespace :growstuff do
         end
       end
     end
+
+    desc "January 2015: fill in si_weight column"
+    task :populate_si_weight => :environment do
+      Harvest.find_each do |h|
+        h.set_si_weight
+      end
+    end
+
+    desc "January 2015: build Elasticsearch index"
+    task :elasticsearch_create_index => :environment do
+      Crop.__elasticsearch__.create_index! force: true
+      Crop.import
+    end
   end # end oneoff section
 
 end

script/deploy-tasks.sh

@@ -18,3 +18,9 @@ rake growstuff:import_crops file=db/seeds/crops-15-squashes.csv
 
 echo "2014-12-01 - load alternate names for crops"
 rake growstuff:oneoff:add_alternate_names
+
+echo "2015-01-28 - populate the harvest si_weight field"
+rake growstuff:oneoff:populate_si_weight
+
+echo "2015-01-30 - build Elasticsearch index"
+rake growstuff:oneoff:elasticsearch_create_index

spec/features/harvests/harvesting_a_crop_spec.rb

@@ -7,12 +7,13 @@ feature "Harvesting a crop", :js => true do
   background do
     login_as(member)
     visit new_harvest_path
+    sync_elasticsearch([maize])
   end
 
   it_behaves_like "crop suggest", "harvest", "crop"
 
   scenario "Creating a new harvest", :js => true do
-    fill_autocomplete "crop", :with => "m"
+    fill_autocomplete "crop", :with => "mai"
     select_from_autocomplete "maize"
     within "form#new_harvest" do
       fill_in "When?", :with => "2014-06-15"

spec/features/plantings/planting_a_crop_spec.rb

@@ -9,12 +9,13 @@ feature "Planting a crop", :js => true do
   background do
     login_as(member)
     visit new_planting_path
+    sync_elasticsearch([maize])
   end
 
   it_behaves_like "crop suggest", "planting"
 
   scenario "Creating a new planting" do
-    fill_autocomplete "crop", :with => "m"
+    fill_autocomplete "crop", :with => "mai"
     select_from_autocomplete "maize"
     within "form#new_planting" do
       fill_in "When", :with => "2014-06-15"
@@ -41,7 +42,7 @@ feature "Planting a crop", :js => true do
   end
 
   scenario "Marking a planting as finished" do
-    fill_autocomplete "crop", :with => "m"
+    fill_autocomplete "crop", :with => "mai"
     select_from_autocomplete "maize"
     within "form#new_planting" do
       fill_in "When?", :with => "2014-07-01"
@@ -77,7 +78,7 @@ feature "Planting a crop", :js => true do
   end
 
   scenario "Marking a planting as finished without a date" do
-    fill_autocomplete "crop", :with => "m"
+    fill_autocomplete "crop", :with => "mai"
     select_from_autocomplete "maize"
     within "form#new_planting" do
       check "Mark as finished"

spec/features/seeds/adding_seeds_spec.rb

@@ -1,18 +1,19 @@
 require 'rails_helper'
 
-feature "Harvesting a crop", :js => true do
+feature "Seeds", :js => true do
   let(:member)   { FactoryGirl.create(:member) }
   let!(:maize)   { FactoryGirl.create(:maize) }
 
   background do
     login_as(member)
     visit new_seed_path
+    sync_elasticsearch([maize])
   end
 
   it_behaves_like "crop suggest", "seed", "crop"
 
   scenario "Adding a new seed", :js => true do
-    fill_autocomplete "crop", :with => "m"
+    fill_autocomplete "crop", :with => "mai"
     select_from_autocomplete "maize"
     within "form#new_seed" do
       fill_in "Quantity:", :with => 42

spec/features/shared_examples/crop_suggest.rb

@@ -1,29 +1,39 @@
 require 'rails_helper'
 
 shared_examples "crop suggest" do |resource|
-  let!(:popcorn) { FactoryGirl.create(:popcorn) }
+  let!(:pea)     { FactoryGirl.create(:crop, :name => 'pea') }
   let!(:pear)    { FactoryGirl.create(:pear) }
   let!(:tomato)  { FactoryGirl.create(:tomato) }
   let!(:roma)    { FactoryGirl.create(:roma) }
 
+  background do
+    sync_elasticsearch([pea, pear, maize, tomato])
+  end
+
   scenario "See text in crop auto suggest field" do
     expect(page).to have_selector("input[placeholder='e.g. lettuce']")
   end
 
   scenario "Typing in the crop name displays suggestions" do
     within "form#new_#{resource}" do
-      fill_autocomplete "crop", :with => "p"
+      fill_autocomplete "crop", :with => "pe"
+    end
+
+    expect(page).to_not have_content("pear")
+    expect(page).to_not have_content("pea")
+
+    within "form#new_#{resource}" do
+      fill_autocomplete "crop", :with => "pea"
     end
 
     expect(page).to have_content("pear")
-    expect(page).to have_content("popcorn")
+    expect(page).to have_content("pea")
 
     within "form#new_#{resource}" do
       fill_autocomplete "crop", :with => "pear"
     end
 
     expect(page).to have_content("pear")
-    expect(page).to_not have_content("popcorn")
 
     select_from_autocomplete("pear")
 
@@ -32,16 +42,16 @@ shared_examples "crop suggest" do |resource|
 
   scenario "Typing and pausing does not affect input" do
     within "form#new_#{resource}" do
-      fill_autocomplete "crop", :with => "p"
+      fill_autocomplete "crop", :with => "pea"
     end
 
     expect(page).to have_content("pear")
-    expect(find_field("crop").value).to eq("p")
+    expect(find_field("crop").value).to eq("pea")
   end
 
   scenario "Searching for a crop casts a wide net on results" do
     within "form#new_#{resource}" do
-      fill_autocomplete "crop", :with => "to"
+      fill_autocomplete "crop", :with => "tom"
     end
 
     expect(page).to have_content("tomato")

spec/models/crop_spec.rb

@@ -328,6 +328,7 @@ describe Crop do
   context "search" do
     before :each do
       @mushroom = FactoryGirl.create(:crop, :name => 'mushroom')
+      sync_elasticsearch([@mushroom])
     end
     it "finds exact matches" do
       Crop.search('mushroom').should eq [@mushroom]

spec/models/harvest_spec.rb

@@ -118,6 +118,26 @@ describe Harvest do
     end
   end
 
+  context "standardized weights" do
+    it 'converts from pounds' do
+      @harvest = FactoryGirl.create(:harvest, :weight_quantity => 2, :weight_unit => "lb")
+      @harvest.should be_valid
+      @harvest.reload.si_weight.should eq 0.907
+    end
+
+    it 'converts from ounces' do
+      @harvest = FactoryGirl.create(:harvest, :weight_quantity => 16, :weight_unit => "oz")
+      @harvest.should be_valid
+      @harvest.reload.si_weight.should eq 0.454
+    end
+
+    it 'leaves kg alone' do
+      @harvest = FactoryGirl.create(:harvest, :weight_quantity => 2, :weight_unit => "kg")
+      @harvest.should be_valid
+      @harvest.reload.si_weight.should eq 2.0
+    end
+  end
+
   context 'ordering' do
     it 'lists most recent harvests first' do
       @h1 = FactoryGirl.create(:harvest, :created_at => 1.day.ago)

spec/support/elasticsearch_helpers.rb

@@ -0,0 +1,18 @@
+module ElasticsearchHelpers
+  def sync_elasticsearch(crops)
+    if ENV['GROWSTUFF_ELASTICSEARCH'] == "true"
+      crops.each {|crop| crop.__elasticsearch__.index_document}
+      Crop.__elasticsearch__.refresh_index!
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.include ElasticsearchHelpers
+
+  config.before(:each) do
+    if ENV['GROWSTUFF_ELASTICSEARCH'] == "true"
+      Crop.__elasticsearch__.create_index! force: true
+    end
+  end
+end

spec/views/crops/search.html.haml_spec.rb

@@ -11,19 +11,18 @@ describe "crops/search" do
       @tomato = FactoryGirl.create(:tomato)
       @roma = FactoryGirl.create(:crop, :name => 'Roma tomato', :parent => @tomato)
       assign(:search, 'tomato')
-      assign(:exact_match, @tomato)
-      assign(:partial_matches, [@roma])
+      assign(:all_matches, [@tomato, @roma])
       render
     end
 
     it "shows exact matches" do
-      assert_select "div#exact_match" do
+      assert_select "div#all_matches" do
         assert_select "a[href=#{crop_path(@tomato)}]"
       end
     end
 
     it "shows partial matches" do
-      assert_select "div#partial_matches" do
+      assert_select "div#all_matches" do
         assert_select "a[href=#{crop_path(@roma)}]"
       end
     end
@@ -31,8 +30,7 @@ describe "crops/search" do
 
   context "no results" do
     before :each do
-      assign(:exact_match, nil)
-      assign(:partial_matches, [])
+      assign(:all_matches, [])
       assign(:search, 'tomato')
       render
     end