From 10e6e7c3cb11cf21a0356c1507d215c8369497b5 Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Thu, 15 Jan 2015 19:05:02 +1100 Subject: [PATCH 01/33] Integrate Elasticsearch and implement crop search against scientific_name --- Gemfile | 5 ++ Gemfile.lock | 20 ++++++++ app/controllers/crops_controller.rb | 4 +- app/models/crop.rb | 79 +++++++++++++++++++++++++++-- spec/models/crop_spec.rb | 3 ++ 5 files changed, 106 insertions(+), 5 deletions(-) diff --git a/Gemfile b/Gemfile index 36da0c2c0..dc3ac9982 100644 --- a/Gemfile +++ b/Gemfile @@ -116,6 +116,11 @@ gem 'omniauth' gem 'omniauth-twitter' gem 'omniauth-flickr', '>= 0.0.15' +# client for elastic search +gem "elasticsearch-model" +gem "elasticsearch-rails" +gem 'bonsai-elasticsearch-rails' + gem 'rake', '>= 10.0.0' group :development, :test do diff --git a/Gemfile.lock b/Gemfile.lock index 8a053b509..5b6d37bee 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -57,6 +57,7 @@ GEM binding_of_caller (0.7.2) debug_inspector (>= 0.0.1) bluecloth (2.2.0) + bonsai-elasticsearch-rails (0.0.4) bootstrap-datepicker-rails (1.3.0.2) railties (>= 3.0) builder (3.0.4) @@ -116,6 +117,19 @@ GEM json thread thread_safe + elasticsearch (1.0.6) + elasticsearch-api (= 1.0.6) + elasticsearch-transport (= 1.0.6) + elasticsearch-api (1.0.6) + multi_json + elasticsearch-model (0.1.6) + activesupport (> 3) + elasticsearch (> 0.4) + hashie + elasticsearch-rails (0.1.6) + elasticsearch-transport (1.0.6) + faraday + multi_json erubis (2.7.0) execjs (2.2.2) factory_girl (4.5.0) @@ -123,6 +137,8 @@ GEM factory_girl_rails (4.5.0) factory_girl (~> 4.5.0) railties (>= 3.0.0) + faraday (0.9.1) + multipart-post (>= 1.2, < 3) figaro (1.0.0) thor (~> 0.14) flickraw (0.9.8) @@ -193,6 +209,7 @@ GEM mini_portile (0.6.1) multi_json (1.10.1) multi_xml (0.5.5) + multipart-post (2.0.0) netrc (0.8.0) newrelic_rpm (3.9.7.266) nokogiri (1.6.5) @@ -327,6 +344,7 @@ DEPENDENCIES better_errors binding_of_caller bluecloth + bonsai-elasticsearch-rails bootstrap-datepicker-rails bundler (>= 1.1.5) byebug @@ -340,6 +358,8 @@ DEPENDENCIES dalli database_cleaner (~> 1.3.0) devise (~> 3.2.0) + elasticsearch-model + elasticsearch-rails factory_girl_rails (~> 4.0) figaro flickraw diff --git a/app/controllers/crops_controller.rb b/app/controllers/crops_controller.rb index be4eccc2e..ad46366f6 100644 --- a/app/controllers/crops_controller.rb +++ b/app/controllers/crops_controller.rb @@ -54,11 +54,11 @@ class CropsController < ApplicationController @search = params[:search] @exact_match = Crop.find_by_name(params[:search]) - @partial_matches = Crop.search(params[:search]) + @partial_matches = Crop.search(params[:search], true) # exclude exact match from partial match list @partial_matches.reject!{ |r| @exact_match && r.eql?(@exact_match) } - @fuzzy = Crop.search(params[:term]) + @fuzzy = Crop.search(params[:term], false) respond_to do |format| format.html diff --git a/app/models/crop.rb b/app/models/crop.rb index b63f9a8a5..45a5c216a 100644 --- a/app/models/crop.rb +++ b/app/models/crop.rb @@ -33,6 +33,42 @@ class Crop < ActiveRecord::Base :with => /^https?:\/\/en\.wikipedia\.org\/wiki/, :message => 'is not a valid English Wikipedia URL' } + + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + index_name [Rails.env, "growstuff"].join('_') + settings index: { number_of_shards: 1}, + analysis: { + tokenizer: { + my_ngram_tokenizer: { + type: "edgeNGram", + min_gram: 1, + max_gram: 20, + token_chars: [ "letter", "digit" ] + } + }, + analyzer: { + my_lowercase_analyzer: { + tokenizer: "my_ngram_tokenizer", + filter: ["lowercase"] + } + }, + } do + mappings dynamic: 'false' do + indexes :id, type: 'long' + indexes :name, type: 'string', analyzer: 'my_lowercase_analyzer' + indexes :scientific_names do + indexes :scientific_name, type: 'string', analyzer: 'my_lowercase_analyzer' + end + end + end + + def as_indexed_json(options={}) + self.as_json( + only: [:id, :name], + include: { scientific_names: { only: :scientific_name } + }) + end def to_s return name @@ -205,8 +241,45 @@ class Crop < ActiveRecord::Base # Crop.search(string) # searches for crops whose names match the string given # just uses SQL LIKE for now, but can be made fancier later - def self.search(query) - where("name ILIKE ?", "%#{query}%") - end + def self.search(query, option=true) + if option + response = __elasticsearch__.search( + { + query: { + bool: { + should: [ + { match: { + name: "#{query}" + } + }, + { match: { + scientific_name: "#{query}" + } + } + ] + } + }, + size: 50 + } + ) + else + response = __elasticsearch__.search( + { + query: { + bool: { + should: [ + { match: { + name: "#{query}" + } + } + ] + } + }, + size: 50 + } + ) + end + response.records.to_a + end end diff --git a/spec/models/crop_spec.rb b/spec/models/crop_spec.rb index 5bd7d01ba..77a6c3688 100644 --- a/spec/models/crop_spec.rb +++ b/spec/models/crop_spec.rb @@ -327,7 +327,10 @@ describe Crop do context "search" do before :each do + Crop.__elasticsearch__.create_index! force: true @mushroom = FactoryGirl.create(:crop, :name => 'mushroom') + @mushroom.__elasticsearch__.index_document + sleep 1 end it "finds exact matches" do Crop.search('mushroom').should eq [@mushroom] From ba3a1f6298758d1fc36a059971f8d2ed02c2fac6 Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Thu, 15 Jan 2015 19:05:02 +1100 Subject: [PATCH 02/33] Integrate Elasticsearch and implement crop search against scientific_name --- Gemfile | 5 ++++ Gemfile.lock | 20 ++++++++++++++ app/models/crop.rb | 60 ++++++++++++++++++++++++++++++++++++++-- spec/models/crop_spec.rb | 3 ++ 4 files changed, 86 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 36da0c2c0..dc3ac9982 100644 --- a/Gemfile +++ b/Gemfile @@ -116,6 +116,11 @@ gem 'omniauth' gem 'omniauth-twitter' gem 'omniauth-flickr', '>= 0.0.15' +# client for elastic search +gem "elasticsearch-model" +gem "elasticsearch-rails" +gem 'bonsai-elasticsearch-rails' + gem 'rake', '>= 10.0.0' group :development, :test do diff --git a/Gemfile.lock b/Gemfile.lock index 8a053b509..5b6d37bee 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -57,6 +57,7 @@ GEM binding_of_caller (0.7.2) debug_inspector (>= 0.0.1) bluecloth (2.2.0) + bonsai-elasticsearch-rails (0.0.4) bootstrap-datepicker-rails (1.3.0.2) railties (>= 3.0) builder (3.0.4) @@ -116,6 +117,19 @@ GEM json thread thread_safe + elasticsearch (1.0.6) + elasticsearch-api (= 1.0.6) + elasticsearch-transport (= 1.0.6) + elasticsearch-api (1.0.6) + multi_json + elasticsearch-model (0.1.6) + activesupport (> 3) + elasticsearch (> 0.4) + hashie + elasticsearch-rails (0.1.6) + elasticsearch-transport (1.0.6) + faraday + multi_json erubis (2.7.0) execjs (2.2.2) factory_girl (4.5.0) @@ -123,6 +137,8 @@ GEM factory_girl_rails (4.5.0) factory_girl (~> 4.5.0) railties (>= 3.0.0) + faraday (0.9.1) + multipart-post (>= 1.2, < 3) figaro (1.0.0) thor (~> 0.14) flickraw (0.9.8) @@ -193,6 +209,7 @@ GEM mini_portile (0.6.1) multi_json (1.10.1) multi_xml (0.5.5) + multipart-post (2.0.0) netrc (0.8.0) newrelic_rpm (3.9.7.266) nokogiri (1.6.5) @@ -327,6 +344,7 @@ DEPENDENCIES better_errors binding_of_caller bluecloth + bonsai-elasticsearch-rails bootstrap-datepicker-rails bundler (>= 1.1.5) byebug @@ -340,6 +358,8 @@ DEPENDENCIES dalli database_cleaner (~> 1.3.0) devise (~> 3.2.0) + elasticsearch-model + elasticsearch-rails factory_girl_rails (~> 4.0) figaro flickraw diff --git a/app/models/crop.rb b/app/models/crop.rb index b63f9a8a5..ace65816b 100644 --- a/app/models/crop.rb +++ b/app/models/crop.rb @@ -33,6 +33,42 @@ class Crop < ActiveRecord::Base :with => /^https?:\/\/en\.wikipedia\.org\/wiki/, :message => 'is not a valid English Wikipedia URL' } + + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + index_name [Rails.env, "growstuff"].join('_') + settings index: { number_of_shards: 1}, + analysis: { + tokenizer: { + my_ngram_tokenizer: { + type: "edgeNGram", + min_gram: 1, + max_gram: 20, + token_chars: [ "letter", "digit" ] + } + }, + analyzer: { + my_lowercase_analyzer: { + tokenizer: "my_ngram_tokenizer", + filter: ["lowercase"] + } + }, + } do + mappings dynamic: 'false' do + indexes :id, type: 'long' + indexes :name, type: 'string', analyzer: 'my_lowercase_analyzer' + indexes :scientific_names do + indexes :scientific_name, type: 'string', analyzer: 'my_lowercase_analyzer' + end + end + end + + def as_indexed_json(options={}) + self.as_json( + only: [:id, :name], + include: { scientific_names: { only: :scientific_name } + }) + end def to_s return name @@ -206,7 +242,27 @@ class Crop < ActiveRecord::Base # searches for crops whose names match the string given # just uses SQL LIKE for now, but can be made fancier later def self.search(query) - where("name ILIKE ?", "%#{query}%") - end + response = __elasticsearch__.search( + { + query: { + bool: { + should: [ + { match: { + name: "#{query}" + } + }, + { match: { + scientific_name: "#{query}" + } + } + ] + } + }, + size: 50 + } + ) + end + response.records.to_a + end end diff --git a/spec/models/crop_spec.rb b/spec/models/crop_spec.rb index 5bd7d01ba..77a6c3688 100644 --- a/spec/models/crop_spec.rb +++ b/spec/models/crop_spec.rb @@ -327,7 +327,10 @@ describe Crop do context "search" do before :each do + Crop.__elasticsearch__.create_index! force: true @mushroom = FactoryGirl.create(:crop, :name => 'mushroom') + @mushroom.__elasticsearch__.index_document + sleep 1 end it "finds exact matches" do Crop.search('mushroom').should eq [@mushroom] From 556ba33172704589c6c87fe3e353bafc395107aa Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Sat, 17 Jan 2015 00:09:39 +1100 Subject: [PATCH 03/33] updated crop search view --- app/controllers/crops_controller.rb | 17 +++++++++-------- app/views/crops/search.html.haml | 17 ++++------------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/app/controllers/crops_controller.rb b/app/controllers/crops_controller.rb index be4eccc2e..7a9e227b9 100644 --- a/app/controllers/crops_controller.rb +++ b/app/controllers/crops_controller.rb @@ -51,18 +51,19 @@ class CropsController < ApplicationController # GET /crops/search def search - @search = params[:search] - @exact_match = Crop.find_by_name(params[:search]) + query = params[:term] || params[:search] + @all_matches = Crop.search(query) - @partial_matches = Crop.search(params[:search]) - # exclude exact match from partial match list - @partial_matches.reject!{ |r| @exact_match && r.eql?(@exact_match) } - - @fuzzy = Crop.search(params[:term]) + if exact_match = Crop.find_by_name(query) + @all_matches.delete(exact_match) + @all_matches.unshift(exact_match) + end respond_to do |format| format.html - format.json { render :json => @fuzzy } + format.json { + render :json => @all_matches + } end end diff --git a/app/views/crops/search.html.haml b/app/views/crops/search.html.haml index e10ae6f76..082bfde9e 100644 --- a/app/views/crops/search.html.haml +++ b/app/views/crops/search.html.haml @@ -1,6 +1,6 @@ - content_for :title, "Crops matching #{@search}" -- unless (@exact_match || @partial_matches.length > 0) +- unless (@all_matches.length > 0) %h2 No results found %p Sorry, we couldn't find any crops that matched your search for "#{@search}". @@ -9,18 +9,9 @@ = link_to "browsing our crop database", crops_path instead. -- if @exact_match - %div#exact_match - %h2 Exact match - +- if ! @all_matches.empty? + %div#all_matches .row - .col-md-2.six-across - = render :partial => "thumbnail", :locals => { :crop => @exact_match } - -- if ! @partial_matches.empty? - %div#partial_matches - %h2 Partial matches - .row - - @partial_matches.each do |c| + - @all_matches.each do |c| .col-md-2.six-across = render :partial => "thumbnail", :locals => { :crop => c } From 88efcf4da6a69acac85d46fb3597cf307138eefa Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Sat, 17 Jan 2015 00:25:34 +1100 Subject: [PATCH 04/33] updated test spec for view change --- spec/views/crops/search.html.haml_spec.rb | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/spec/views/crops/search.html.haml_spec.rb b/spec/views/crops/search.html.haml_spec.rb index c186c6c4c..4462df9a5 100644 --- a/spec/views/crops/search.html.haml_spec.rb +++ b/spec/views/crops/search.html.haml_spec.rb @@ -11,8 +11,7 @@ describe "crops/search" do @tomato = FactoryGirl.create(:tomato) @roma = FactoryGirl.create(:crop, :name => 'Roma tomato', :parent => @tomato) assign(:search, 'tomato') - assign(:exact_match, @tomato) - assign(:partial_matches, [@roma]) + assign(:all_matches, [@tomato, @roma]) render end @@ -22,8 +21,8 @@ describe "crops/search" do end end - it "shows partial matches" do - assert_select "div#partial_matches" do + it "shows all matches" do + assert_select "div#all_matches" do assert_select "a[href=#{crop_path(@roma)}]" end end @@ -31,8 +30,7 @@ describe "crops/search" do context "no results" do before :each do - assign(:exact_match, nil) - assign(:partial_matches, []) + assign(:all_matches, []) assign(:search, 'tomato') render end From 619e8590c8884a8f1cc7dc0056e38b21cceae98d Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Sat, 17 Jan 2015 00:29:11 +1100 Subject: [PATCH 05/33] updated test spec for view change --- spec/views/crops/search.html.haml_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/views/crops/search.html.haml_spec.rb b/spec/views/crops/search.html.haml_spec.rb index 4462df9a5..f3fd5dd55 100644 --- a/spec/views/crops/search.html.haml_spec.rb +++ b/spec/views/crops/search.html.haml_spec.rb @@ -16,7 +16,7 @@ describe "crops/search" do end it "shows exact matches" do - assert_select "div#exact_match" do + assert_select "div#all_matches" do assert_select "a[href=#{crop_path(@tomato)}]" end end From 06c77036287c77cc41b76210ce142225aa2f9cae Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Sat, 17 Jan 2015 00:31:01 +1100 Subject: [PATCH 06/33] auto-suggest to use existing search --- app/controllers/crops_controller.rb | 2 +- app/models/crop.rb | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/controllers/crops_controller.rb b/app/controllers/crops_controller.rb index 7a9e227b9..4bf2e1b9c 100644 --- a/app/controllers/crops_controller.rb +++ b/app/controllers/crops_controller.rb @@ -62,7 +62,7 @@ class CropsController < ApplicationController respond_to do |format| format.html format.json { - render :json => @all_matches + render :json => Crop.autosuggest(params[:term]) } end end diff --git a/app/models/crop.rb b/app/models/crop.rb index f43ba1894..371fcb263 100644 --- a/app/models/crop.rb +++ b/app/models/crop.rb @@ -42,7 +42,7 @@ class Crop < ActiveRecord::Base tokenizer: { gs_edgeNGram_tokenizer: { type: "edgeNGram", - min_gram: 1, + min_gram: 2, max_gram: 20, token_chars: [ "letter", "digit" ] } @@ -261,4 +261,8 @@ class Crop < ActiveRecord::Base ) response.records.to_a end + + def self.autosuggest(term) + where("name ILIKE ?", "%#{term}%") + end end From f4511c79e6c078071d0c67bd257d4779ac83b1ed Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Sat, 17 Jan 2015 00:35:57 +1100 Subject: [PATCH 07/33] cleaning up the code --- app/controllers/crops_controller.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/controllers/crops_controller.rb b/app/controllers/crops_controller.rb index 4bf2e1b9c..e852123a7 100644 --- a/app/controllers/crops_controller.rb +++ b/app/controllers/crops_controller.rb @@ -51,10 +51,9 @@ class CropsController < ApplicationController # GET /crops/search def search - query = params[:term] || params[:search] - @all_matches = Crop.search(query) + @all_matches = Crop.search(params[:search]) - if exact_match = Crop.find_by_name(query) + if exact_match = Crop.find_by_name(params[:search]) @all_matches.delete(exact_match) @all_matches.unshift(exact_match) end From ce265e281afd9421543eb2fc75de4291a3fcf916 Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Sat, 17 Jan 2015 00:35:57 +1100 Subject: [PATCH 08/33] cleaning up the code --- app/controllers/crops_controller.rb | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/app/controllers/crops_controller.rb b/app/controllers/crops_controller.rb index 4bf2e1b9c..06176d267 100644 --- a/app/controllers/crops_controller.rb +++ b/app/controllers/crops_controller.rb @@ -51,19 +51,16 @@ class CropsController < ApplicationController # GET /crops/search def search - query = params[:term] || params[:search] - @all_matches = Crop.search(query) + @all_matches = Crop.search(params[:search]) - if exact_match = Crop.find_by_name(query) + if exact_match = Crop.find_by_name(params[:search]) @all_matches.delete(exact_match) @all_matches.unshift(exact_match) end respond_to do |format| format.html - format.json { - render :json => Crop.autosuggest(params[:term]) - } + format.json { render :json => Crop.autosuggest(params[:term]) } end end From 4237dfb2699bbf508c0e86f31014bd287ce35caa Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Sat, 17 Jan 2015 18:28:45 +1100 Subject: [PATCH 09/33] cleaning up the code --- app/models/crop.rb | 9 ++++----- app/views/crops/search.html.haml | 5 +++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/models/crop.rb b/app/models/crop.rb index 6c32b0049..38781333b 100644 --- a/app/models/crop.rb +++ b/app/models/crop.rb @@ -245,19 +245,18 @@ class Crop < ActiveRecord::Base # searches for crops whose names match the string given # just uses SQL LIKE for now, but can be made fancier later def self.search(query) - search_str = query.nil? ? "" : "#{query.downcase}" - response = __elasticsearch__.search( - { + search_str = query.nil? ? "" : query.downcase + response = __elasticsearch__.search( { query: { multi_match: { - query: search_str, + query: "#{search_str}", fields: ["name", "scientific_names.scientific_name", "alternate_names.name"] } }, size: 50 } ) - response.records.to_a + return response.records.to_a end def self.autosuggest(term) diff --git a/app/views/crops/search.html.haml b/app/views/crops/search.html.haml index 082bfde9e..2ed095402 100644 --- a/app/views/crops/search.html.haml +++ b/app/views/crops/search.html.haml @@ -1,6 +1,6 @@ - content_for :title, "Crops matching #{@search}" -- unless (@all_matches.length > 0) +- if @all_matches.empty? %h2 No results found %p Sorry, we couldn't find any crops that matched your search for "#{@search}". @@ -9,9 +9,10 @@ = link_to "browsing our crop database", crops_path instead. -- if ! @all_matches.empty? +- else %div#all_matches .row - @all_matches.each do |c| .col-md-2.six-across = render :partial => "thumbnail", :locals => { :crop => c } + From ad7cfdabd04cbcae8373695d95ccda3fed08c439 Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Sat, 17 Jan 2015 18:28:45 +1100 Subject: [PATCH 10/33] cleaning up the code --- app/models/crop.rb | 9 ++++----- app/views/crops/search.html.haml | 5 +++-- spec/views/crops/search.html.haml_spec.rb | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/models/crop.rb b/app/models/crop.rb index 6c32b0049..38781333b 100644 --- a/app/models/crop.rb +++ b/app/models/crop.rb @@ -245,19 +245,18 @@ class Crop < ActiveRecord::Base # searches for crops whose names match the string given # just uses SQL LIKE for now, but can be made fancier later def self.search(query) - search_str = query.nil? ? "" : "#{query.downcase}" - response = __elasticsearch__.search( - { + search_str = query.nil? ? "" : query.downcase + response = __elasticsearch__.search( { query: { multi_match: { - query: search_str, + query: "#{search_str}", fields: ["name", "scientific_names.scientific_name", "alternate_names.name"] } }, size: 50 } ) - response.records.to_a + return response.records.to_a end def self.autosuggest(term) diff --git a/app/views/crops/search.html.haml b/app/views/crops/search.html.haml index 082bfde9e..2ed095402 100644 --- a/app/views/crops/search.html.haml +++ b/app/views/crops/search.html.haml @@ -1,6 +1,6 @@ - content_for :title, "Crops matching #{@search}" -- unless (@all_matches.length > 0) +- if @all_matches.empty? %h2 No results found %p Sorry, we couldn't find any crops that matched your search for "#{@search}". @@ -9,9 +9,10 @@ = link_to "browsing our crop database", crops_path instead. -- if ! @all_matches.empty? +- else %div#all_matches .row - @all_matches.each do |c| .col-md-2.six-across = render :partial => "thumbnail", :locals => { :crop => c } + diff --git a/spec/views/crops/search.html.haml_spec.rb b/spec/views/crops/search.html.haml_spec.rb index 8878f01c0..8458f254e 100644 --- a/spec/views/crops/search.html.haml_spec.rb +++ b/spec/views/crops/search.html.haml_spec.rb @@ -21,7 +21,7 @@ describe "crops/search" do end end - it "shows all matches" do + it "shows partial matches" do assert_select "div#all_matches" do assert_select "a[href=#{crop_path(@roma)}]" end From 2a184bcb2ecc0d385cf9944711557902e7d6122c Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Sun, 18 Jan 2015 00:20:50 +1100 Subject: [PATCH 11/33] updated test spec for improved elasticsearch syncing method --- spec/models/crop_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/models/crop_spec.rb b/spec/models/crop_spec.rb index 3a24c339c..012a74ed5 100644 --- a/spec/models/crop_spec.rb +++ b/spec/models/crop_spec.rb @@ -329,7 +329,7 @@ describe Crop do before :each do @mushroom = FactoryGirl.create(:crop, :name => 'mushroom') @mushroom.__elasticsearch__.index_document - sleep 1 + Crop.__elasticsearch__.refresh_index! end it "finds exact matches" do Crop.search('mushroom').should eq [@mushroom] From cd57c9cd34733b210325ce902cb0e2e2cc1330fe Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Sun, 18 Jan 2015 00:24:51 +1100 Subject: [PATCH 12/33] updated test so elasticsearch index is deleted before every test --- spec/support/database_cleaner.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/support/database_cleaner.rb b/spec/support/database_cleaner.rb index b3e908378..9b5ec34cd 100644 --- a/spec/support/database_cleaner.rb +++ b/spec/support/database_cleaner.rb @@ -14,6 +14,7 @@ RSpec.configure do |config| config.before(:each) do DatabaseCleaner.start + Crop.__elasticsearch__.create_index! force: true end config.after(:each) do From 99eb33ccbb918c04af89e753083ed7349f0382ba Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Sun, 18 Jan 2015 12:23:54 +1100 Subject: [PATCH 13/33] clearning up the code --- Gemfile | 6 ++++-- app/controllers/crops_controller.rb | 4 ++-- app/models/crop.rb | 7 +++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Gemfile b/Gemfile index bbce8f785..a1a6f1a18 100644 --- a/Gemfile +++ b/Gemfile @@ -76,10 +76,11 @@ gem 'omniauth' gem 'omniauth-twitter' gem 'omniauth-flickr', '>= 0.0.15' -# client for elastic search +# client for Elastic search. Elasticsearch is a flexible +# and powerful, distributed, real-time search and analytics engine. +# An example of the use in the project is fuzzy crop search. gem "elasticsearch-model" gem "elasticsearch-rails" -gem 'bonsai-elasticsearch-rails' gem 'rake', '>= 10.0.0' @@ -88,6 +89,7 @@ group :production, :staging do gem 'dalli' gem 'memcachier' gem 'rails_12factor' # supresses heroku plugin injection + gem 'bonsai-elasticsearch-rails' # Integration with Bonsa-Elasticsearch on heroku end group :development do diff --git a/app/controllers/crops_controller.rb b/app/controllers/crops_controller.rb index aaaca5c27..5d71b21dc 100644 --- a/app/controllers/crops_controller.rb +++ b/app/controllers/crops_controller.rb @@ -50,8 +50,8 @@ class CropsController < ApplicationController # GET /crops/search def search @all_matches = Crop.search(params[:search]) - - if exact_match = Crop.find_by_name(params[:search]) + exact_match = Crop.find_by_name(params[:search]) + if exact_match @all_matches.delete(exact_match) @all_matches.unshift(exact_match) end diff --git a/app/models/crop.rb b/app/models/crop.rb index 38781333b..7c0461e39 100644 --- a/app/models/crop.rb +++ b/app/models/crop.rb @@ -242,8 +242,6 @@ class Crop < ActiveRecord::Base end # Crop.search(string) - # searches for crops whose names match the string given - # just uses SQL LIKE for now, but can be made fancier later def self.search(query) search_str = query.nil? ? "" : query.downcase response = __elasticsearch__.search( { @@ -259,6 +257,11 @@ class Crop < ActiveRecord::Base return response.records.to_a end + # Crop.autosuggest(string) + # ActiveRecord search is used for auto-suggest instead of Elasticsearch. + # This is to avoid confusing users since the drop-down list for auto-suggest + # only caters for crop names at this time, not including scientific names or + # alternative names. def self.autosuggest(term) where("name ILIKE ?", "%#{term}%") end From 666d6dac48c7d9447ea3d94b105abcb13e87508d Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Mon, 19 Jan 2015 15:24:11 +1100 Subject: [PATCH 14/33] adding elasticsearch service to Travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index af5732a70..4351f77fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,3 +10,5 @@ before_script: script: - bundle exec rake db:migrate --trace - bundle exec rspec spec/ +services: + - elasticsearch \ No newline at end of file From 79e1835216af60707fb283be08b2d88c91c614e2 Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Mon, 19 Jan 2015 15:45:59 +1100 Subject: [PATCH 15/33] use elasticsearch for auto-suggest --- app/controllers/crops_controller.rb | 2 +- app/models/crop.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/crops_controller.rb b/app/controllers/crops_controller.rb index 5d71b21dc..07c4ccf39 100644 --- a/app/controllers/crops_controller.rb +++ b/app/controllers/crops_controller.rb @@ -58,7 +58,7 @@ class CropsController < ApplicationController respond_to do |format| format.html - format.json { render :json => Crop.autosuggest(params[:term]) } + format.json { render :json => Crop.search(params[:term]) } end end diff --git a/app/models/crop.rb b/app/models/crop.rb index 7c0461e39..c8678c458 100644 --- a/app/models/crop.rb +++ b/app/models/crop.rb @@ -40,8 +40,8 @@ class Crop < ActiveRecord::Base tokenizer: { gs_edgeNGram_tokenizer: { type: "edgeNGram", - min_gram: 2, - max_gram: 20, + min_gram: 4, + max_gram: 10, token_chars: [ "letter", "digit" ] } }, From f910fdfa73efa6790079f112f11361ff85b662c0 Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Mon, 19 Jan 2015 15:45:59 +1100 Subject: [PATCH 16/33] use elasticsearch for auto-suggest --- app/controllers/crops_controller.rb | 2 +- app/models/crop.rb | 13 ++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/app/controllers/crops_controller.rb b/app/controllers/crops_controller.rb index 5d71b21dc..07c4ccf39 100644 --- a/app/controllers/crops_controller.rb +++ b/app/controllers/crops_controller.rb @@ -58,7 +58,7 @@ class CropsController < ApplicationController respond_to do |format| format.html - format.json { render :json => Crop.autosuggest(params[:term]) } + format.json { render :json => Crop.search(params[:term]) } end end diff --git a/app/models/crop.rb b/app/models/crop.rb index 7c0461e39..d6937b838 100644 --- a/app/models/crop.rb +++ b/app/models/crop.rb @@ -40,8 +40,8 @@ class Crop < ActiveRecord::Base tokenizer: { gs_edgeNGram_tokenizer: { type: "edgeNGram", - min_gram: 2, - max_gram: 20, + min_gram: 4, + max_gram: 10, token_chars: [ "letter", "digit" ] } }, @@ -256,13 +256,4 @@ class Crop < ActiveRecord::Base ) return response.records.to_a end - - # Crop.autosuggest(string) - # ActiveRecord search is used for auto-suggest instead of Elasticsearch. - # This is to avoid confusing users since the drop-down list for auto-suggest - # only caters for crop names at this time, not including scientific names or - # alternative names. - def self.autosuggest(term) - where("name ILIKE ?", "%#{term}%") - end end From d12ec968c41233a51aec6307cb901400675eff44 Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Wed, 21 Jan 2015 16:44:59 +1100 Subject: [PATCH 17/33] elasticsearch to be turned off as a default in dev and test --- app/models/crop.rb | 30 +++++++++++-------- config/application.yml.example | 9 ++++++ .../harvests/harvesting_a_crop_spec.rb | 3 +- .../plantings/planting_a_crop_spec.rb | 7 +++-- spec/features/seeds/adding_seeds_spec.rb | 3 +- spec/features/shared_examples/crop_suggest.rb | 24 ++++++++++----- spec/models/crop_spec.rb | 3 +- spec/support/database_cleaner.rb | 1 - spec/support/elastisearch_helpers.rb | 18 +++++++++++ 9 files changed, 70 insertions(+), 28 deletions(-) create mode 100644 spec/support/elastisearch_helpers.rb diff --git a/app/models/crop.rb b/app/models/crop.rb index d6937b838..fcaf84147 100644 --- a/app/models/crop.rb +++ b/app/models/crop.rb @@ -40,7 +40,7 @@ class Crop < ActiveRecord::Base tokenizer: { gs_edgeNGram_tokenizer: { type: "edgeNGram", - min_gram: 4, + min_gram: 3, max_gram: 10, token_chars: [ "letter", "digit" ] } @@ -243,17 +243,21 @@ class Crop < ActiveRecord::Base # Crop.search(string) def self.search(query) - search_str = query.nil? ? "" : query.downcase - response = __elasticsearch__.search( { - query: { - multi_match: { - query: "#{search_str}", - fields: ["name", "scientific_names.scientific_name", "alternate_names.name"] - } - }, - size: 50 - } - ) - return response.records.to_a + if ENV['GROWSTUFF_ELASTICSEARCH'] == "true" + search_str = query.nil? ? "" : query.downcase + response = __elasticsearch__.search( { + query: { + multi_match: { + query: "#{search_str}", + fields: ["name", "scientific_names.scientific_name", "alternate_names.name"] + } + }, + size: 50 + } + ) + return response.records.to_a + else + where("name ILIKE ?", "%#{query}%") + end end end diff --git a/config/application.yml.example b/config/application.yml.example index f00978be0..03e8afe2c 100644 --- a/config/application.yml.example +++ b/config/application.yml.example @@ -62,6 +62,13 @@ GROWSTUFF_PAYPAL_USERNAME: "dummy" GROWSTUFF_PAYPAL_PASSWORD: "dummy" GROWSTUFF_PAYPAL_SIGNATURE: "dummy" +# Elasticsearch is used for flexible search and it requires another component +# to be installed. To make it easy for people who don't need to test this feature +# it's been turned off for test and development environment as a default. +# If you want to test this functionality, install elasticsearch and +# set this flag to "true". +GROWSTUFF_ELASTICSEARCH: "false" + ############################################################################## # Other environments # You can override the above for staging, production, etc. @@ -78,6 +85,8 @@ test: staging: GROWSTUFF_SITE_NAME: Growstuff (staging) + GROWSTUFF_ELASTICSEARCH: "true" production: GROWSTUFF_SITE_NAME: Growstuff + GROWSTUFF_ELASTICSEARCH: "true" diff --git a/spec/features/harvests/harvesting_a_crop_spec.rb b/spec/features/harvests/harvesting_a_crop_spec.rb index 40c0eb8f8..c1148cf4b 100644 --- a/spec/features/harvests/harvesting_a_crop_spec.rb +++ b/spec/features/harvests/harvesting_a_crop_spec.rb @@ -7,12 +7,13 @@ feature "Harvesting a crop", :js => true do background do login_as(member) visit new_harvest_path + sync_elasticsearch([maize]) end it_behaves_like "crop suggest", "harvest", "crop" scenario "Creating a new harvest", :js => true do - fill_autocomplete "crop", :with => "m" + fill_autocomplete "crop", :with => "mai" select_from_autocomplete "maize" within "form#new_harvest" do fill_in "When?", :with => "2014-06-15" diff --git a/spec/features/plantings/planting_a_crop_spec.rb b/spec/features/plantings/planting_a_crop_spec.rb index 16b47973f..bda85fadc 100644 --- a/spec/features/plantings/planting_a_crop_spec.rb +++ b/spec/features/plantings/planting_a_crop_spec.rb @@ -9,12 +9,13 @@ feature "Planting a crop", :js => true do background do login_as(member) visit new_planting_path + sync_elasticsearch([maize]) end it_behaves_like "crop suggest", "planting" scenario "Creating a new planting" do - fill_autocomplete "crop", :with => "m" + fill_autocomplete "crop", :with => "mai" select_from_autocomplete "maize" within "form#new_planting" do fill_in "When", :with => "2014-06-15" @@ -41,7 +42,7 @@ feature "Planting a crop", :js => true do end scenario "Marking a planting as finished" do - fill_autocomplete "crop", :with => "m" + fill_autocomplete "crop", :with => "mai" select_from_autocomplete "maize" within "form#new_planting" do fill_in "When?", :with => "2014-07-01" @@ -77,7 +78,7 @@ feature "Planting a crop", :js => true do end scenario "Marking a planting as finished without a date" do - fill_autocomplete "crop", :with => "m" + fill_autocomplete "crop", :with => "mai" select_from_autocomplete "maize" within "form#new_planting" do check "Mark as finished" diff --git a/spec/features/seeds/adding_seeds_spec.rb b/spec/features/seeds/adding_seeds_spec.rb index 668771850..f42c05758 100644 --- a/spec/features/seeds/adding_seeds_spec.rb +++ b/spec/features/seeds/adding_seeds_spec.rb @@ -7,12 +7,13 @@ feature "Harvesting a crop", :js => true do background do login_as(member) visit new_seed_path + sync_elasticsearch([maize]) end it_behaves_like "crop suggest", "seed", "crop" scenario "Adding a new seed", :js => true do - fill_autocomplete "crop", :with => "m" + fill_autocomplete "crop", :with => "mai" select_from_autocomplete "maize" within "form#new_seed" do fill_in "Quantity:", :with => 42 diff --git a/spec/features/shared_examples/crop_suggest.rb b/spec/features/shared_examples/crop_suggest.rb index 141dfbabf..9856282ca 100644 --- a/spec/features/shared_examples/crop_suggest.rb +++ b/spec/features/shared_examples/crop_suggest.rb @@ -1,29 +1,39 @@ require 'rails_helper' shared_examples "crop suggest" do |resource| - let!(:popcorn) { FactoryGirl.create(:popcorn) } + let!(:pea) { FactoryGirl.create(:crop, :name => 'pea') } let!(:pear) { FactoryGirl.create(:pear) } let!(:tomato) { FactoryGirl.create(:tomato) } let!(:roma) { FactoryGirl.create(:roma) } + background do + sync_elasticsearch([pea, pear, maize, tomato]) + end + scenario "See text in crop auto suggest field" do expect(page).to have_selector("input[placeholder='e.g. lettuce']") end scenario "Typing in the crop name displays suggestions" do within "form#new_#{resource}" do - fill_autocomplete "crop", :with => "p" + fill_autocomplete "crop", :with => "pe" + end + + expect(page).to_not have_content("pear") + expect(page).to_not have_content("pea") + + within "form#new_#{resource}" do + fill_autocomplete "crop", :with => "pea" end expect(page).to have_content("pear") - expect(page).to have_content("popcorn") + expect(page).to have_content("pea") within "form#new_#{resource}" do fill_autocomplete "crop", :with => "pear" end expect(page).to have_content("pear") - expect(page).to_not have_content("popcorn") select_from_autocomplete("pear") @@ -32,16 +42,16 @@ shared_examples "crop suggest" do |resource| scenario "Typing and pausing does not affect input" do within "form#new_#{resource}" do - fill_autocomplete "crop", :with => "p" + fill_autocomplete "crop", :with => "pea" end expect(page).to have_content("pear") - expect(find_field("crop").value).to eq("p") + expect(find_field("crop").value).to eq("pea") end scenario "Searching for a crop casts a wide net on results" do within "form#new_#{resource}" do - fill_autocomplete "crop", :with => "to" + fill_autocomplete "crop", :with => "tom" end expect(page).to have_content("tomato") diff --git a/spec/models/crop_spec.rb b/spec/models/crop_spec.rb index 012a74ed5..9cda818e8 100644 --- a/spec/models/crop_spec.rb +++ b/spec/models/crop_spec.rb @@ -328,8 +328,7 @@ describe Crop do context "search" do before :each do @mushroom = FactoryGirl.create(:crop, :name => 'mushroom') - @mushroom.__elasticsearch__.index_document - Crop.__elasticsearch__.refresh_index! + sync_elasticsearch([@mushroom]) end it "finds exact matches" do Crop.search('mushroom').should eq [@mushroom] diff --git a/spec/support/database_cleaner.rb b/spec/support/database_cleaner.rb index 9b5ec34cd..b3e908378 100644 --- a/spec/support/database_cleaner.rb +++ b/spec/support/database_cleaner.rb @@ -14,7 +14,6 @@ RSpec.configure do |config| config.before(:each) do DatabaseCleaner.start - Crop.__elasticsearch__.create_index! force: true end config.after(:each) do diff --git a/spec/support/elastisearch_helpers.rb b/spec/support/elastisearch_helpers.rb new file mode 100644 index 000000000..0e80ea321 --- /dev/null +++ b/spec/support/elastisearch_helpers.rb @@ -0,0 +1,18 @@ +module ElasticsearchHelpers + def sync_elasticsearch(crops) + if ENV['GROWSTUFF_ELASTICSEARCH'] == "true" + crops.each {|crop| crop.__elasticsearch__.index_document} + Crop.__elasticsearch__.refresh_index! + end + end +end + +RSpec.configure do |config| + config.include ElasticsearchHelpers + + config.before(:each) do + if ENV['GROWSTUFF_ELASTICSEARCH'] == "true" + Crop.__elasticsearch__.create_index! force: true + end + end +end \ No newline at end of file From 018b2b4711a8aa737deb7ca69f14aaa595f41440 Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Wed, 21 Jan 2015 16:44:59 +1100 Subject: [PATCH 18/33] elasticsearch to be turned off as a default in dev and test --- app/models/crop.rb | 30 +++++++++++-------- config/application.yml.example | 9 ++++++ .../harvests/harvesting_a_crop_spec.rb | 3 +- .../plantings/planting_a_crop_spec.rb | 7 +++-- spec/features/seeds/adding_seeds_spec.rb | 3 +- spec/features/shared_examples/crop_suggest.rb | 24 ++++++++++----- spec/models/crop_spec.rb | 3 +- spec/support/database_cleaner.rb | 1 - spec/support/elasticsearch_helpers.rb | 18 +++++++++++ 9 files changed, 70 insertions(+), 28 deletions(-) create mode 100644 spec/support/elasticsearch_helpers.rb diff --git a/app/models/crop.rb b/app/models/crop.rb index d6937b838..fcaf84147 100644 --- a/app/models/crop.rb +++ b/app/models/crop.rb @@ -40,7 +40,7 @@ class Crop < ActiveRecord::Base tokenizer: { gs_edgeNGram_tokenizer: { type: "edgeNGram", - min_gram: 4, + min_gram: 3, max_gram: 10, token_chars: [ "letter", "digit" ] } @@ -243,17 +243,21 @@ class Crop < ActiveRecord::Base # Crop.search(string) def self.search(query) - search_str = query.nil? ? "" : query.downcase - response = __elasticsearch__.search( { - query: { - multi_match: { - query: "#{search_str}", - fields: ["name", "scientific_names.scientific_name", "alternate_names.name"] - } - }, - size: 50 - } - ) - return response.records.to_a + if ENV['GROWSTUFF_ELASTICSEARCH'] == "true" + search_str = query.nil? ? "" : query.downcase + response = __elasticsearch__.search( { + query: { + multi_match: { + query: "#{search_str}", + fields: ["name", "scientific_names.scientific_name", "alternate_names.name"] + } + }, + size: 50 + } + ) + return response.records.to_a + else + where("name ILIKE ?", "%#{query}%") + end end end diff --git a/config/application.yml.example b/config/application.yml.example index f00978be0..03e8afe2c 100644 --- a/config/application.yml.example +++ b/config/application.yml.example @@ -62,6 +62,13 @@ GROWSTUFF_PAYPAL_USERNAME: "dummy" GROWSTUFF_PAYPAL_PASSWORD: "dummy" GROWSTUFF_PAYPAL_SIGNATURE: "dummy" +# Elasticsearch is used for flexible search and it requires another component +# to be installed. To make it easy for people who don't need to test this feature +# it's been turned off for test and development environment as a default. +# If you want to test this functionality, install elasticsearch and +# set this flag to "true". +GROWSTUFF_ELASTICSEARCH: "false" + ############################################################################## # Other environments # You can override the above for staging, production, etc. @@ -78,6 +85,8 @@ test: staging: GROWSTUFF_SITE_NAME: Growstuff (staging) + GROWSTUFF_ELASTICSEARCH: "true" production: GROWSTUFF_SITE_NAME: Growstuff + GROWSTUFF_ELASTICSEARCH: "true" diff --git a/spec/features/harvests/harvesting_a_crop_spec.rb b/spec/features/harvests/harvesting_a_crop_spec.rb index 40c0eb8f8..c1148cf4b 100644 --- a/spec/features/harvests/harvesting_a_crop_spec.rb +++ b/spec/features/harvests/harvesting_a_crop_spec.rb @@ -7,12 +7,13 @@ feature "Harvesting a crop", :js => true do background do login_as(member) visit new_harvest_path + sync_elasticsearch([maize]) end it_behaves_like "crop suggest", "harvest", "crop" scenario "Creating a new harvest", :js => true do - fill_autocomplete "crop", :with => "m" + fill_autocomplete "crop", :with => "mai" select_from_autocomplete "maize" within "form#new_harvest" do fill_in "When?", :with => "2014-06-15" diff --git a/spec/features/plantings/planting_a_crop_spec.rb b/spec/features/plantings/planting_a_crop_spec.rb index 16b47973f..bda85fadc 100644 --- a/spec/features/plantings/planting_a_crop_spec.rb +++ b/spec/features/plantings/planting_a_crop_spec.rb @@ -9,12 +9,13 @@ feature "Planting a crop", :js => true do background do login_as(member) visit new_planting_path + sync_elasticsearch([maize]) end it_behaves_like "crop suggest", "planting" scenario "Creating a new planting" do - fill_autocomplete "crop", :with => "m" + fill_autocomplete "crop", :with => "mai" select_from_autocomplete "maize" within "form#new_planting" do fill_in "When", :with => "2014-06-15" @@ -41,7 +42,7 @@ feature "Planting a crop", :js => true do end scenario "Marking a planting as finished" do - fill_autocomplete "crop", :with => "m" + fill_autocomplete "crop", :with => "mai" select_from_autocomplete "maize" within "form#new_planting" do fill_in "When?", :with => "2014-07-01" @@ -77,7 +78,7 @@ feature "Planting a crop", :js => true do end scenario "Marking a planting as finished without a date" do - fill_autocomplete "crop", :with => "m" + fill_autocomplete "crop", :with => "mai" select_from_autocomplete "maize" within "form#new_planting" do check "Mark as finished" diff --git a/spec/features/seeds/adding_seeds_spec.rb b/spec/features/seeds/adding_seeds_spec.rb index 668771850..f42c05758 100644 --- a/spec/features/seeds/adding_seeds_spec.rb +++ b/spec/features/seeds/adding_seeds_spec.rb @@ -7,12 +7,13 @@ feature "Harvesting a crop", :js => true do background do login_as(member) visit new_seed_path + sync_elasticsearch([maize]) end it_behaves_like "crop suggest", "seed", "crop" scenario "Adding a new seed", :js => true do - fill_autocomplete "crop", :with => "m" + fill_autocomplete "crop", :with => "mai" select_from_autocomplete "maize" within "form#new_seed" do fill_in "Quantity:", :with => 42 diff --git a/spec/features/shared_examples/crop_suggest.rb b/spec/features/shared_examples/crop_suggest.rb index 141dfbabf..9856282ca 100644 --- a/spec/features/shared_examples/crop_suggest.rb +++ b/spec/features/shared_examples/crop_suggest.rb @@ -1,29 +1,39 @@ require 'rails_helper' shared_examples "crop suggest" do |resource| - let!(:popcorn) { FactoryGirl.create(:popcorn) } + let!(:pea) { FactoryGirl.create(:crop, :name => 'pea') } let!(:pear) { FactoryGirl.create(:pear) } let!(:tomato) { FactoryGirl.create(:tomato) } let!(:roma) { FactoryGirl.create(:roma) } + background do + sync_elasticsearch([pea, pear, maize, tomato]) + end + scenario "See text in crop auto suggest field" do expect(page).to have_selector("input[placeholder='e.g. lettuce']") end scenario "Typing in the crop name displays suggestions" do within "form#new_#{resource}" do - fill_autocomplete "crop", :with => "p" + fill_autocomplete "crop", :with => "pe" + end + + expect(page).to_not have_content("pear") + expect(page).to_not have_content("pea") + + within "form#new_#{resource}" do + fill_autocomplete "crop", :with => "pea" end expect(page).to have_content("pear") - expect(page).to have_content("popcorn") + expect(page).to have_content("pea") within "form#new_#{resource}" do fill_autocomplete "crop", :with => "pear" end expect(page).to have_content("pear") - expect(page).to_not have_content("popcorn") select_from_autocomplete("pear") @@ -32,16 +42,16 @@ shared_examples "crop suggest" do |resource| scenario "Typing and pausing does not affect input" do within "form#new_#{resource}" do - fill_autocomplete "crop", :with => "p" + fill_autocomplete "crop", :with => "pea" end expect(page).to have_content("pear") - expect(find_field("crop").value).to eq("p") + expect(find_field("crop").value).to eq("pea") end scenario "Searching for a crop casts a wide net on results" do within "form#new_#{resource}" do - fill_autocomplete "crop", :with => "to" + fill_autocomplete "crop", :with => "tom" end expect(page).to have_content("tomato") diff --git a/spec/models/crop_spec.rb b/spec/models/crop_spec.rb index 012a74ed5..9cda818e8 100644 --- a/spec/models/crop_spec.rb +++ b/spec/models/crop_spec.rb @@ -328,8 +328,7 @@ describe Crop do context "search" do before :each do @mushroom = FactoryGirl.create(:crop, :name => 'mushroom') - @mushroom.__elasticsearch__.index_document - Crop.__elasticsearch__.refresh_index! + sync_elasticsearch([@mushroom]) end it "finds exact matches" do Crop.search('mushroom').should eq [@mushroom] diff --git a/spec/support/database_cleaner.rb b/spec/support/database_cleaner.rb index 9b5ec34cd..b3e908378 100644 --- a/spec/support/database_cleaner.rb +++ b/spec/support/database_cleaner.rb @@ -14,7 +14,6 @@ RSpec.configure do |config| config.before(:each) do DatabaseCleaner.start - Crop.__elasticsearch__.create_index! force: true end config.after(:each) do diff --git a/spec/support/elasticsearch_helpers.rb b/spec/support/elasticsearch_helpers.rb new file mode 100644 index 000000000..0e80ea321 --- /dev/null +++ b/spec/support/elasticsearch_helpers.rb @@ -0,0 +1,18 @@ +module ElasticsearchHelpers + def sync_elasticsearch(crops) + if ENV['GROWSTUFF_ELASTICSEARCH'] == "true" + crops.each {|crop| crop.__elasticsearch__.index_document} + Crop.__elasticsearch__.refresh_index! + end + end +end + +RSpec.configure do |config| + config.include ElasticsearchHelpers + + config.before(:each) do + if ENV['GROWSTUFF_ELASTICSEARCH'] == "true" + Crop.__elasticsearch__.create_index! force: true + end + end +end \ No newline at end of file From 5a96b7efd6417b52e9d94a5d073bebc25db06e55 Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Sun, 25 Jan 2015 15:47:40 +1100 Subject: [PATCH 19/33] adding a rake task to generate elasticsearch index --- lib/tasks/growstuff.rake | 7 +++++++ spec/support/elasticsearch_helpers.rb | 18 ------------------ 2 files changed, 7 insertions(+), 18 deletions(-) delete mode 100644 spec/support/elasticsearch_helpers.rb diff --git a/lib/tasks/growstuff.rake b/lib/tasks/growstuff.rake index 656961fd3..5db6f631b 100644 --- a/lib/tasks/growstuff.rake +++ b/lib/tasks/growstuff.rake @@ -314,6 +314,13 @@ namespace :growstuff do end end end + + desc "January 2015: build Elasticsearch index" + task :elasticsearch_create_index => :environment do + Crop.__elasticsearch__.create_index! force: true + Crop.import + end + end # end oneoff section end diff --git a/spec/support/elasticsearch_helpers.rb b/spec/support/elasticsearch_helpers.rb deleted file mode 100644 index 0e80ea321..000000000 --- a/spec/support/elasticsearch_helpers.rb +++ /dev/null @@ -1,18 +0,0 @@ -module ElasticsearchHelpers - def sync_elasticsearch(crops) - if ENV['GROWSTUFF_ELASTICSEARCH'] == "true" - crops.each {|crop| crop.__elasticsearch__.index_document} - Crop.__elasticsearch__.refresh_index! - end - end -end - -RSpec.configure do |config| - config.include ElasticsearchHelpers - - config.before(:each) do - if ENV['GROWSTUFF_ELASTICSEARCH'] == "true" - Crop.__elasticsearch__.create_index! force: true - end - end -end \ No newline at end of file From 04d61bd04035d6d88ab45a843b8e47929f21546e Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Sun, 25 Jan 2015 16:18:26 +1100 Subject: [PATCH 20/33] deployment sript --- lib/tasks/growstuff.rake | 1 - script/deploy-tasks.sh | 5 +++++ spec/support/elastisearch_helpers.rb | 18 ------------------ 3 files changed, 5 insertions(+), 19 deletions(-) delete mode 100644 spec/support/elastisearch_helpers.rb diff --git a/lib/tasks/growstuff.rake b/lib/tasks/growstuff.rake index 5db6f631b..db0b928d3 100644 --- a/lib/tasks/growstuff.rake +++ b/lib/tasks/growstuff.rake @@ -320,7 +320,6 @@ namespace :growstuff do Crop.__elasticsearch__.create_index! force: true Crop.import end - end # end oneoff section end diff --git a/script/deploy-tasks.sh b/script/deploy-tasks.sh index 3d427ed85..5684afbbf 100755 --- a/script/deploy-tasks.sh +++ b/script/deploy-tasks.sh @@ -18,3 +18,8 @@ rake growstuff:import_crops file=db/seeds/crops-15-squashes.csv echo "2014-12-01 - load alternate names for crops" rake growstuff:oneoff:add_alternate_names + +echo "2015-01-30 - build Elasticsearch index" +rake growstuff:oneoff:elasticsearch_create_index + + diff --git a/spec/support/elastisearch_helpers.rb b/spec/support/elastisearch_helpers.rb deleted file mode 100644 index 0e80ea321..000000000 --- a/spec/support/elastisearch_helpers.rb +++ /dev/null @@ -1,18 +0,0 @@ -module ElasticsearchHelpers - def sync_elasticsearch(crops) - if ENV['GROWSTUFF_ELASTICSEARCH'] == "true" - crops.each {|crop| crop.__elasticsearch__.index_document} - Crop.__elasticsearch__.refresh_index! - end - end -end - -RSpec.configure do |config| - config.include ElasticsearchHelpers - - config.before(:each) do - if ENV['GROWSTUFF_ELASTICSEARCH'] == "true" - Crop.__elasticsearch__.create_index! force: true - end - end -end \ No newline at end of file From 2c4e768a3afddd2d78fb7233cd9207c1d885d65f Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Mon, 26 Jan 2015 22:30:19 +1100 Subject: [PATCH 21/33] fixing wrong file name --- spec/support/elasticsearch_helpers.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 spec/support/elasticsearch_helpers.rb diff --git a/spec/support/elasticsearch_helpers.rb b/spec/support/elasticsearch_helpers.rb new file mode 100644 index 000000000..0d89eb33b --- /dev/null +++ b/spec/support/elasticsearch_helpers.rb @@ -0,0 +1,18 @@ +module ElasticsearchHelpers + def sync_elasticsearch(crops) + if ENV['GROWSTUFF_ELASTICSEARCH'] == "true" + crops.each {|crop| crop.__elasticsearch__.index_document} + Crop.__elasticsearch__.refresh_index! + end + end +end + +RSpec.configure do |config| + config.include ElasticsearchHelpers + + config.before(:each) do + if ENV['GROWSTUFF_ELASTICSEARCH'] == "true" + Crop.__elasticsearch__.create_index! force: true + end + end +end From b414598b070857a9a0f58a7da57cd21d90fdc61e Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Mon, 26 Jan 2015 22:50:47 +1100 Subject: [PATCH 22/33] adding comments for search options --- app/models/crop.rb | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/app/models/crop.rb b/app/models/crop.rb index fcaf84147..f9c21fa2d 100644 --- a/app/models/crop.rb +++ b/app/models/crop.rb @@ -32,17 +32,23 @@ class Crop < ActiveRecord::Base :message => 'is not a valid English Wikipedia URL' } + #################################### + # Elastic search configuration include Elasticsearch::Model include Elasticsearch::Model::Callbacks + # In order to avoid clashing between different environments, + # use Rails.env as a part of index name (eg. development_growstuff) index_name [Rails.env, "growstuff"].join('_') - settings index: { number_of_shards: 1}, + settings index: { number_of_shards: 1 }, analysis: { tokenizer: { gs_edgeNGram_tokenizer: { - type: "edgeNGram", + type: "edgeNGram", # edgeNGram: NGram match from the start of a token min_gram: 3, max_gram: 10, - token_chars: [ "letter", "digit" ] + # token_chars: Elasticsearch will split on characters + # that don’t belong to any of these classes + token_chars: [ "letter", "digit" ] } }, analyzer: { @@ -56,7 +62,12 @@ class Crop < ActiveRecord::Base indexes :id, type: 'long' indexes :name, type: 'string', analyzer: 'gs_edgeNGram_analyzer' indexes :scientific_names do - indexes :scientific_name, type: 'string', analyzer: 'gs_edgeNGram_analyzer', norms: { enabled: false } + indexes :scientific_name, + type: 'string', + analyzer: 'gs_edgeNGram_analyzer', + # Disabling field-length norm (norm). If the norm option is turned on(by default), + # higher weigh would be given for shorter fields, which in our case is irrelevant. + norms: { enabled: false } end indexes :alternate_names do indexes :name, type: 'string', analyzer: 'gs_edgeNGram_analyzer' @@ -72,6 +83,7 @@ class Crop < ActiveRecord::Base alternate_names: { only: :name } }) end + #################################### def to_s return name @@ -246,6 +258,8 @@ class Crop < ActiveRecord::Base if ENV['GROWSTUFF_ELASTICSEARCH'] == "true" search_str = query.nil? ? "" : query.downcase response = __elasticsearch__.search( { + # Finds documents which match any field, but uses the _score from + # the best field insead of adding up _score from each field. query: { multi_match: { query: "#{search_str}", From 675ac5a03f2bef48173f85a559de0e30efc24423 Mon Sep 17 00:00:00 2001 From: Mackenzie Morgan Date: Mon, 26 Jan 2015 22:26:21 -0500 Subject: [PATCH 23/33] add before_save for harvest weight and index for harvest photos --- app/models/harvest.rb | 11 ++++++++++- .../20140905001730_add_harvests_photos_table.rb | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/app/models/harvest.rb b/app/models/harvest.rb index 48e9eac19..a90a031d6 100644 --- a/app/models/harvest.rb +++ b/app/models/harvest.rb @@ -4,7 +4,7 @@ class Harvest < ActiveRecord::Base friendly_id :harvest_slug, use: :slugged attr_accessible :crop_id, :harvested_at, :description, :owner_id, - :quantity, :unit, :weight_quantity, :weight_unit, :plant_part_id, :slug + :quantity, :unit, :weight_quantity, :weight_unit, :plant_part_id, :slug, :si_weight belongs_to :crop belongs_to :owner, :class_name => 'Member' @@ -62,6 +62,15 @@ class Harvest < ActiveRecord::Base after_validation :cleanup_quantities + before_save { + if :weight_unit == "kg" + :si_weight = :weight_quantity + elsif :weight_unit == "lb" + :si_weight = :weight_quantity * 0.45 + elsif :weight_unit == "oz" + :si_weight = 0.028 + } + def cleanup_quantities if quantity == 0 self.quantity = nil diff --git a/db/migrate/20140905001730_add_harvests_photos_table.rb b/db/migrate/20140905001730_add_harvests_photos_table.rb index edacd061b..299d6653e 100644 --- a/db/migrate/20140905001730_add_harvests_photos_table.rb +++ b/db/migrate/20140905001730_add_harvests_photos_table.rb @@ -5,4 +5,6 @@ class AddHarvestsPhotosTable < ActiveRecord::Migration t.integer :harvest_id end end + + add_index(:harvests_photos, [:harvest_id, :photo_id]) end From 0bdab0d9fcd81a7d3e6732e8f801414ad9d15539 Mon Sep 17 00:00:00 2001 From: Mackenzie Morgan Date: Wed, 28 Jan 2015 23:00:16 -0500 Subject: [PATCH 24/33] basics of harvest standardized weights --- app/controllers/harvests_controller.rb | 2 +- .../20150129034206_add_si_weight_to_harvest.rb | 5 +++++ lib/tasks/growstuff.rake | 15 +++++++++++++++ script/deploy-tasks.sh | 3 +++ 4 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20150129034206_add_si_weight_to_harvest.rb diff --git a/app/controllers/harvests_controller.rb b/app/controllers/harvests_controller.rb index 3bc151d8c..23cf3cc0b 100644 --- a/app/controllers/harvests_controller.rb +++ b/app/controllers/harvests_controller.rb @@ -107,6 +107,6 @@ class HarvestsController < ApplicationController def harvest_params params.require(:harvest).permit(:crop_id, :harvested_at, :description, :owner_id, - :quantity, :unit, :weight_quantity, :weight_unit, :plant_part_id, :slug) + :quantity, :unit, :weight_quantity, :weight_unit, :plant_part_id, :slug, :si_weight) end end diff --git a/db/migrate/20150129034206_add_si_weight_to_harvest.rb b/db/migrate/20150129034206_add_si_weight_to_harvest.rb new file mode 100644 index 000000000..b1e532d8d --- /dev/null +++ b/db/migrate/20150129034206_add_si_weight_to_harvest.rb @@ -0,0 +1,5 @@ +class AddSiWeightToHarvest < ActiveRecord::Migration + def change + add_column :harvests, :si_weight, :float + end +end diff --git a/lib/tasks/growstuff.rake b/lib/tasks/growstuff.rake index 656961fd3..ab04b58c2 100644 --- a/lib/tasks/growstuff.rake +++ b/lib/tasks/growstuff.rake @@ -314,6 +314,21 @@ namespace :growstuff do end end end + + desc "January 2015: fill in si_weight column" + task :populate_si_weight => :environment do + Harvest.find_each do |h| + if h.weight_unit == "lb" + h.si_weight = h.weight_quantity * 0.453592 + elsif h.weight_unit == "oz" + h.si_weight = h.weight_quantity * 0.0283495 + else # kg is all that's left, and that's SI + h.si_weight = h.weight_quantity + end + h.save + end + end + end # end oneoff section end diff --git a/script/deploy-tasks.sh b/script/deploy-tasks.sh index 3d427ed85..c2eaf559f 100755 --- a/script/deploy-tasks.sh +++ b/script/deploy-tasks.sh @@ -18,3 +18,6 @@ rake growstuff:import_crops file=db/seeds/crops-15-squashes.csv echo "2014-12-01 - load alternate names for crops" rake growstuff:oneoff:add_alternate_names + +echo "2015-01-28 - populate the harvest si_weight field" +rake growstuff:populate_si_weight From 2567e7cd747441ee47619800264f1d1115e71770 Mon Sep 17 00:00:00 2001 From: Mackenzie Morgan Date: Thu, 29 Jan 2015 00:27:53 -0500 Subject: [PATCH 25/33] adding tests (not passing, need to figure out saving mid-test), fixing before_save, adding CSV column, and adding task to set the si_weight throughout the db --- app/models/harvest.rb | 19 +++++++++++-------- app/views/harvests/index.csv.shaper | 2 ++ db/schema.rb | 3 ++- lib/tasks/growstuff.rake | 4 ++-- spec/models/harvest_spec.rb | 20 ++++++++++++++++++++ 5 files changed, 37 insertions(+), 11 deletions(-) diff --git a/app/models/harvest.rb b/app/models/harvest.rb index 49cd44cae..b949b6f68 100644 --- a/app/models/harvest.rb +++ b/app/models/harvest.rb @@ -59,14 +59,17 @@ class Harvest < ActiveRecord::Base after_validation :cleanup_quantities - before_save { - if :weight_unit == "kg" - :si_weight = :weight_quantity - elsif :weight_unit == "lb" - :si_weight = :weight_quantity * 0.45 - elsif :weight_unit == "oz" - :si_weight = 0.028 - } + before_save :set_si_weight + + def set_si_weight + if self.weight_unit == "kg" + self.si_weight = self.weight_quantity + elsif self.weight_unit == "lb" + self.si_weight = self.weight_quantity * 0.45 + elsif self.weight_unit == "oz" + self.si_weight = self.weight_quantity * 0.028 + end + end def cleanup_quantities if quantity == 0 diff --git a/app/views/harvests/index.csv.shaper b/app/views/harvests/index.csv.shaper index 4560dde80..712c827dd 100644 --- a/app/views/harvests/index.csv.shaper +++ b/app/views/harvests/index.csv.shaper @@ -10,6 +10,7 @@ csv.headers :id, :unit, :weight_quantity, :weight_unit, + :si_weight, :date_harvested, :description, :date_added, @@ -32,6 +33,7 @@ csv.headers :id, csv.cell :plant_part_name, h.plant_part ? h.plant_part.to_s : '' csv.cells :quantity, :unit, :weight_quantity, :weight_unit + csv.cell :si_weight, :si_weight csv.cell :date_harvested, h.created_at.to_s(:db) diff --git a/db/schema.rb b/db/schema.rb index 9e593a422..bf80810c7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20150127043022) do +ActiveRecord::Schema.define(version: 20150129034206) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -139,6 +139,7 @@ ActiveRecord::Schema.define(version: 20150127043022) do t.decimal "weight_quantity" t.string "weight_unit" t.integer "plant_part_id" + t.float "si_weight" end create_table "harvests_photos", id: false, force: true do |t| diff --git a/lib/tasks/growstuff.rake b/lib/tasks/growstuff.rake index ab04b58c2..7ffaba0d3 100644 --- a/lib/tasks/growstuff.rake +++ b/lib/tasks/growstuff.rake @@ -319,9 +319,9 @@ namespace :growstuff do task :populate_si_weight => :environment do Harvest.find_each do |h| if h.weight_unit == "lb" - h.si_weight = h.weight_quantity * 0.453592 + h.si_weight = h.weight_quantity * 0.45 elsif h.weight_unit == "oz" - h.si_weight = h.weight_quantity * 0.0283495 + h.si_weight = h.weight_quantity * 0.028 else # kg is all that's left, and that's SI h.si_weight = h.weight_quantity end diff --git a/spec/models/harvest_spec.rb b/spec/models/harvest_spec.rb index 504c8c414..f90a18a8a 100644 --- a/spec/models/harvest_spec.rb +++ b/spec/models/harvest_spec.rb @@ -118,6 +118,26 @@ describe Harvest do end end + context "standardized weights" do + it 'converts from pounds' do + @harvest = FactoryGirl.build(:harvest, :weight_quantity => 2, :weight_unit => "lb") + @harvest.should be_valid + @harvest.reload.si_weight.should eq 0.9 + end + + it 'converts from ounces' do + @harvest = FactoryGirl.build(:harvest, :weight_quantity => 16, :weight_unit => "oz") + @harvest.should be_valid + @harvest.reload.si_weight.should eq 0.9 + end + + it 'leaves kg alone' do + @harvest = FactoryGirl.build(:harvest, :weight_quantity => 2, :weight_unit => "kg") + @harvest.should be_valid + @harvest.reload.si_weight.should eq 2 + end + end + context 'ordering' do it 'lists most recent harvests first' do @h1 = FactoryGirl.create(:harvest, :created_at => 1.day.ago) From 3f393b093739556f8cbd539764c5cc75ad1b9bfe Mon Sep 17 00:00:00 2001 From: Mackenzie Morgan Date: Fri, 30 Jan 2015 23:17:42 -0500 Subject: [PATCH 26/33] moving si_weight population to one-off --- script/deploy-tasks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/deploy-tasks.sh b/script/deploy-tasks.sh index c2eaf559f..e5f95aed2 100755 --- a/script/deploy-tasks.sh +++ b/script/deploy-tasks.sh @@ -20,4 +20,4 @@ echo "2014-12-01 - load alternate names for crops" rake growstuff:oneoff:add_alternate_names echo "2015-01-28 - populate the harvest si_weight field" -rake growstuff:populate_si_weight +rake growstuff:oneoff:populate_si_weight From d436fd86f8694abb1311c146a530d1770a20bb55 Mon Sep 17 00:00:00 2001 From: Mackenzie Morgan Date: Sat, 31 Jan 2015 00:22:35 -0500 Subject: [PATCH 27/33] use ruby-units to handle conversion. note: the deploy task isn't working and i don't know why --- Gemfile | 1 + Gemfile.lock | 2 ++ app/models/harvest.rb | 11 ++++------- lib/tasks/growstuff.rake | 10 +++------- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/Gemfile b/Gemfile index fcf7b39d4..87d18f8a1 100644 --- a/Gemfile +++ b/Gemfile @@ -31,6 +31,7 @@ gem 'figaro' # for handling config via ENV variables gem 'cancancan', '~> 1.9' # for checking member privileges gem 'gibbon' # for Mailchimp newsletter subscriptions gem 'csv_shaper' # CSV export +gem 'ruby-units' # for unit conversion # vendored activemerchant for testing- needed for bogus paypal # gateway monkeypatch diff --git a/Gemfile.lock b/Gemfile.lock index d54873d81..d867e16f5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -275,6 +275,7 @@ GEM rspec-mocks (~> 3.1.0) rspec-support (~> 3.1.0) rspec-support (3.1.2) + ruby-units (1.4.5) ruby_parser (3.1.3) sexp_processor (~> 4.1) sass (3.2.19) @@ -388,6 +389,7 @@ DEPENDENCIES rake (>= 10.0.0) rspec-activemodel-mocks rspec-rails (~> 3.1.0) + ruby-units sass-rails (~> 4.0.4) therubyracer (~> 0.12) uglifier (~> 2.5.3) diff --git a/app/models/harvest.rb b/app/models/harvest.rb index b949b6f68..ce44f97c2 100644 --- a/app/models/harvest.rb +++ b/app/models/harvest.rb @@ -61,14 +61,11 @@ class Harvest < ActiveRecord::Base before_save :set_si_weight + # we're storing the harvest weight in grams in the db too + # to make data manipulation easier def set_si_weight - if self.weight_unit == "kg" - self.si_weight = self.weight_quantity - elsif self.weight_unit == "lb" - self.si_weight = self.weight_quantity * 0.45 - elsif self.weight_unit == "oz" - self.si_weight = self.weight_quantity * 0.028 - end + weight_string = "#{self.weight_quantity} #{self.weight_unit}" + self.si_weight = Unit(weight_string).convert_to("g").to_s("%0.2f").delete(" g").to_f end def cleanup_quantities diff --git a/lib/tasks/growstuff.rake b/lib/tasks/growstuff.rake index 7ffaba0d3..2c8ad585c 100644 --- a/lib/tasks/growstuff.rake +++ b/lib/tasks/growstuff.rake @@ -318,13 +318,9 @@ namespace :growstuff do desc "January 2015: fill in si_weight column" task :populate_si_weight => :environment do Harvest.find_each do |h| - if h.weight_unit == "lb" - h.si_weight = h.weight_quantity * 0.45 - elsif h.weight_unit == "oz" - h.si_weight = h.weight_quantity * 0.028 - else # kg is all that's left, and that's SI - h.si_weight = h.weight_quantity - end + weight_string = "#{h.weight_quantity} #{h.weight_unit}" + print weight_string + h.si_weight = Unit(weight_string).convert_to("g").to_s("%0.2f").delete(" g").to_f h.save end end From bb9695b27267d9267aa99255dafdb9e30e29e819 Mon Sep 17 00:00:00 2001 From: Mackenzie Morgan Date: Sat, 31 Jan 2015 00:45:31 -0500 Subject: [PATCH 28/33] add check for nil to set_si_weights and the corresponding deploy script and update test --- app/models/harvest.rb | 6 ++++-- lib/tasks/growstuff.rake | 9 +++++---- spec/models/harvest_spec.rb | 12 ++++++------ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/app/models/harvest.rb b/app/models/harvest.rb index ce44f97c2..135ed9f3c 100644 --- a/app/models/harvest.rb +++ b/app/models/harvest.rb @@ -64,8 +64,10 @@ class Harvest < ActiveRecord::Base # we're storing the harvest weight in grams in the db too # to make data manipulation easier def set_si_weight - weight_string = "#{self.weight_quantity} #{self.weight_unit}" - self.si_weight = Unit(weight_string).convert_to("g").to_s("%0.2f").delete(" g").to_f + if self.weight_unit != nil + weight_string = "#{self.weight_quantity} #{self.weight_unit}" + self.si_weight = Unit(weight_string).convert_to("g").to_s("%0.2f").delete(" g").to_f + end end def cleanup_quantities diff --git a/lib/tasks/growstuff.rake b/lib/tasks/growstuff.rake index 2c8ad585c..6bc4258fd 100644 --- a/lib/tasks/growstuff.rake +++ b/lib/tasks/growstuff.rake @@ -318,10 +318,11 @@ namespace :growstuff do desc "January 2015: fill in si_weight column" task :populate_si_weight => :environment do Harvest.find_each do |h| - weight_string = "#{h.weight_quantity} #{h.weight_unit}" - print weight_string - h.si_weight = Unit(weight_string).convert_to("g").to_s("%0.2f").delete(" g").to_f - h.save + if h.weight_unit != nil + weight_string = "#{h.weight_quantity} #{h.weight_unit}" + h.si_weight = Unit(weight_string).convert_to("g").to_s("%0.2f").delete(" g").to_f + h.save + end end end diff --git a/spec/models/harvest_spec.rb b/spec/models/harvest_spec.rb index f90a18a8a..a7ec2114c 100644 --- a/spec/models/harvest_spec.rb +++ b/spec/models/harvest_spec.rb @@ -120,21 +120,21 @@ describe Harvest do context "standardized weights" do it 'converts from pounds' do - @harvest = FactoryGirl.build(:harvest, :weight_quantity => 2, :weight_unit => "lb") + @harvest = FactoryGirl.create(:harvest, :weight_quantity => 2, :weight_unit => "lb") @harvest.should be_valid - @harvest.reload.si_weight.should eq 0.9 + @harvest.reload.si_weight.should eq 907.18 end it 'converts from ounces' do - @harvest = FactoryGirl.build(:harvest, :weight_quantity => 16, :weight_unit => "oz") + @harvest = FactoryGirl.create(:harvest, :weight_quantity => 16, :weight_unit => "oz") @harvest.should be_valid - @harvest.reload.si_weight.should eq 0.9 + @harvest.reload.si_weight.should eq 453.59 end it 'leaves kg alone' do - @harvest = FactoryGirl.build(:harvest, :weight_quantity => 2, :weight_unit => "kg") + @harvest = FactoryGirl.create(:harvest, :weight_quantity => 2, :weight_unit => "kg") @harvest.should be_valid - @harvest.reload.si_weight.should eq 2 + @harvest.reload.si_weight.should eq 2000 end end From 9a5e15b29212694eec4ca7a955daf7f957304333 Mon Sep 17 00:00:00 2001 From: Shiho Takagi Date: Sat, 31 Jan 2015 23:59:46 +1100 Subject: [PATCH 29/33] index when sci name or alt name is updated --- app/models/alternate_name.rb | 1 + app/models/crop.rb | 8 ++++++-- app/models/scientific_name.rb | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/models/alternate_name.rb b/app/models/alternate_name.rb index 408740680..cb6f79e9d 100644 --- a/app/models/alternate_name.rb +++ b/app/models/alternate_name.rb @@ -1,4 +1,5 @@ class AlternateName < ActiveRecord::Base + after_commit { |an| an.crop.__elasticsearch__.index_document if an.crop } belongs_to :crop belongs_to :creator, :class_name => 'Member' end diff --git a/app/models/crop.rb b/app/models/crop.rb index f9c21fa2d..82d1e24a0 100644 --- a/app/models/crop.rb +++ b/app/models/crop.rb @@ -2,12 +2,12 @@ class Crop < ActiveRecord::Base extend FriendlyId friendly_id :name, use: [:slugged, :finders] - has_many :scientific_names + has_many :scientific_names, after_add: :update_index, after_remove: :update_index accepts_nested_attributes_for :scientific_names, :allow_destroy => true, :reject_if => :all_blank - has_many :alternate_names + has_many :alternate_names, after_add: :update_index, after_remove: :update_index has_many :plantings has_many :photos, :through => :plantings has_many :seeds @@ -83,6 +83,10 @@ class Crop < ActiveRecord::Base alternate_names: { only: :name } }) end + + def update_index(name_obj) + __elasticsearch__.index_document + end #################################### def to_s diff --git a/app/models/scientific_name.rb b/app/models/scientific_name.rb index 1a585e713..7469b3946 100644 --- a/app/models/scientific_name.rb +++ b/app/models/scientific_name.rb @@ -1,4 +1,5 @@ class ScientificName < ActiveRecord::Base + after_commit { |sn| sn.crop.__elasticsearch__.index_document if sn.crop } belongs_to :crop belongs_to :creator, :class_name => 'Member' end From de5b16e384baa7933a9c22643331d44f648d6d46 Mon Sep 17 00:00:00 2001 From: Mackenzie Morgan Date: Sat, 31 Jan 2015 18:35:29 -0500 Subject: [PATCH 30/33] back to kg for si weight, also now adding si_weight to the csv correctly --- app/models/harvest.rb | 4 ++-- app/views/harvests/index.csv.shaper | 3 +-- lib/tasks/growstuff.rake | 2 +- spec/models/harvest_spec.rb | 6 +++--- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/app/models/harvest.rb b/app/models/harvest.rb index 135ed9f3c..57e542fd2 100644 --- a/app/models/harvest.rb +++ b/app/models/harvest.rb @@ -61,12 +61,12 @@ class Harvest < ActiveRecord::Base before_save :set_si_weight - # we're storing the harvest weight in grams in the db too + # we're storing the harvest weight in kilograms in the db too # to make data manipulation easier def set_si_weight if self.weight_unit != nil weight_string = "#{self.weight_quantity} #{self.weight_unit}" - self.si_weight = Unit(weight_string).convert_to("g").to_s("%0.2f").delete(" g").to_f + self.si_weight = Unit(weight_string).convert_to("kg").to_s("%0.2f").delete(" kg").to_f end end diff --git a/app/views/harvests/index.csv.shaper b/app/views/harvests/index.csv.shaper index 712c827dd..b75b7a879 100644 --- a/app/views/harvests/index.csv.shaper +++ b/app/views/harvests/index.csv.shaper @@ -32,8 +32,7 @@ csv.headers :id, csv.cell :plant_part_id, h.plant_part ? h.plant_part.id : '' csv.cell :plant_part_name, h.plant_part ? h.plant_part.to_s : '' - csv.cells :quantity, :unit, :weight_quantity, :weight_unit - csv.cell :si_weight, :si_weight + csv.cells :quantity, :unit, :weight_quantity, :weight_unit, :si_weight csv.cell :date_harvested, h.created_at.to_s(:db) diff --git a/lib/tasks/growstuff.rake b/lib/tasks/growstuff.rake index 6bc4258fd..0e77c6fdd 100644 --- a/lib/tasks/growstuff.rake +++ b/lib/tasks/growstuff.rake @@ -320,7 +320,7 @@ namespace :growstuff do Harvest.find_each do |h| if h.weight_unit != nil weight_string = "#{h.weight_quantity} #{h.weight_unit}" - h.si_weight = Unit(weight_string).convert_to("g").to_s("%0.2f").delete(" g").to_f + h.si_weight = Unit(weight_string).convert_to("kg").to_s("%0.2f").delete(" kg").to_f h.save end end diff --git a/spec/models/harvest_spec.rb b/spec/models/harvest_spec.rb index a7ec2114c..9cbf6a1d8 100644 --- a/spec/models/harvest_spec.rb +++ b/spec/models/harvest_spec.rb @@ -122,19 +122,19 @@ describe Harvest do it 'converts from pounds' do @harvest = FactoryGirl.create(:harvest, :weight_quantity => 2, :weight_unit => "lb") @harvest.should be_valid - @harvest.reload.si_weight.should eq 907.18 + @harvest.reload.si_weight.should eq 0.91 end it 'converts from ounces' do @harvest = FactoryGirl.create(:harvest, :weight_quantity => 16, :weight_unit => "oz") @harvest.should be_valid - @harvest.reload.si_weight.should eq 453.59 + @harvest.reload.si_weight.should eq 0.45 end it 'leaves kg alone' do @harvest = FactoryGirl.create(:harvest, :weight_quantity => 2, :weight_unit => "kg") @harvest.should be_valid - @harvest.reload.si_weight.should eq 2000 + @harvest.reload.si_weight.should eq 2.0 end end From 2c94f61843b223c881f5867b2579fd850c0e1b97 Mon Sep 17 00:00:00 2001 From: Mackenzie Morgan Date: Sun, 1 Feb 2015 21:59:33 -0500 Subject: [PATCH 31/33] use %0.3f instead of %0.2f so we get to gram-granularity, and don't duplicate code --- app/models/harvest.rb | 2 +- lib/tasks/growstuff.rake | 6 +----- spec/models/harvest_spec.rb | 4 ++-- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/app/models/harvest.rb b/app/models/harvest.rb index 57e542fd2..c565b2c91 100644 --- a/app/models/harvest.rb +++ b/app/models/harvest.rb @@ -66,7 +66,7 @@ class Harvest < ActiveRecord::Base def set_si_weight if self.weight_unit != nil weight_string = "#{self.weight_quantity} #{self.weight_unit}" - self.si_weight = Unit(weight_string).convert_to("kg").to_s("%0.2f").delete(" kg").to_f + self.si_weight = Unit(weight_string).convert_to("kg").to_s("%0.3f").delete(" kg").to_f end end diff --git a/lib/tasks/growstuff.rake b/lib/tasks/growstuff.rake index 0e77c6fdd..d347bfaa3 100644 --- a/lib/tasks/growstuff.rake +++ b/lib/tasks/growstuff.rake @@ -318,11 +318,7 @@ namespace :growstuff do desc "January 2015: fill in si_weight column" task :populate_si_weight => :environment do Harvest.find_each do |h| - if h.weight_unit != nil - weight_string = "#{h.weight_quantity} #{h.weight_unit}" - h.si_weight = Unit(weight_string).convert_to("kg").to_s("%0.2f").delete(" kg").to_f - h.save - end + h.set_si_weight end end diff --git a/spec/models/harvest_spec.rb b/spec/models/harvest_spec.rb index 9cbf6a1d8..cd6d5f1ef 100644 --- a/spec/models/harvest_spec.rb +++ b/spec/models/harvest_spec.rb @@ -122,13 +122,13 @@ describe Harvest do it 'converts from pounds' do @harvest = FactoryGirl.create(:harvest, :weight_quantity => 2, :weight_unit => "lb") @harvest.should be_valid - @harvest.reload.si_weight.should eq 0.91 + @harvest.reload.si_weight.should eq 0.907 end it 'converts from ounces' do @harvest = FactoryGirl.create(:harvest, :weight_quantity => 16, :weight_unit => "oz") @harvest.should be_valid - @harvest.reload.si_weight.should eq 0.45 + @harvest.reload.si_weight.should eq 0.454 end it 'leaves kg alone' do From af39df5e0c45bc3343a3754943aeb402cc5953a1 Mon Sep 17 00:00:00 2001 From: Skud Date: Mon, 2 Feb 2015 18:04:56 +1100 Subject: [PATCH 32/33] Fixed test failure if elasticsearch isn't present in test env --- app/models/crop.rb | 13 +++++-- db/schema.rb | 97 +++++++++++++++++++++++----------------------- 2 files changed, 58 insertions(+), 52 deletions(-) diff --git a/app/models/crop.rb b/app/models/crop.rb index 82d1e24a0..00f08f528 100644 --- a/app/models/crop.rb +++ b/app/models/crop.rb @@ -31,7 +31,7 @@ class Crop < ActiveRecord::Base :with => /\Ahttps?:\/\/en\.wikipedia\.org\/wiki/, :message => 'is not a valid English Wikipedia URL' } - + #################################### # Elastic search configuration include Elasticsearch::Model @@ -81,13 +81,18 @@ class Crop < ActiveRecord::Base include: { scientific_names: { only: :scientific_name }, alternate_names: { only: :name } - }) + }) end + # update the Elasticsearch index (only if we're using it in this + # environment) def update_index(name_obj) - __elasticsearch__.index_document + if ENV["GROWSTUFF_ELASTICSEARCH"] == "true" + __elasticsearch__.index_document + end end - #################################### + + # End Elasticsearch section def to_s return name diff --git a/db/schema.rb b/db/schema.rb index bf80810c7..e54fc8911 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -20,16 +20,16 @@ ActiveRecord::Schema.define(version: 20150129034206) do t.string "name", null: false t.boolean "is_paid" t.boolean "is_permanent_paid" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" end create_table "accounts", force: true do |t| t.integer "member_id", null: false t.integer "account_type_id" t.datetime "paid_until" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" end create_table "alternate_names", force: true do |t| @@ -46,8 +46,8 @@ ActiveRecord::Schema.define(version: 20150129034206) do t.string "uid" t.string "token" t.string "secret" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.string "name" end @@ -57,15 +57,15 @@ ActiveRecord::Schema.define(version: 20150129034206) do t.integer "post_id", null: false t.integer "author_id", null: false t.text "body", null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" end create_table "crops", force: true do |t| t.string "name", null: false t.string "en_wikipedia_url" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.string "slug" t.integer "parent_id" t.integer "plantings_count", default: 0 @@ -94,8 +94,8 @@ ActiveRecord::Schema.define(version: 20150129034206) do t.string "name", null: false t.text "description", null: false t.integer "owner_id", null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.string "slug" end @@ -105,8 +105,8 @@ ActiveRecord::Schema.define(version: 20150129034206) do t.string "name", null: false t.integer "owner_id" t.string "slug", null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.text "description" t.boolean "active", default: true t.string "location" @@ -116,7 +116,7 @@ ActiveRecord::Schema.define(version: 20150129034206) do t.string "area_unit" end - add_index "gardens", ["owner_id"], name: "index_gardens_on_user_id", using: :btree + add_index "gardens", ["owner_id"], name: "index_gardens_on_owner_id", using: :btree add_index "gardens", ["slug"], name: "index_gardens_on_slug", unique: true, using: :btree create_table "gardens_photos", id: false, force: true do |t| @@ -133,8 +133,8 @@ ActiveRecord::Schema.define(version: 20150129034206) do t.decimal "quantity" t.string "unit" t.text "description" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.string "slug" t.decimal "weight_quantity" t.string "weight_unit" @@ -167,8 +167,8 @@ ActiveRecord::Schema.define(version: 20150129034206) do t.integer "failed_attempts", default: 0 t.string "unlock_token" t.datetime "locked_at" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.string "login_name" t.string "slug" t.boolean "tos_agreement" @@ -183,11 +183,11 @@ ActiveRecord::Schema.define(version: 20150129034206) do t.boolean "send_planting_reminder", default: true end - add_index "members", ["confirmation_token"], name: "index_users_on_confirmation_token", unique: true, using: :btree - add_index "members", ["email"], name: "index_users_on_email", unique: true, using: :btree - add_index "members", ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true, using: :btree - add_index "members", ["slug"], name: "index_users_on_slug", unique: true, using: :btree - add_index "members", ["unlock_token"], name: "index_users_on_unlock_token", unique: true, using: :btree + add_index "members", ["confirmation_token"], name: "index_members_on_confirmation_token", unique: true, using: :btree + add_index "members", ["email"], name: "index_members_on_email", unique: true, using: :btree + add_index "members", ["reset_password_token"], name: "index_members_on_reset_password_token", unique: true, using: :btree + add_index "members", ["slug"], name: "index_members_on_slug", unique: true, using: :btree + add_index "members", ["unlock_token"], name: "index_members_on_unlock_token", unique: true, using: :btree create_table "members_roles", id: false, force: true do |t| t.integer "member_id" @@ -201,8 +201,8 @@ ActiveRecord::Schema.define(version: 20150129034206) do t.text "body" t.boolean "read", default: false t.integer "post_id" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" end create_table "order_items", force: true do |t| @@ -210,13 +210,13 @@ ActiveRecord::Schema.define(version: 20150129034206) do t.integer "product_id" t.integer "price" t.integer "quantity" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" end create_table "orders", force: true do |t| - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.datetime "completed_at" t.integer "member_id" t.string "paypal_express_token" @@ -233,8 +233,8 @@ ActiveRecord::Schema.define(version: 20150129034206) do t.integer "owner_id", null: false t.string "thumbnail_url", null: false t.string "fullsize_url", null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.string "title", null: false t.string "license_name", null: false t.string "license_url" @@ -249,8 +249,8 @@ ActiveRecord::Schema.define(version: 20150129034206) do create_table "plant_parts", force: true do |t| t.string "name" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.string "slug" end @@ -260,8 +260,8 @@ ActiveRecord::Schema.define(version: 20150129034206) do t.date "planted_at" t.integer "quantity" t.text "description" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.string "slug" t.string "sunniness" t.string "planted_from" @@ -276,21 +276,22 @@ ActiveRecord::Schema.define(version: 20150129034206) do t.integer "author_id", null: false t.string "subject", null: false t.text "body", null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.string "slug" t.integer "forum_id" + t.integer "parent_id" end - add_index "posts", ["created_at", "author_id"], name: "index_updates_on_created_at_and_user_id", using: :btree - add_index "posts", ["slug"], name: "index_updates_on_slug", unique: true, using: :btree + add_index "posts", ["created_at", "author_id"], name: "index_posts_on_created_at_and_author_id", using: :btree + add_index "posts", ["slug"], name: "index_posts_on_slug", unique: true, using: :btree create_table "products", force: true do |t| t.string "name", null: false t.text "description", null: false t.integer "min_price", null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.integer "account_type_id" t.integer "paid_months" t.integer "recommended_price" @@ -299,8 +300,8 @@ ActiveRecord::Schema.define(version: 20150129034206) do create_table "roles", force: true do |t| t.string "name", null: false t.text "description" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.string "slug" end @@ -309,8 +310,8 @@ ActiveRecord::Schema.define(version: 20150129034206) do create_table "scientific_names", force: true do |t| t.string "scientific_name", null: false t.integer "crop_id", null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.integer "creator_id" end @@ -320,8 +321,8 @@ ActiveRecord::Schema.define(version: 20150129034206) do t.text "description" t.integer "quantity" t.date "plant_before" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.string "tradable_to", default: "nowhere" t.string "slug" end From 87bfceb0353c7bd2acfc5e736fe2baac67825cfe Mon Sep 17 00:00:00 2001 From: Mackenzie Morgan Date: Mon, 2 Feb 2015 23:49:07 -0500 Subject: [PATCH 33/33] make the one-off save the si weight --- lib/tasks/growstuff.rake | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/tasks/growstuff.rake b/lib/tasks/growstuff.rake index 35365ae1e..bfb7b8642 100644 --- a/lib/tasks/growstuff.rake +++ b/lib/tasks/growstuff.rake @@ -319,6 +319,7 @@ namespace :growstuff do task :populate_si_weight => :environment do Harvest.find_each do |h| h.set_si_weight + h.save end end