From 929049abad4953cf01f52a98d3c722763cdf26ec Mon Sep 17 00:00:00 2001 From: Brenda Wallace Date: Sat, 25 Jan 2020 18:57:32 +1300 Subject: [PATCH 1/8] Note about tech used --- TECH. md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 TECH. md diff --git a/TECH. md b/TECH. md new file mode 100644 index 000000000..9df85715a --- /dev/null +++ b/TECH. md @@ -0,0 +1,33 @@ + +## Elastic search + +Elasticsearch is used to retrieve data used in many pages. When data is +created, updated, or deleted, and index of this data is saved into elastic +search. This is denormlised data, which for us means it's saved into elastic +search in a structure ready to be displayed onto our pages. Data in postgres, +on the other hand, is saved in a different structure, that ensures data +integrity. + +For example, a crop needs a default photo. The photo used is the most "liked" +photo of plantings, seeds or harvests for that crop. To get this from +postgresql you'd need to look up the crops tables, the +plantings,seeds,harvests, the photos, and then the likes. Instead of doing +that on every display of a crop photo, we calculate this at data change and +saved into the Elasticsearch index for crops. + +### Installing Elasticsearch + +Currently we use elastic search 7. You can check if this is still true by +looking for the `ELASTIC_SEARCH_VERSION` variable in the `.travis.yml` file + +To install ES on a debian/ubuntu machine you can use the same script used in +travis-ci: + +``` +export ELASTIC_SEARCH_VERSION="7.5.1-amd64" ./script/install_elasticsearch.sh +``` + +You can check that it started happily using: +``` +./script/check_elasticsearch.sh +``` From 1d710abb77d8249f0e6a0fcb142ab8d961635930 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2020 08:12:30 +0000 Subject: [PATCH 2/8] Bump rubocop-rails from 2.4.1 to 2.4.2 Bumps [rubocop-rails](https://github.com/rubocop-hq/rubocop-rails) from 2.4.1 to 2.4.2. - [Release notes](https://github.com/rubocop-hq/rubocop-rails/releases) - [Changelog](https://github.com/rubocop-hq/rubocop-rails/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop-hq/rubocop-rails/compare/v2.4.1...v2.4.2) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 86eaa8f94..cb0725069 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -460,7 +460,7 @@ GEM rainbow (>= 2.2.2, < 4.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 1.7) - rubocop-rails (2.4.1) + rubocop-rails (2.4.2) rack (>= 1.1) rubocop (>= 0.72.0) rubocop-rspec (1.37.1) @@ -532,7 +532,7 @@ GEM thread_safe (~> 0.1) uglifier (4.2.0) execjs (>= 0.3.0, < 3) - unicode-display_width (1.6.0) + unicode-display_width (1.6.1) unicorn (5.5.2) kgio (~> 2.6) raindrops (~> 0.7) From 530217552ef9e54af889cf7377b61fa475aec924 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2020 07:20:12 +0000 Subject: [PATCH 3/8] Bump capybara from 3.30.0 to 3.31.0 Bumps [capybara](https://github.com/teamcapybara/capybara) from 3.30.0 to 3.31.0. - [Release notes](https://github.com/teamcapybara/capybara/releases) - [Changelog](https://github.com/teamcapybara/capybara/blob/master/History.md) - [Commits](https://github.com/teamcapybara/capybara/compare/3.30.0...3.31.0) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index cb0725069..b84839020 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -91,7 +91,7 @@ GEM uniform_notifier (~> 1.11) byebug (11.1.1) cancancan (3.0.2) - capybara (3.30.0) + capybara (3.31.0) addressable mini_mime (>= 0.1.3) nokogiri (~> 1.8) @@ -354,11 +354,11 @@ GEM moneta (~> 1.0.0) popper_js (1.14.5) procto (0.0.3) - public_suffix (4.0.2) + public_suffix (4.0.3) puma (4.3.1) nio4r (~> 2.0) query_diet (0.6.2) - rack (2.1.1) + rack (2.1.2) rack-protection (2.0.8.1) rack rack-test (1.1.0) From a0e1184cda26b8fa1cb4c4b73e58a4a9272f8340 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2020 21:50:24 +0000 Subject: [PATCH 4/8] Bump bootstrap-datepicker-rails from 1.8.0.1 to 1.9.0.1 Bumps [bootstrap-datepicker-rails](https://github.com/Nerian/bootstrap-datepicker-rails) from 1.8.0.1 to 1.9.0.1. - [Release notes](https://github.com/Nerian/bootstrap-datepicker-rails/releases) - [Commits](https://github.com/Nerian/bootstrap-datepicker-rails/compare/1.8.0.1...1.9.0.1) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index b84839020..60d2d29bf 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -80,7 +80,7 @@ GEM autoprefixer-rails (>= 9.1.0) popper_js (>= 1.14.3, < 2) sassc-rails (>= 2.0.0) - bootstrap-datepicker-rails (1.8.0.1) + bootstrap-datepicker-rails (1.9.0.1) railties (>= 3.0) bootstrap_form (4.3.0) actionpack (>= 5.0) From 42aeba9eebce80048acb16549abf5c6b8fa41d16 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 29 Jan 2020 18:33:46 +0000 Subject: [PATCH 5/8] Bump searchkick from 4.2.0 to 4.2.1 Bumps [searchkick](https://github.com/ankane/searchkick) from 4.2.0 to 4.2.1. - [Release notes](https://github.com/ankane/searchkick/releases) - [Changelog](https://github.com/ankane/searchkick/blob/master/CHANGELOG.md) - [Commits](https://github.com/ankane/searchkick/compare/v4.2.0...v4.2.1) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 60d2d29bf..671924328 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -491,7 +491,7 @@ GEM tilt scout_apm (2.6.6) parser - searchkick (4.2.0) + searchkick (4.2.1) activemodel (>= 5) elasticsearch (>= 6) hashie From 42cce210c090a7a3fabc3a6f2fad6752eb449f38 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 31 Jan 2020 07:58:17 +0000 Subject: [PATCH 6/8] Bump database_cleaner from 1.7.0 to 1.8.1 Bumps [database_cleaner](https://github.com/DatabaseCleaner/database_cleaner) from 1.7.0 to 1.8.1. - [Release notes](https://github.com/DatabaseCleaner/database_cleaner/releases) - [Changelog](https://github.com/DatabaseCleaner/database_cleaner/blob/master/History.rdoc) - [Commits](https://github.com/DatabaseCleaner/database_cleaner/commits) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 671924328..77055482f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -149,7 +149,7 @@ GEM csv_shaper (1.3.1) activesupport (>= 3.0.0) dalli (2.7.10) - database_cleaner (1.7.0) + database_cleaner (1.8.1) devise (4.7.1) bcrypt (~> 3.0) orm_adapter (~> 0.1) From 5fbdd577d682469702e4a55db89b73ec0e822f2f Mon Sep 17 00:00:00 2001 From: Brenda Wallace Date: Mon, 3 Feb 2020 11:54:26 +1300 Subject: [PATCH 7/8] One per person on home page --- app/models/concerns/search_harvests.rb | 24 ++++++--- app/models/concerns/search_plantings.rb | 72 +++++++++++++++---------- 2 files changed, 63 insertions(+), 33 deletions(-) diff --git a/app/models/concerns/search_harvests.rb b/app/models/concerns/search_harvests.rb index 79b7c9796..43ae17587 100644 --- a/app/models/concerns/search_harvests.rb +++ b/app/models/concerns/search_harvests.rb @@ -49,12 +49,24 @@ module SearchHarvests end def self.homepage_records(limit) - search('*', limit: limit, - where: { - photos_count: { gt: 0 } - }, - boost_by: [:created_at], - load: false) + records = [] + owners = [] + 1..limit.times do + where = { + photos_count: { gt: 0 }, + owner_id: { not: owners } + } + one_record = search('*', + limit: 1, + where: where, + boost_by: [:created_at], + load: false).first + return records if one_record.nil? + + owners << one_record.owner_id + records << one_record + end + records end end end diff --git a/app/models/concerns/search_plantings.rb b/app/models/concerns/search_plantings.rb index 680b4a1d7..61fd4fc2c 100644 --- a/app/models/concerns/search_plantings.rb +++ b/app/models/concerns/search_plantings.rb @@ -18,46 +18,64 @@ module SearchPlantings def search_data { - slug: slug, - active: active, - finished: finished?, - has_photos: photos.size.positive?, - location: location, - percentage_grown: percentage_grown.to_i, - planted_at: planted_at, - planted_from: planted_from, - quantity: quantity, - sunniness: sunniness, + slug: slug, + active: active, + finished: finished?, + has_photos: photos.size.positive?, + location: location, + percentage_grown: percentage_grown.to_i, + planted_at: planted_at, + planted_from: planted_from, + planted_year: planted_at.year, + quantity: quantity, + sunniness: sunniness, + garden_id: garden_id, + + first_harvest_predicted_at: first_harvest_predicted_at, + finish_predicted_at: finish_predicted_at, # crops - crop_id: crop_id, - crop_name: crop.name, - crop_slug: crop.slug, + crop_id: crop_id, + crop_name: crop_name, + crop_slug: crop_slug, + crop_perennial: crop_perennial, # owner - owner_id: owner_id, - owner_location: owner_location, - owner_login_name: owner_login_name, - owner_slug: owner_slug, + owner_id: owner_id, + owner_location: owner_location, + owner_login_name: owner_login_name, + owner_slug: owner_slug, # photos - thumbnail_url: default_photo&.thumbnail_url || crop.default_photo&.thumbnail_url, + thumbnail_url: default_photo&.thumbnail_url || crop.default_photo&.thumbnail_url, # counts - photos_count: photos.size, - harvests_count: harvests_count, + photos_count: photos.size, + harvests_count: harvests_count, # timestamps - created_at: created_at.to_i + created_at: created_at.to_i } end def self.homepage_records(limit) - search('*', limit: limit, - where: { - photos_count: { gt: 0 } - }, - boost_by: [:created_at], - load: false) + records = [] + owners = [] + 1..limit.times do + where = { + photos_count: { gt: 0 }, + owner_id: { not: owners } + } + one_record = search('*', + limit: 1, + where: where, + boost_by: [:created_at], + load: false).first + return records if one_record.nil? + + owners << one_record.owner_id + records << one_record + end + records end end end From d15df948fff2e7f3f431c75190946a4b173adb7e Mon Sep 17 00:00:00 2001 From: Brenda Wallace Date: Mon, 3 Feb 2020 15:06:45 +1300 Subject: [PATCH 8/8] Added perennial to plantings --- app/models/planting.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/planting.rb b/app/models/planting.rb index c45c89c39..7c216bd03 100644 --- a/app/models/planting.rb +++ b/app/models/planting.rb @@ -56,7 +56,7 @@ class Planting < ApplicationRecord ## ## Delegations - delegate :name, :slug, :en_wikipedia_url, :default_scientific_name, :plantings_count, + delegate :name, :slug, :en_wikipedia_url, :default_scientific_name, :plantings_count, :perennial, to: :crop, prefix: true delegate :login_name, :slug, :location, to: :owner, prefix: true delegate :slug, to: :planting, prefix: true