From a98cb3116153cd292fa4a552bcda639cc0cf16c7 Mon Sep 17 00:00:00 2001 From: Skud Date: Thu, 17 Oct 2013 22:08:54 +1100 Subject: [PATCH 1/3] Added rake task to import new crops from CSV --- db/seeds.rb | 7 ----- db/seeds/peppers.csv | 61 ++++++++++++++++++++++++++++++++++++++++ lib/tasks/growstuff.rake | 51 +++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 7 deletions(-) create mode 100644 db/seeds/peppers.csv diff --git a/db/seeds.rb b/db/seeds.rb index e0f9c3792..5e7085ead 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -1,12 +1,5 @@ # This file should contain all the record creation needed to seed the database with its default values. # The data can then be loaded with the rake db:seed (or created alongside the db with db:setup). -# -# Examples: -# -# cities = City.create([{ name: 'Chicago' }, { name: 'Copenhagen' }]) -# Mayor.create(name: 'Emanuel', city: cities.first) - -# import crops from CSV require 'csv' diff --git a/db/seeds/peppers.csv b/db/seeds/peppers.csv new file mode 100644 index 000000000..55c3aff62 --- /dev/null +++ b/db/seeds/peppers.csv @@ -0,0 +1,61 @@ +Capsicum,Capsicum,http://en.wikipedia.org/wiki/Capsicum,,This is the top parent variety -- the entire family of all kinds of peppers +Capsicum annuum,Capsicum annuum,http://en.wikipedia.org/wiki/Capsicum_annuum,Capsicum, +Capsicum baccatum,Capsicum baccatum,http://en.wikipedia.org/wiki/Capsicum_baccatum,Capsicum, +Capsicum chinense,Capsicum chinense,http://en.wikipedia.org/wiki/Capsicum_chinense,Capsicum, +rocoto,Capsicum pubescens,http://en.wikipedia.org/wiki/Capsicum_pubescens,Capsicum,AKA locoto +Capsicum frutescens,Capsicum frutescens,http://en.wikipedia.org/wiki/Capsicum_frutescens,Capsicum, +Aleppo pepper,,http://en.wikipedia.org/wiki/Aleppo_pepper,Capsicum annuum,"AKA pul biber, halaby pepper" +Anaheim pepper,,http://en.wikipedia.org/wiki/Anaheim_pepper,Capsicum annuum,"AKA New Mexico pepper, California chili, Magdalena chili" +banana pepper,,http://en.wikipedia.org/wiki/Banana_pepper,Capsicum annuum,"AKA yellow wax pepper, banana chili" +bell pepper,,http://en.wikipedia.org/wiki/Bell_pepper,Capsicum annuum,"AKA capsicum, sweet pepper, pepper, poivron, peperoni, piment" +bird's eye chili,,http://en.wikipedia.org/wiki/Bird%27s_eye_chili,Capsicum annuum,"AKA Thai chili pepper, bird's chili" +cascabel chili,,http://en.wikipedia.org/wiki/Cascabel_chili,Capsicum annuum,"AKA rattle chile, bola chile, chile bola" +cayenne pepper,,http://en.wikipedia.org/wiki/Cayenne_pepper,Capsicum annuum,"AKA Guinea spice, cow-horn pepper, aleva, bird pepper, red pepper" +pasilla pepper,,http://en.wikipedia.org/wiki/Pasilla,Capsicum annuum,"AKA chilaca pepper, chile negro" +chiltepin pepper,Capsicum annuum var. glabriusculum,http://en.wikipedia.org/wiki/Capsicum_annuum_var._glabriusculum,Capsicum annuum,"AKA wild chiltepin, chiltepe, chile tepin, turkey pepper, bird’s eye pepper, bird pepper" +Chungyang red pepper,,http://en.wikipedia.org/wiki/Chungyang_Red_Pepper,Capsicum annuum, +cubanelle pepper,,http://en.wikipedia.org/wiki/Cubanelle,Capsicum annuum, +chile de árbol pepper,,http://en.wikipedia.org/wiki/Chile_de_%C3%A1rbol,Capsicum annuum,"AKA bird's beak chile, rat's tail chile" +dundicut,,http://en.wikipedia.org/wiki/Dundicut,Capsicum annuum,AKA lal mirch +facing heaven pepper,Capsicum annuum var. conoides,http://en.wikipedia.org/wiki/Facing_heaven_pepper,Capsicum annuum, +Fresno pepper,,http://en.wikipedia.org/wiki/Fresno_pepper,Capsicum annuum, +guajillo pepper,,http://en.wikipedia.org/wiki/Guajillo_chili,Capsicum annuum,"AKA guajillo chili, guajillo chilli, chile guajillo" +Guntur Sannam chili,Capsicum Annuum var. Longhum,http://en.wikipedia.org/wiki/Guntur_Sannam,Capsicum annuum, +Hungarian wax pepper,,http://en.wikipedia.org/wiki/Hungarian_wax_pepper,Capsicum annuum, +Italian sweet pepper,,http://en.wikipedia.org/wiki/Italian_sweet_pepper,Capsicum annuum, +jalapeño pepper,,http://en.wikipedia.org/wiki/Jalape%C3%B1o,Capsicum annuum, +shishito pepper,,http://en.wikipedia.org/wiki/Shishito,Capsicum annuum, +medusa pepper,,http://en.wikipedia.org/wiki/Medusa_pepper,Capsicum annuum, +mulato pepper,,http://en.wikipedia.org/wiki/Mulato_pepper,Capsicum annuum, +peter pepper,Capsicum annuum var. annuum,http://en.wikipedia.org/wiki/Peter_pepper,Capsicum annuum,AKA chilli willy pepper +peperoncini pepper,,http://en.wikipedia.org/wiki/Peperoncini,Capsicum annuum,"AKA friggitello, peperone" +pequin pepper,"Capsicum annuum var. glabriusculum +",http://en.wikipedia.org/wiki/Pequin_pepper,Capsicum annuum,AKA piquin +pimento pepper,,http://en.wikipedia.org/wiki/Pimiento,Capsicum annuum,"AKA cherry pepper, pimiento pepper" +poblano pepper,,http://en.wikipedia.org/wiki/Poblano,Capsicum annuum, +Santa Fe Grande pepper,,http://en.wikipedia.org/wiki/Santa_Fe_Grande_pepper,Capsicum annuum,"AKA yellow hot chili pepper, guero chili pepper" +serrano pepper,,http://en.wikipedia.org/wiki/Serrano_pepper,Capsicum annuum, +bishop's crown pepper,Capsicum baccatum var. pendulum,http://en.wikipedia.org/wiki/Bishop%27s_Crown,Capsicum baccatum,AKA Christmas bell +lemon drop pepper,,http://en.wikipedia.org/wiki/Lemon_drop_%28pepper%29,Capsicum baccatum,"AKA ají limon, kellu uchu, hot lemon" +peppadew pepper,,http://en.wikipedia.org/wiki/Peppadew,Capsicum baccatum, +Brazilian starfish,,http://en.wikipedia.org/wiki/Capsicum_baccatum,Capsicum baccatum, +wild baccatum,,http://en.wikipedia.org/wiki/Capsicum_baccatum,Capsicum baccatum, +adjuma pepper,,http://en.wikipedia.org/wiki/Adjuma,Capsicum chinense,"AKA adjoema, aji umba, ojemma" +ají dulce pepper,,http://en.wikipedia.org/wiki/Aj%C3%AD_dulce,Capsicum chinense,"AKA ajicito, ajice, rubra, biquinho" +datil pepper,,http://en.wikipedia.org/wiki/Datil_pepper,Capsicum chinense, +fatalii pepper,,http://en.wikipedia.org/wiki/Fatalii,Capsicum chinense, +Madame Jeanette pepper,,http://en.wikipedia.org/wiki/Madame_Jeanette,Capsicum chinense, +habanero pepper,,http://en.wikipedia.org/wiki/Habanero,Capsicum chinense,AKA habañero +black habanero pepper,,http://en.wikipedia.org/wiki/Habanero,Capsicum chinense, +red savino pepper,,http://en.wikipedia.org/wiki/Red_Savina_pepper,Capsicum chinense, +bhut jolokia pepper,,http://en.wikipedia.org/wiki/Bhut_Jolokia,Capsicum chinense,"AKA ghost pepper, ghost chili pepper, red naga chilli, ghost chilli, naga jolokia" +Scotch bonnet pepper,,http://en.wikipedia.org/wiki/Scotch_bonnet_%28pepper%29,Capsicum chinense,"AKA boabs bonnet, Scotty bons, Bonney peppers, Caribbean red peppers, ball of fire" +cachucha pepper,,http://en.wikipedia.org/wiki/Scotch_bonnet_%28pepper%29,Capsicum chinense, +Trinidad scorpion butch T pepper,,http://en.wikipedia.org/wiki/Trinidad_Scorpion_Butch_T_pepper,Capsicum chinense, +Trinidad Moruga scorpion pepper,,http://en.wikipedia.org/wiki/Trinidad_Moruga_Scorpion,Capsicum chinense, +Hainan yellow lantern,,http://en.wikipedia.org/wiki/Hainan_Yellow_Lantern_Chili,Capsicum chinense,AKA yellow emperor chili +piri-piri pepper,,http://en.wikipedia.org/wiki/Piri_piri,Capsicum frutescens,"AKA African bird's eye pepper, peri peri, pili pili" +Tabasco pepper,,http://en.wikipedia.org/wiki/Tabasco_pepper,Capsicum frutescens, +malagueta pepper,,http://en.wikipedia.org/wiki/Malagueta_pepper,Capsicum frutescens, +siling labuyo pepper,,http://en.wikipedia.org/wiki/Siling_labuyo,Capsicum frutescens,"AKA chileng bundok, siling palay, pasitis, pasite, katumbal, kitikot, siling kolikot, silit-diablo, lada, rimorimo, paktin" +kambuzi,,http://en.wikipedia.org/wiki/Kambuzi,Capsicum frutescens, diff --git a/lib/tasks/growstuff.rake b/lib/tasks/growstuff.rake index f1e63c5cb..ab0983922 100644 --- a/lib/tasks/growstuff.rake +++ b/lib/tasks/growstuff.rake @@ -10,6 +10,57 @@ namespace :growstuff do member.roles << admin end + desc "Upload crops from a CSV file" + # usage: rake growstuff:import_crops file=filename.csv + + task :import_crops => :environment do + require 'csv' + + @cropbot = Member.find_by_login_name('cropbot') + raise "cropbot account not found: run rake db:seed" unless @cropbot + @file = ENV['file'] or raise "Usage: rake growstuff:import_crops file=file.csv" + + puts "Loading crops from #{@file}..." + + CSV.foreach(@file) do |row| + system_name,scientific_name,en_wikipedia_url,parent = row + @crop = Crop.find_or_create_by_system_name(system_name) + puts "Loading #{system_name}..." + @crop.update_attributes( + :en_wikipedia_url => en_wikipedia_url, + :creator_id => @cropbot.id + ) + if parent + @parent = Crop.find_by_system_name(parent) + if @parent + @crop.update_attributes(:parent_id => @parent.id) + else + puts "Warning: parent crop #{parent} not found" + end + end + + unless @crop.scientific_names.exists?(:scientific_name => scientific_name) + @sn = '' + if scientific_name + @sn = scientific_name + elsif @crop.parent + @sn = @crop.parent.scientific_names.first.scientific_name + end + + if @sn + @crop.scientific_names.create( + :scientific_name => @sn, + :creator_id => @cropbot.id + ) + else + puts "Warning: no scientific name (not even on parent crop) for #{@crop}" + end + + end + end + puts "Finished loading crops" + end + desc "One-off tasks needed at various times and kept for posterity" namespace :oneoff do From c46e19e43e9a9931e76c6c277aa812c864e17936 Mon Sep 17 00:00:00 2001 From: Skud Date: Thu, 17 Oct 2013 22:45:09 +1100 Subject: [PATCH 2/3] Load multiple crop CSV files from seeds.rb We factored out a Crop.create_from_csv method to avoid duplicated code. Note: naming convention with db/seeds/crop*.csv is to use a 2-digit number for ordering. 00 and 10 chosen (with a gap in between) because I learned to program in BASIC and it seemed like a good idea just in case. --- app/models/crop.rb | 40 +++++++++++++++++++ db/seeds.rb | 18 +++------ db/seeds/{crops.csv => crops-00.csv} | 0 .../{peppers.csv => crops-10-peppers.csv} | 0 lib/tasks/growstuff.rake | 39 +----------------- 5 files changed, 48 insertions(+), 49 deletions(-) rename db/seeds/{crops.csv => crops-00.csv} (100%) rename db/seeds/{peppers.csv => crops-10-peppers.csv} (100%) diff --git a/app/models/crop.rb b/app/models/crop.rb index e8d332220..84ea43679 100644 --- a/app/models/crop.rb +++ b/app/models/crop.rb @@ -99,4 +99,44 @@ class Crop < ActiveRecord::Base return interesting_crops end + def Crop.create_from_csv(row) + system_name,scientific_name,en_wikipedia_url,parent = row + + @cropbot = Member.find_by_login_name('cropbot') + raise "cropbot account not found: run rake db:seed" unless @cropbot + + @crop = Crop.find_or_create_by_system_name(system_name) + @crop.update_attributes( + :en_wikipedia_url => en_wikipedia_url, + :creator_id => @cropbot.id + ) + if parent + @parent = Crop.find_by_system_name(parent) + if @parent + @crop.update_attributes(:parent_id => @parent.id) + else + logger.warn("Warning: parent crop #{parent} not found") + end + end + + unless @crop.scientific_names.exists?(:scientific_name => scientific_name) + @sn = '' + if scientific_name + @sn = scientific_name + elsif @crop.parent + @sn = @crop.parent.scientific_names.first.scientific_name + end + + if @sn + @crop.scientific_names.create( + :scientific_name => @sn, + :creator_id => @cropbot.id + ) + else + logger.warn("Warning: no scientific name (not even on parent crop) for #{@crop}") + end + + end + end + end diff --git a/db/seeds.rb b/db/seeds.rb index 5e7085ead..c025a0295 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -22,18 +22,12 @@ def load_data end def load_crops - puts "Loading crops..." - CSV.foreach(Rails.root.join('db', 'seeds', 'crops.csv')) do |row| - system_name,scientific_name,en_wikipedia_url = row - @crop = Crop.create( - :system_name => system_name, - :en_wikipedia_url => en_wikipedia_url, - :creator_id => @cropbot_user.id - ) - @crop.scientific_names.create( - :scientific_name => scientific_name, - :creator_id => @cropbot_user.id - ) + source_path = Rails.root.join('db', 'seeds') + Dir.glob("#{source_path}/crops*.csv").each do |crop_file| + puts "Loading crops from #{crop_file}..." + CSV.foreach(crop_file) do |row| + Crop.create_from_csv(row) + end end puts "Finished loading crops" end diff --git a/db/seeds/crops.csv b/db/seeds/crops-00.csv similarity index 100% rename from db/seeds/crops.csv rename to db/seeds/crops-00.csv diff --git a/db/seeds/peppers.csv b/db/seeds/crops-10-peppers.csv similarity index 100% rename from db/seeds/peppers.csv rename to db/seeds/crops-10-peppers.csv diff --git a/lib/tasks/growstuff.rake b/lib/tasks/growstuff.rake index ab0983922..b215b38e1 100644 --- a/lib/tasks/growstuff.rake +++ b/lib/tasks/growstuff.rake @@ -16,49 +16,14 @@ namespace :growstuff do task :import_crops => :environment do require 'csv' - @cropbot = Member.find_by_login_name('cropbot') - raise "cropbot account not found: run rake db:seed" unless @cropbot @file = ENV['file'] or raise "Usage: rake growstuff:import_crops file=file.csv" puts "Loading crops from #{@file}..." - CSV.foreach(@file) do |row| - system_name,scientific_name,en_wikipedia_url,parent = row - @crop = Crop.find_or_create_by_system_name(system_name) - puts "Loading #{system_name}..." - @crop.update_attributes( - :en_wikipedia_url => en_wikipedia_url, - :creator_id => @cropbot.id - ) - if parent - @parent = Crop.find_by_system_name(parent) - if @parent - @crop.update_attributes(:parent_id => @parent.id) - else - puts "Warning: parent crop #{parent} not found" - end - end - - unless @crop.scientific_names.exists?(:scientific_name => scientific_name) - @sn = '' - if scientific_name - @sn = scientific_name - elsif @crop.parent - @sn = @crop.parent.scientific_names.first.scientific_name - end - - if @sn - @crop.scientific_names.create( - :scientific_name => @sn, - :creator_id => @cropbot.id - ) - else - puts "Warning: no scientific name (not even on parent crop) for #{@crop}" - end - - end + Crop.create_from_csv(row) end puts "Finished loading crops" + end From ed01c6fb29fd1aad05f66dbd734b6c0ea7cdee18 Mon Sep 17 00:00:00 2001 From: Skud Date: Thu, 17 Oct 2013 22:49:57 +1100 Subject: [PATCH 3/3] Added some comments/docs to Crop.create_from_csv --- app/models/crop.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/models/crop.rb b/app/models/crop.rb index 84ea43679..36ea98584 100644 --- a/app/models/crop.rb +++ b/app/models/crop.rb @@ -99,6 +99,14 @@ class Crop < ActiveRecord::Base return interesting_crops end +# Crop.create_from_csv(row) +# used by db/seeds.rb and rake growstuff:import_crops +# CSV fields: +# - system_name (required) +# - scientific name (optional, can be picked up from parent if it has one) +# - en_wikipedia_url (required) +# - parent (system name, optional) + def Crop.create_from_csv(row) system_name,scientific_name,en_wikipedia_url,parent = row