diff --git a/Gemfile b/Gemfile index 29589a662..38fea0f5b 100644 --- a/Gemfile +++ b/Gemfile @@ -117,6 +117,7 @@ gem 'recaptcha' gem "gbifrb" gem "msgpack" +gem "herringbone" group :production do gem 'dalli' diff --git a/Gemfile.lock b/Gemfile.lock index b9209f51d..f65f58bbf 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -235,6 +235,8 @@ GEM geocoder (1.8.6) base64 (>= 0.1.0) csv (>= 3.0.0) + herringbone (0.9.1) + bigdecimal globalid (1.4.0) activesupport (>= 6.1) graphiti (1.13.5) @@ -676,6 +678,7 @@ DEPENDENCIES haml-rails haml_lint (>= 0.25.1) hashie (>= 3.5.3) + herringbone i18n-tasks i18n_data (~> 1.1) icalendar diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 08e47626c..6fbdc9779 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -329,7 +329,7 @@ namespace :import do task ecocrop_data: :environment do require 'open-uri' require 'tempfile' - require 'json' + require 'herringbone' url = ENV.fetch('ECOCROP_URL', 'https://digital-atlas.s3.amazonaws.com/cdh/data/ecocrop/ecocrop.parquet') puts "Downloading EcoCrop data from #{url}..." @@ -342,31 +342,24 @@ namespace :import do end temp_file.flush - py_script = <<~PYTHON - import sys, json, pyarrow.parquet as pq - table = pq.read_table(sys.argv[1]) - print(json.dumps(table.to_pylist())) - PYTHON - - rows = nil - IO.popen(['python3', '-c', py_script, temp_file.path]) do |io| - rows = JSON.parse(io.read) - end - - puts "Importing #{rows.length} EcoCrop records into ecocrop_data..." - now = Time.current attribute_keys = EcocropData.column_names.map(&:to_sym) - %i(id created_at updated_at) + records_to_upsert = [] - records_to_upsert = rows.map do |row| - attrs = {} - attribute_keys.each do |key| - key_s = key.to_s - attrs[key] = row[key_s] if row.key?(key_s) + File.open(temp_file.path, 'rb') do |file| + reader = Herringbone::Reader.new(file) + puts "Importing #{reader.num_rows} EcoCrop records into ecocrop_data..." + + reader.each_row do |row| + attrs = {} + attribute_keys.each do |key| + key_s = key.to_s + attrs[key] = row[key_s] if row.key?(key_s) + end + attrs[:created_at] = now + attrs[:updated_at] = now + records_to_upsert << attrs end - attrs[:created_at] = now - attrs[:updated_at] = now - attrs end EcocropData.upsert_all( # rubocop:disable Rails/SkipsModelValidations