Refactor import:ecocrop_data to avoid python (#5255)

- Add herringbone gem for pure-Ruby Apache Parquet parsing
- Update import:ecocrop_data Rake task to use Herringbone::Reader
  instead of executing an external python script via IO.popen
- Remove python and pyarrow dependencies for EcoCrop data import

Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com>

Gemfile

@@ -117,6 +117,7 @@ gem 'recaptcha'
 gem "gbifrb"
 
 gem "msgpack"
+gem "herringbone"
 
 group :production do
   gem 'dalli'

Gemfile.lock

@@ -235,6 +235,8 @@ GEM
     geocoder (1.8.6)
       base64 (>= 0.1.0)
       csv (>= 3.0.0)
+    herringbone (0.9.1)
+      bigdecimal
     globalid (1.4.0)
       activesupport (>= 6.1)
     graphiti (1.13.5)
@@ -676,6 +678,7 @@ DEPENDENCIES
   haml-rails
   haml_lint (>= 0.25.1)
   hashie (>= 3.5.3)
+  herringbone
   i18n-tasks
   i18n_data (~> 1.1)
   icalendar

lib/tasks/import.rake

@@ -329,7 +329,7 @@ namespace :import do
   task ecocrop_data: :environment do
     require 'open-uri'
     require 'tempfile'
-    require 'json'
+    require 'herringbone'
 
     url = ENV.fetch('ECOCROP_URL', 'https://digital-atlas.s3.amazonaws.com/cdh/data/ecocrop/ecocrop.parquet')
     puts "Downloading EcoCrop data from #{url}..."
@@ -342,31 +342,24 @@ namespace :import do
       end
       temp_file.flush
 
-      py_script = <<~PYTHON
-        import sys, json, pyarrow.parquet as pq
-        table = pq.read_table(sys.argv[1])
-        print(json.dumps(table.to_pylist()))
-      PYTHON
-
-      rows = nil
-      IO.popen(['python3', '-c', py_script, temp_file.path]) do |io|
-        rows = JSON.parse(io.read)
-      end
-
-      puts "Importing #{rows.length} EcoCrop records into ecocrop_data..."
-
       now = Time.current
       attribute_keys = EcocropData.column_names.map(&:to_sym) - %i(id created_at updated_at)
+      records_to_upsert = []
 
-      records_to_upsert = rows.map do |row|
-        attrs = {}
-        attribute_keys.each do |key|
-          key_s = key.to_s
-          attrs[key] = row[key_s] if row.key?(key_s)
+      File.open(temp_file.path, 'rb') do |file|
+        reader = Herringbone::Reader.new(file)
+        puts "Importing #{reader.num_rows} EcoCrop records into ecocrop_data..."
+
+        reader.each_row do |row|
+          attrs = {}
+          attribute_keys.each do |key|
+            key_s = key.to_s
+            attrs[key] = row[key_s] if row.key?(key_s)
+          end
+          attrs[:created_at] = now
+          attrs[:updated_at] = now
+          records_to_upsert << attrs
         end
-        attrs[:created_at] = now
-        attrs[:updated_at] = now
-        attrs
       end
 
       EcocropData.upsert_all( # rubocop:disable Rails/SkipsModelValidations