Implement normalized name handling and search improvements in Postgres

app/models/alternate_name.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 class AlternateName < ApplicationRecord
+  # Only crop search reads it (see SearchCrops).
+  self.ignored_columns += %w(normalised_name)
+
   belongs_to :crop
   belongs_to :creator, class_name: 'Member', inverse_of: :created_alternate_names
   validates :name, presence: true

app/models/concerns/search_crops.rb

@@ -3,6 +3,17 @@
 # Crop search, done in Postgres. A crop matches when a word of its name, an
 # alternate name or a scientific name starts with the term, or when one of them
 # is similar to it (pg_trgm), which is what forgives typos and plurals.
+#
+# Names are compared lower-cased with accents folded, so "jalapeno" finds
+# "jalape\u00f1o". Each name table has a stored `normalised_name` column holding
+# that (see db/migrate/20261021000000_add_crop_name_search_indexes.rb), with a
+# trigram index that the queries below use. The folding is done with
+# translate() because unaccent() is not immutable. ACCENTED and PLAIN must match
+# the migration, and the term is folded the same way by `normalise`.
+#
+# Similarity uses the `<%` operator so that the indexes apply. It compares
+# against pg_trgm.word_similarity_threshold, which config/database.yml sets to
+# MIN_SIMILARITY for every connection.
 module SearchCrops
   extend ActiveSupport::Concern
 
@@ -10,13 +21,24 @@ module SearchCrops
   NAME_WEIGHT = 5
   PLANTED_BOOST = 5
 
-  NAMES_SQL = <<~SQL.squish
-    SELECT id AS crop_id, name, #{NAME_WEIGHT} AS weight FROM crops
-    UNION ALL SELECT crop_id, name, 1 FROM alternate_names
-    UNION ALL SELECT crop_id, name, 1 FROM scientific_names
-  SQL
+  ACCENTED = %w(
+    àáâãäåāçćčèéêëēė
+    ęìíîïīįłñńòóôõöø
+    ōùúûüūýÿžźż
+  ).join.freeze
+  PLAIN = "#{'a' * 7}ccc#{'e' * 7}#{'i' * 6}lnn#{'o' * 7}#{'u' * 5}yyzzz".freeze
+
+  def self.normalise(text)
+    text.to_s.downcase.tr(ACCENTED, PLAIN)
+  end
+
+  # [table, crop id column, weight]: a crop's own name counts for more than its other names.
+  NAME_SOURCES = [['crops', 'id', NAME_WEIGHT], ['alternate_names', 'crop_id', 1], ['scientific_names', 'crop_id', 1]].freeze
 
   included do
+    # Only the search queries read it.
+    self.ignored_columns += %w(normalised_name)
+
     ####################################
     # Elastic search configuration. Nothing searches the index any more, but it
     # is still kept up to date.
@@ -74,15 +96,24 @@ module SearchCrops
   end
 
   def self.matches_sql(term)
-    like = ApplicationRecord.sanitize_sql_like(term.downcase)
-    ApplicationRecord.sanitize_sql_array([<<~SQL.squish, { term: term, starts: "#{like}%", word: "% #{like}%", hyphen: "%-#{like}%", min: MIN_SIMILARITY }])
+    normalised = normalise(term)
+    like = ApplicationRecord.sanitize_sql_like(normalised)
+    binds = { term: normalised, contains: "%#{like}%", starts: "#{like}%", word: "% #{like}%", hyphen: "%-#{like}%", min: MIN_SIMILARITY }
+
+    selects = NAME_SOURCES.map do |table, crop_id, weight|
+      name = "#{table}.normalised_name"
+      <<~SQL.squish
+        SELECT #{crop_id} AS crop_id, #{weight} AS weight,
+               CASE WHEN #{name} LIKE :starts OR #{name} LIKE :word OR #{name} LIKE :hyphen
+                    THEN 1.0 ELSE word_similarity(:term, #{name}) END AS score
+        FROM #{table}
+        WHERE #{name} LIKE :contains OR :term <% #{name}
+      SQL
+    end
+
+    ApplicationRecord.sanitize_sql_array([<<~SQL.squish, binds])
       SELECT crop_id, MAX(weight * score) AS score
-      FROM (
-        SELECT crop_id, weight,
-               CASE WHEN name_norm LIKE :starts OR name_norm LIKE :word OR name_norm LIKE :hyphen
-                    THEN 1.0 ELSE word_similarity(unaccent(lower(:term)), name_norm) END AS score
-        FROM (SELECT crop_id, weight, unaccent(lower(name)) AS name_norm FROM (#{NAMES_SQL}) names) normalised
-      ) scored
+      FROM (#{selects.join(' UNION ALL ')}) scored
       WHERE score >= :min
       GROUP BY crop_id
     SQL

app/models/scientific_name.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 class ScientificName < ApplicationRecord
+  # Only crop search reads it (see SearchCrops).
+  self.ignored_columns += %w(normalised_name)
+
   belongs_to :crop
   belongs_to :creator, class_name: 'Member', inverse_of: :created_scientific_names
   validates :name, presence: true

config/database.yml

@@ -4,6 +4,9 @@ development:
   user: postgres
   password: postgres
   host: localhost
+  variables:
+    # Crop search (SearchCrops) relies on this threshold for the <% operator.
+    pg_trgm.word_similarity_threshold: 0.35
 
 test:
   adapter: postgresql
@@ -11,6 +14,9 @@ test:
   user: postgres
   password: postgres
   host: localhost
+  variables:
+    # Crop search (SearchCrops) relies on this threshold for the <% operator.
+    pg_trgm.word_similarity_threshold: 0.35
 
 production:
   adapter: postgresql
@@ -20,6 +26,9 @@ production:
   username: growstuff
   host: localhost
   password: thisisnottherealpassword
+  variables:
+    # Crop search (SearchCrops) relies on this threshold for the <% operator.
+    pg_trgm.word_similarity_threshold: 0.35
 
 staging:
   adapter: postgresql
@@ -29,3 +38,6 @@ staging:
   username: growstuff
   host: localhost
   password: thisisnottherealpassword
+  variables:
+    # Crop search (SearchCrops) relies on this threshold for the <% operator.
+    pg_trgm.word_similarity_threshold: 0.35

config/database.yml.example

@@ -4,6 +4,9 @@ development:
   host: localhost
   user: postgres
   password: postgres
+  variables:
+    # Crop search (SearchCrops) relies on this threshold for the <% operator.
+    pg_trgm.word_similarity_threshold: 0.35
 
 test:
   adapter: postgresql
@@ -11,6 +14,9 @@ test:
   host: localhost
   user: postgres
   password: postgres
+  variables:
+    # Crop search (SearchCrops) relies on this threshold for the <% operator.
+    pg_trgm.word_similarity_threshold: 0.35
 
 production:
   adapter: postgresql
@@ -20,6 +26,9 @@ production:
   username: growstuff
   host: localhost
   password: thisisnottherealpassword
+  variables:
+    # Crop search (SearchCrops) relies on this threshold for the <% operator.
+    pg_trgm.word_similarity_threshold: 0.35
 
 staging:
   adapter: postgresql
@@ -29,3 +38,6 @@ staging:
   username: growstuff
   host: localhost
   password: thisisnottherealpassword
+  variables:
+    # Crop search (SearchCrops) relies on this threshold for the <% operator.
+    pg_trgm.word_similarity_threshold: 0.35

db/migrate/20261020000000_enable_search_extensions.rb

@@ -1,10 +1,8 @@
 # frozen_string_literal: true
 
-# Crop search runs in Postgres: pg_trgm for typo-tolerant matching and
-# unaccent so "jalapeno" finds "jalapeño".
+# Crop search runs in Postgres: pg_trgm for typo-tolerant matching.
 class EnableSearchExtensions < ActiveRecord::Migration[8.1]
   def change
     enable_extension 'pg_trgm'
-    enable_extension 'unaccent'
   end
 end

db/migrate/20261021000000_add_crop_name_search_indexes.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Trigram indexes for crop search (see SearchCrops). Each name table gets a
+# stored column holding the name lower-cased with accents folded (translate(),
+# because unaccent() is not immutable and so cannot be used here), and a GIN
+# index on it.
+class AddCropNameSearchIndexes < ActiveRecord::Migration[8.1]
+  ACCENTED = "àáâãäåāçćčèéêëēėęìíîïīįłñńòóôõöøōùúûüūýÿžźż"
+  PLAIN = "#{'a' * 7}ccc#{'e' * 7}#{'i' * 6}lnn#{'o' * 7}#{'u' * 5}yyzzz".freeze
+  NORMALISED = "translate(lower(name), '#{ACCENTED}', '#{PLAIN}')".freeze
+
+  TABLES = %i(crops alternate_names scientific_names).freeze
+
+  def change
+    TABLES.each do |table|
+      add_column table, :normalised_name, :virtual, type: :text, as: NORMALISED, stored: true
+      add_index table, :normalised_name, using: :gin, opclass: :gin_trgm_ops
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.1].define(version: 2026_10_20_000000) do
+ActiveRecord::Schema[8.1].define(version: 2026_10_21_000000) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_catalog.plpgsql"
   enable_extension "pg_trgm"
@@ -69,10 +69,12 @@ ActiveRecord::Schema[8.1].define(version: 2026_10_20_000000) do
     t.integer "crop_id", null: false
     t.string "language"
     t.string "name", null: false
+    t.virtual "normalised_name", type: :text, as: "translate(lower((name)::text), 'àáâãäåāçćčèéêëēėęìíîïīįłñńòóôõöøōùúûüūýÿžźż'::text, 'aaaaaaaccceeeeeeeiiiiiilnnooooooouuuuuyyzzz'::text)", stored: true
     t.datetime "updated_at", precision: nil
     t.index ["creator_id"], name: "index_alternate_names_on_creator_id"
     t.index ["crop_id"], name: "index_alternate_names_on_crop_id"
     t.index ["language"], name: "index_alternate_names_on_language"
+    t.index ["normalised_name"], name: "index_alternate_names_on_normalised_name", opclass: :gin_trgm_ops, using: :gin
   end
 
   create_table "australian_food_classification_data", force: :cascade do |t|
@@ -568,6 +570,7 @@ ActiveRecord::Schema[8.1].define(version: 2026_10_20_000000) do
     t.integer "median_days_to_last_harvest"
     t.integer "median_lifespan"
     t.string "name", null: false
+    t.virtual "normalised_name", type: :text, as: "translate(lower((name)::text), 'àáâãäåāçćčèéêëēėęìíîïīįłñńòóôõöøōùúûüūýÿžźż'::text, 'aaaaaaaccceeeeeeeiiiiiilnnooooooouuuuuyyzzz'::text)", stored: true
     t.jsonb "openfarm_data"
     t.integer "parent_id"
     t.boolean "perennial", default: false
@@ -586,6 +589,7 @@ ActiveRecord::Schema[8.1].define(version: 2026_10_20_000000) do
     t.datetime "updated_at", precision: nil
     t.index ["creator_id"], name: "index_crops_on_creator_id"
     t.index ["name"], name: "index_crops_on_name"
+    t.index ["normalised_name"], name: "index_crops_on_normalised_name", opclass: :gin_trgm_ops, using: :gin
     t.index ["parent_id"], name: "index_crops_on_parent_id"
     t.index ["public_food_key"], name: "index_crops_on_public_food_key"
     t.index ["requester_id"], name: "index_crops_on_requester_id"
@@ -994,10 +998,12 @@ ActiveRecord::Schema[8.1].define(version: 2026_10_20_000000) do
     t.string "gbif_rank"
     t.string "gbif_status"
     t.string "name", null: false
+    t.virtual "normalised_name", type: :text, as: "translate(lower((name)::text), 'àáâãäåāçćčèéêëēėęìíîïīįłñńòóôõöøōùúûüūýÿžźż'::text, 'aaaaaaaccceeeeeeeiiiiiilnnooooooouuuuuyyzzz'::text)", stored: true
     t.datetime "updated_at", precision: nil
     t.string "wikidata_id"
     t.index ["creator_id"], name: "index_scientific_names_on_creator_id"
     t.index ["crop_id"], name: "index_scientific_names_on_crop_id"
+    t.index ["normalised_name"], name: "index_scientific_names_on_normalised_name", opclass: :gin_trgm_ops, using: :gin
   end
 
   create_table "seeds", id: :serial, force: :cascade do |t|

spec/models/concerns/search_crops_spec.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe SearchCrops do
+  describe 'ACCENTED and PLAIN' do
+    it 'have the same length, as translate() pairs them up' do
+      expect(described_class::ACCENTED.length).to eq described_class::PLAIN.length
+    end
+  end
+
+  describe '.normalise' do
+    it 'folds accents and case' do
+      expect(described_class.normalise('Jalapeño')).to eq 'jalapeno'
+    end
+  end
+
+  describe 'database settings' do
+    it 'uses the similarity threshold the queries are written for' do
+      threshold = ActiveRecord::Base.connection.select_value('SHOW pg_trgm.word_similarity_threshold')
+      expect(threshold.to_f).to eq described_class::MIN_SIMILARITY
+    end
+  end
+
+  describe 'Crop.search_by_name' do
+    subject(:names) { Crop.search_by_name(term).map(&:name) }
+
+    let!(:jalapeno) { create(:crop, name: 'jalapeño') }
+    let!(:cucumber) { create(:crop, name: 'cucumber') }
+    let!(:eggplant) { create(:crop, name: 'eggplant') }
+
+    before { create(:alternate_name, crop: eggplant, name: 'aubergine') }
+
+    context 'with a term without the accent' do
+      let(:term) { 'jalapeno' }
+
+      it { is_expected.to eq ['jalapeño'] }
+    end
+
+    context 'with a term with the accent' do
+      let(:term) { 'jalapeño' }
+
+      it { is_expected.to eq ['jalapeño'] }
+    end
+
+    context 'with the start of a word' do
+      let(:term) { 'cucum' }
+
+      it { is_expected.to eq ['cucumber'] }
+    end
+
+    context 'with a typo' do
+      let(:term) { 'cucmber' }
+
+      it { is_expected.to eq ['cucumber'] }
+    end
+
+    context 'with an alternate name' do
+      let(:term) { 'aubergine' }
+
+      it { is_expected.to eq ['eggplant'] }
+    end
+
+    context 'with text unlike any name' do
+      let(:term) { 'zzzzzz' }
+
+      it { is_expected.to be_empty }
+    end
+
+    context 'with a blank term' do
+      let(:term) { '' }
+
+      it { is_expected.to match_array %w(jalapeño cucumber eggplant) }
+    end
+  end
+
+  describe 'trigram indexes' do
+    let(:term) { 'cucumber' }
+
+    %w(crops alternate_names scientific_names).each do |table|
+      it "serve the search of #{table}" do
+        plan = ActiveRecord::Base.transaction do
+          ActiveRecord::Base.connection.execute('SET LOCAL enable_seqscan = off')
+          ActiveRecord::Base.connection.select_values("EXPLAIN #{Crop.search_by_name(term).to_sql}").join("\n")
+        end
+        expect(plan).to include "Bitmap Index Scan on index_#{table}_on_normalised_name"
+      end
+    end
+  end
+end