diff --git a/app/models/alternate_name.rb b/app/models/alternate_name.rb index e133d89e6..b6724ea84 100644 --- a/app/models/alternate_name.rb +++ b/app/models/alternate_name.rb @@ -1,6 +1,9 @@ # frozen_string_literal: true class AlternateName < ApplicationRecord + # Only crop search reads it (see SearchCrops). + self.ignored_columns += %w(normalised_name) + belongs_to :crop belongs_to :creator, class_name: 'Member', inverse_of: :created_alternate_names validates :name, presence: true diff --git a/app/models/concerns/search_crops.rb b/app/models/concerns/search_crops.rb index 3ab2fffea..8b611954a 100644 --- a/app/models/concerns/search_crops.rb +++ b/app/models/concerns/search_crops.rb @@ -3,6 +3,17 @@ # Crop search, done in Postgres. A crop matches when a word of its name, an # alternate name or a scientific name starts with the term, or when one of them # is similar to it (pg_trgm), which is what forgives typos and plurals. +# +# Names are compared lower-cased with accents folded, so "jalapeno" finds +# "jalape\u00f1o". Each name table has a stored `normalised_name` column holding +# that (see db/migrate/20261021000000_add_crop_name_search_indexes.rb), with a +# trigram index that the queries below use. The folding is done with +# translate() because unaccent() is not immutable. ACCENTED and PLAIN must match +# the migration, and the term is folded the same way by `normalise`. +# +# Similarity uses the `<%` operator so that the indexes apply. It compares +# against pg_trgm.word_similarity_threshold, which config/database.yml sets to +# MIN_SIMILARITY for every connection. module SearchCrops extend ActiveSupport::Concern @@ -10,13 +21,24 @@ module SearchCrops NAME_WEIGHT = 5 PLANTED_BOOST = 5 - NAMES_SQL = <<~SQL.squish - SELECT id AS crop_id, name, #{NAME_WEIGHT} AS weight FROM crops - UNION ALL SELECT crop_id, name, 1 FROM alternate_names - UNION ALL SELECT crop_id, name, 1 FROM scientific_names - SQL + ACCENTED = %w( + àáâãäåāçćčèéêëēė + ęìíîïīįłñńòóôõöø + ōùúûüūýÿžźż + ).join.freeze + PLAIN = "#{'a' * 7}ccc#{'e' * 7}#{'i' * 6}lnn#{'o' * 7}#{'u' * 5}yyzzz".freeze + + def self.normalise(text) + text.to_s.downcase.tr(ACCENTED, PLAIN) + end + + # [table, crop id column, weight]: a crop's own name counts for more than its other names. + NAME_SOURCES = [['crops', 'id', NAME_WEIGHT], ['alternate_names', 'crop_id', 1], ['scientific_names', 'crop_id', 1]].freeze included do + # Only the search queries read it. + self.ignored_columns += %w(normalised_name) + #################################### # Elastic search configuration. Nothing searches the index any more, but it # is still kept up to date. @@ -74,15 +96,24 @@ module SearchCrops end def self.matches_sql(term) - like = ApplicationRecord.sanitize_sql_like(term.downcase) - ApplicationRecord.sanitize_sql_array([<<~SQL.squish, { term: term, starts: "#{like}%", word: "% #{like}%", hyphen: "%-#{like}%", min: MIN_SIMILARITY }]) + normalised = normalise(term) + like = ApplicationRecord.sanitize_sql_like(normalised) + binds = { term: normalised, contains: "%#{like}%", starts: "#{like}%", word: "% #{like}%", hyphen: "%-#{like}%", min: MIN_SIMILARITY } + + selects = NAME_SOURCES.map do |table, crop_id, weight| + name = "#{table}.normalised_name" + <<~SQL.squish + SELECT #{crop_id} AS crop_id, #{weight} AS weight, + CASE WHEN #{name} LIKE :starts OR #{name} LIKE :word OR #{name} LIKE :hyphen + THEN 1.0 ELSE word_similarity(:term, #{name}) END AS score + FROM #{table} + WHERE #{name} LIKE :contains OR :term <% #{name} + SQL + end + + ApplicationRecord.sanitize_sql_array([<<~SQL.squish, binds]) SELECT crop_id, MAX(weight * score) AS score - FROM ( - SELECT crop_id, weight, - CASE WHEN name_norm LIKE :starts OR name_norm LIKE :word OR name_norm LIKE :hyphen - THEN 1.0 ELSE word_similarity(unaccent(lower(:term)), name_norm) END AS score - FROM (SELECT crop_id, weight, unaccent(lower(name)) AS name_norm FROM (#{NAMES_SQL}) names) normalised - ) scored + FROM (#{selects.join(' UNION ALL ')}) scored WHERE score >= :min GROUP BY crop_id SQL diff --git a/app/models/scientific_name.rb b/app/models/scientific_name.rb index 1c8160bcb..d8a14585d 100644 --- a/app/models/scientific_name.rb +++ b/app/models/scientific_name.rb @@ -1,6 +1,9 @@ # frozen_string_literal: true class ScientificName < ApplicationRecord + # Only crop search reads it (see SearchCrops). + self.ignored_columns += %w(normalised_name) + belongs_to :crop belongs_to :creator, class_name: 'Member', inverse_of: :created_scientific_names validates :name, presence: true diff --git a/config/database.yml b/config/database.yml index 0add4c8f9..3fbeb58bd 100644 --- a/config/database.yml +++ b/config/database.yml @@ -4,6 +4,9 @@ development: user: postgres password: postgres host: localhost + variables: + # Crop search (SearchCrops) relies on this threshold for the <% operator. + pg_trgm.word_similarity_threshold: 0.35 test: adapter: postgresql @@ -11,6 +14,9 @@ test: user: postgres password: postgres host: localhost + variables: + # Crop search (SearchCrops) relies on this threshold for the <% operator. + pg_trgm.word_similarity_threshold: 0.35 production: adapter: postgresql @@ -20,6 +26,9 @@ production: username: growstuff host: localhost password: thisisnottherealpassword + variables: + # Crop search (SearchCrops) relies on this threshold for the <% operator. + pg_trgm.word_similarity_threshold: 0.35 staging: adapter: postgresql @@ -29,3 +38,6 @@ staging: username: growstuff host: localhost password: thisisnottherealpassword + variables: + # Crop search (SearchCrops) relies on this threshold for the <% operator. + pg_trgm.word_similarity_threshold: 0.35 diff --git a/config/database.yml.example b/config/database.yml.example index eb75b3791..1da97ccb3 100644 --- a/config/database.yml.example +++ b/config/database.yml.example @@ -4,6 +4,9 @@ development: host: localhost user: postgres password: postgres + variables: + # Crop search (SearchCrops) relies on this threshold for the <% operator. + pg_trgm.word_similarity_threshold: 0.35 test: adapter: postgresql @@ -11,6 +14,9 @@ test: host: localhost user: postgres password: postgres + variables: + # Crop search (SearchCrops) relies on this threshold for the <% operator. + pg_trgm.word_similarity_threshold: 0.35 production: adapter: postgresql @@ -20,6 +26,9 @@ production: username: growstuff host: localhost password: thisisnottherealpassword + variables: + # Crop search (SearchCrops) relies on this threshold for the <% operator. + pg_trgm.word_similarity_threshold: 0.35 staging: adapter: postgresql @@ -29,3 +38,6 @@ staging: username: growstuff host: localhost password: thisisnottherealpassword + variables: + # Crop search (SearchCrops) relies on this threshold for the <% operator. + pg_trgm.word_similarity_threshold: 0.35 diff --git a/db/migrate/20261020000000_enable_search_extensions.rb b/db/migrate/20261020000000_enable_search_extensions.rb index 0cca55fa4..5859bcedf 100644 --- a/db/migrate/20261020000000_enable_search_extensions.rb +++ b/db/migrate/20261020000000_enable_search_extensions.rb @@ -1,10 +1,8 @@ # frozen_string_literal: true -# Crop search runs in Postgres: pg_trgm for typo-tolerant matching and -# unaccent so "jalapeno" finds "jalapeño". +# Crop search runs in Postgres: pg_trgm for typo-tolerant matching. class EnableSearchExtensions < ActiveRecord::Migration[8.1] def change enable_extension 'pg_trgm' - enable_extension 'unaccent' end end diff --git a/db/migrate/20261021000000_add_crop_name_search_indexes.rb b/db/migrate/20261021000000_add_crop_name_search_indexes.rb new file mode 100644 index 000000000..c747bae66 --- /dev/null +++ b/db/migrate/20261021000000_add_crop_name_search_indexes.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# Trigram indexes for crop search (see SearchCrops). Each name table gets a +# stored column holding the name lower-cased with accents folded (translate(), +# because unaccent() is not immutable and so cannot be used here), and a GIN +# index on it. +class AddCropNameSearchIndexes < ActiveRecord::Migration[8.1] + ACCENTED = "àáâãäåāçćčèéêëēėęìíîïīįłñńòóôõöøōùúûüūýÿžźż" + PLAIN = "#{'a' * 7}ccc#{'e' * 7}#{'i' * 6}lnn#{'o' * 7}#{'u' * 5}yyzzz".freeze + NORMALISED = "translate(lower(name), '#{ACCENTED}', '#{PLAIN}')".freeze + + TABLES = %i(crops alternate_names scientific_names).freeze + + def change + TABLES.each do |table| + add_column table, :normalised_name, :virtual, type: :text, as: NORMALISED, stored: true + add_index table, :normalised_name, using: :gin, opclass: :gin_trgm_ops + end + end +end diff --git a/db/schema.rb b/db/schema.rb index ad5d81f64..521ab5bf3 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_10_20_000000) do +ActiveRecord::Schema[8.1].define(version: 2026_10_21_000000) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" enable_extension "pg_trgm" @@ -69,10 +69,12 @@ ActiveRecord::Schema[8.1].define(version: 2026_10_20_000000) do t.integer "crop_id", null: false t.string "language" t.string "name", null: false + t.virtual "normalised_name", type: :text, as: "translate(lower((name)::text), 'àáâãäåāçćčèéêëēėęìíîïīįłñńòóôõöøōùúûüūýÿžźż'::text, 'aaaaaaaccceeeeeeeiiiiiilnnooooooouuuuuyyzzz'::text)", stored: true t.datetime "updated_at", precision: nil t.index ["creator_id"], name: "index_alternate_names_on_creator_id" t.index ["crop_id"], name: "index_alternate_names_on_crop_id" t.index ["language"], name: "index_alternate_names_on_language" + t.index ["normalised_name"], name: "index_alternate_names_on_normalised_name", opclass: :gin_trgm_ops, using: :gin end create_table "australian_food_classification_data", force: :cascade do |t| @@ -568,6 +570,7 @@ ActiveRecord::Schema[8.1].define(version: 2026_10_20_000000) do t.integer "median_days_to_last_harvest" t.integer "median_lifespan" t.string "name", null: false + t.virtual "normalised_name", type: :text, as: "translate(lower((name)::text), 'àáâãäåāçćčèéêëēėęìíîïīįłñńòóôõöøōùúûüūýÿžźż'::text, 'aaaaaaaccceeeeeeeiiiiiilnnooooooouuuuuyyzzz'::text)", stored: true t.jsonb "openfarm_data" t.integer "parent_id" t.boolean "perennial", default: false @@ -586,6 +589,7 @@ ActiveRecord::Schema[8.1].define(version: 2026_10_20_000000) do t.datetime "updated_at", precision: nil t.index ["creator_id"], name: "index_crops_on_creator_id" t.index ["name"], name: "index_crops_on_name" + t.index ["normalised_name"], name: "index_crops_on_normalised_name", opclass: :gin_trgm_ops, using: :gin t.index ["parent_id"], name: "index_crops_on_parent_id" t.index ["public_food_key"], name: "index_crops_on_public_food_key" t.index ["requester_id"], name: "index_crops_on_requester_id" @@ -994,10 +998,12 @@ ActiveRecord::Schema[8.1].define(version: 2026_10_20_000000) do t.string "gbif_rank" t.string "gbif_status" t.string "name", null: false + t.virtual "normalised_name", type: :text, as: "translate(lower((name)::text), 'àáâãäåāçćčèéêëēėęìíîïīįłñńòóôõöøōùúûüūýÿžźż'::text, 'aaaaaaaccceeeeeeeiiiiiilnnooooooouuuuuyyzzz'::text)", stored: true t.datetime "updated_at", precision: nil t.string "wikidata_id" t.index ["creator_id"], name: "index_scientific_names_on_creator_id" t.index ["crop_id"], name: "index_scientific_names_on_crop_id" + t.index ["normalised_name"], name: "index_scientific_names_on_normalised_name", opclass: :gin_trgm_ops, using: :gin end create_table "seeds", id: :serial, force: :cascade do |t| diff --git a/spec/models/concerns/search_crops_spec.rb b/spec/models/concerns/search_crops_spec.rb new file mode 100644 index 000000000..9bd57eef4 --- /dev/null +++ b/spec/models/concerns/search_crops_spec.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe SearchCrops do + describe 'ACCENTED and PLAIN' do + it 'have the same length, as translate() pairs them up' do + expect(described_class::ACCENTED.length).to eq described_class::PLAIN.length + end + end + + describe '.normalise' do + it 'folds accents and case' do + expect(described_class.normalise('Jalapeño')).to eq 'jalapeno' + end + end + + describe 'database settings' do + it 'uses the similarity threshold the queries are written for' do + threshold = ActiveRecord::Base.connection.select_value('SHOW pg_trgm.word_similarity_threshold') + expect(threshold.to_f).to eq described_class::MIN_SIMILARITY + end + end + + describe 'Crop.search_by_name' do + subject(:names) { Crop.search_by_name(term).map(&:name) } + + let!(:jalapeno) { create(:crop, name: 'jalapeño') } + let!(:cucumber) { create(:crop, name: 'cucumber') } + let!(:eggplant) { create(:crop, name: 'eggplant') } + + before { create(:alternate_name, crop: eggplant, name: 'aubergine') } + + context 'with a term without the accent' do + let(:term) { 'jalapeno' } + + it { is_expected.to eq ['jalapeño'] } + end + + context 'with a term with the accent' do + let(:term) { 'jalapeño' } + + it { is_expected.to eq ['jalapeño'] } + end + + context 'with the start of a word' do + let(:term) { 'cucum' } + + it { is_expected.to eq ['cucumber'] } + end + + context 'with a typo' do + let(:term) { 'cucmber' } + + it { is_expected.to eq ['cucumber'] } + end + + context 'with an alternate name' do + let(:term) { 'aubergine' } + + it { is_expected.to eq ['eggplant'] } + end + + context 'with text unlike any name' do + let(:term) { 'zzzzzz' } + + it { is_expected.to be_empty } + end + + context 'with a blank term' do + let(:term) { '' } + + it { is_expected.to match_array %w(jalapeño cucumber eggplant) } + end + end + + describe 'trigram indexes' do + let(:term) { 'cucumber' } + + %w(crops alternate_names scientific_names).each do |table| + it "serve the search of #{table}" do + plan = ActiveRecord::Base.transaction do + ActiveRecord::Base.connection.execute('SET LOCAL enable_seqscan = off') + ActiveRecord::Base.connection.select_values("EXPLAIN #{Crop.search_by_name(term).to_sql}").join("\n") + end + expect(plan).to include "Bitmap Index Scan on index_#{table}_on_normalised_name" + end + end + end +end