Pre compute compatibility scores for faster profile lookup

backend/api/src/compatible-profiles.ts

@@ -1,62 +1,29 @@
-import {groupBy, sortBy} from 'lodash'
-import {APIError, type APIHandler} from 'api/helpers/endpoint'
-import {getCompatibilityScore, hasAnsweredQuestions} from 'common/profiles/compatibility-score'
-import {getCompatibilityAnswers, getGenderCompatibleProfiles, getProfile,} from 'shared/profiles/supabase'
-import {log} from 'shared/utils'
+import {type APIHandler} from 'api/helpers/endpoint'
+import {createSupabaseDirectClient} from "shared/supabase/init";
 
 export const getCompatibleProfilesHandler: APIHandler<'compatible-profiles'> = async (props) => {
-  return getCompatibleProfiles(props.userId, false)
+  return getCompatibleProfiles(props.userId)
 }
 
 export const getCompatibleProfiles = async (
   userId: string,
-  includeProfilesWithoutPromptAnswers: boolean = true,
 ) => {
-  const profile = await getProfile(userId)
-
-  log('got profile', {
-    id: profile?.id,
-    userId: profile?.user_id,
-    username: profile?.user?.username,
-  })
-
-  if (!profile) throw new APIError(404, 'Profile not found')
-
-  let profiles = await getGenderCompatibleProfiles(profile)
-
-  const profileAnswers = await getCompatibilityAnswers([
-    userId,
-    ...profiles.map((l) => l.user_id),
-  ])
-  log('got profile answers ' + profileAnswers.length)
-
-  const answersByUserId = groupBy(profileAnswers, 'creator_id')
-  if (!includeProfilesWithoutPromptAnswers) {
-    profiles = profiles.filter((l) => hasAnsweredQuestions(answersByUserId[l.user_id]))
-    if (!hasAnsweredQuestions(answersByUserId[profile.user_id])) profiles = []
-  }
-  const profileCompatibilityScores = Object.fromEntries(
-    profiles.map(
-      (l) =>
-        [
-          l.user_id,
-          getCompatibilityScore(
-            answersByUserId[profile.user_id] ?? [],
-            answersByUserId[l.user_id] ?? []
-          ),
-        ] as const
-    )
+  const pg = createSupabaseDirectClient()
+  const scores = await pg.map(
+    `select *
+     from compatibility_scores
+     where score is not null
+       and (user_id_1 = $1 or user_id_2 = $1)`,
+    [userId],
+    (r) => [r.user_id_1 == userId ? r.user_id_2 : r.user_id_1, {score: r.score}] as const
   )
 
-  const sortedCompatibleProfiles = sortBy(
-    profiles,
-    (l) => profileCompatibilityScores[l.user_id].score
-  ).reverse()
+  const profileCompatibilityScores = Object.fromEntries(scores)
+
+  // console.log('scores', profileCompatibilityScores)
 
   return {
     status: 'success',
-    profile,
-    compatibleProfiles: sortedCompatibleProfiles,
     profileCompatibilityScores,
   }
 }

backend/api/src/delete-compatibility-answer.ts

@@ -1,6 +1,7 @@
 import {APIHandler} from 'api/helpers/endpoint'
 import {createSupabaseDirectClient} from 'shared/supabase/init'
 import {APIError} from 'common/api/utils'
+import {recomputeCompatibilityScoresForUser} from 'shared/compatibility/compute-scores'
 
 export const deleteCompatibilityAnswer: APIHandler<'delete-compatibility-answer'> = async (
   {id}, auth) => {
@@ -27,4 +28,14 @@ export const deleteCompatibilityAnswer: APIHandler<'delete-compatibility-answer'
        AND creator_id = $2`,
     [id, auth.uid]
   )
+
+  const continuation = async () => {
+    // Recompute precomputed compatibility scores for this user
+    await recomputeCompatibilityScoresForUser(auth.uid, pg)
+  }
+
+  return {
+    status: 'success',
+    continue: continuation,
+  }
 }

backend/api/src/get-profiles.ts

@@ -1,10 +1,9 @@
 import {type APIHandler} from 'api/helpers/endpoint'
 import {convertRow} from 'shared/profiles/supabase'
-import {createSupabaseDirectClient} from 'shared/supabase/init'
+import {createSupabaseDirectClient, pgp} from 'shared/supabase/init'
 import {from, join, leftJoin, limit, orderBy, renderSql, select, where,} from 'shared/supabase/sql-builder'
-import {getCompatibleProfiles} from 'api/compatible-profiles'
-import {intersection} from 'lodash'
-import {MAX_INT, MIN_BIO_LENGTH, MIN_INT} from "common/constants";
+import {MIN_BIO_LENGTH} from "common/constants";
+import {compact} from "lodash";
 
 export type profileQueryType = {
   limit?: number | undefined,
@@ -40,7 +39,7 @@ export type profileQueryType = {
   lastModificationWithin?: string | undefined,
 }
 
-const userActivityColumns = ['last_online_time']
+// const userActivityColumns = ['last_online_time']
 
 
 export const loadProfiles = async (props: profileQueryType) => {
@@ -84,86 +83,35 @@ export const loadProfiles = async (props: profileQueryType) => {
   const keywords = name ? name.split(",").map(q => q.trim()).filter(Boolean) : []
   // console.debug('keywords:', keywords)
 
-  // TODO: do this in SQL for better performance. Precompute compatibility scores:
-  // - Have a table compatibility_scores(user_id_1, user_id_2, score) that updates whenever answers from either user change.
-  // - Query this table directly with "ORDER BY score DESC LIMIT {limit}".
-  if (orderByParam === 'compatibility_score') {
-    if (!compatibleWithUserId) {
-      console.error('Incompatible with user ID')
-      throw Error('Incompatible with user ID')
-    }
-
-    const {compatibleProfiles} = await getCompatibleProfiles(compatibleWithUserId)
-    let profiles = compatibleProfiles.filter(
-      (l) =>
-        (!name || l.user.name.toLowerCase().includes(name.toLowerCase())) &&
-        (!genders || genders.includes(l.gender ?? '')) &&
-        (!education_levels || education_levels.includes(l.education_level ?? '')) &&
-        (!mbti || mbti.includes(l.mbti ?? '')) &&
-        (!pref_gender || intersection(pref_gender, l.pref_gender).length) &&
-        (!pref_age_min || (l.age ?? MAX_INT) >= pref_age_min) &&
-        (!pref_age_max || (l.age ?? MIN_INT) <= pref_age_max) &&
-        (!drinks_min || (l.drinks_per_month ?? MAX_INT) >= drinks_min) &&
-        (!drinks_max || (l.drinks_per_month ?? MIN_INT) <= drinks_max) &&
-        (!pref_relation_styles ||
-          intersection(pref_relation_styles, l.pref_relation_styles).length) &&
-        (!pref_romantic_styles ||
-          intersection(pref_romantic_styles, l.pref_romantic_styles).length) &&
-        (!diet ||
-          intersection(diet, l.diet).length) &&
-        (!political_beliefs ||
-          intersection(political_beliefs, l.political_beliefs).length) &&
-        (!relationship_status ||
-          intersection(relationship_status, l.relationship_status).length) &&
-        (!languages ||
-          intersection(languages, l.languages).length) &&
-        (!religion ||
-          intersection(religion, l.religion).length) &&
-        (!wants_kids_strength ||
-          wants_kids_strength == -1 ||
-          !l.wants_kids_strength ||
-          l.wants_kids_strength == -1 ||
-          (wants_kids_strength >= 2
-            ? l.wants_kids_strength >= wants_kids_strength
-            : l.wants_kids_strength <= wants_kids_strength)) &&
-        (has_kids == undefined ||
-          has_kids == -1 ||
-          (has_kids == 0 && !l.has_kids) ||
-          (l.has_kids && l.has_kids > 0)) &&
-        (is_smoker === undefined || l.is_smoker === is_smoker) &&
-        (!l.disabled) &&
-        (l.id.toString() != skipId) &&
-        (!geodbCityIds ||
-          (l.geodb_city_id && geodbCityIds.includes(l.geodb_city_id))) &&
-        (!filterLocation || (
-          l.city_latitude && l.city_longitude &&
-          Math.abs(l.city_latitude - lat) < radius / 69.0 &&
-          Math.abs(l.city_longitude - lon) < radius / (69.0 * Math.cos(lat * Math.PI / 180)) &&
-          Math.pow(l.city_latitude - lat, 2) + Math.pow((l.city_longitude - lon) * Math.cos(lat * Math.PI / 180), 2) < Math.pow(radius / 69.0, 2)
-        )) &&
-        (shortBio || (l.bio_length ?? 0) >= MIN_BIO_LENGTH)
-    )
-
-    const count = profiles.length
-
-    const cursor = after
-      ? profiles.findIndex((l) => l.id.toString() === after) + 1
-      : 0
-    console.debug(cursor)
-
-    if (limitParam) profiles = profiles.slice(cursor, cursor + limitParam)
-
-    return {profiles, count}
+  if (orderByParam === 'compatibility_score' && !compatibleWithUserId) {
+    console.error('Incompatible with user ID')
+    throw Error('Incompatible with user ID')
   }
 
-  const tablePrefix = userActivityColumns.includes(orderByParam) ? 'user_activity' : 'profiles'
-  const userActivityJoin = 'user_activity on user_activity.user_id = profiles.user_id'
+  const tablePrefix = orderByParam === 'compatibility_score'
+    ? 'compatibility_scores'
+    : orderByParam === 'last_online_time'
+      ? 'user_activity'
+      : 'profiles'
 
-  const tableSelection = [
+  const userActivityJoin = 'user_activity on user_activity.user_id = profiles.user_id'
+  const compatibilityScoreJoin = pgp.as.format(`compatibility_scores cs on (cs.user_id_1 = LEAST(profiles.user_id, $(compatibleWithUserId)) and cs.user_id_2 = GREATEST(profiles.user_id, $(compatibleWithUserId)))`, {compatibleWithUserId})
+
+  const _orderBy = orderByParam === 'compatibility_score' ? 'cs.score' : `${tablePrefix}.${orderByParam}`
+  const afterFilter = renderSql(
+    select(_orderBy),
+    from('profiles'),
+    orderByParam === 'last_online_time' && leftJoin(userActivityJoin),
+    orderByParam === 'compatibility_score' && compatibleWithUserId && join(compatibilityScoreJoin),
+    where('profiles.id = $(after)', {after}),
+  )
+
+  const tableSelection = compact([
     from('profiles'),
     join('users on users.id = profiles.user_id'),
-    leftJoin(userActivityJoin),
-  ]
+    orderByParam === 'last_online_time' && leftJoin(userActivityJoin),
+    orderByParam === 'compatibility_score' && compatibleWithUserId && join(compatibilityScoreJoin),
+  ])
 
   const filters = [
     where('looking_for_matches = true'),
@@ -281,29 +229,27 @@ export const loadProfiles = async (props: profileQueryType) => {
     lastModificationWithin && where(`last_modification_time >= NOW() - INTERVAL $(lastModificationWithin)`, {lastModificationWithin}),
   ]
 
+  let selectCols = 'profiles.*, name, username, users.data as user'
+  if (orderByParam === 'compatibility_score') {
+    selectCols += ', cs.score as compatibility_score'
+  } else if (orderByParam === 'last_online_time') {
+    selectCols += ', user_activity.last_online_time'
+  }
+
   const query = renderSql(
-    select('profiles.*, name, username, users.data as user, user_activity.last_online_time'),
+    select(selectCols),
     ...tableSelection,
     ...filters,
-
-    orderBy(`${tablePrefix}.${orderByParam} DESC`),
-
-    after && where(
-      `${tablePrefix}.${orderByParam} < (
-      SELECT ${tablePrefix}.${orderByParam}
-      FROM profiles
-      LEFT JOIN ${userActivityJoin}
-      WHERE profiles.id = $(after)
-    )`, {after}),
-
+    orderBy(`${_orderBy} DESC`),
+    after && where(`${_orderBy} < (${afterFilter})`),
     limitParam && limit(limitParam),
   )
 
-  // console.debug('query:', query)
+  console.debug('query:', query)
 
   const profiles = await pg.map(query, [], convertRow)
 
-  console.debug('profiles:', profiles)
+  // console.debug('profiles:', profiles)
 
   const countQuery = renderSql(
     select(`count(*) as count`),

backend/api/src/set-compatibility-answer.ts

@@ -1,6 +1,7 @@
 import {APIHandler} from './helpers/endpoint'
 import {createSupabaseDirectClient} from 'shared/supabase/init'
 import {Row} from 'common/supabase/utils'
+import {recomputeCompatibilityScoresForUser} from 'shared/compatibility/compute-scores'
 
 export const setCompatibilityAnswer: APIHandler<'set-compatibility-answer'> = async (
   {questionId, multipleChoice, prefChoices, importance, explanation},
@@ -30,5 +31,13 @@ export const setCompatibilityAnswer: APIHandler<'set-compatibility-answer'> = as
     ],
   })
 
-  return result
+  const continuation = async () => {
+    // Recompute precomputed compatibility scores for this user
+    await recomputeCompatibilityScoresForUser(auth.uid, pg)
+  }
+
+  return {
+    result: result,
+    continue: continuation,
+  }
 }