From cff703fbfe44f37d5fe2abe607cc44b72f2f05c4 Mon Sep 17 00:00:00 2001 From: Kirk van Gorkom Date: Sun, 2 Mar 2025 23:40:47 -0800 Subject: [PATCH] Use commonAncestor from offline model Closes MOB-373 Change the schema of offline predictions to include results array and seaprate commonAncestor, matching vision api responses. Configure offline image predictions to set common ancestor mode. --- src/sharedHelpers/mlModel.ts | 9 ++- .../useSuggestions/useOfflineSuggestions.ts | 55 ++++++++++++++----- .../useSuggestions/useSuggestions.js | 19 +++++-- 3 files changed, 61 insertions(+), 22 deletions(-) diff --git a/src/sharedHelpers/mlModel.ts b/src/sharedHelpers/mlModel.ts index f3f561479..81cf56469 100644 --- a/src/sharedHelpers/mlModel.ts +++ b/src/sharedHelpers/mlModel.ts @@ -3,7 +3,11 @@ import { Alert, Platform } from "react-native"; import Config from "react-native-config"; import RNFS from "react-native-fs"; import type { Location } from "vision-camera-plugin-inatvision"; -import { getPredictionsForImage, getPredictionsForLocation } from "vision-camera-plugin-inatvision"; +import { + getPredictionsForImage, + getPredictionsForLocation, + MODE +} from "vision-camera-plugin-inatvision"; const modelFiles = { // The iOS model and taxonomy files always have to be referenced in the @@ -62,7 +66,8 @@ export const predictImage = ( uri: string, location: Location ) => { geomodelPath, location: hasLocation ? location - : undefined + : undefined, + mode: MODE.COMMON_ANCESTOR } ); }; diff --git a/src/sharedHooks/useSuggestions/useOfflineSuggestions.ts b/src/sharedHooks/useSuggestions/useOfflineSuggestions.ts index c8aabed99..29d7caba6 100644 --- a/src/sharedHooks/useSuggestions/useOfflineSuggestions.ts +++ b/src/sharedHooks/useSuggestions/useOfflineSuggestions.ts @@ -30,10 +30,16 @@ const useOfflineSuggestions = ( tryOfflineSuggestions: boolean } ): { - offlineSuggestions: OfflineSuggestion[]; + offlineSuggestions: { + results: OfflineSuggestion[], + commonAncestor: OfflineSuggestion | undefined + }; } => { const realm = useRealm( ); - const [offlineSuggestions, setOfflineSuggestions] = useState( [] ); + const [offlineSuggestions, setOfflineSuggestions] = useState<{ + results: OfflineSuggestion[], + commonAncestor: OfflineSuggestion | undefined + }>( { results: [], commonAncestor: undefined } ); const [error, setError] = useState( null ); const { @@ -43,10 +49,14 @@ const useOfflineSuggestions = ( useEffect( ( ) => { const predictOffline = async ( ) => { let rawPredictions = []; + let commonAncestor; try { const location = { latitude, longitude }; const result = await predictImage( photoUri, location ); rawPredictions = result.predictions; + // Destructuring here leads to different errors from the linter. + // eslint-disable-next-line prefer-destructuring + commonAncestor = result.commonAncestor; } catch ( predictImageError ) { onFetchError( { isOnline: false } ); logger.error( "Error predicting image offline", predictImageError ); @@ -56,27 +66,44 @@ const useOfflineSuggestions = ( // but we're offline so we only need the local list from realm // and don't need to fetch taxon from the API const iconicTaxa = realm?.objects( "Taxon" ).filtered( "isIconic = true" ); - const branchIDs = rawPredictions.map( t => t.taxon_id ); + const branchIDs = [...rawPredictions.map( t => t.taxon_id ), ...( commonAncestor + ? [commonAncestor.taxon_id] + : [] )]; const iconicTaxonName = iconicTaxa?.find( t => branchIDs.indexOf( t.id ) >= 0 )?.name; + // This function handles either regular or common ancestor predictions as input objects. I'm + // not going to define an interface for them in the middle of refactoring and changing logic. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const formatPrediction = ( prediction: any ): OfflineSuggestion => ( { + combined_score: prediction.combined_score, + taxon: { + id: Number( prediction.taxon_id ), + name: prediction.name, + rank_level: prediction.rank_level, + iconic_taxon_name: iconicTaxonName + } + } ); + // using the same rank level for displaying predictions in AI Camera // this is all temporary, since we ultimately want predictions // returned similarly to how we return them on web; this is returning a // single branch like on the AI Camera 2023-12-08 const formattedPredictions = rawPredictions?.reverse( ) .filter( prediction => prediction.rank_level <= 40 ) - .map( prediction => ( { - combined_score: prediction.combined_score, - taxon: { - id: Number( prediction.taxon_id ), - name: prediction.name, - rank_level: prediction.rank_level, - iconic_taxon_name: iconicTaxonName - } - } ) ); - setOfflineSuggestions( formattedPredictions ); + .map( prediction => formatPrediction( prediction ) ); + + const commonAncestorSuggestion = commonAncestor + ? formatPrediction( commonAncestor ) + : undefined; + + const returnValue = { + results: formattedPredictions, + commonAncestor: commonAncestorSuggestion + }; + + setOfflineSuggestions( returnValue ); onFetched( { isOnline: false } ); - return formattedPredictions; + return returnValue; }; if ( photoUri && tryOfflineSuggestions ) { diff --git a/src/sharedHooks/useSuggestions/useSuggestions.js b/src/sharedHooks/useSuggestions/useSuggestions.js index 6c6a2b884..3ec0d5a47 100644 --- a/src/sharedHooks/useSuggestions/useSuggestions.js +++ b/src/sharedHooks/useSuggestions/useSuggestions.js @@ -63,27 +63,34 @@ export const useSuggestions = ( photoUri, options ) => { } ); const usingOfflineSuggestions = tryOfflineSuggestions || ( - offlineSuggestions.length > 0 + offlineSuggestions?.results?.length > 0 && ( !onlineSuggestions || onlineSuggestions?.results?.length === 0 ) ); const hasOnlineSuggestionResults = onlineSuggestions?.results?.length > 0; - const unfilteredSuggestions = hasOnlineSuggestionResults - ? onlineSuggestions.results - : offlineSuggestions; + const unfilteredSuggestions = useMemo( + ( ) => ( hasOnlineSuggestionResults + ? onlineSuggestions.results || [] + : offlineSuggestions.results || [] ), + [hasOnlineSuggestionResults, onlineSuggestions, offlineSuggestions] + ); + + const commonAncestor = hasOnlineSuggestionResults + ? onlineSuggestions?.commonAncestor + : offlineSuggestions?.commonAncestor; // since we can calculate this, there's no need to store it in state const suggestions = useMemo( ( ) => filterSuggestions( unfilteredSuggestions, usingOfflineSuggestions, - onlineSuggestions?.common_ancestor + commonAncestor ), [ unfilteredSuggestions, usingOfflineSuggestions, - onlineSuggestions?.common_ancestor + commonAncestor ] );