From cff703fbfe44f37d5fe2abe607cc44b72f2f05c4 Mon Sep 17 00:00:00 2001 From: Kirk van Gorkom Date: Sun, 2 Mar 2025 23:40:47 -0800 Subject: [PATCH 1/5] Use commonAncestor from offline model Closes MOB-373 Change the schema of offline predictions to include results array and seaprate commonAncestor, matching vision api responses. Configure offline image predictions to set common ancestor mode. --- src/sharedHelpers/mlModel.ts | 9 ++- .../useSuggestions/useOfflineSuggestions.ts | 55 ++++++++++++++----- .../useSuggestions/useSuggestions.js | 19 +++++-- 3 files changed, 61 insertions(+), 22 deletions(-) diff --git a/src/sharedHelpers/mlModel.ts b/src/sharedHelpers/mlModel.ts index f3f561479..81cf56469 100644 --- a/src/sharedHelpers/mlModel.ts +++ b/src/sharedHelpers/mlModel.ts @@ -3,7 +3,11 @@ import { Alert, Platform } from "react-native"; import Config from "react-native-config"; import RNFS from "react-native-fs"; import type { Location } from "vision-camera-plugin-inatvision"; -import { getPredictionsForImage, getPredictionsForLocation } from "vision-camera-plugin-inatvision"; +import { + getPredictionsForImage, + getPredictionsForLocation, + MODE +} from "vision-camera-plugin-inatvision"; const modelFiles = { // The iOS model and taxonomy files always have to be referenced in the @@ -62,7 +66,8 @@ export const predictImage = ( uri: string, location: Location ) => { geomodelPath, location: hasLocation ? location - : undefined + : undefined, + mode: MODE.COMMON_ANCESTOR } ); }; diff --git a/src/sharedHooks/useSuggestions/useOfflineSuggestions.ts b/src/sharedHooks/useSuggestions/useOfflineSuggestions.ts index c8aabed99..29d7caba6 100644 --- a/src/sharedHooks/useSuggestions/useOfflineSuggestions.ts +++ b/src/sharedHooks/useSuggestions/useOfflineSuggestions.ts @@ -30,10 +30,16 @@ const useOfflineSuggestions = ( tryOfflineSuggestions: boolean } ): { - offlineSuggestions: OfflineSuggestion[]; + offlineSuggestions: { + results: OfflineSuggestion[], + commonAncestor: OfflineSuggestion | undefined + }; } => { const realm = useRealm( ); - const [offlineSuggestions, setOfflineSuggestions] = useState( [] ); + const [offlineSuggestions, setOfflineSuggestions] = useState<{ + results: OfflineSuggestion[], + commonAncestor: OfflineSuggestion | undefined + }>( { results: [], commonAncestor: undefined } ); const [error, setError] = useState( null ); const { @@ -43,10 +49,14 @@ const useOfflineSuggestions = ( useEffect( ( ) => { const predictOffline = async ( ) => { let rawPredictions = []; + let commonAncestor; try { const location = { latitude, longitude }; const result = await predictImage( photoUri, location ); rawPredictions = result.predictions; + // Destructuring here leads to different errors from the linter. + // eslint-disable-next-line prefer-destructuring + commonAncestor = result.commonAncestor; } catch ( predictImageError ) { onFetchError( { isOnline: false } ); logger.error( "Error predicting image offline", predictImageError ); @@ -56,27 +66,44 @@ const useOfflineSuggestions = ( // but we're offline so we only need the local list from realm // and don't need to fetch taxon from the API const iconicTaxa = realm?.objects( "Taxon" ).filtered( "isIconic = true" ); - const branchIDs = rawPredictions.map( t => t.taxon_id ); + const branchIDs = [...rawPredictions.map( t => t.taxon_id ), ...( commonAncestor + ? [commonAncestor.taxon_id] + : [] )]; const iconicTaxonName = iconicTaxa?.find( t => branchIDs.indexOf( t.id ) >= 0 )?.name; + // This function handles either regular or common ancestor predictions as input objects. I'm + // not going to define an interface for them in the middle of refactoring and changing logic. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const formatPrediction = ( prediction: any ): OfflineSuggestion => ( { + combined_score: prediction.combined_score, + taxon: { + id: Number( prediction.taxon_id ), + name: prediction.name, + rank_level: prediction.rank_level, + iconic_taxon_name: iconicTaxonName + } + } ); + // using the same rank level for displaying predictions in AI Camera // this is all temporary, since we ultimately want predictions // returned similarly to how we return them on web; this is returning a // single branch like on the AI Camera 2023-12-08 const formattedPredictions = rawPredictions?.reverse( ) .filter( prediction => prediction.rank_level <= 40 ) - .map( prediction => ( { - combined_score: prediction.combined_score, - taxon: { - id: Number( prediction.taxon_id ), - name: prediction.name, - rank_level: prediction.rank_level, - iconic_taxon_name: iconicTaxonName - } - } ) ); - setOfflineSuggestions( formattedPredictions ); + .map( prediction => formatPrediction( prediction ) ); + + const commonAncestorSuggestion = commonAncestor + ? formatPrediction( commonAncestor ) + : undefined; + + const returnValue = { + results: formattedPredictions, + commonAncestor: commonAncestorSuggestion + }; + + setOfflineSuggestions( returnValue ); onFetched( { isOnline: false } ); - return formattedPredictions; + return returnValue; }; if ( photoUri && tryOfflineSuggestions ) { diff --git a/src/sharedHooks/useSuggestions/useSuggestions.js b/src/sharedHooks/useSuggestions/useSuggestions.js index 6c6a2b884..3ec0d5a47 100644 --- a/src/sharedHooks/useSuggestions/useSuggestions.js +++ b/src/sharedHooks/useSuggestions/useSuggestions.js @@ -63,27 +63,34 @@ export const useSuggestions = ( photoUri, options ) => { } ); const usingOfflineSuggestions = tryOfflineSuggestions || ( - offlineSuggestions.length > 0 + offlineSuggestions?.results?.length > 0 && ( !onlineSuggestions || onlineSuggestions?.results?.length === 0 ) ); const hasOnlineSuggestionResults = onlineSuggestions?.results?.length > 0; - const unfilteredSuggestions = hasOnlineSuggestionResults - ? onlineSuggestions.results - : offlineSuggestions; + const unfilteredSuggestions = useMemo( + ( ) => ( hasOnlineSuggestionResults + ? onlineSuggestions.results || [] + : offlineSuggestions.results || [] ), + [hasOnlineSuggestionResults, onlineSuggestions, offlineSuggestions] + ); + + const commonAncestor = hasOnlineSuggestionResults + ? onlineSuggestions?.commonAncestor + : offlineSuggestions?.commonAncestor; // since we can calculate this, there's no need to store it in state const suggestions = useMemo( ( ) => filterSuggestions( unfilteredSuggestions, usingOfflineSuggestions, - onlineSuggestions?.common_ancestor + commonAncestor ), [ unfilteredSuggestions, usingOfflineSuggestions, - onlineSuggestions?.common_ancestor + commonAncestor ] ); From 2c8d4d3e9f9136396f6259ecf8df5e7d90042ea4 Mon Sep 17 00:00:00 2001 From: Kirk van Gorkom Date: Sun, 2 Mar 2025 23:43:01 -0800 Subject: [PATCH 2/5] Remap score to combined_score for common ancestor in prediction results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Duplicative with existing change in MOB-513’s confidence calculation fix, but makes a combined score available for sorting or filtering. --- src/sharedHooks/useSuggestions/filterSuggestions.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/sharedHooks/useSuggestions/filterSuggestions.js b/src/sharedHooks/useSuggestions/filterSuggestions.js index 945106505..b969fd7b2 100644 --- a/src/sharedHooks/useSuggestions/filterSuggestions.js +++ b/src/sharedHooks/useSuggestions/filterSuggestions.js @@ -78,11 +78,16 @@ const filterSuggestions = ( suggestionsToFilter, usingOfflineSuggestions, common }; } - // online common ancestor + // online or offline common ancestor if ( commonAncestor ) { + const sortableCommonAncestor = { + ...commonAncestor, + // temp patch to let calling code sort online common ancestor with other suggestions + combined_score: commonAncestor.combined_score ?? commonAncestor.score + }; return { ...newSuggestions, - topSuggestion: commonAncestor, + topSuggestion: sortableCommonAncestor, topSuggestionType: TOP_SUGGESTION_COMMON_ANCESTOR }; } From c2e1249a39cbe0bf9b81af659e48a1dffe4d8906 Mon Sep 17 00:00:00 2001 From: Kirk van Gorkom Date: Sun, 2 Mar 2025 23:47:21 -0800 Subject: [PATCH 3/5] Remove early rank filtering of offline suggestions This is likely no longer necessary now that offline scoring is based on common ancestor rollup instead of best branch. If we want to have a rank ceiling, it can be in filterSuggestions or at the UI level, applying both to offline and vision api results. --- src/sharedHooks/useSuggestions/useOfflineSuggestions.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/sharedHooks/useSuggestions/useOfflineSuggestions.ts b/src/sharedHooks/useSuggestions/useOfflineSuggestions.ts index 29d7caba6..a24ea7d4d 100644 --- a/src/sharedHooks/useSuggestions/useOfflineSuggestions.ts +++ b/src/sharedHooks/useSuggestions/useOfflineSuggestions.ts @@ -84,12 +84,7 @@ const useOfflineSuggestions = ( } } ); - // using the same rank level for displaying predictions in AI Camera - // this is all temporary, since we ultimately want predictions - // returned similarly to how we return them on web; this is returning a - // single branch like on the AI Camera 2023-12-08 const formattedPredictions = rawPredictions?.reverse( ) - .filter( prediction => prediction.rank_level <= 40 ) .map( prediction => formatPrediction( prediction ) ); const commonAncestorSuggestion = commonAncestor From 375937bd6bef932a6421ce5c8731483fe25bb855 Mon Sep 17 00:00:00 2001 From: Kirk van Gorkom Date: Mon, 3 Mar 2025 11:02:36 -0800 Subject: [PATCH 4/5] Add common ancestor mode to vision camera plugin mock --- __mocks__/vision-camera-plugin-inatvision.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/__mocks__/vision-camera-plugin-inatvision.ts b/__mocks__/vision-camera-plugin-inatvision.ts index 55711bac3..51a90048a 100644 --- a/__mocks__/vision-camera-plugin-inatvision.ts +++ b/__mocks__/vision-camera-plugin-inatvision.ts @@ -1,4 +1,7 @@ -export const getPredictionsForImage = jest.fn( () => Promise.resolve( { predictions: [] } ) ); +export const getPredictionsForImage = jest.fn( () => Promise.resolve( { + predictions: [], + commonAncestor: undefined +} ) ); export const getPredictionsForLocation = jest.fn( () => Promise.resolve( { predictions: [] } ) ); export const removeLogListener = jest.fn( ); export const resetStoredResults = jest.fn( ); @@ -6,3 +9,8 @@ export const getCellLocation = jest.fn( location => ( { ...location, elevation: 12 } ) ); + +export const MODE = { + BEST_BRANCH: "BEST_BRANCH", + COMMON_ANCESTOR: "COMMON_ANCESTOR" +}; From 8d86e411e07974c4f93d2819b37b548970fc4a18 Mon Sep 17 00:00:00 2001 From: Kirk van Gorkom Date: Mon, 3 Mar 2025 17:47:27 -0800 Subject: [PATCH 5/5] Fix property naming between online/offline Snakes and camels. --- src/sharedHooks/useSuggestions/useSuggestions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sharedHooks/useSuggestions/useSuggestions.js b/src/sharedHooks/useSuggestions/useSuggestions.js index 3ec0d5a47..b3119d4bc 100644 --- a/src/sharedHooks/useSuggestions/useSuggestions.js +++ b/src/sharedHooks/useSuggestions/useSuggestions.js @@ -77,7 +77,7 @@ export const useSuggestions = ( photoUri, options ) => { ); const commonAncestor = hasOnlineSuggestionResults - ? onlineSuggestions?.commonAncestor + ? onlineSuggestions?.common_ancestor : offlineSuggestions?.commonAncestor; // since we can calculate this, there's no need to store it in state